can't keept up w/ the version numbers

SPARK-1535

Approved-by: Eric Harding <eric.harding@bluetradesys.com>

release_notes.txt

@@ -1,10 +1,35 @@
-1.1.0
-- Added per user self match protection, location and self match protection FIX tag settings
-- Implemented a new "Trigger Order" algo
-- Corrected tick formatting for a number of products
-- Added the hability to recreate canceled orders
-- Added leg fills to facilitate futured based PnL
-- PnL report now has PnL columns calculated based on your start of day, current day, and total position
-- Start, Day and Total PnL columns are painted with a red or green background color based on if the pnl value is negative or positive respectively
-- Corrected cash per point values, used in pnl calculations, for a number of products
-- PnL report will now remember any customizations you make to filtering and/or sorting criteria 
+1.1.22
+- Orderbook: Orders in Error status can now be recreated
+- Orderbook: Orders containing child orders will now be collapsed by default.
+- Orderbook: Expand / Collapse All button added to the right-click menu and window action pane, denoted by ▲ icon
+- Orderbook: Symbol column added to allow for filtering based on group symbol
+- Time filters now have a quick Set Last Hour hyperlink which will set the Period Begin filter to (Now - 1 hour)
+
+1.1.20
+- Update CME market data handler for price precision extension.
+  https://www.cmegroup.com/confluence/display/EPICSANDBOX/Price+Precision+Extension
+
+1.1.8
+- Enabled wild cards in risk management UI
+- Fixed spreader order updates
+
+1.1.7
+- Fixed spread leg price fill
+- Fixed HO cash per point value
+
+1.1.6
+- Fixed occasional crash when filtering fills
+- Fixed several cash per point multipliers
+- Colored time of sales row with aggressor side
+- Iceberg orders now cancel (rathern than error) when a child is canceled
+
+1.1.0
+- Added per user self match protection, location and self match protection FIX tag settings
+- Implemented a new "Trigger Order" algo
+- Corrected tick formatting for a number of products
+- Added the hability to recreate canceled orders
+- Added leg fills to facilitate futured based PnL
+- PnL report now has PnL columns calculated based on your start of day, current day, and total position
+- Start, Day and Total PnL columns are painted with a red or green background color based on if the pnl value is negative or positive respectively
+- Corrected cash per point values, used in pnl calculations, for a number of products
+- PnL report will now remember any customizations you make to filtering and/or sorting criteria 

src/client/spark/Images/Icons.cs

@@ -107,6 +107,7 @@ namespace mbts.ui
       public static ImageSource Tetris { get { return (DrawingImage)_resources["Tetris"]; } }
       public static ImageSource ViewSettings => (DrawingImage)_resources["ViewSettings"];
       public static ImageSource Wrench { get { return (DrawingImage)_resources["Wrench"]; } }
+      public static ImageSource UpArrow { get { return (DrawingImage)_resources["UpArrow"]; } }
 
       public static ImageSource Support => (DrawingImage)_resources["SupportIcon"];
    }

src/client/spark/Images/Vector/Icons.xaml

@@ -753,6 +753,19 @@ M0,0L63.999001,0 63.999001,48.792999 0,48.792999z">
     </DrawingImage.Drawing>
   </DrawingImage>
 
+  <DrawingImage x:Key="UpArrow" x:Shared="False">
+    <DrawingImage.Drawing>
+      <DrawingGroup>
+        <DrawingGroup.Children>
+          <GeometryDrawing
+            Brush="Black"
+            Geometry="M16,0L32,19.745 0,19.745z"
+          />
+        </DrawingGroup.Children>
+      </DrawingGroup>
+    </DrawingImage.Drawing>
+  </DrawingImage>
+
   <!-- Pin 
         <Path Data="F1M3,11L3,7 0,7 0,6 1,6 1,1 1,0 6,0 6,1 6,6 7,6 7,7 4,7 4,11 3,11z M2,1L2,6 4,6 4,1 2,1z" />
         Unpin 

src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs

@@ -6,11 +6,14 @@ using mbts.ui.controls;
 using mbts.ui.spark;
 using mbts.ui.utility;
 using System;
+using System.Linq;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using System.Threading.Tasks;
 using System.Windows.Input;
+using System.Windows.Media;
 using static bts.shell.TimeAndSalesWindowFactory;
+using static mbts.ui.controls.DIYGrid;
 
 namespace mbts.spark {
    class TimeAndSalesVM : SecurityGrid<TimeAndSalesVM.Row, TradeInfo> {
@@ -26,23 +29,32 @@ namespace mbts.spark {
             security_filter.load_state(filters.security);
             price_filter.load_state(filters.price);
             size_filter.load_state(filters.size);
+            aggressor_filter.load_state(filters.aggressor_sides);
             trade_key_filter.load_state(filters.trade_key);
             security_key_filter.load_state(filters.security_key);
          }
          // Columns
-         add_column(TimeAndSalesView.Columns.Security, "Security", DIYGrid.DefaultColumnWidths.Security, security_filter, DataGridFilterKnownTemplates.Regex);
-         add_column(TimeAndSalesView.Columns.Time, "Time", DIYGrid.DefaultColumnWidths.Time, time_filter, DataGridFilterKnownTemplates.TimeFilter);
-         add_column(TimeAndSalesView.Columns.Price, "Price", DIYGrid.DefaultColumnWidths.Price, price_filter, DataGridFilterKnownTemplates.DoubleMinMax);
-         add_column(TimeAndSalesView.Columns.Size, "Size", DIYGrid.DefaultColumnWidths.Size, size_filter, DataGridFilterKnownTemplates.UIntMinMax);
-         add_column(TimeAndSalesView.Columns.TradeKey, "Trade Key", DIYGrid.DefaultColumnWidths.TradeKey, trade_key_filter, DataGridFilterKnownTemplates.Regex);
-         add_column(TimeAndSalesView.Columns.SecurityKey, "Security Key", DIYGrid.DefaultColumnWidths.SecurityKey, security_key_filter, DataGridFilterKnownTemplates.Regex);
-         void add_column(TimeAndSalesView.Columns col_key, string header, double width, IDataGridColumnFilter filter, string filter_template) =>
-            columns.Add(new DIYGrid.RenderTextColumn<Row>(r => r[col_key], filter, filter_template) { Header = header, Width = width });
-         //
+         add_column(TimeAndSalesView.Columns.Security, DefaultColumnHeaders.Security, DefaultColumnWidths.Security, security_filter, DataGridFilterKnownTemplates.Regex);
+         add_column(TimeAndSalesView.Columns.Time, DefaultColumnHeaders.Time, DefaultColumnWidths.Time, time_filter, DataGridFilterKnownTemplates.TimeFilter);
+         add_column(TimeAndSalesView.Columns.Price, DefaultColumnHeaders.Price, DefaultColumnWidths.Price, price_filter, DataGridFilterKnownTemplates.DoubleMinMax, System.Windows.HorizontalAlignment.Right);
+         add_column(TimeAndSalesView.Columns.Size, DefaultColumnHeaders.Size, DefaultColumnWidths.Size, size_filter, DataGridFilterKnownTemplates.UIntMinMax, System.Windows.HorizontalAlignment.Right);
+         add_column(TimeAndSalesView.Columns.Aggressor, DefaultColumnHeaders.Aggressor, DIYGrid.DefaultColumnWidths.Aggressor, aggressor_filter, DataGridFilterKnownTemplates.EnumValues);
+         add_column(TimeAndSalesView.Columns.TradeKey, DefaultColumnHeaders.TradeKey, DIYGrid.DefaultColumnWidths.TradeKey, trade_key_filter, DataGridFilterKnownTemplates.Regex);
+         add_column(TimeAndSalesView.Columns.SecurityKey, DefaultColumnHeaders.SecurityKey, DIYGrid.DefaultColumnWidths.SecurityKey, security_key_filter, DataGridFilterKnownTemplates.Regex);
+         void add_column(TimeAndSalesView.Columns col_key, string header, double width, IDataGridColumnFilter filter, string filter_template, System.Windows.HorizontalAlignment h_alignment = System.Windows.HorizontalAlignment.Left) =>
+            columns.Add(new TimeAndSalesColumn(
+               r => r[col_key]
+               , (r) => _get_aggressor_brush(r)
+               , filter
+               , filter_template
+            ) 
+            { Header = header, Width = width, horizontal_alignment = h_alignment  }
+         );
+         
          _resources.AddRange(
             show_ladder,
             DataGridFilterHelper.hook_filters(
-               rows_view, time_filter, security_filter, price_filter, size_filter, trade_key_filter, security_key_filter
+               rows_view, time_filter, security_filter, price_filter, size_filter, aggressor_filter, trade_key_filter, security_key_filter
              )
           );
       }
@@ -50,11 +62,22 @@ namespace mbts.spark {
       public ObservableCollection<DIYGrid.ColumnBase> columns { get; } = new ObservableCollection<DIYGrid.ColumnBase>();
       public DelegateCommandParamObserver<Row> show_ladder { get; }
 
+      private Brush _get_aggressor_brush(Row r) {
+         BTSColor c;
+         Brush b;
+         if (r.aggressor_side == AggressorSide.Buy) c = BTSColors.MarketData.bid;
+         else if (r.aggressor_side == AggressorSide.Sell) c = BTSColors.MarketData.ask;
+         else c = Brushes.Transparent;
+         (b = new SolidColorBrush(c) { Opacity = 0.33 }).Freeze();
+         return b;
+      }
+
       #region Filters
       public IDataGridColumnPersistentFilter<Row, TimeFilterTuple> time_filter { get; } = new TimeFilterVM<Row>(r => r.time, nameof(Row.time));
       public IDataGridColumnPersistentFilter<Row, string> security_filter { get; }
       public IDataGridColumnPersistentFilter<Row, DoubleMinMaxState> price_filter { get; } = new DoubleMinMax<Row>(r => r.price, nameof(Row.price));
       public IDataGridColumnPersistentFilter<Row, UIntMinMaxState> size_filter { get; } = new UIntMinMax<Row>(r => r.size, nameof(Row.size));
+      public IDataGridColumnPersistentFilter<Row, AggressorSide[]> aggressor_filter { get; } = DiscreteValuesFilterVM.Create<Row, AggressorSide>(EnumHelper.get_values<AggressorSide>().ToArray(), r => r.aggressor_side, nameof(Row.aggressor_side));
       public IDataGridColumnPersistentFilter<Row, string> security_key_filter { get; } = new RegexFilterVM<Row>(r => r.security_key.ToString(), nameof(Row.security_key));
       public IDataGridColumnPersistentFilter<Row, string> trade_key_filter { get; } = new RegexFilterVM<Row>(r => r.trade_key.ToString(), nameof(Row.trade_key));
 
@@ -65,7 +88,8 @@ namespace mbts.spark {
             price = price_filter.get_state(),
             size = size_filter.get_state(),
             trade_key = trade_key_filter.get_state(),
-            security_key = security_key_filter.get_state()
+            security_key = security_key_filter.get_state(),
+            aggressor_sides = aggressor_filter.get_state()
          };
       #endregion
 
@@ -90,6 +114,7 @@ namespace mbts.spark {
             _time_text,
             _price_text,
             _size_text,
+            _aggressor_text,
             _trade_key_text,
             _security_key_text;
 
@@ -98,11 +123,13 @@ namespace mbts.spark {
             time = trade.Time.lldatetime();
             price = trade.Mantissa * Math.Pow(10, trade.Exponent);
             size = trade.Size;
+            aggressor_side = trade.AggressorSide;
             show_ladder = new DelegateCommand(() => parent._show_ladder_async(this).ignore());
             _security_text = security.ToString();
             _time_text = time.ToString("M/d h:mm:ss tt");
             _price_text = security.ContractSpec.format(price);
             _size_text = size.ToString();
+            _aggressor_text = aggressor_side.ToString();
             _trade_key_text = _trade_key.ToString();
             _security_key_text = security_key.ToString();
          }
@@ -113,6 +140,7 @@ namespace mbts.spark {
          public DateTime time { get; }
          public double price { get; }
          public uint size { get; }
+         public AggressorSide aggressor_side { get; }
          public ICommand show_ladder { get; }
 
          public string this[TimeAndSalesView.Columns col] {
@@ -122,6 +150,7 @@ namespace mbts.spark {
                   case TimeAndSalesView.Columns.Time: return _time_text;
                   case TimeAndSalesView.Columns.Price: return _price_text;
                   case TimeAndSalesView.Columns.Size: return _size_text;
+                  case TimeAndSalesView.Columns.Aggressor: return _aggressor_text;
                   case TimeAndSalesView.Columns.TradeKey: return _trade_key_text;
                   case TimeAndSalesView.Columns.SecurityKey: return _security_key_text;
                   default: return null;

src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesView.xaml.cs

@@ -7,7 +7,7 @@ namespace mbts.ui.spark {
    /// Interaction logic for TimeAndSalesView.xaml
    /// </summary>
    partial class TimeAndSalesView : UserControl {
-      public enum Columns { Time, Security, Price, Size, OrderType, TradeKey, SecurityKey }
+      public enum Columns { Time, Security, Price, Size, OrderType, TradeKey, SecurityKey, Aggressor }
 
       public TimeAndSalesView() =>
          InitializeComponent();

src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesWindow.cs

@@ -12,6 +12,7 @@ using System.Reactive.Linq;
 using System.Windows;
 using System.Windows.Media;
 using static bts.shell.TimeAndSalesWindowFactory;
+using static mbts.ui.controls.DIYGrid;
 
 namespace mbts.spark {
    class TimeAndSalesWindow : SparkWindow<StartupData>, ISlowWindow {
@@ -36,7 +37,16 @@ namespace mbts.spark {
       public override string window_type => _window_type;
       public override IEnumerable<SparkCommand> Commands => _commands;
       public override UIElement Content => _view;
-      public override double MinWidth => 444 + SPARK_MENU_COLLAPSED_WIDTH + LAUNCHER_BUTTON_BORDER_WIDTH;
+      public override double MinWidth => DefaultColumnWidths.Security
+         + DefaultColumnWidths.Time
+         + DefaultColumnWidths.Price
+         + DefaultColumnWidths.Size
+         + DefaultColumnWidths.Aggressor
+         + DefaultColumnWidths.TradeKey
+         + DefaultColumnWidths.SecurityKey
+         + SPARK_MENU_COLLAPSED_WIDTH
+         + LAUNCHER_BUTTON_BORDER_WIDTH;
+
       public override StartupData SaveState => _state = new StartupData(_state.security_criteria, _vm.value.rows_view.GetSortingCriteria(), _vm.value.get_filter_states());
 
       public override void Dispose() =>

src/client/spark/Modules/Market/TimeAndSalesWindowFactory.cs

@@ -70,6 +70,8 @@ namespace bts.shell {
          public string trade_key;
          [ProtoMember(6)]
          public string security_key;
+         [ProtoMember(7)]
+         public co2.AggressorSide[] aggressor_sides;
       }
    }
 }

src/client/spark/Modules/Trading/OrderBookWindowFactory.cs

@@ -87,6 +87,8 @@ namespace bts.shell {
          public string security_regex;
          [ProtoMember(19)]
          public api.orderman.OrderSubType[] order_sub_types;
+         [ProtoMember(20)]
+         public string[] symbols;
       }
    }
 }

src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs

@@ -19,10 +19,12 @@ namespace mbts.ui.controls {
             SecurityKey = 90,
             Side = 47,
             Size = 48,
+            Symbol = 48,
             Time = 100,
             TimeInForce = 47,
             TradeKey = 76,
-            User = 53;
+            User = 53,
+            Aggressor = 80;
       }
 
       public static class DefaultColumnHeaders {
@@ -38,9 +40,12 @@ namespace mbts.ui.controls {
             SecurityKey = "Security Key",
             Side = "Side",
             Size = "Qty",
+            Symbol = "Symbol",
             TimeInForce = "TIF",
             TradeKey = "Trade Key",
-            User = "User";
+            User = "User",
+            Time = "Time",
+            Aggressor = "Aggressor";
       }
 
       internal interface IKeyedColumn<TColKey> {

src/client/spark/view/Controls/DIYGrid/Columns/FrameworkElementColumns/TimeAndSalesColumn.cs

@@ -0,0 +1,36 @@
+using mbts.spark;
+using mbts.spark.filters;
+using mbts.ui.utility.text;
+using System;
+using System.Windows;
+using System.Windows.Media;
+
+namespace mbts.ui.controls {
+   public partial class DIYGrid {
+      internal class TimeAndSalesColumn : RenderTextColumn<TimeAndSalesVM.Row> {
+
+         readonly Func<TimeAndSalesVM.Row, string> _get_text;
+         readonly Func<TimeAndSalesVM.Row, Brush> _get_brush;
+
+         public TimeAndSalesColumn(Func<TimeAndSalesVM.Row, string> get_text, Func<TimeAndSalesVM.Row, Brush> get_brush, IDataGridColumnFilter filter = null, string filter_template = null, GlyphContext glyph_context = null) : base(get_text, filter, filter_template, glyph_context) {
+            _get_text = get_text;
+            _get_brush = get_brush;
+         }
+
+         public override void draw(TimeAndSalesVM.Row row, DrawingContext dc, ColumnGeometry position, double row_height) {
+            CellRender.draw_text(dc
+               , glyph_context
+               , font_size
+               , _get_text(row)
+               , position
+               , row_height
+               , foreground
+               , _get_brush(row)
+               , horizontal_alignment
+               , vertical_alignment
+               , padding);
+         }
+
+      }
+   }
+}

src/client/spark/view/Controls/DataGridEx/Filters/Views/TimeFilterControl.xaml

@@ -42,9 +42,12 @@
                   <ColumnDefinition/>
                 </Grid.ColumnDefinitions>
                 <TextBlock VerticalAlignment="Center">From</TextBlock>
-                <TextBlock Grid.Column="2" HorizontalAlignment="Left">
-                  <Hyperlink x:Name="ClearBeginLink" Focusable="False">(Set Anytime)</Hyperlink>
-                </TextBlock>
+                <Grid Grid.Column="2">
+                  <TextBlock HorizontalAlignment="Left">
+                    <Hyperlink x:Name="ClearBeginLink" Focusable="False">(Set Anytime)</Hyperlink>
+                    <Hyperlink x:Name="SetLastHourLink" Focusable="False">(Set Last Hour)</Hyperlink>
+                  </TextBlock>
+                </Grid>
                 <c:DateTimePicker
                   x:Name="PART_Begin_DateTimePicker"
                   Grid.Row="2"

src/client/spark/view/Controls/DataGridEx/Filters/Views/TimeFilterControl.xaml.cs

@@ -44,8 +44,10 @@ namespace mbts.ui.controls {
       public override void OnApplyTemplate() {
          var clear_begin_time = this.assert_template_child<Hyperlink>("ClearBeginLink");
          var clear_end_time = this.assert_template_child<Hyperlink>("ClearEndLink");
+         var set_last_hour = this.assert_template_child<Hyperlink>("SetLastHourLink");
          clear_begin_time.Command = new DelegateCommand(() => Begin = default);
          clear_end_time.Command = new DelegateCommand(() => End = default);
+         set_last_hour.Command = new DelegateCommand(() => Begin = TimeTextTuple.FromDateTime(DateTime.Now.AddHours(-1)));
       }
 
       class TimeFilterErrorAdorner : NotifyDataErrorAdorner<TimeFilterControl, ITimeFilterVM>, IDisposable {

src/client/spark/view/Controls/ListBoxEx/ListBoxEx.xaml.cs

@@ -190,17 +190,21 @@ namespace mbts.ui.controls
 			}
 		}
 
-		void item_Unselected(object sender, RoutedEventArgs e) {
-			if (_selection_changing) return;
-			var item = ((ListBoxItem)sender).DataContext;
-			var selected_items = Selection;
-			if (selected_items.Contains(item)) {
-				using (_selection_change()) {
-					selected_items.Remove(item);
-					_update_is_all_selected();
-				}
-			}
-		}
+      void item_Unselected(object sender, RoutedEventArgs e) {
+         if (!_selection_changing && sender is ListBoxItem container) {
+            // the [ListBoxItem] sender may be "Disconnected" https://stackoverflow.com/questions/14282894/wpf-listbox-virtualization-creates-disconnecteditems/14358688
+            var item = ItemContainerGenerator.ItemFromContainer(container);
+            if (item != DependencyProperty.UnsetValue) {
+               var selected_items = Selection;
+               if (selected_items.Contains(item)) {
+                  using (_selection_change()) {
+                     selected_items.Remove(item);
+                     _update_is_all_selected();
+                  }
+               }
+            }
+         }
+      }
 
 		void item_DataContextChanged(object sender, DependencyPropertyChangedEventArgs e) {
 			using (_selection_change()) ((ListBoxItem)sender).IsSelected = Selection.Contains(e.NewValue);
@@ -368,4 +372,4 @@ namespace mbts.ui.controls
 			}
 		}
 	}
-}
+}

src/client/spark/view/Spark/Shell/CVOrderBook/CVOrderBookColumns.cs

@@ -54,6 +54,19 @@ namespace mbts.spark {
                   EqualityComparer.Create<SecurityImmutable, uint>(s => s.Id),
                   SecurityHelper.Comparer.InstanceImmutable
                )));
+            // symbol
+            Add(symbol_column = new SimpleTextColumn<string[]>(
+               Columns.Symbol,
+               DefaultColumnHeaders.Symbol,
+               nameof(CVOrderBookRowVM.symbol),
+               r => r.symbol.value,
+               DefaultColumnWidths.Symbol,
+               DiscreteValuesFilterVM.Create<CVOrderBookRowVM, string>(
+                  mbts.co2.UserSymbol.get_all().Select(u => u.symbol).Distinct().ToList(),
+                  r => r.symbol.value,
+                  value_path(nameof(CVOrderBookRowVM.symbol))
+               ),
+               DataGridFilterKnownTemplates.DiscreteValues));
             // time_created
             Add(time_created_column = new TimeCreatedColumn(
                new TimeFilterVM<CVOrderBookRowVM>(r => r.time_created.value, value_path(nameof(OrderBookRowVM.time_created)))));
@@ -223,6 +236,7 @@ namespace mbts.spark {
                status_column.filter.load_state(fs.statuses);
                last_message_column.filter.load_state(fs.exchange_message_regex);
                last_change_column.filter.load_state(fs.last_change);
+               symbol_column.filter.load_state(fs.symbols);
             }
             if (startup_data?.sort_criteria is List<SortingCriteria> lsc) {
                foreach (var sc in lsc)
@@ -238,6 +252,7 @@ namespace mbts.spark {
          public event Action filter_changed;
 
          public SecurityColumn security_column { get; }
+         public SimpleTextColumn<string[]> symbol_column { get; }
          public TimeCreatedColumn time_created_column { get; }
          public LastChangeTimeColumn last_change_column { get; }
          public SimpleTextColumn<string[]> user_column { get; }
@@ -279,7 +294,8 @@ namespace mbts.spark {
                statuses = status_column.filter.get_state(),
                exchange_message_regex = last_message_column.filter.get_state(),
                last_change = last_change_column.filter.get_state(),
-               security_criteria = security_column.filter.get_state()
+               security_criteria = security_column.filter.get_state(),
+               symbols = symbol_column.filter.get_state()
             };
 
          public void Dispose() {

src/client/spark/view/Spark/Shell/CVOrderBook/CVOrderBookRowVM.cs

@@ -31,6 +31,7 @@ namespace mbts.spark {
       IOrderBookCellStore<uint> _security_key_store;
       IOrderBookCellStore<SecurityImmutable?> _security_store;
       IOrderBookCellStore<Venue> _exchange_store;
+      IOrderBookCellStore<string> _symbol_store;
       // Def
       IOrderBookCellStore<CustomOrderDefinition.CustomOneofCase> _order_type_store;
       IOrderBookCellStore<Side> _side_store;
@@ -59,6 +60,7 @@ namespace mbts.spark {
          _security_key_store = CellFactory.create(this, _book_vm.columns.security_key_column, nameof(security_key), r => r._order.security_key(), r => r._security_key_store);
          _parent_order_key_store = CellFactory.create(this, _book_vm.columns.parent_key_column, nameof(parent_order_key), r => r._order.ParentKey, r => r._parent_order_key_store);
          _order_key_store = CellFactory.create(this, _book_vm.columns.order_key_column, nameof(order_key), r => r._order.Key, r => r._order_key_store);
+         _symbol_store = CellFactory.create(this, _book_vm.columns.symbol_column, nameof(symbol), r => r._security.HasValue ? _security.Value.Symbol : "unknown symbol", r => r._symbol_store);
          _time_created_store = CellFactory.create(this, _book_vm.columns.time_created_column, nameof(time_created), r => r._order.created(), r => r._time_created_store, format_date);
          _user_store = CellFactory.create(this, _book_vm.columns.user_column, nameof(user), r => _order.user(), r => r._user_store);
          _account_store = CellFactory.create(this, _book_vm.columns.account_column, nameof(account), r => r._order.account(), r => r._account_store);
@@ -88,7 +90,7 @@ namespace mbts.spark {
          }
          //
          _visible = backing(nameof(is_visible), true);
-         _expanded = backing(nameof(is_expanded), true);
+         _expanded = backing(nameof(is_expanded), false);
          _depth_level = backing(nameof(depth_level), () => {
             int i = 0;
             var curr = this;
@@ -120,6 +122,7 @@ namespace mbts.spark {
       public IOrderBookCell<uint> security_key => _security_key_store.current;
       public IOrderBookCell<SecurityImmutable?> security => _security_store.current;
       public IOrderBookCell<Venue> exchange => _exchange_store.current;
+      public IOrderBookCell<string> symbol => _symbol_store.current;
       // Def
       public IOrderBookCell<CustomOrderDefinition.CustomOneofCase> order_type => _order_type_store.current;
       public IOrderBookCell<Side> side => _side_store.current;
@@ -190,6 +193,7 @@ namespace mbts.spark {
             yield return _price_store;
             yield return _avg_price_store;
             yield return _last_price_store;
+            yield return _symbol_store;
          }
       }
 
@@ -282,6 +286,7 @@ namespace mbts.spark {
             case Columns.ParentKey: return row._parent_order_key_store;
             case Columns.SecurityKey: return row._security_key_store;
             case Columns.Security: return row._security_store;
+            case Columns.Symbol: return row._symbol_store;
             case Columns.Exchange: return row._exchange_store;
             case Columns.OrderType: return row._order_type_store;
             case Columns.Side: return row._side_store;

src/client/spark/view/Spark/Shell/CVOrderBook/CVOrderBookVM.cs

@@ -38,6 +38,8 @@ namespace mbts.spark {
       SetMap<uint, uint> _parent_to_children = new SetMap<uint, uint>();
       SetMap<uint, uint> _security_orders = new SetMap<uint, uint>();
 
+      bool _expand_collapse_all_status = false;
+
       public CVOrderBookVM(IBus bus, OrdersModel model, OrderBookStartupData state) {
          columns = new CVOrderBookColumnCollection(bus, model, state, _order_securities_subject);
          foreach (ColumnBase col in columns)
@@ -69,6 +71,15 @@ namespace mbts.spark {
          SetMap<uint, uint> missing_security_orders = new SetMap<uint, uint>();
          IReadOnlyDictionary<uint, SecurityMD> security_dict = new Dictionary<uint, SecurityMD>();
 
+         expand_collapse_all_rows_command = new DelegateCommand(
+            () => {
+               _order_rows.Where(r => r.is_visible && r.passes_filter).ForEach(r =>
+                  r.is_expanded = !_expand_collapse_all_status
+               );
+               _expand_collapse_all_status = !_expand_collapse_all_status;
+            }
+         );
+
          _dispose = Disposable.Create(
             bus.subscribe(SparkTags.SecuritiesTag).OnDispatcher().Subscribe(on_securities),
             model.on_orders.OnDispatcher().Subscribe(on_orders),
@@ -232,6 +243,8 @@ namespace mbts.spark {
       public CVOrderBookRowVM current_row { get => _current_row.Value; set => _current_row.Value = value ?? _selected_rows.FirstOrDefault(); }
       public OrderRecordImmutable? current_order => current_row?.order;
       public IEnumerable<OrderRecordImmutable> selected_orders => _selected_rows.Select(r => r.order);
+
+      public System.Windows.Input.ICommand expand_collapse_all_rows_command { get; }
       void IOrderBookVM.on_sorting(DIYGridSortingEventArgs e) { }
 
       public void row_selection_changed(SelectionChangedEventArgs e) {

src/client/spark/view/Spark/Shell/OrderBook/OrderBookVM.cs

@@ -18,6 +18,7 @@ using System.Linq;
 using System.Reactive.Linq;
 using System.Windows.Controls;
 using System.Windows.Data;
+using System.Windows.Input;
 using static bts.shell.OrderBookWindowFactory;
 using static mbts.spark.OrderBookView;
 using static mbts.ui.controls.DIYGrid;
@@ -427,6 +428,9 @@ namespace mbts.spark {
       public ObservableCollection<OrderBookRowVM> selected_rows => _selected_rows;
       public IEnumerable<OrderRecordImmutable> selected_orders => _selected_rows.Select(r => r.order);
 
+      public ICommand expand_collapse_all_rows_command => throw new NotImplementedException();
+
+
       #region Filters
       public IDataGridColumnPersistentFilter<OrderBookRowVM, DoubleMinMaxState> price_filter { get; } = new DoubleMinMax<OrderBookRowVM>(r => r.price, nameof(OrderBookRowVM.price));
       public IDataGridColumnPersistentFilter<OrderBookRowVM, UIntMinMaxState> qty_filter { get; } = new UIntMinMax<OrderBookRowVM>(r => r.quantity, nameof(OrderBookRowVM.quantity));
@@ -448,7 +452,6 @@ namespace mbts.spark {
       public IDataGridColumnPersistentFilter<OrderBookRowVM, string> exchange_message_filter { get; }
       public IDataGridColumnPersistentFilter<OrderBookRowVM, string> security_filter { get; } = new RegexFilterVM<OrderBookRowVM>(r => r.security_text, nameof(OrderBookRowVM.security_text));
 
-
       public FilterStates get_filter_states() =>
          new FilterStates {
             order_key_regex = order_key_filter.get_state(),

src/client/spark/view/Spark/Shell/OrderBook/OrderBookView.xaml.cs

@@ -17,10 +17,11 @@ namespace mbts.spark {
       void on_sorting(DIYGridSortingEventArgs e);
       List<SortingCriteria> get_sorting_states();
       OrderBookWindowFactory.FilterStates get_filter_states();
+      System.Windows.Input.ICommand expand_collapse_all_rows_command { get; }
    }
 
    internal partial class OrderBookView : UserControl {
-      public enum Columns { Security, Time, LastChange, User, Account, Exchange, TimeInForce, OrderType, SubType, Side, Size, Price, Filled, AvgFilled, LastPrice, Status, LastMessage, SecurityKey, OrderKey, ParentKey }
+      public enum Columns { Security, Time, LastChange, User, Account, Exchange, TimeInForce, OrderType, SubType, Side, Size, Price, Filled, AvgFilled, LastPrice, Status, LastMessage, SecurityKey, OrderKey, ParentKey, Symbol }
       public const string
          SubTypeFilterTemplateKey = "SubTypeFilterTemplate",
          ExpandCollapseToggleStyleKey = "ExpandCollapseToggleStyle",

src/client/spark/view/Spark/Shell/OrderBook/OrderBookWindow.cs

@@ -28,7 +28,8 @@ namespace mbts.spark {
          , _cancel_sell_orders_command
          , _cancel_all_orders_command
          , _cancel_selected_orders_command
-         , _recreate_selected_orders_command;
+         , _recreate_selected_orders_command
+         , _toggle_collapse_expand_all_command;
 
 
       public OrderBookWindow(OrdersModel model, OrderBookWindowFactory factory, IBus bus, OrderBookWindowFactory.OrderBookStartupData state, ImageSource icon) : base("Order Book", bus, factory, SparkCommands.ShowOrders.icon) {
@@ -115,6 +116,10 @@ namespace mbts.spark {
             can_execute: () => _vm.selected_orders.Any(x => _passes_recreate_search_criteria(x))
          );
 
+         _toggle_collapse_expand_all_command = new SparkCommandCommand("Expand / Collapse All", 
+            Icons.UpArrow,
+            _vm.expand_collapse_all_rows_command);
+
          _commands = new SparkCommand[] {
             _show_fills_command,
             _modify_order_command,
@@ -122,13 +127,14 @@ namespace mbts.spark {
             _cancel_buy_orders_command,
             _cancel_sell_orders_command,
             _cancel_all_orders_command,
-            _recreate_selected_orders_command
+            _recreate_selected_orders_command,
+            _toggle_collapse_expand_all_command
          };
          _vm.selected_order_changed += _vm_selected_order_changed;
       }
 
       private bool _passes_recreate_search_criteria(OrderRecordImmutable record) {
-         return record.status() == OrderStatus.Canceled 
+         return (record.status() == OrderStatus.Canceled || record.status() == OrderStatus.Error)
             && !record.has_parent() 
             && record.State.Quantity > record.State.FilledQuantity;
       }

src/client/spark_core/Co2/MDConsumerHandle.cs

@@ -124,8 +124,8 @@ namespace mbts.co2
       public double session_high { get { return _cold->session_high;} }
       public double session_low { get {  return _cold->session_low; } }
       public double open_interest { get {  return _cold->open_interest; } }
-      public double session_high_bid { get {  return _cold->session_high_bid; } }
-      public double session_low_offer { get {  return _cold->session_low_offer; } }
+      // public double session_high_bid { get {  return _cold->session_high_bid; } }
+      // public double session_low_offer { get {  return _cold->session_low_offer; } }
       public double fixing_price { get {  return _cold->fixing_price; } }
 
       //

src/lib/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cpp_config.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cpp_config.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/quickbus_cfg.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp)
+set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cpp_config.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cpp_config.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/quickbus_cfg.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp)

src/lib/config/protos/MarketData.proto

@@ -145,4 +145,12 @@ message CMETradingSchedule {
    // we can elaborate later, but for now it's very simple:
    // this is "hours after midnight, in Central Time"
    optional double gen_open_hour = 1 [default = 8.5]; // 8:30 am CT
+}
+
+message SimSource {
+    oneof source {
+        string replay_url = 1 [default = "http://mdreplay.bluetradesys.local/spark/futures.coconut"];
+        string replay_file = 2 [default = "futures.coconut"];
+        bool local_relay = 3;
+    }
 }

src/lib/md/Diag/ShmDumper.cpp

@@ -28,8 +28,10 @@ double dump(TimeStamp time) {
 string dump(PriceSize ps) {
    stringstream res;
    res << "{";
-   res << "'price': " << ps.price() << ", ";
-   res << "'size': " << ps.size();
+   res << "'price': " << std::setprecision(11) << ps.price() << ", ";
+   res << "'size': " << ps.size() << " ";
+   // res << "'mant': " << ps.mantissa() << ", ";
+   // res << "'exp': " << (int)ps.exponent() << " ";
    res << "}";
    return res.str();
 }

src/lib/md/MDProducerHandle.cpp

@@ -115,7 +115,7 @@ void MDProducerHandle::set_status(CoconutSecurityStatus status) {
    }
 }
 
-void MDProducerHandle::add_trade(TimeStamp time, const PriceSize price, LLDateTime tradetime, uint32_t volume) {
+void MDProducerHandle::add_trade(TimeStamp time, const PriceSize price, LLDateTime tradetime, co2::AggressorSide agside, uint32_t volume) {
    cdata().last_trade.store(price);
    cdata().last_trade_time = tradetime;
    cdata().total_volume = volume;
@@ -127,6 +127,7 @@ void MDProducerHandle::add_trade(TimeStamp time, const PriceSize price, LLDateTi
       trade.set_mantissa(price.mantissa());
       trade.set_size(price.size());
       trade.set_time(tradetime);
+      trade.set_aggressor_side(agside);
       if (volume > 0) trade.set_total_volume(volume);
       trade.set_key(key());
       _channel.add_trade(trade);

src/lib/md/MDProducerHandle.h

@@ -56,7 +56,7 @@ public:
    void set_status(co2::CoconutSecurityStatus status); // hot
 
    // Updates trade log and last trade
-   void add_trade(TimeStamp time, const PriceSize pricesize, datetime::LLDateTime date, uint32_t total_volume=0); // cold
+   void add_trade(TimeStamp time, const PriceSize pricesize, datetime::LLDateTime date, co2::AggressorSide agside, uint32_t total_volume=0); // cold
    // Update last_trade without updating trade log
    void set_last_trade(TimeStamp time, const PriceSize ps, datetime::LLDateTime date, uint32_t total_volume);
    void add_rfq(TimeStamp time, const co2::QuoteRequestMessage& rfq); // cold

src/lib/md/protos/co2.proto

@@ -119,6 +119,12 @@ message MiscMessage {
    double session_low_offer    = 12;
    double fixing_price		    = 13;
 }
+   
+enum AggressorSide {
+    AGGRESSOR_SIDE_NONE = 0;
+    AGGRESSOR_SIDE_BUY = 1;
+    AGGRESSOR_SIDE_SELL = 2;
+}
 
 message TradeMessage {
    uint32 key = 1;
@@ -130,6 +136,7 @@ message TradeMessage {
    uint32 size = 6; 
    uint32 total_volume = 7;
    bool gap = 8;
+   AggressorSide aggressor_side = 9;
 }
 message QuoteRequestMessage {
     enum Side {

src/lib/md/relay/RelayClient.cpp

@@ -283,15 +283,38 @@ struct RelayScheduler {
 
 void relay_everything() {
    api::ServiceContext ctx;
-   auto relay = ctx.locate<MarketDataRelay>();
    std::set<UserSymbol> system_symbols;
    get_system_md_symbols(system_symbols);
    RelayProducer producer(true);
-   RelayClient client(relay, nullptr, system_symbols, producer);
-   RelayScheduler scheduler(ctx, config::get<RelayOptions>().naggle_ms(), client);
-   ctx.post([&]() { scheduler.run(); });
+   relay_selected(ctx, system_symbols, producer);
 }
 
+void relay_selected(api::ServiceContext& ctx, const std::set<UserSymbol>& symbols, IRelayWriter& writer) {
+   auto relay = ctx.locate<MarketDataRelay>();
+   RelayClient client(relay, nullptr, symbols, writer);
+   RelayScheduler scheduler(ctx, config::get<RelayOptions>().naggle_ms(), client);
+   ctx.post([&]() { scheduler.run(); });
+
+   // block forever, I guess
+   utils::Dispatcher::current().run();
+}
+void local_relay(IRelayWriter& writer) {
+   api::ServiceContext ctx;
+   auto relay = ctx.locate<MarketDataRelay>();
+   std::set<UserSymbol> symbols;
+   {
+      std::set<UserSymbol> system_symbols;
+      get_system_md_symbols(system_symbols);
+
+      // Filter out VENUE_SIM
+      for (const auto& s : system_symbols) {
+         if (s.exchange() != api::VENUE_SIM) {
+            symbols.insert(s);
+         }
+      }
+   }
+   relay_selected(ctx, symbols, writer);
+}
 
 void record_everything(std::string path, std::string filename) {
    auto options = config::get<RecordOptions>();

src/lib/md/relay/RelayClient.h

@@ -14,6 +14,9 @@ namespace bts {
 namespace pubsub {
    class PubsubHub;
 }
+namespace api {
+   class ServiceContext;
+}
 namespace md {
 
 struct RelayControl;
@@ -26,6 +29,9 @@ struct IRelayWriter {
 
 utils::Dispatcher& scan_securities_thread();
 
+void relay_selected(api::ServiceContext& ctx, const std::set<UserSymbol>& symbols, IRelayWriter& writer);
+void local_relay(IRelayWriter& writer);
+
 extern "C" {
 BTS_EXPORT void relay_everything();
 

src/lib/md/relay/RelayRecorder.cpp

@@ -157,7 +157,7 @@ void replay_recording(istream& in, IRelayWriter& writer, int32_t naggle_ms) {
    }
 }
 
-void replay_file(const std::string& filename, IRelayWriter& writer, int32_t naggle_ms, bool loop = false) {
+void replay_file(const std::string& filename, IRelayWriter& writer, int32_t naggle_ms, bool loop) {
    do {
       ifstream in;
       in.open(filename, ios::binary | ios::in);

src/lib/md/relay/RelayRecorder.h

@@ -30,6 +30,7 @@ void unprint_recording(const std::string& src, const std::string& dest);
 void trace_recording(const std::string& file, security_key key);
 void replay_recording(const std::string&, int32_t delay_ms);
 void replay_url(const std::string& url, IRelayWriter& writer, int32_t naggle_ms);
+void replay_file(const std::string& filename, IRelayWriter& writer, int32_t naggle_ms, bool loop=false);
 
 }}//namespace
 

src/server/sparkd/CMakeLists.txt

@@ -24,7 +24,7 @@ target_link_libraries( sparkd
   tbb
   double-conversion
   pugixml
-  onixs-cmemdh-all-4.5.2
+  onixs-cmemdh-all-4.8.0
   quickfix
   benchmark
   lttng-ust

src/server/sparkd/exchange_sim/ExchangeSim.cpp

@@ -104,11 +104,27 @@ int start_exchange_sim()
       acceptor.start();
       
       std::map<uint32_t,md::Security> known_securities;
+      const auto& conf = config::get<md::SimSource>(); 
       while ( true ) {
          MDPlayer player(application,known_securities);
-         md::replay_url("http://mdreplay.bluetradesys.local/spark/futures.coconut",
-                        player,
-                        config::get<md::RelayOptions>().naggle_ms());
+         switch(conf.source_case()) {
+            case md::SimSource::SOURCE_NOT_SET:
+            case md::SimSource::kReplayUrl:
+               md::replay_url(conf.replay_url(),
+                  player,
+                  config::get<md::RelayOptions>().naggle_ms());
+            break;
+            case md::SimSource::kReplayFile:
+               md::replay_file(conf.replay_file(),
+                  player,
+                  config::get<md::RelayOptions>().naggle_ms(), true);
+            break;
+            case md::SimSource::kLocalRelay: {
+               md::local_relay(player);
+            }
+            break;
+
+         }
       }
       acceptor.stop();
       return 0;

src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp

@@ -474,7 +474,8 @@ void ExchangeSimApplication::update_md(std::string const &sec_desc) {
              utils::TimeStamp::now(),
              md::PriceSize::from_double<7>(t.price.value() * price_factor,
                                            t.quantity.value()),
-             datetime::ll_now());
+             datetime::ll_now(),
+             co2::AGGRESSOR_SIDE_NONE);
    }
 }
 

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp

@@ -3,6 +3,7 @@
 #include <vector>
 #include "lib/utils/helper_macros.h"
 #include "lib/config/protos/MarketData.pb.h"
+#include "lib/md/protos/co2.pb.h"
 
 using namespace std;
 using namespace bts::co2;
@@ -10,74 +11,6 @@ using namespace bts::co2;
 namespace bts {
 namespace md {
 
-void decode_ticks(int vtt, double mintick, FixedSecurity& dest) {
-   switch(vtt) {
-      default:
-      case 0:
-         dest.low_tick = mintick;
-         dest.high_tick = mintick;
-         dest.price_break = 0;
-         break;
-      case 1:
-         dest.price_break = 500;
-         dest.low_tick = 5;
-         dest.high_tick = 10;
-         break;
-      case 2:
-         dest.price_break = 5;
-         dest.low_tick = 0.5;
-         dest.high_tick = 1;
-         break;
-      case 3:
-         dest.price_break = 10;
-         dest.low_tick = 1;
-         dest.high_tick = 2;
-         break;
-      case 4:
-         dest.price_break = 500;
-         dest.low_tick = 5;
-         dest.high_tick = 25;
-         break;
-      case 10:
-         dest.price_break = 300;
-         dest.low_tick = 5;
-         dest.high_tick = 25;
-         break;
-      case 11:
-         dest.price_break = 300;
-         dest.low_tick = 5;
-         dest.high_tick = 10;
-         break;
-      case 12:
-         dest.price_break = 5;
-         dest.low_tick = 0.25;
-         dest.high_tick = 0.5;
-         break;
-   }
-}
-
-int floorlog10(int64_t m) {
-   bts_assert(m >= 0);
-   if (m < 10) return 0;
-   if (m < 100) return 1;
-   if (m < 1000) return 2;
-   if (m < 10000) return 3;
-   if (m < 100000) return 4;
-   if (m < 1000000) return 5;
-   if (m < 10000000) return 6;
-   if (m < 100000000) return 7;
-   if (m < 1000000000) return 8;
-   if (m < 10000000000) return 9;
-   if (m < 100000000000) return 10;
-   if (m < 1000000000000) return 11;
-   if (m < 10000000000000) return 12;
-   if (m < 100000000000000) return 13;
-   if (m < 1000000000000000) return 14;
-   if (m < 10000000000000000) return 15;
-   if (m < 100000000000000000) return 16;
-   if (m < 1000000000000000000) return 17;
-   return 19;
-}
 
 MDP3Listener::MDP3Listener(
       CMESecurityIndex& index,
@@ -181,159 +114,6 @@ void MDP3Listener::onEndOfPacket(onixs::Handler&, const onixs::PacketArgs&) {
 //
 // Security discovery
 
-template <class Price>
-double double_of_price(const Price& price, int8_t display_factor) {
-   // order of multiplication is important because of floating point error
-   return (price.mantissa() * pow10(price.exponent())) * pow10(display_factor);
-}
-
-inline datetime::LLDateTime ll_of_timestamp(const onixs::Timestamp& ts) {
-   return datetime::ll_from_parts(ts.year(), (int)ts.month(), ts.day(), ts.hour(), ts.minute(), ts.second(), ts.millisecond());
-}
-
-datetime::Date date_of_timestamp(const onixs::Timestamp& ts) {
-   return datetime::Date(ts.year(), (int)ts.month(), ts.day());
-}
-
-template <class Definition>
-datetime::Date date_of_maturity(const Definition& def) {
-   for (const auto& event : def.events()) {
-      if (event.eventType() == onixs::EventType::LastEligibleTradeDate) {
-         return date_of_timestamp(event.eventTime());
-      }
-   }
-   return datetime::Date();
-}
-
-
-template <class Definition>
-void set_security_common(const Definition& def, FixedSecurity& fsec) {
-   fsec.numeric_id = def.securityId();
-   fsec.symbol(def.asset().items(), def.asset().size());
-   fsec.cfi_code(def.cfiCode().items(), def.cfiCode().size());
-   fsec.alt_symbol(def.securityGroup().items(), def.securityGroup().size());
-   fsec.security_desc(def.symbol().items(), def.symbol().size());
-   fsec.exchange = api::Venue::VENUE_GLBX;
-   // tickRule not present in futures
-   fsec.cme_market_segment_id = def.marketSegmentId();
-   fsec.display_factor = def.displayFactor().exponent() + floorlog10(def.displayFactor().mantissa());
-
-   def.mainFraction(fsec.cme_main_fraction);
-   def.subFraction(fsec.cme_sub_fraction);
-   def.priceDisplayFormat(fsec.cme_price_display_format);
-}
-
-void MDP3Listener::onDefinition(onixs::Handler&, const onixs::Security& osec, const onixs::InstrumentDefinitionFutureArgs& def) {
-   tracepoint (bts_trace,trace_order,(char*) "onDefinition.future", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   const auto& sec = def.message();
-   auto exchange_id = sec.securityId();
-   // LOG(md, Info, "onDefinition future {} {}",  
-   //    std::string(sec.asset().items(), sec.asset().size()),
-   //    std::string(sec.symbol().items(), sec.symbol().size()));
-
-   if (filter_security(exchange_id, sec) != ADD) return;
-   bts_assert(exchange_id == osec.id());
-
-   FixedFutureSecurity fsec;
-
-   set_security_common(sec, fsec);
-   fsec.price_break = 0;
-   // low/high tick are stored non-premultiplied and multiplied in conversion to api::Secutity
-   double min_tick = double_of_price(sec.minPriceIncrement(), 0);
-   fsec.low_tick = fsec.high_tick = min_tick;
-   onixs::PRICE min_tick_amount;
-   sec.minPriceIncrementAmount(min_tick_amount);
-   double min_tick_dbl = double_of_price(min_tick_amount, 0);
-   fsec.cash_per_point = min_tick_dbl / min_tick / pow10(fsec.display_factor);
-   auto expiration = date_of_maturity(sec);
-   fsec.expiration_year = expiration.year;
-   fsec.expiration_month = expiration.month;
-   fsec.expiration_day = expiration.day;
-   add_security(fsec);
-}
-void MDP3Listener::onDefinition(onixs::Handler&, const onixs::Security&, const onixs::InstrumentDefinitionOptionArgs& def) {
-   tracepoint (bts_trace,trace_order,(char*) "onDefinition.option", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   const auto& sec = def.message();
-   auto exchange_id = sec.securityId();
-   // LOG(Info, MarketData) << "onDefinition option " << exchange_id;
-   if (filter_security(exchange_id, sec) != ADD) return;
-
-   FixedOptionSecurity fsec;
-   set_security_common(sec, fsec);
-   auto expiration = date_of_maturity(sec);
-   fsec.expiration_year = expiration.year;
-   fsec.expiration_month = expiration.month;
-   fsec.expiration_day = expiration.day;
-   
-   onixs::PRICE strike_price;
-   bts_assert(sec.strikePrice(strike_price));
-   fsec.strike = double_of_price(strike_price, fsec.display_factor);
-
-   onixs::PRICE min_tick, min_tick_amount;
-   sec.minPriceIncrement(min_tick);
-   sec.minPriceIncrementAmount(min_tick_amount);
-
-   int8_t tick_rule;
-   if (sec.tickRule(tick_rule)) {
-      decode_ticks(tick_rule, double_of_price(min_tick, 0), fsec);
-   }
-   fsec.cash_per_point = double_of_price(min_tick_amount, 0) / fsec.low_tick / pow10(fsec.display_factor);
-
-   fsec.call_put = sec.putOrCall() == onixs::PutOrCall::Call ? api::PUTORCALL_CALL : api::PUTORCALL_PUT;
-
-   add_security(fsec);
-}
-
-void MDP3Listener::onDefinition(onixs::Handler&, const onixs::Security&, const onixs::InstrumentDefinitionSpreadArgs& spread_args) {
-   tracepoint (bts_trace,trace_order,(char*) "onDefinition.spread", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   const auto& sec = spread_args.message();
-   auto exchange_id = sec.securityId();
-   // LOG(md, Info, "onDefinition spread {} {}",  
-   //    std::string(sec.asset().items(), sec.asset().size()),
-   //    std::string(sec.symbol().items(), sec.symbol().size()));
-   if (filter_security(exchange_id, sec) != ADD) return;
-   
-   // Filter out any already expired spreads
-   auto expiration = date_of_maturity(sec);
-   if (expiration < datetime::today()) return;
-
-   FixedSpreadSecurityL fsec;
-   set_security_common(sec, fsec);
-   vector<uint32_t> leg_exchange_ids;
-   leg_exchange_ids.reserve(sec.legs().size());
-   
-   onixs::PRICE min_tick = sec.minPriceIncrement();
-   
-   int8_t tick_rule;
-   if (sec.tickRule(tick_rule)) {
-      decode_ticks(tick_rule, double_of_price(min_tick, 0), fsec);
-   }
-   else {
-      double mintick = double_of_price(min_tick, 0);
-      fsec.low_tick = mintick;
-      fsec.high_tick = mintick;
-      fsec.price_break = 0;
-   }
-
-   fsec.set_leg_count(sec.legs().size());
-   for(size_t x = 0; x < fsec.leg_count(); ++x) {
-      const auto& leg = sec.legs()[x];
-      onixs::PRICE price;
-      if (leg.legPrice(price)) {
-         fsec.legs[x].leg_price = double_of_price(price, fsec.display_factor);
-      }
-      onixs::DecimalQty qty;
-      if (leg.legOptionDelta(qty)) {
-         fsec.legs[x].leg_delta = double_of_price(qty, 0);
-      }
-      fsec.legs[x].leg_quantity = leg.legRatioQty();
-      fsec.legs[x].leg_side = leg.legSide() == onixs::LegSide::BuySide ? api::SIDE_BUY : api::SIDE_SELL;
-      leg_exchange_ids.push_back(leg.legSecurityId());
-   }
-   add_security(fsec);
-   _index.add_spread_legs(fsec.numeric_id, leg_exchange_ids);
-}
-
 void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::DirectBook& book) {
    if (_booktype == CMEOptions::BOOKTYPE_DIRECT) {
       onBookUpdate(sec, book);
@@ -349,48 +129,28 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con
 //
 // Trades
 
-void on_trade(SecurityData* data, const onixs::PRICE& price, int size, const onixs::Timestamp& onixs_date) {
+AggressorSide map_aggressor(onixs::AggressorSide::Enum agside) {
+   switch(agside) {
+      default:
+      case onixs::AggressorSide::NoAggressor:
+         return AGGRESSOR_SIDE_NONE;
+      case onixs::AggressorSide::Buy:
+         return AGGRESSOR_SIDE_BUY;
+      case onixs::AggressorSide::Sell:
+         return AGGRESSOR_SIDE_SELL;
+   }
+}
+
+void on_trade(SecurityData* data, const onixs::Decimal& price, int size, onixs::AggressorSide::Enum agside, const onixs::Timestamp& onixs_date) {
    auto ts = utils::TimeStamp::now();
    PriceSize ps = pricesize_of_price_size(price, data->handle.security().display_factor, size);
    auto date = ll_of_timestamp(onixs_date);
-   data->handle.add_trade(ts, ps, date, data->electronic_volume + size);
-}
-
-void MDP3Listener::onTrade(onixs::Handler&, const onixs::Security& sec, const onixs::RecoveryEntryArgs& arg) {
-   tracepoint (bts_trace,trace_order,(char*) "onTrade.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != NULL) {
-      int size = 0;
-      arg.entry().entrySize(size);
-      onixs::PRICE price;
-      arg.entry().entryPx(price);
-      on_trade(data, price, size, arg.message().transactTime());
-   }
-}
-void MDP3Listener::onTrade(onixs::Handler&, const onixs::Security& sec, const onixs::TradeSummaryArgs& arg) {
-   tracepoint (bts_trace,trace_order,(char*) "onTrade.summary", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != NULL) {
-      auto size = arg.entry().entrySize();
-      onixs::PRICE price = arg.entry().entryPx();
-      on_trade(data, price, size, arg.message().transactTime());
-   }
-}
-void MDP3Listener::onTrade(onixs::Handler&, const onixs::Security& sec, const onixs::TradeArgs& arg) {
-   tracepoint (bts_trace,trace_order,(char*) "onTrade.trade_args", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   if (arg.entry().updateAction() == onixs::UpdateAction::New) {
-      auto* data = _index.get_security(sec.id());
-      if (data != nullptr) {
-         int size = arg.entry().entrySize();
-         onixs::PRICE price = arg.entry().entryPx();
-         on_trade(data, price, size, arg.message().transactTime());
-      }
-   }
+   data->handle.add_trade(ts, ps, date, map_aggressor(agside), data->electronic_volume + size);
 }
 
 //
 // RFQ
-void MDP3Listener::onQuoteRequest(onixs::Handler&, const onixs::Security& sec, const onixs::QuoteRequestRelatedSymArgs& arg) {
+void MDP3Listener::onQuoteRequest(onixs::Handler&, const onixs::Security& sec, const onixs::QuoteRequest39RelatedSymArgs& arg) {
    tracepoint (bts_trace,trace_order,(char*) "onQuoteRequest", _latest_sequence, (uint16_t) _latest_sequence, 0);
    auto* data = _index.get_security(sec.id());
    if (data != nullptr) {
@@ -439,7 +199,7 @@ void MDP3Listener::onQuoteRequest(onixs::Handler&, const onixs::Security& sec, c
 
 //
 // Volume
-void MDP3Listener::onElectronicVolume(onixs::Handler&, const onixs::Security& sec, const onixs::VolumeArgs& args) {
+void MDP3Listener::onElectronicVolume(onixs::Handler&, const onixs::Security& sec, const onixs::Volume37Args& args) {
    tracepoint (bts_trace,trace_order,(char*) "onElectronicVolume.volume", _latest_sequence, (uint16_t) _latest_sequence, 0);
    auto* data = _index.get_security(sec.id());
    if (data != nullptr) {
@@ -449,162 +209,6 @@ void MDP3Listener::onElectronicVolume(onixs::Handler&, const onixs::Security& se
       data->electronic_volume = args.entry().entrySize();
    }
 }
-void MDP3Listener::onElectronicVolume(onixs::Handler&, const onixs::Security& sec, const onixs::RecoveryEntryArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onElectronicVolume.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data) {
-      int32_t size;
-      if (args.entry().entrySize(size)) {
-         data->electronic_volume = size;
-      }
-   }
-}
-
-//
-// Misc
-void MDP3Listener::onSettlementPrice(onixs::Handler&, const onixs::Security& sec, const onixs::DailyStatisticsArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onSettlementPrice.daily", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      onixs::PRICE price;
-      onixs::Timestamp time;
-      args.entry().entryPx(price);
-      args.entry().tradingReferenceDate(time);
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_settlement(
-         utils::TimeStamp::now(), 
-         double_of_price(price, display_factor),
-         date_of_timestamp(time));
-   }
-}
-void MDP3Listener::onSettlementPrice(onixs::Handler&, const onixs::Security& sec, const onixs::RecoveryEntryArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onSettlementPrice.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      onixs::PRICE price;
-      onixs::Timestamp time;
-      args.entry().entryPx(price);
-      args.entry().tradingReferenceDate(time);
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_settlement(
-         utils::TimeStamp::now(), 
-         double_of_price(price, display_factor),
-         date_of_timestamp(time));
-   }
-}
-void MDP3Listener::onOpenInterest(onixs::Handler&, const onixs::Security& sec, const onixs::DailyStatisticsArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onOpenInterest.daily", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      onixs::PRICE price;
-      args.entry().entryPx(price);
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_open_interest(utils::TimeStamp::now(), double_of_price(price, display_factor));
-   }
-}
-void MDP3Listener::onOpenInterest(onixs::Handler&, const onixs::Security& sec, const onixs::RecoveryEntryArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onOpenInterest.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      onixs::PRICE price;
-      args.entry().entryPx(price);
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_open_interest(utils::TimeStamp::now(), double_of_price(price, display_factor));
-   }
-}
-void MDP3Listener::onOpeningPrice(onixs::Handler&, const onixs::Security& sec, const onixs::SessionStatisticsArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onOpeningPrice.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      onixs::PRICE price = args.entry().entryPx();
-      onixs::Timestamp time = args.message().transactTime();
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_opening(
-         utils::TimeStamp::now(), 
-         double_of_price(price, display_factor),
-         date_of_timestamp(time));
-   }
-}
-void MDP3Listener::onOpeningPrice(onixs::Handler&, const onixs::Security& sec, const onixs::RecoveryEntryArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onOpeningPrice.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      onixs::PRICE price; 
-      args.entry().entryPx(price);
-      onixs::Timestamp time;
-      args.entry().tradingReferenceDate(time);
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_opening(
-         utils::TimeStamp::now(), 
-         double_of_price(price, display_factor),
-         date_of_timestamp(time));
-   }
-}
-void MDP3Listener::onHighTradePrice(onixs::Handler&, const onixs::Security& sec, const onixs::SessionStatisticsArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onHighTradePrice.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_session_high(
-         utils::TimeStamp::now(),
-         double_of_price(args.entry().entryPx(), display_factor));
-   }
-}
-void MDP3Listener::onHighTradePrice(onixs::Handler&, const onixs::Security& sec, const onixs::RecoveryEntryArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onHighTradePrice.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      auto display_factor = data->handle.security().display_factor;
-      onixs::PRICE price; 
-      args.entry().entryPx(price);
-      data->handle.update_session_high(
-         utils::TimeStamp::now(),
-         double_of_price(price, display_factor));
-   }
-}
-void MDP3Listener::onLowTradePrice(onixs::Handler&, const onixs::Security& sec, const onixs::SessionStatisticsArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onLowTradePrice.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_session_low(
-         utils::TimeStamp::now(),
-         double_of_price(args.entry().entryPx(), display_factor));
-   }
-}
-void MDP3Listener::onLowTradePrice(onixs::Handler&, const onixs::Security& sec, const onixs::RecoveryEntryArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onLowTradePrice.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      auto display_factor = data->handle.security().display_factor;
-      onixs::PRICE price; 
-      args.entry().entryPx(price);
-      data->handle.update_session_low(
-         utils::TimeStamp::now(),
-         double_of_price(price, display_factor));
-   }
-}
-void MDP3Listener::onHighestBid(onixs::Handler&, const onixs::Security& sec, const onixs::SessionStatisticsArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onHighestBid.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_session_high_bid(
-         utils::TimeStamp::now(),
-         double_of_price(args.entry().entryPx(), display_factor));
-   }
-}
-void MDP3Listener::onLowestOffer(onixs::Handler&, const onixs::Security& sec, const onixs::SessionStatisticsArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onLowestOffer.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_session_low_offer(
-         utils::TimeStamp::now(),
-         double_of_price(args.entry().entryPx(), display_factor));
-   }
-}
-
 
 //
 // Non-onixs calls

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h

@@ -5,7 +5,9 @@
 #include "CMESecurityIndex.h"
 #include "server/sparkd/tracing/bts_trace.h"
 
+#include "MDP3ListenerHelpers.h"
 #include "onixs.h"
+
 #include <tbb/concurrent_queue.h>
 #include <vector>
 #include <atomic>
@@ -14,11 +16,6 @@
 namespace bts {
 namespace md {
 
-inline PriceSize pricesize_of_price_size(const onixs::PRICE price, int8_t display_factor, Size size) {
-   PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
-   return result;
-}
-
 class MDP3Listener :
    public onixs::HandlerListener,
    public onixs::MarketDataListener,
@@ -54,9 +51,21 @@ public:
 
    //
    // Security discovery
-   void onDefinition(onixs::Handler&, const onixs::Security&, const onixs::InstrumentDefinitionFutureArgs&) override;
-   void onDefinition(onixs::Handler&, const onixs::Security&, const onixs::InstrumentDefinitionOptionArgs&) override;
-   void onDefinition(onixs::Handler&, const onixs::Security&, const onixs::InstrumentDefinitionSpreadArgs&) override;
+   template <typename InstrumentDefinitionFuture>
+   void onFuture(onixs::Handler&, const onixs::Security&, const InstrumentDefinitionFuture&);
+   template <typename InstrumentDefinition>
+   void onOption(onixs::Handler&, const onixs::Security&, const InstrumentDefinition&);
+   template <typename InstrumentDefinition>
+   void onSpread(onixs::Handler&, const onixs::Security&, const InstrumentDefinition&);
+
+   void onDefinition(onixs::Handler& h, const onixs::Security& s, const onixs::InstrumentDefinitionFuture27Args& def) override { onFuture(h, s, def); }
+   void onDefinition(onixs::Handler& h, const onixs::Security& s, const onixs::InstrumentDefinitionFuture54Args& def) override { onFuture(h, s, def); }
+
+   void onDefinition(onixs::Handler& h, const onixs::Security& s, const onixs::InstrumentDefinitionOption41Args& def) override { onOption(h, s, def); }
+   void onDefinition(onixs::Handler& h, const onixs::Security& s, const onixs::InstrumentDefinitionOption55Args& def) override { onOption(h, s, def); }
+
+   void onDefinition(onixs::Handler& h, const onixs::Security& s, const onixs::InstrumentDefinitionSpread29Args& def) override { onSpread(h, s, def); }
+   void onDefinition(onixs::Handler& h, const onixs::Security& s, const onixs::InstrumentDefinitionSpread56Args& def) override { onSpread(h, s, def); }
 
    //
    // Book
@@ -66,79 +75,74 @@ public:
    void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ConsolidatedBook&) override;
 
    template <class TBook>
-   void onBookUpdate(const onixs::Security& sec, const TBook& book) {
-      tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", _latest_sequence, (uint16_t) _latest_sequence, 0);
-
-      auto data = _index.get_security(sec.id());
-      if (data == nullptr) return;
-      bts_assert(data->id == (uint32_t)sec.id());
-      auto display_factor = data->handle.security().display_factor;
-
-      // Put the data into the cache. Flush at the end of the packet.
-      auto key = data->handle.key();
-      auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) {
-         return key == entry.handle.key();
-      });
-      bool found = find_iter != _book_updates.end();
-      if (!found) {
-         _book_updates.emplace_back(data->handle);
-      }
-      BookData& cache = found ? *find_iter : _book_updates.back();
-
-      // Overwrite all entries every time
-      size_t bids_size = book.bids().size();
-      size_t asks_size = book.offers().size();
-      if (!data->handle.has_depth()) {
-         bids_size = std::min(bids_size, size_t { 1 });
-         asks_size = std::min(asks_size, size_t { 1 });
-      }
-      for (size_t x = 0; x < bids_size; ++x) {
-         cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
-      }
-      for (size_t x = 0; x < asks_size; ++x) {
-         cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
-      }
-      // NaN out the remaining depth
-      for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
-         cache.bids[x] = PriceSize::NO_MARKET;
-      }
-      for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
-         cache.asks[x] = PriceSize::NO_MARKET;
-      }
-   }
+   void onBookUpdate(const onixs::Security& sec, const TBook& book);
 
    //
    // Trades
-   void onTrade(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
-   void onTrade(onixs::Handler&, const onixs::Security&, const onixs::TradeArgs&) override;
-   void onTrade(onixs::Handler&, const onixs::Security&, const onixs::TradeSummaryArgs&) override;
+   template <typename TradeRecovery>
+   void onTradeRecovery(const onixs::Security&, const TradeRecovery&);
+   void onTrade(onixs::Handler&, const onixs::Security& s, const onixs::Recovery52EntryArgs& arg) override { onTradeRecovery(s, arg); }
+   void onTrade(onixs::Handler&, const onixs::Security& s, const onixs::Recovery38EntryArgs& arg) override { onTradeRecovery(s, arg); }
+
+   template <typename Trade>
+   void onTradeSummary(const onixs::Security&, const Trade&);
+   void onTrade(onixs::Handler&, const onixs::Security& s, const onixs::TradeSummary48Args& arg) override { onTradeSummary(s, arg); }
+   void onTrade(onixs::Handler&, const onixs::Security& s, const onixs::TradeSummary42Args& arg) override { onTradeSummary(s, arg); }
 
    //
    // RFQ
-   void onQuoteRequest(onixs::Handler&, const onixs::Security&, const onixs::QuoteRequestRelatedSymArgs&) override;
+   void onQuoteRequest(onixs::Handler&, const onixs::Security&, const onixs::QuoteRequest39RelatedSymArgs&) override;
 
    //
    // Volume
-   void onElectronicVolume(onixs::Handler&, const onixs::Security&, const onixs::VolumeArgs&) override;
-   void onElectronicVolume(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
+   void onElectronicVolume(onixs::Handler&, const onixs::Security&, const onixs::Volume37Args&) override;
+   template <typename RecoveryEntry>
+   void onElectronicVolume(const onixs::Security&, const RecoveryEntry&);
+   void onElectronicVolume(onixs::Handler&, const onixs::Security& s, const onixs::Recovery52EntryArgs& arg) override { onElectronicVolume(s, arg); }
+   void onElectronicVolume(onixs::Handler&, const onixs::Security& s, const onixs::Recovery38EntryArgs& arg) override { onElectronicVolume(s, arg); }
 
    //
    // Misc
-   void onSettlementPrice(onixs::Handler&, const onixs::Security&, const onixs::DailyStatisticsArgs&) override;
-   void onSettlementPrice(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
-   void onOpenInterest(onixs::Handler&, const onixs::Security&, const onixs::DailyStatisticsArgs&) override;
-   void onOpenInterest(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
-   void onOpeningPrice(onixs::Handler&, const onixs::Security&, const onixs::SessionStatisticsArgs&) override;
-   void onOpeningPrice(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
-   void onHighTradePrice(onixs::Handler&, const onixs::Security&, const onixs::SessionStatisticsArgs&) override;
-   void onHighTradePrice(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
-   void onLowTradePrice(onixs::Handler&, const onixs::Security&, const onixs::SessionStatisticsArgs&) override;
-   void onLowTradePrice(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
+   template <typename StatisticsArgs>
+   void onSettlementPrice(const onixs::Security& sec, const StatisticsArgs& args);
+   void onSettlementPrice(onixs::Handler&, const onixs::Security& s, const onixs::DailyStatistics49Args& arg) override { onSettlementPrice(s, arg); }
+   void onSettlementPrice(onixs::Handler&, const onixs::Security& s, const onixs::DailyStatistics33Args& arg) override { onSettlementPrice(s, arg); }
+   void onSettlementPrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery52EntryArgs& arg) override { onSettlementPrice(s, arg); }
+   void onSettlementPrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery38EntryArgs& arg) override { onSettlementPrice(s, arg); }
 
-   void onHighestBid(onixs::Handler&, const onixs::Security&, const onixs::SessionStatisticsArgs&) override;
-   void onLowestOffer(onixs::Handler&, const onixs::Security&, const onixs::SessionStatisticsArgs&) override;
-   
+   template <typename InterestArgs>
+   void onOpenInterest(const onixs::Security& sec, const InterestArgs& args);
+   void onOpenInterest(onixs::Handler&, const onixs::Security& s, const onixs::DailyStatistics49Args& arg) override { onOpenInterest(s, arg); }
+   void onOpenInterest(onixs::Handler&, const onixs::Security& s, const onixs::DailyStatistics33Args& arg) override { onOpenInterest(s, arg); }
+   void onOpenInterest(onixs::Handler&, const onixs::Security& s, const onixs::Recovery52EntryArgs& arg) override { onOpenInterest(s, arg); }
+   void onOpenInterest(onixs::Handler&, const onixs::Security& s, const onixs::Recovery38EntryArgs& arg) override { onOpenInterest(s, arg); }
 
+   template <typename SessionArgs>
+   void onSessionOpeningPrice(const onixs::Security&, const SessionArgs&);
+   template <typename RecoveryArgs>
+   void onRecoveryOpeningPrice(const onixs::Security& sec, const RecoveryArgs& args);
+   void onOpeningPrice(onixs::Handler&, const onixs::Security& s, const onixs::SessionStatistics51Args& arg) override { onSessionOpeningPrice(s, arg); }
+   void onOpeningPrice(onixs::Handler&, const onixs::Security& s, const onixs::SessionStatistics35Args& arg) override { onSessionOpeningPrice(s, arg); }
+   void onOpeningPrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery52EntryArgs& arg) override { onRecoveryOpeningPrice(s, arg); }
+   void onOpeningPrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery38EntryArgs& arg) override { onRecoveryOpeningPrice(s, arg); }
+
+   template <typename SessionArgs>
+   void onSessionHighTradePrice(const onixs::Security&, const SessionArgs&);
+   template <typename RecoveryArgs>
+   void onRecoveryHighTradePrice(const onixs::Security&, const RecoveryArgs&);
+   void onHighTradePrice(onixs::Handler&, const onixs::Security& s, const onixs::SessionStatistics51Args& arg) override { onSessionHighTradePrice(s, arg); }
+   void onHighTradePrice(onixs::Handler&, const onixs::Security& s, const onixs::SessionStatistics35Args& arg) override { onSessionHighTradePrice(s, arg); }
+   void onHighTradePrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery52EntryArgs& arg) override { onRecoveryHighTradePrice(s, arg); }
+   void onHighTradePrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery38EntryArgs& arg) override { onRecoveryHighTradePrice(s, arg); }
+
+   template <typename SessionArgs>
+   void onSessionLowTradePrice(const onixs::Security&, const SessionArgs&);
+   template <typename RecoveryArgs>
+   void onRecoveryLowTradePrice(const onixs::Security&, const RecoveryArgs&);
+   void onLowTradePrice(onixs::Handler&, const onixs::Security& s, const onixs::SessionStatistics51Args& arg) override { onSessionLowTradePrice(s, arg); }
+   void onLowTradePrice(onixs::Handler&, const onixs::Security& s, const onixs::SessionStatistics35Args& arg) override { onSessionLowTradePrice(s, arg); }
+   void onLowTradePrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery52EntryArgs& arg) override { onRecoveryLowTradePrice(s, arg); }
+   void onLowTradePrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery38EntryArgs& arg) override { onRecoveryLowTradePrice(s, arg); }
 
    //
    // Discover securities
@@ -190,4 +194,345 @@ private:
 
 };
 
+//////////////////////////////////////////
+// Template Member Function Definitions
+//////////////////////////////////////////
+
+template <class Definition>
+void set_security_common(const Definition& def, FixedSecurity& fsec) {
+   fsec.numeric_id = def.securityId();
+   fsec.symbol(def.asset().items(), def.asset().size());
+   fsec.cfi_code(def.cfiCode().items(), def.cfiCode().size());
+   fsec.alt_symbol(def.securityGroup().items(), def.securityGroup().size());
+   fsec.security_desc(def.symbol().items(), def.symbol().size());
+   fsec.exchange = api::Venue::VENUE_GLBX;
+   // tickRule not present in futures
+   fsec.cme_market_segment_id = def.marketSegmentId();
+   fsec.display_factor = def.displayFactor().exponent() + floorlog10(def.displayFactor().mantissa());
+
+   def.mainFraction(fsec.cme_main_fraction);
+   def.subFraction(fsec.cme_sub_fraction);
+   def.priceDisplayFormat(fsec.cme_price_display_format);
+}
+
+
+template <typename InstrumentDefinition>
+void MDP3Listener::onFuture(onixs::Handler&, const onixs::Security& osec, const InstrumentDefinition& def) {
+   tracepoint (bts_trace,trace_order,(char*) "onDefinition.future", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   const auto& sec = def.message();
+   auto exchange_id = sec.securityId();
+   // LOG(md, Info, "onDefinition future {} {}",  
+   //    std::string(sec.asset().items(), sec.asset().size()),
+   //    std::string(sec.symbol().items(), sec.symbol().size()));
+
+   if (filter_security(exchange_id, sec) != ADD) return;
+   bts_assert(exchange_id == osec.id());
+
+   FixedFutureSecurity fsec;
+
+   set_security_common(sec, fsec);
+   fsec.price_break = 0;
+   // low/high tick are stored non-premultiplied and multiplied in conversion to api::Secutity
+   double min_tick = double_of_price(sec.minPriceIncrement(), 0);
+   fsec.low_tick = fsec.high_tick = min_tick;
+   onixs::Decimal min_tick_amount;
+   sec.minPriceIncrementAmount(min_tick_amount);
+   double min_tick_dbl = double_of_price(min_tick_amount, 0);
+   fsec.cash_per_point = min_tick_dbl / min_tick / pow10(fsec.display_factor);
+   auto expiration = date_of_maturity(sec);
+   fsec.expiration_year = expiration.year;
+   fsec.expiration_month = expiration.month;
+   fsec.expiration_day = expiration.day;
+   add_security(fsec);
+}
+
+template <typename InstrumentDefinition>
+void MDP3Listener::onOption(onixs::Handler&, const onixs::Security&, const InstrumentDefinition& def) {
+   tracepoint (bts_trace,trace_order,(char*) "onDefinition.option", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   const auto& sec = def.message();
+   auto exchange_id = sec.securityId();
+   // LOG(Info, MarketData) << "onDefinition option " << exchange_id;
+   if (filter_security(exchange_id, sec) != ADD) return;
+
+   FixedOptionSecurity fsec;
+   set_security_common(sec, fsec);
+   auto expiration = date_of_maturity(sec);
+   fsec.expiration_year = expiration.year;
+   fsec.expiration_month = expiration.month;
+   fsec.expiration_day = expiration.day;
+   
+   onixs::Decimal strike_price;
+   bts_assert(sec.strikePrice(strike_price));
+   fsec.strike = double_of_price(strike_price, fsec.display_factor);
+
+   onixs::Decimal min_tick, min_tick_amount;
+   sec.minPriceIncrement(min_tick);
+   sec.minPriceIncrementAmount(min_tick_amount);
+
+   int8_t tick_rule;
+   if (sec.tickRule(tick_rule)) {
+      decode_ticks(tick_rule, double_of_price(min_tick, 0), fsec);
+   }
+   fsec.cash_per_point = double_of_price(min_tick_amount, 0) / fsec.low_tick / pow10(fsec.display_factor);
+
+   fsec.call_put = sec.putOrCall() == onixs::PutOrCall::Call ? api::PUTORCALL_CALL : api::PUTORCALL_PUT;
+
+   add_security(fsec);
+}
+
+// min_tick specializations
+inline onixs::Decimal tick_rule(const onixs::InstrumentDefinitionSpread29& sec) {
+   return sec.minPriceIncrement();
+}
+inline onixs::Decimal tick_rule(const onixs::InstrumentDefinitionSpread56& sec) {
+   onixs::Decimal min_tick;
+   sec.minPriceIncrement(min_tick);
+   return min_tick;
+}
+
+template <typename InstrumentDefinition>
+void MDP3Listener::onSpread(onixs::Handler&, const onixs::Security&, const InstrumentDefinition& spread_args) {
+   tracepoint (bts_trace,trace_order,(char*) "onDefinition.spread", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   const auto& sec = spread_args.message();
+   auto exchange_id = sec.securityId();
+   // LOG(md, Info, "onDefinition spread {} {}",  
+   //    std::string(sec.asset().items(), sec.asset().size()),
+   //    std::string(sec.symbol().items(), sec.symbol().size()));
+   if (filter_security(exchange_id, sec) != ADD) return;
+   
+   // Filter out any already expired spreads
+   auto expiration = date_of_maturity(sec);
+   if (expiration < datetime::today()) return;
+
+   FixedSpreadSecurityL fsec;
+   set_security_common(sec, fsec);
+   std::vector<uint32_t> leg_exchange_ids;
+   leg_exchange_ids.reserve(sec.legs().size());
+
+   onixs::Decimal min_tick = tick_rule(sec);
+   
+   int8_t tick_rule;
+   if (sec.tickRule(tick_rule)) {
+      decode_ticks(tick_rule, double_of_price(min_tick, 0), fsec);
+   }
+   else {
+      double mintick = double_of_price(min_tick, 0);
+      fsec.low_tick = mintick;
+      fsec.high_tick = mintick;
+      fsec.price_break = 0;
+   }
+
+   fsec.set_leg_count(sec.legs().size());
+   for(size_t x = 0; x < fsec.leg_count(); ++x) {
+      const auto& leg = sec.legs()[x];
+      onixs::Decimal price;
+      if (leg.legPrice(price)) {
+         fsec.legs[x].leg_price = double_of_price(price, fsec.display_factor);
+      }
+      onixs::Decimal qty;
+      if (leg.legOptionDelta(qty)) {
+         fsec.legs[x].leg_delta = double_of_price(qty, 0);
+      }
+      fsec.legs[x].leg_quantity = leg.legRatioQty();
+      fsec.legs[x].leg_side = leg.legSide() == onixs::LegSide::BuySide ? api::SIDE_BUY : api::SIDE_SELL;
+      leg_exchange_ids.push_back(leg.legSecurityId());
+   }
+   add_security(fsec);
+   _index.add_spread_legs(fsec.numeric_id, leg_exchange_ids);
+}
+
+
+template <class TBook>
+void MDP3Listener::onBookUpdate(const onixs::Security& sec, const TBook& book) {
+   tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", _latest_sequence, (uint16_t) _latest_sequence, 0);
+
+   auto data = _index.get_security(sec.id());
+   if (data == nullptr) return;
+   bts_assert(data->id == (uint32_t)sec.id());
+   auto display_factor = data->handle.security().display_factor;
+
+   // Put the data into the cache. Flush at the end of the packet.
+   auto key = data->handle.key();
+   auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) {
+      return key == entry.handle.key();
+   });
+   bool found = find_iter != _book_updates.end();
+   if (!found) {
+      _book_updates.emplace_back(data->handle);
+   }
+   BookData& cache = found ? *find_iter : _book_updates.back();
+
+   // Overwrite all entries every time
+   size_t bids_size = book.bids().size();
+   size_t asks_size = book.offers().size();
+   if (!data->handle.has_depth()) {
+      bids_size = std::min(bids_size, size_t { 1 });
+      asks_size = std::min(asks_size, size_t { 1 });
+   }
+   for (size_t x = 0; x < bids_size; ++x) {
+      cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
+   }
+   for (size_t x = 0; x < asks_size; ++x) {
+      cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
+   }
+   // NaN out the remaining depth
+   for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
+      cache.bids[x] = PriceSize::NO_MARKET;
+   }
+   for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
+      cache.asks[x] = PriceSize::NO_MARKET;
+   }
+}
+
+void on_trade(SecurityData* data, const onixs::Decimal& price, int size, onixs::AggressorSide::Enum agside, const onixs::Timestamp& onixs_date);
+
+template <typename TradeRecovery>
+void MDP3Listener::onTradeRecovery(const onixs::Security& sec, const TradeRecovery& arg) {
+   tracepoint (bts_trace,trace_order,(char*) "onTrade.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != NULL) {
+      int size = 0;
+      arg.entry().entrySize(size);
+      onixs::Decimal price;
+      arg.entry().entryPx(price);
+      on_trade(data, price, size, onixs::AggressorSide::NoAggressor, arg.message().transactTime());
+   }
+}
+
+template <typename TradeSummary>
+void MDP3Listener::onTradeSummary(const onixs::Security& sec, const TradeSummary& arg) {
+   tracepoint (bts_trace,trace_order,(char*) "onTrade.summary", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != NULL) {
+      auto size = arg.entry().entrySize();
+      onixs::Decimal price = arg.entry().entryPx();
+      onixs::AggressorSide::Enum agside = onixs::AggressorSide::NoAggressor;
+      arg.entry().aggressorSide(agside);
+      on_trade(data, price, size, agside, arg.message().transactTime());
+   }
+}
+
+
+template <typename RecoveryEntryArgs>
+void MDP3Listener::onElectronicVolume(const onixs::Security& sec, const RecoveryEntryArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onElectronicVolume.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data) {
+      int32_t size;
+      if (args.entry().entrySize(size)) {
+         data->electronic_volume = size;
+      }
+   }
+}
+
+template <typename StatisticsArgs>
+void MDP3Listener::onSettlementPrice(const onixs::Security& sec, const StatisticsArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onSettlementPrice.daily", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != nullptr) {
+      onixs::Decimal price;
+      onixs::Timestamp time;
+      args.entry().entryPx(price);
+      args.entry().tradingReferenceDate(time);
+      auto display_factor = data->handle.security().display_factor;
+      data->handle.update_settlement(
+         utils::TimeStamp::now(), 
+         double_of_price(price, display_factor),
+         date_of_timestamp(time));
+   }
+}
+
+template <typename InterestArgs>
+void MDP3Listener::onOpenInterest(const onixs::Security& sec, const InterestArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onOpenInterest.daily", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != nullptr) {
+      onixs::Decimal price;
+      args.entry().entryPx(price);
+      auto display_factor = data->handle.security().display_factor;
+      data->handle.update_open_interest(utils::TimeStamp::now(), double_of_price(price, display_factor));
+   }
+}
+
+template <typename SessionArgs>
+void MDP3Listener::onSessionOpeningPrice(const onixs::Security& sec, const SessionArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onOpeningPrice.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != nullptr) {
+      onixs::Decimal price = args.entry().entryPx();
+      onixs::Timestamp time = args.message().transactTime();
+      auto display_factor = data->handle.security().display_factor;
+      data->handle.update_opening(
+         utils::TimeStamp::now(), 
+         double_of_price(price, display_factor),
+         date_of_timestamp(time));
+   }
+}
+template <typename RecoveryArgs>
+void MDP3Listener::onRecoveryOpeningPrice(const onixs::Security& sec, const RecoveryArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onOpeningPrice.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != nullptr) {
+      onixs::Decimal price; 
+      args.entry().entryPx(price);
+      onixs::Timestamp time;
+      args.entry().tradingReferenceDate(time);
+      auto display_factor = data->handle.security().display_factor;
+      data->handle.update_opening(
+         utils::TimeStamp::now(), 
+         double_of_price(price, display_factor),
+         date_of_timestamp(time));
+   }
+}
+
+template <typename SessionArgs>
+void MDP3Listener::onSessionHighTradePrice(const onixs::Security& sec, const SessionArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onHighTradePrice.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != nullptr) {
+      auto display_factor = data->handle.security().display_factor;
+      data->handle.update_session_high(
+         utils::TimeStamp::now(),
+         double_of_price(args.entry().entryPx(), display_factor));
+   }
+}
+template <typename RecoveryArgs>
+void MDP3Listener::onRecoveryHighTradePrice(const onixs::Security& sec, const RecoveryArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onHighTradePrice.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != nullptr) {
+      auto display_factor = data->handle.security().display_factor;
+      onixs::Decimal price; 
+      args.entry().entryPx(price);
+      data->handle.update_session_high(
+         utils::TimeStamp::now(),
+         double_of_price(price, display_factor));
+   }
+}
+template <typename SessionArgs>
+void MDP3Listener::onSessionLowTradePrice(const onixs::Security& sec, const SessionArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onLowTradePrice.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != nullptr) {
+      auto display_factor = data->handle.security().display_factor;
+      data->handle.update_session_low(
+         utils::TimeStamp::now(),
+         double_of_price(args.entry().entryPx(), display_factor));
+   }
+}
+template <typename RecoveryArgs>
+void MDP3Listener::onRecoveryLowTradePrice(const onixs::Security& sec, const RecoveryArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onLowTradePrice.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != nullptr) {
+      auto display_factor = data->handle.security().display_factor;
+      onixs::Decimal price; 
+      args.entry().entryPx(price);
+      data->handle.update_session_low(
+         utils::TimeStamp::now(),
+         double_of_price(price, display_factor));
+   }
+}
+
+
 }}//namespace

src/server/sparkd/md/platforms/mdp3_onixs/MDP3ListenerHelpers.h

@@ -0,0 +1,112 @@
+#include "lib/utils/bts_assert.h"
+#include "lib/md/coconut2.h"
+
+#include "onixs.h"
+
+#include <vector>
+#include <atomic>
+#include <array>  
+
+namespace bts {
+namespace md {
+
+inline PriceSize pricesize_of_price_size(const onixs::Decimal price, int8_t display_factor, Size size) {
+   PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
+   return result;
+}
+
+template <class Price>
+double double_of_price(const Price& price, int8_t display_factor) {
+   // order of multiplication is important because of floating point error
+   return (price.mantissa() * pow10(price.exponent())) * pow10(display_factor);
+}
+
+inline void decode_ticks(int vtt, double mintick, FixedSecurity& dest) {
+   switch(vtt) {
+      default:
+      case 0:
+         dest.low_tick = mintick;
+         dest.high_tick = mintick;
+         dest.price_break = 0;
+         break;
+      case 1:
+         dest.price_break = 500;
+         dest.low_tick = 5;
+         dest.high_tick = 10;
+         break;
+      case 2:
+         dest.price_break = 5;
+         dest.low_tick = 0.5;
+         dest.high_tick = 1;
+         break;
+      case 3:
+         dest.price_break = 10;
+         dest.low_tick = 1;
+         dest.high_tick = 2;
+         break;
+      case 4:
+         dest.price_break = 500;
+         dest.low_tick = 5;
+         dest.high_tick = 25;
+         break;
+      case 10:
+         dest.price_break = 300;
+         dest.low_tick = 5;
+         dest.high_tick = 25;
+         break;
+      case 11:
+         dest.price_break = 300;
+         dest.low_tick = 5;
+         dest.high_tick = 10;
+         break;
+      case 12:
+         dest.price_break = 5;
+         dest.low_tick = 0.25;
+         dest.high_tick = 0.5;
+         break;
+   }
+}
+
+inline int floorlog10(int64_t m) {
+   bts_assert(m >= 0);
+   if (m < 10) return 0;
+   if (m < 100) return 1;
+   if (m < 1000) return 2;
+   if (m < 10000) return 3;
+   if (m < 100000) return 4;
+   if (m < 1000000) return 5;
+   if (m < 10000000) return 6;
+   if (m < 100000000) return 7;
+   if (m < 1000000000) return 8;
+   if (m < 10000000000) return 9;
+   if (m < 100000000000) return 10;
+   if (m < 1000000000000) return 11;
+   if (m < 10000000000000) return 12;
+   if (m < 100000000000000) return 13;
+   if (m < 1000000000000000) return 14;
+   if (m < 10000000000000000) return 15;
+   if (m < 100000000000000000) return 16;
+   if (m < 1000000000000000000) return 17;
+   return 19;
+}
+
+inline datetime::LLDateTime ll_of_timestamp(const onixs::Timestamp& ts) {
+   return datetime::ll_from_parts(ts.year(), (int)ts.month(), ts.day(), ts.hour(), ts.minute(), ts.second(), ts.millisecond());
+}
+
+inline datetime::Date date_of_timestamp(const onixs::Timestamp& ts) {
+   return datetime::Date(ts.year(), (int)ts.month(), ts.day());
+}
+
+template <class Definition>
+datetime::Date date_of_maturity(const Definition& def) {
+   for (const auto& event : def.events()) {
+      if (event.eventType() == onixs::EventType::LastEligibleTradeDate) {
+         return date_of_timestamp(event.eventTime());
+      }
+   }
+   return datetime::Date();
+}
+
+
+}}//namespace

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp

@@ -299,6 +299,12 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(const string& channel_id) {
       loggerSettings.filename(log_path() + "/log_" + channel_id);
       loggerSettings.severityLevel((onixs::LogSeverity::Enum)options.onixs_log_mode());
       logger = new onixs::FileLogger(loggerSettings);
+      if (options.onixs_log_mode() == CMEOptions::Debug) {
+         // trace all the things
+         settings.debugLogging().traceBooks(true);
+         settings.debugLogging().traceFeeds(true);
+         settings.debugLogging().traceMessages(onixs::MessageTracing::FixFormat);
+      }
    }
 
 

src/server/sparkd/orderman/actors/ExecutionGateway.cpp

@@ -218,9 +218,14 @@ void ExecutionGateway::on_exe_report(
               : api::orderman::MANUAL_ORDER_INDICATOR_YES;
       str_to(msg.get<fix::OrderID>(), &data->exchange_order_id);
 
-      if (msg.has<fix::Price>())
-         data->price = securities::decode_exchange_price(
-             *sec, Price(msg.get<fix::Price>()));
+      if (msg.has<fix::Price>()) {
+         if (data->leg_security)
+            data->price = securities::decode_exchange_price(
+               *data->leg_security, Price(msg.get<fix::Price>()));
+         else
+            data->price = securities::decode_exchange_price(
+               *sec, Price(msg.get<fix::Price>()));
+      }
       else
          data->price = Price(os::nan());
 

src/server/sparkd/orderman/limiter/LimiterUtils.h

@@ -21,14 +21,14 @@ public:
          net_pos_new = net_pos + quantity.value();
       else if(side == api::Side::SIDE_SELL)
          net_pos_new = net_pos - quantity.value();
-      if (net_pos_new > long_pos_limit){
+      if (net_pos_new > long_pos_limit && side==api::Side::SIDE_BUY){
          out_res.ok = false;
          if (is_security_level)
             out_res.error_msg = fmt::format("Security Long Position (Max {} Current {})", long_pos_limit, net_pos);
          else
             out_res.error_msg = fmt::format("Product Long Position (Max {} Current {})", long_pos_limit, net_pos);
       }
-      else if (net_pos_new < short_pos_limit){
+      else if (net_pos_new < short_pos_limit && side==api::Side::SIDE_SELL){
          out_res.ok = false;
          if (is_security_level)
             out_res.error_msg = fmt::format("Security Short Position (Max {} Current {})", short_pos_limit, net_pos);

src/server/sparkd/orderman/orders/IcebergOrder.cpp

@@ -271,9 +271,17 @@ void IcebergOrder::on_child_order_update(OrderStateUpdateEvent const &ev) {
             break;
          }
 
+         case ORDER_STATUS_CANCELED:
+            LOG(OrderMan, Info,
+                "Unexpected cancel of child {} in Iceberg Order {}. Canceling "
+                "parent",
+                ev.key.value(),
+                key().value());
+            transition(ORDER_STATUS_CANCELED);
+            break;
+
          case ORDER_STATUS_DISCONNECTED:
          case ORDER_STATUS_REJECTED_BY_SPARK:
-         case ORDER_STATUS_CANCELED:
          case ORDER_STATUS_ERROR:
             me->child.clear();
             transition(ORDER_STATUS_ERROR, "Unexpected child order state");

src/server/sparkd/orderman/orders/NativeOrder.cpp

@@ -588,6 +588,12 @@ void NativeOrder::on_exe_report(const NewExecutionReportEvent &er) {
    auto &state = me->state;
    auto &native_state = get_native_state(state);
 
+   if (er.data->leg_security) {
+      LOG(OrderMan, Diag,
+          "Ignoring leg execution report for order {}", key().value());
+      return;
+   }
+
    if (native_state.last_msg_seq_num() > er.data->msg_seq_num) {
       LOG(OrderMan, Warn,
           "Ignoring already seen execution report {} in order {}",

src/web/ui/sass/main.scss

@@ -139,7 +139,7 @@ body {
 .is-main-control-bar {
   margin:2rem 3rem 0 0;
   overflow: hidden;
-  text-align: center;
+  //text-align: center;
   .button {
     margin-left: 1em;
   }

src/web/ui/src/Accounts/State.fs

@@ -24,8 +24,8 @@ let init (context:ServiceContext.ServiceContext) =
     mdRelayExpNameData = Array.empty<ExpirationNameList>
     scratch = emptyAccount
     previousScratch = emptyAccount
-    AllValue = ""
-    AllValueDisplay = ""
+    AllValue = "*"
+    AllValueDisplay = "All"
     OptionsMenuOpen = ""
     ConfirmModal = "", (Msg.SelectTab General)
     SelectedClipLimits = []

src/web/ui/src/Accounts/View.fs

@@ -69,10 +69,12 @@ let GetSymbolArray (model:Model) =
   |> Array.distinct
 
 let GetFilteredSymbolArray (model:Model) (remove:string []) =
-  Array.map<bts.api.Product, string> (fun a -> a.symbol ) model.mdRelayProductData
-  |> Array.filter (fun elm -> (not <| Array.exists (fun e -> elm = e) remove))
-  |> Array.distinct
-
+  let a = (Array.map<bts.api.Product, string> (fun a -> a.symbol ) model.mdRelayProductData
+          |> Array.filter (fun elm -> (not <| Array.exists (fun e -> elm = e) remove))
+          |> Array.distinct)
+  match a.Length with
+  | 0 -> a
+  | _ -> [ a; [|model.AllValueDisplay|] ] |> Array.concat
 
 let limitFilter (positionLimit:AccountSecurityPositionLimitUnit option) (clipLimit:AccountSecurityClipLimitUnit option) allValueDisplay (array:'a []) =
   match positionLimit, clipLimit with
@@ -193,7 +195,11 @@ let clipLimitsView model dispatch =
                                                               && (elm.security.securityType = limit.security.securityType)
                                                             )
                                                             model.scratch.securityClipLimits.limit
-    |> Array.map (fun (elm:AccountSecurityClipLimitUnit) -> elm.security.securitySymbol)
+    |> Array.map (fun (elm:AccountSecurityClipLimitUnit) ->
+              match (elm.security.securitySymbol = model.AllValue) with
+              | true -> model.AllValueDisplay
+              | false -> elm.security.securitySymbol
+            )
 
 
 
@@ -223,7 +229,7 @@ let clipLimitsView model dispatch =
             )
             TableControl.Cell (Fields.enumSelect<string>
                           ""
-                          limit.security.securitySymbol
+                          (match (limit.security.securitySymbol = model.AllValue) with | true -> model.AllValueDisplay | false -> limit.security.securitySymbol)
                           (GetFilteredSymbolArray model <| (getUsedSymbolNames limit))
                           []
                           (fun value -> ((limit, (changeSymbol limit value)) |> ClipLimits |> dispatch))
@@ -262,7 +268,7 @@ let clipLimitsView model dispatch =
             div [ ClassName "column is-size-7 has-text-grey-mid is-help-text";  ]
               [str "Clip limits can be set for one security, for all securities in a product (by selecting \"All\" for securities), or for all products within a security type (by selecting \"All\" for both symbol and security)"]
       ]
-      TableControl.createDataTable (Some headerRow) rws (Button.button "Add" "" "plus" Button.Enabled addRow)
+      TableControl.createDataTable (Some headerRow) rws (Button.button "Add" "" "plus-circle" Button.Enabled addRow)
     ]
 
 let productLimits model dispatch =
@@ -362,7 +368,11 @@ let productLimits model dispatch =
                                                                 && (elm.product.securityType = limit.product.securityType)
                                                             )
                                                             model.scratch.productPositionLimits
-    |> Array.map (fun (elm:AccountProductPositionLimitUnit) -> elm.product.securitySymbol)
+    |> Array.map (fun (elm:AccountProductPositionLimitUnit) ->
+            match (elm.product.securitySymbol = model.AllValue) with
+            | true -> model.AllValueDisplay
+            | false -> elm.product.securitySymbol
+            )
 
 
   let rws:TableControl.row list =
@@ -390,7 +400,7 @@ let productLimits model dispatch =
             )
             TableControl.Cell (Fields.enumSelect<string>
                           ""
-                          limit.product.securitySymbol
+                          (match (limit.product.securitySymbol = model.AllValue) with | true -> model.AllValueDisplay | false -> limit.product.securitySymbol)
                           (GetFilteredSymbolArray model (getUsedSymbolNames limit))
                           []
                           (fun value -> ((limit, (changeSymbol limit value)) |> ProductLimits |> dispatch))
@@ -433,7 +443,7 @@ let productLimits model dispatch =
             div [ ClassName "column is-size-7 has-text-grey-mid is-help-text" ]
               [str "This limits the net position for a product (i.e. accumulated position over all securities belonging to a product). This can be set for one product, or for all products within a security type by selecting \"All\" for symbol"]
       ]
-      TableControl.createDataTable (Some headerRow) rws (Button.button "Add" "" "plus" Button.Enabled addRow)
+      TableControl.createDataTable (Some headerRow) rws (Button.button "Add" "" "plus-circle" Button.Enabled addRow)
       br []
     ]
 
@@ -551,7 +561,7 @@ let securityLimitsView model dispatch =
             div [ ClassName "column is-size-7 has-text-grey-mid is-help-text" ]
               [str "Optionally, the net position for one or more securities can also be limited. For example, the Product Limits can be set to restrict the Max Net Long Position across all ES futures to 10,000 contracts, but Security Limits can be used to restrict the ES Jun future to 500 only"]
       ]
-      TableControl.createDataTable (Some headerRow) rws (Button.button "Add" "" "plus" Button.Enabled addRow)
+      TableControl.createDataTable (Some headerRow) rws (Button.button "Add" "" "plus-circle" Button.Enabled addRow)
     ]
 
 let LimitsView model dispatch =

tools/cpp_helper/Generate-Cpps-reqs.ps1

@@ -0,0 +1,36 @@
+$known_good = @{
+	"6A" = 1000
+	"6B" = 625
+	"6D" = 1000
+	"6E" = 1250
+	"6J" = 1250
+	"CL" = 1000
+	"EMD" = 100
+	"ES" = 50
+	"GC" = 100
+	"GE" = 2500
+	"HE" = 400
+	"HG" = 5000
+	"HO" = 42000
+	"LE" = 400
+	"M6E" = 125
+	"NG" = 100
+	"NQ" = 20
+	"QM" = 500
+	"RB" = 42000
+	"RTY" = 50
+	"SI" = 5000
+	"VX" = 1000
+	"YI" = 10
+	"YM" = 5
+	"ZB" = 1000
+	"ZC" = 50
+	"ZF" = 1000
+	"ZL" = 600
+	"ZM" = 100
+	"ZN" = 1000
+	"ZQ" = 4167
+	"ZS" = 50
+	"ZT" = 2000
+	"ZW" = 50
+}

tools/cpp_helper/Generate-Cpps.ps1

@@ -1,43 +1,9 @@
-$known_good = @{
-	"6A" = 1000
-	"6B" = 625
-	"6D" = 1000
-	"6E" = 1250
-	"6J" = 1250
-	"M6E" = 125
-	"NQ" = 20
-	"ES" = 50
-	"YM" = 5
-	"RTY" = 50
-	"VX" = 1000
-	"EMD" = 100
-	"GE" = 2500
-	"ZB" = 1000
-	"ZN" = 1000
-	"ZF" = 1000
-	"ZT" = 2000
-	"ZQ" = 4167
-	"SI" = 50
-	"YI" = 10
-	"GC" = 100
-	"HG" = 250
-	"CL" = 1000
-	"NG" = 100
-	"HO" = 420
-	"QM" = 500
-	"RB" = 420
-	"ZC" = 50
-	"ZS" = 50
-	"ZW" = 50
-	"LE" = 400
-	"HE" = 400
-	"ZM" = 100
-	"ZL" = 600
-}
+# Dot sourcing the reqs file will make 
+. .\Generate-Cpps-reqs.ps1
 
 $cme_file_url = "ftp://ftp.cmegroup.com/pub/fprf/csv/cmeg.fut.prf.csv"
 $csv_save_path = ".\cmeg.fut.prf.csv"
-$output_file = ".\cpp_out.txt"
+$output_file = ".\cpp_config.conf"
 
 # turn off console update progress while downloading
 $CurrentProgressPref = $ProgressPreference;
@@ -47,11 +13,23 @@ $ProgressPreference = $CurrentProgressPref
 
 $a = Import-Csv $csv_save_path
 $b = $a | select Sym, Mult | Group Sym | Sort Name
+
+@"
+# Generated by automated script.  DO NOT EDIT!
+# source: tools/cpp_helper/generate-cpps.ps1
+"@ | out-file -Encoding ascii $output_file 
+"CppConfig`n{`n`tvalue`n`t{" | out-file -Encoding ascii $output_file -Append
 $b | % { 
 	if ($known_good.ContainsKey($_.Group[0].Sym)) { 
 		$cpp = $known_good[$_.Group[0].Sym] 
 	} else { 
 		$cpp = $_.Group[0].Mult 
 	}
-	"cpp_entry {{ `n`tkey: `"{0}`"`n`tvalue: {1}`n}}" -f $_.Group[0].Sym, $cpp } `
-	| out-file -Encoding ascii $output_file
+	"`t`tcpp_entry {{ key: `"{0}`" value: {1} }}" -f $_.Group[0].Sym, $cpp } `
+	| out-file -Encoding ascii $output_file -Append
+"`t}`n}" | out-file -Encoding ascii $output_file -Append
+Remove-Item $csv_save_path -Force
+
+# You can now copy the output file to defaults assuming all looks good
+# Copy-Item .\cpp_config.conf ..\..\src\lib\config\defaults\ -Force
+# Remove-Item cpp-config.conf