Merge branch 'prod' of bitbucket.org:bluetradesys/spark into prod

Implemented Jake's note on spread leg last px.

Approved-by: Jake Norbut <jake.norbut@bluetradesys.com>

release_notes.txt

@@ -1,10 +1,35 @@
-1.1.0
-- Added per user self match protection, location and self match protection FIX tag settings
-- Implemented a new "Trigger Order" algo
-- Corrected tick formatting for a number of products
-- Added the hability to recreate canceled orders
-- Added leg fills to facilitate futured based PnL
-- PnL report now has PnL columns calculated based on your start of day, current day, and total position
-- Start, Day and Total PnL columns are painted with a red or green background color based on if the pnl value is negative or positive respectively
-- Corrected cash per point values, used in pnl calculations, for a number of products
-- PnL report will now remember any customizations you make to filtering and/or sorting criteria 
+1.1.22
+- Orderbook: Orders in Error status can now be recreated
+- Orderbook: Orders containing child orders will now be collapsed by default.
+- Orderbook: Expand / Collapse All button added to the right-click menu and window action pane, denoted by ▲ icon
+- Orderbook: Symbol column added to allow for filtering based on group symbol
+- Time filters now have a quick Set Last Hour hyperlink which will set the Period Begin filter to (Now - 1 hour)
+
+1.1.20
+- Update CME market data handler for price precision extension.
+  https://www.cmegroup.com/confluence/display/EPICSANDBOX/Price+Precision+Extension
+
+1.1.8
+- Enabled wild cards in risk management UI
+- Fixed spreader order updates
+
+1.1.7
+- Fixed spread leg price fill
+- Fixed HO cash per point value
+
+1.1.6
+- Fixed occasional crash when filtering fills
+- Fixed several cash per point multipliers
+- Colored time of sales row with aggressor side
+- Iceberg orders now cancel (rathern than error) when a child is canceled
+
+1.1.0
+- Added per user self match protection, location and self match protection FIX tag settings
+- Implemented a new "Trigger Order" algo
+- Corrected tick formatting for a number of products
+- Added the hability to recreate canceled orders
+- Added leg fills to facilitate futured based PnL
+- PnL report now has PnL columns calculated based on your start of day, current day, and total position
+- Start, Day and Total PnL columns are painted with a red or green background color based on if the pnl value is negative or positive respectively
+- Corrected cash per point values, used in pnl calculations, for a number of products
+- PnL report will now remember any customizations you make to filtering and/or sorting criteria 

src/client/spark/Images/Icons.cs

@@ -107,6 +107,7 @@ namespace mbts.ui
       public static ImageSource Tetris { get { return (DrawingImage)_resources["Tetris"]; } }
       public static ImageSource ViewSettings => (DrawingImage)_resources["ViewSettings"];
       public static ImageSource Wrench { get { return (DrawingImage)_resources["Wrench"]; } }
+      public static ImageSource UpArrow { get { return (DrawingImage)_resources["UpArrow"]; } }
 
       public static ImageSource Support => (DrawingImage)_resources["SupportIcon"];
    }

src/client/spark/Images/Vector/Icons.xaml

@@ -753,6 +753,19 @@ M0,0L63.999001,0 63.999001,48.792999 0,48.792999z">
     </DrawingImage.Drawing>
   </DrawingImage>
 
+  <DrawingImage x:Key="UpArrow" x:Shared="False">
+    <DrawingImage.Drawing>
+      <DrawingGroup>
+        <DrawingGroup.Children>
+          <GeometryDrawing
+            Brush="Black"
+            Geometry="M16,0L32,19.745 0,19.745z"
+          />
+        </DrawingGroup.Children>
+      </DrawingGroup>
+    </DrawingImage.Drawing>
+  </DrawingImage>
+
   <!-- Pin 
         <Path Data="F1M3,11L3,7 0,7 0,6 1,6 1,1 1,0 6,0 6,1 6,6 7,6 7,7 4,7 4,11 3,11z M2,1L2,6 4,6 4,1 2,1z" />
         Unpin 

src/client/spark/Modules/Market/MarketView/MarketViewRow.cs

@@ -78,7 +78,7 @@ namespace mbts.spark {
             _animation.AnimationsChanged += on_animations_changed;
             void on_animations_changed() => RaisePropertyChanged(nameof(animations));
             _dispose = Disposable.Create(
-               _orders_model.on_security_orders(security.Id).OnDispatcher().Subscribe(_on_orders),
+               _orders_model.on_security_root_orders(security.Id).OnDispatcher().Subscribe(_on_orders),
                _position.OnDispatcher().Subscribe(v => _animation.animate_position(v.newval > v.oldval)),
                _inside_bids.OnDispatcher().Subscribe(v => _animation.animate_working_bids_inside(v.newval)),
                _inside_asks.OnDispatcher().Subscribe(v => _animation.animate_working_asks_inside(v.newval)),
@@ -250,9 +250,6 @@ namespace mbts.spark {
             public int position => _position;
 
             public void add_order(OrderRecordImmutable o) {
-               // This is a market data form - not an orders form - so just show data for root/non-child orders.
-               if (o.has_parent())
-                  return;
                switch (o.side()) {
                   case Side.Buy:
                      _position += (int)o.filled();
@@ -274,9 +271,6 @@ namespace mbts.spark {
             }
 
             public void remove_order(OrderRecordImmutable o) {
-               // This is a market data form - not an orders form - so just show data for root/non-child orders.
-               if (o.has_parent())
-                  return;
                switch (o.side()) {
                   case Side.Buy:
                      _position -= (int)o.filled();

src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs

@@ -6,11 +6,14 @@ using mbts.ui.controls;
 using mbts.ui.spark;
 using mbts.ui.utility;
 using System;
+using System.Linq;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using System.Threading.Tasks;
 using System.Windows.Input;
+using System.Windows.Media;
 using static bts.shell.TimeAndSalesWindowFactory;
+using static mbts.ui.controls.DIYGrid;
 
 namespace mbts.spark {
    class TimeAndSalesVM : SecurityGrid<TimeAndSalesVM.Row, TradeInfo> {
@@ -26,23 +29,32 @@ namespace mbts.spark {
             security_filter.load_state(filters.security);
             price_filter.load_state(filters.price);
             size_filter.load_state(filters.size);
+            aggressor_filter.load_state(filters.aggressor_sides);
             trade_key_filter.load_state(filters.trade_key);
             security_key_filter.load_state(filters.security_key);
          }
          // Columns
-         add_column(TimeAndSalesView.Columns.Security, "Security", DIYGrid.DefaultColumnWidths.Security, security_filter, DataGridFilterKnownTemplates.Regex);
-         add_column(TimeAndSalesView.Columns.Time, "Time", DIYGrid.DefaultColumnWidths.Time, time_filter, DataGridFilterKnownTemplates.TimeFilter);
-         add_column(TimeAndSalesView.Columns.Price, "Price", DIYGrid.DefaultColumnWidths.Price, price_filter, DataGridFilterKnownTemplates.DoubleMinMax);
-         add_column(TimeAndSalesView.Columns.Size, "Size", DIYGrid.DefaultColumnWidths.Size, size_filter, DataGridFilterKnownTemplates.UIntMinMax);
-         add_column(TimeAndSalesView.Columns.TradeKey, "Trade Key", DIYGrid.DefaultColumnWidths.TradeKey, trade_key_filter, DataGridFilterKnownTemplates.Regex);
-         add_column(TimeAndSalesView.Columns.SecurityKey, "Security Key", DIYGrid.DefaultColumnWidths.SecurityKey, security_key_filter, DataGridFilterKnownTemplates.Regex);
-         void add_column(TimeAndSalesView.Columns col_key, string header, double width, IDataGridColumnFilter filter, string filter_template) =>
-            columns.Add(new DIYGrid.RenderTextColumn<Row>(r => r[col_key], filter, filter_template) { Header = header, Width = width });
-         //
+         add_column(TimeAndSalesView.Columns.Security, DefaultColumnHeaders.Security, DefaultColumnWidths.Security, security_filter, DataGridFilterKnownTemplates.Regex);
+         add_column(TimeAndSalesView.Columns.Time, DefaultColumnHeaders.Time, DefaultColumnWidths.Time, time_filter, DataGridFilterKnownTemplates.TimeFilter);
+         add_column(TimeAndSalesView.Columns.Price, DefaultColumnHeaders.Price, DefaultColumnWidths.Price, price_filter, DataGridFilterKnownTemplates.DoubleMinMax, System.Windows.HorizontalAlignment.Right);
+         add_column(TimeAndSalesView.Columns.Size, DefaultColumnHeaders.Size, DefaultColumnWidths.Size, size_filter, DataGridFilterKnownTemplates.UIntMinMax, System.Windows.HorizontalAlignment.Right);
+         add_column(TimeAndSalesView.Columns.Aggressor, DefaultColumnHeaders.Aggressor, DIYGrid.DefaultColumnWidths.Aggressor, aggressor_filter, DataGridFilterKnownTemplates.EnumValues);
+         add_column(TimeAndSalesView.Columns.TradeKey, DefaultColumnHeaders.TradeKey, DIYGrid.DefaultColumnWidths.TradeKey, trade_key_filter, DataGridFilterKnownTemplates.Regex);
+         add_column(TimeAndSalesView.Columns.SecurityKey, DefaultColumnHeaders.SecurityKey, DIYGrid.DefaultColumnWidths.SecurityKey, security_key_filter, DataGridFilterKnownTemplates.Regex);
+         void add_column(TimeAndSalesView.Columns col_key, string header, double width, IDataGridColumnFilter filter, string filter_template, System.Windows.HorizontalAlignment h_alignment = System.Windows.HorizontalAlignment.Left) =>
+            columns.Add(new TimeAndSalesColumn(
+               r => r[col_key]
+               , (r) => _get_aggressor_brush(r)
+               , filter
+               , filter_template
+            ) 
+            { Header = header, Width = width, horizontal_alignment = h_alignment  }
+         );
+         
          _resources.AddRange(
             show_ladder,
             DataGridFilterHelper.hook_filters(
-               rows_view, time_filter, security_filter, price_filter, size_filter, trade_key_filter, security_key_filter
+               rows_view, time_filter, security_filter, price_filter, size_filter, aggressor_filter, trade_key_filter, security_key_filter
              )
           );
       }
@@ -50,11 +62,22 @@ namespace mbts.spark {
       public ObservableCollection<DIYGrid.ColumnBase> columns { get; } = new ObservableCollection<DIYGrid.ColumnBase>();
       public DelegateCommandParamObserver<Row> show_ladder { get; }
 
+      private Brush _get_aggressor_brush(Row r) {
+         BTSColor c;
+         Brush b;
+         if (r.aggressor_side == AggressorSide.Buy) c = BTSColors.MarketData.bid;
+         else if (r.aggressor_side == AggressorSide.Sell) c = BTSColors.MarketData.ask;
+         else c = Brushes.Transparent;
+         (b = new SolidColorBrush(c) { Opacity = 0.33 }).Freeze();
+         return b;
+      }
+
       #region Filters
       public IDataGridColumnPersistentFilter<Row, TimeFilterTuple> time_filter { get; } = new TimeFilterVM<Row>(r => r.time, nameof(Row.time));
       public IDataGridColumnPersistentFilter<Row, string> security_filter { get; }
       public IDataGridColumnPersistentFilter<Row, DoubleMinMaxState> price_filter { get; } = new DoubleMinMax<Row>(r => r.price, nameof(Row.price));
       public IDataGridColumnPersistentFilter<Row, UIntMinMaxState> size_filter { get; } = new UIntMinMax<Row>(r => r.size, nameof(Row.size));
+      public IDataGridColumnPersistentFilter<Row, AggressorSide[]> aggressor_filter { get; } = DiscreteValuesFilterVM.Create<Row, AggressorSide>(EnumHelper.get_values<AggressorSide>().ToArray(), r => r.aggressor_side, nameof(Row.aggressor_side));
       public IDataGridColumnPersistentFilter<Row, string> security_key_filter { get; } = new RegexFilterVM<Row>(r => r.security_key.ToString(), nameof(Row.security_key));
       public IDataGridColumnPersistentFilter<Row, string> trade_key_filter { get; } = new RegexFilterVM<Row>(r => r.trade_key.ToString(), nameof(Row.trade_key));
 
@@ -65,7 +88,8 @@ namespace mbts.spark {
             price = price_filter.get_state(),
             size = size_filter.get_state(),
             trade_key = trade_key_filter.get_state(),
-            security_key = security_key_filter.get_state()
+            security_key = security_key_filter.get_state(),
+            aggressor_sides = aggressor_filter.get_state()
          };
       #endregion
 
@@ -90,6 +114,7 @@ namespace mbts.spark {
             _time_text,
             _price_text,
             _size_text,
+            _aggressor_text,
             _trade_key_text,
             _security_key_text;
 
@@ -98,11 +123,13 @@ namespace mbts.spark {
             time = trade.Time.lldatetime();
             price = trade.Mantissa * Math.Pow(10, trade.Exponent);
             size = trade.Size;
+            aggressor_side = trade.AggressorSide;
             show_ladder = new DelegateCommand(() => parent._show_ladder_async(this).ignore());
             _security_text = security.ToString();
             _time_text = time.ToString("M/d h:mm:ss tt");
             _price_text = security.ContractSpec.format(price);
             _size_text = size.ToString();
+            _aggressor_text = aggressor_side.ToString();
             _trade_key_text = _trade_key.ToString();
             _security_key_text = security_key.ToString();
          }
@@ -113,6 +140,7 @@ namespace mbts.spark {
          public DateTime time { get; }
          public double price { get; }
          public uint size { get; }
+         public AggressorSide aggressor_side { get; }
          public ICommand show_ladder { get; }
 
          public string this[TimeAndSalesView.Columns col] {
@@ -122,6 +150,7 @@ namespace mbts.spark {
                   case TimeAndSalesView.Columns.Time: return _time_text;
                   case TimeAndSalesView.Columns.Price: return _price_text;
                   case TimeAndSalesView.Columns.Size: return _size_text;
+                  case TimeAndSalesView.Columns.Aggressor: return _aggressor_text;
                   case TimeAndSalesView.Columns.TradeKey: return _trade_key_text;
                   case TimeAndSalesView.Columns.SecurityKey: return _security_key_text;
                   default: return null;

src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesView.xaml.cs

@@ -7,7 +7,7 @@ namespace mbts.ui.spark {
    /// Interaction logic for TimeAndSalesView.xaml
    /// </summary>
    partial class TimeAndSalesView : UserControl {
-      public enum Columns { Time, Security, Price, Size, OrderType, TradeKey, SecurityKey }
+      public enum Columns { Time, Security, Price, Size, OrderType, TradeKey, SecurityKey, Aggressor }
 
       public TimeAndSalesView() =>
          InitializeComponent();

src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesWindow.cs

@@ -12,6 +12,7 @@ using System.Reactive.Linq;
 using System.Windows;
 using System.Windows.Media;
 using static bts.shell.TimeAndSalesWindowFactory;
+using static mbts.ui.controls.DIYGrid;
 
 namespace mbts.spark {
    class TimeAndSalesWindow : SparkWindow<StartupData>, ISlowWindow {
@@ -36,7 +37,16 @@ namespace mbts.spark {
       public override string window_type => _window_type;
       public override IEnumerable<SparkCommand> Commands => _commands;
       public override UIElement Content => _view;
-      public override double MinWidth => 444 + SPARK_MENU_COLLAPSED_WIDTH + LAUNCHER_BUTTON_BORDER_WIDTH;
+      public override double MinWidth => DefaultColumnWidths.Security
+         + DefaultColumnWidths.Time
+         + DefaultColumnWidths.Price
+         + DefaultColumnWidths.Size
+         + DefaultColumnWidths.Aggressor
+         + DefaultColumnWidths.TradeKey
+         + DefaultColumnWidths.SecurityKey
+         + SPARK_MENU_COLLAPSED_WIDTH
+         + LAUNCHER_BUTTON_BORDER_WIDTH;
+
       public override StartupData SaveState => _state = new StartupData(_state.security_criteria, _vm.value.rows_view.GetSortingCriteria(), _vm.value.get_filter_states());
 
       public override void Dispose() =>

src/client/spark/Modules/Market/TimeAndSalesWindowFactory.cs

@@ -70,6 +70,8 @@ namespace bts.shell {
          public string trade_key;
          [ProtoMember(6)]
          public string security_key;
+         [ProtoMember(7)]
+         public co2.AggressorSide[] aggressor_sides;
       }
    }
 }

src/client/spark/Modules/Trading/OrderBookWindowFactory.cs

@@ -87,6 +87,8 @@ namespace bts.shell {
          public string security_regex;
          [ProtoMember(19)]
          public api.orderman.OrderSubType[] order_sub_types;
+         [ProtoMember(20)]
+         public string[] symbols;
       }
    }
 }

src/client/spark/Modules/Trading/PAndLWindowFactory.cs

@@ -84,8 +84,11 @@ namespace bts.shell {
          public DoubleMinMaxState total_pnl;
          [ProtoMember(12)]
          public DoubleMinMaxState pnl_price;
-         [ProtoMember(13)]
-         public Fill.Types.FillKind[] fill_kind;
+
+         // [ProtoMember(13)]
+         // ProtoMember 13 cannot be used further
+         [ProtoMember(14)]
+         public SecurityType[] security_types;
       }
 
    }

src/client/spark/Orders/OrdersModel.cs

@@ -2,6 +2,7 @@
 using bts.api.core;
 using bts.api.orderman;
 using bts.spark;
+using mbts.collections;
 using mbts.spark.utilities;
 using mbts.utils;
 using System;
@@ -173,6 +174,69 @@ namespace mbts.spark {
          .Select(t => t.state.Any(kv => kv.Value.is_active()))
          .StartWith(false);
 
+      public static IObservable<ITableUpdateTransform<OrderRecordImmutable, uint>> on_security_root_orders(this OrdersModel orders_model, uint security_key) =>
+         orders_model.on_security_orders(security_key).Scan(new RootOrderData(), (rod, table) => {
+            // ** using HashSet/List pairs, rather than a Dictionary, because TableUpdateTransform calls for IReadOnlyLists, while Dictionary.Values only implements IReadOnlyCollection.
+            // (this way, we avoid copying both added/removed Dictionary.Values to new Lists at the end, see comment at bottom of method if this is unclear) **
+            var added_root_keys = new HashSet<uint>();
+            var added_roots = new List<OrderRecordImmutable>();
+            var removed_root_keys = new HashSet<uint>();
+            var removed_roots = new List<OrderRecordImmutable>();
+            foreach (var o in table.removed) {
+               if (rod.last_update.state.ContainsKey(o.Key)) {
+                  // order was a root, needs to be removed
+                  removed_roots.Add(o);
+                  if (table.state.TryGetValue(o.Key, out OrderRecordImmutable newval)) {
+                     // order was changed, if its an actual root, or its parent isn't present, the newval needs to be added
+                     if (!o.has_parent() || !table.state.ContainsKey(o.ParentKey)) {
+                        added_roots.Add(newval);
+                        added_root_keys.Add(o.Key);
+                     }
+                  } else {
+                     // order was indeed removed, not just changed - if it has a parent, remove its corresponding child-entry
+                     if (o.has_parent()) {
+                        rod.parent_to_children.remove(o.ParentKey, o.Key);
+                     }
+                     if (rod.parent_to_children.try_lookup(o.Key, out ISet<uint> children)) {
+                        // order has left some orphans behind, any children that haven't been removed (i.e. still exist in table.state) are now roots
+                        foreach (uint ck in children) {
+                           if (table.state.TryGetValue(ck, out OrderRecordImmutable child)) {
+                              added_roots.Add(child);
+                              added_root_keys.Add(ck);
+                           }
+                        }
+                     }
+                  }
+               } else if (!table.state.ContainsKey(o.Key)) {
+                  // order was indeed removed, not just changed - if it has a parent, remove its corresponding child-entry
+                  rod.parent_to_children.remove(o.ParentKey, o.Key);
+               }
+            }
+            foreach (var o in table.added) {
+               if (added_root_keys.Contains(o.Key))
+                  continue;
+               if (rod.parent_to_children.try_lookup(o.Key, out ISet<uint> children)) {
+                  // children were previously functioning as roots, need to remove them
+                  foreach (uint ck in children) {
+                     if (rod.last_update.state.TryGetValue(ck, out OrderRecordImmutable child))
+                        removed_roots.Add(child);
+                  }
+               }
+               if (o.has_parent()) {
+                  rod.parent_to_children.add(o.ParentKey, o.Key);
+                  if (table.state.ContainsKey(o.ParentKey)) {
+                     // not a root order
+                     continue;
+                  }
+               }
+               added_roots.Add(o);
+            }
+            // ** [continue comment at top of method] this call would otherwise be:
+            // rod.last_update = rod.last_update.update(removed_dict.Values.ToList(), added_dict.Values.ToList(), o => o.Key) **
+            rod.last_update = rod.last_update.update(removed_roots, added_roots, o => o.Key);
+            return rod;
+         }).Select(r => r.last_update);
+
       public static async Task<OrderRecordImmutable?> order_from_key(this OrdersModel orders_model, uint order_key) {
          (var tu, var success) = await orders_model.on_orders.FirstAsyncTask(t => t.state.ContainsKey(order_key), TimeSpan.FromSeconds(3));
          if (success && tu.state.TryGetValue(order_key, out OrderRecordImmutable rec)) {
@@ -213,5 +277,10 @@ namespace mbts.spark {
          }
          return false;
       }
+
+      class RootOrderData {
+         public readonly SetMap<uint, uint> parent_to_children = new SetMap<uint, uint>();
+         public TableUpdateTransform<OrderRecordImmutable, uint> last_update { get; set; } = new TableUpdateTransform<OrderRecordImmutable, uint>();
+      }
    }
 }

src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs

@@ -19,10 +19,12 @@ namespace mbts.ui.controls {
             SecurityKey = 90,
             Side = 47,
             Size = 48,
+            Symbol = 48,
             Time = 100,
             TimeInForce = 47,
             TradeKey = 76,
-            User = 53;
+            User = 53,
+            Aggressor = 80;
       }
 
       public static class DefaultColumnHeaders {
@@ -38,9 +40,12 @@ namespace mbts.ui.controls {
             SecurityKey = "Security Key",
             Side = "Side",
             Size = "Qty",
+            Symbol = "Symbol",
             TimeInForce = "TIF",
             TradeKey = "Trade Key",
-            User = "User";
+            User = "User",
+            Time = "Time",
+            Aggressor = "Aggressor";
       }
 
       internal interface IKeyedColumn<TColKey> {

src/client/spark/view/Controls/DIYGrid/Columns/FrameworkElementColumns/TimeAndSalesColumn.cs

@@ -0,0 +1,36 @@
+using mbts.spark;
+using mbts.spark.filters;
+using mbts.ui.utility.text;
+using System;
+using System.Windows;
+using System.Windows.Media;
+
+namespace mbts.ui.controls {
+   public partial class DIYGrid {
+      internal class TimeAndSalesColumn : RenderTextColumn<TimeAndSalesVM.Row> {
+
+         readonly Func<TimeAndSalesVM.Row, string> _get_text;
+         readonly Func<TimeAndSalesVM.Row, Brush> _get_brush;
+
+         public TimeAndSalesColumn(Func<TimeAndSalesVM.Row, string> get_text, Func<TimeAndSalesVM.Row, Brush> get_brush, IDataGridColumnFilter filter = null, string filter_template = null, GlyphContext glyph_context = null) : base(get_text, filter, filter_template, glyph_context) {
+            _get_text = get_text;
+            _get_brush = get_brush;
+         }
+
+         public override void draw(TimeAndSalesVM.Row row, DrawingContext dc, ColumnGeometry position, double row_height) {
+            CellRender.draw_text(dc
+               , glyph_context
+               , font_size
+               , _get_text(row)
+               , position
+               , row_height
+               , foreground
+               , _get_brush(row)
+               , horizontal_alignment
+               , vertical_alignment
+               , padding);
+         }
+
+      }
+   }
+}

src/client/spark/view/Controls/DataGridEx/Filters/Views/TimeFilterControl.xaml

@@ -42,9 +42,12 @@
                   <ColumnDefinition/>
                 </Grid.ColumnDefinitions>
                 <TextBlock VerticalAlignment="Center">From</TextBlock>
-                <TextBlock Grid.Column="2" HorizontalAlignment="Left">
-                  <Hyperlink x:Name="ClearBeginLink" Focusable="False">(Set Anytime)</Hyperlink>
-                </TextBlock>
+                <Grid Grid.Column="2">
+                  <TextBlock HorizontalAlignment="Left">
+                    <Hyperlink x:Name="ClearBeginLink" Focusable="False">(Set Anytime)</Hyperlink>
+                    <Hyperlink x:Name="SetLastHourLink" Focusable="False">(Set Last Hour)</Hyperlink>
+                  </TextBlock>
+                </Grid>
                 <c:DateTimePicker
                   x:Name="PART_Begin_DateTimePicker"
                   Grid.Row="2"

src/client/spark/view/Controls/DataGridEx/Filters/Views/TimeFilterControl.xaml.cs

@@ -44,8 +44,10 @@ namespace mbts.ui.controls {
       public override void OnApplyTemplate() {
          var clear_begin_time = this.assert_template_child<Hyperlink>("ClearBeginLink");
          var clear_end_time = this.assert_template_child<Hyperlink>("ClearEndLink");
+         var set_last_hour = this.assert_template_child<Hyperlink>("SetLastHourLink");
          clear_begin_time.Command = new DelegateCommand(() => Begin = default);
          clear_end_time.Command = new DelegateCommand(() => End = default);
+         set_last_hour.Command = new DelegateCommand(() => Begin = TimeTextTuple.FromDateTime(DateTime.Now.AddHours(-1)));
       }
 
       class TimeFilterErrorAdorner : NotifyDataErrorAdorner<TimeFilterControl, ITimeFilterVM>, IDisposable {

src/client/spark/view/Controls/ListBoxEx/ListBoxEx.xaml.cs

@@ -190,17 +190,21 @@ namespace mbts.ui.controls
 			}
 		}
 
-		void item_Unselected(object sender, RoutedEventArgs e) {
-			if (_selection_changing) return;
-			var item = ((ListBoxItem)sender).DataContext;
-			var selected_items = Selection;
-			if (selected_items.Contains(item)) {
-				using (_selection_change()) {
-					selected_items.Remove(item);
-					_update_is_all_selected();
-				}
-			}
-		}
+      void item_Unselected(object sender, RoutedEventArgs e) {
+         if (!_selection_changing && sender is ListBoxItem container) {
+            // the [ListBoxItem] sender may be "Disconnected" https://stackoverflow.com/questions/14282894/wpf-listbox-virtualization-creates-disconnecteditems/14358688
+            var item = ItemContainerGenerator.ItemFromContainer(container);
+            if (item != DependencyProperty.UnsetValue) {
+               var selected_items = Selection;
+               if (selected_items.Contains(item)) {
+                  using (_selection_change()) {
+                     selected_items.Remove(item);
+                     _update_is_all_selected();
+                  }
+               }
+            }
+         }
+      }
 
 		void item_DataContextChanged(object sender, DependencyPropertyChangedEventArgs e) {
 			using (_selection_change()) ((ListBoxItem)sender).IsSelected = Selection.Contains(e.NewValue);
@@ -368,4 +372,4 @@ namespace mbts.ui.controls
 			}
 		}
 	}
-}
+}

src/client/spark/view/Extensions/RandomExtensions.cs

@@ -14,6 +14,9 @@ namespace mbts.ui
       {
          return random.NextDouble() * (maxValue - minValue) + minValue;
       }
+      public static bool NextBool(this Random random, int truePercentage = 50) {
+         return random.NextDouble() < (truePercentage / 100.0);
+      }
 
    }
 

src/client/spark/view/Spark/ContractPicker/ContractPickerModel.cs

@@ -376,7 +376,7 @@ namespace mbts.spark
          return x is SecurityImmutable && y is SecurityImmutable ? Compare((SecurityImmutable)x, (SecurityImmutable)y) : -1;
       }
       public int Compare(SecurityImmutable x, SecurityImmutable y) {
-         return x.EarliestExpiration().CompareTo(y.EarliestExpiration());
+         return SecurityHelper.Comparer.Compare(x, y);
       }
 
 

src/client/spark/view/Spark/PAndL/PAndLVM.cs

@@ -59,7 +59,7 @@ namespace mbts.spark {
             total_net_position_filter.load_state(fs.total_net_pos);
             total_pnl_filter.load_state(fs.total_pnl);
             pnl_price_filter.load_state(fs.pnl_price);
-            fill_kind_filter.load_state(fs.fill_kind);
+            security_type_filter.load_state(fs.security_types);
          }
          // restore sort state or supply default value
          if (startup_data?.sort_criteria?.Any() == true) {
@@ -92,7 +92,7 @@ namespace mbts.spark {
                pnl_price_filter,
                weighted_average_buy_filter,
                weighted_average_sell_filter,
-               fill_kind_filter),
+               security_type_filter),
             WorkLoop.Start("PAndL", _UPDATE_PERIOD, _calculate_pnl)
          );
       }
@@ -197,7 +197,7 @@ namespace mbts.spark {
       public IDataGridColumnPersistentFilter<PnLSummaryRow, DoubleMinMaxState> pnl_price_filter { get; } = new DoubleMinMax<PnLSummaryRow>(r => r.pnl_price, nameof(PnLSummaryRow.pnl_price));
       public IDataGridColumnPersistentFilter<PnLSummaryRow, DoubleMinMaxState> weighted_average_buy_filter { get; } = new DoubleMinMax<PnLSummaryRow>(r => r.weighted_average_buy, nameof(PnLSummaryRow.weighted_average_buy));
       public IDataGridColumnPersistentFilter<PnLSummaryRow, DoubleMinMaxState> weighted_average_sell_filter { get; } = new DoubleMinMax<PnLSummaryRow>(r => r.weighted_average_sell, nameof(PnLSummaryRow.weighted_average_sell));
-      public IDataGridColumnPersistentFilter<PnLSummaryRow, FillKind[]> fill_kind_filter { get; } = DiscreteValuesFilterVM.Create<PnLSummaryRow, FillKind>(EnumHelper.get_values<FillKind>().ToArray(), r => r.fill_kind, nameof(PnLSummaryRow.fill_kind));
+      public IDataGridColumnPersistentFilter<PnLSummaryRow, SecurityType[]> security_type_filter { get; } = DiscreteValuesFilterVM.Create<PnLSummaryRow, SecurityType>(EnumHelper.get_values_except<SecurityType>(SecurityType.Option, SecurityType.Stock, SecurityType.Synthetic).ToArray(), r => r.security_type, nameof(PnLSummaryRow.security_type));
 
       public FilterStates get_filters_states() => new FilterStates {
          account_regex = account_filter.get_state(),
@@ -212,7 +212,7 @@ namespace mbts.spark {
          total_net_pos = total_net_position_filter.get_state(),
          total_pnl = total_pnl_filter.get_state(),
          pnl_price = pnl_price_filter.get_state(),
-         fill_kind = fill_kind_filter.get_state()
+         security_types = security_type_filter.get_state()
       };
 
       #endregion

src/client/spark/view/Spark/PAndL/PAndLView.xaml

@@ -19,7 +19,7 @@
             <Setter Property="TextAlignment" Value="Right" />
          </Style>
          <convert:NumericSignBrushConverter x:Key="PnLBackgroundColorConverter" Opacity="0.33"/>
-         <convert:DoubleTextConverter x:Key="DoubleWithNaNHiddenConverter" Format="F2" />
+         <convert:DoubleTextConverter x:Key="DoubleWithNaNHiddenConverter" Format="N2" />
       </ResourceDictionary>
    </UserControl.Resources>
    <Grid>
@@ -64,12 +64,12 @@
                c:DataGridFilterProperties.FilterOpen="{Binding DataContext.security_filter.open, Source={uc:DirectReference _grid}}"
                c:DataGridFilterProperties.FilterTemplate="{StaticResource RegexTemplate}"
             />
-            <DataGridTextColumn x:Name="fill_kind" Header="Fill Kind" Binding="{Binding fill_kind}" IsReadOnly="true"
-               c:DataGridFilterProperties.Filter="{Binding DataContext.fill_kind_filter, Source={uc:DirectReference _grid}}"
-               c:DataGridFilterProperties.FilterActive="{Binding DataContext.fill_kind_filter.active, Source={uc:DirectReference _grid}}"
-               c:DataGridFilterProperties.FilterOpen="{Binding DataContext.fill_kind_filter.open, Source={uc:DirectReference _grid}}"
+            <DataGridTextColumn x:Name="security_type" Header="Security Type" Binding="{Binding security_type}" IsReadOnly="true"
+               c:DataGridFilterProperties.Filter="{Binding DataContext.security_type_filter, Source={uc:DirectReference _grid}}"
+               c:DataGridFilterProperties.FilterActive="{Binding DataContext.security_type_filter.active, Source={uc:DirectReference _grid}}"
+               c:DataGridFilterProperties.FilterOpen="{Binding DataContext.security_type_filter.open, Source={uc:DirectReference _grid}}"
                c:DataGridFilterProperties.FilterTemplate="{StaticResource DiscreteValuesTemplate}"
-            />
+            />           
             <DataGridTextColumn x:Name="start_net_pos" Header="Start Net Pos" Binding="{Binding start_net_position}" IsReadOnly="true" 
                ElementStyle="{StaticResource RightAlignedColumn}"
                c:DataGridFilterProperties.Filter="{Binding DataContext.start_position_filter, Source={uc:DirectReference _grid}}"

src/client/spark/view/Spark/PAndL/PnLCalculator.cs

@@ -95,7 +95,7 @@ namespace mbts.spark {
 
       public IEnumerable<PnLSummaryRow> get_summary_position() {
          return (from pnl_item in _data
-                 group pnl_item by (pnl_item.account, pnl_item.exchange, pnl_item.symbol, pnl_item.security, pnl_item.sec_md, pnl_item.fill_kind) into grouping
+                 group pnl_item by (pnl_item.account, pnl_item.exchange, pnl_item.symbol, pnl_item.security, pnl_item.sec_md, pnl_item.security_type) into grouping
                  /* let start_of_day = _get_start_of_day(grouping.Key.symbol) */
                  select new PnLSummaryRow(_calc_type, _start_of_trading_day_utc, grouping.Key,
                     grouping.Where(y => y.fill.Timestamp.ToDateTime() >= _start_of_trading_day_utc).Sum(y => (uint)y.buy_quantity),

src/client/spark/view/Spark/PAndL/PnLData.cs

@@ -42,7 +42,7 @@ namespace mbts.spark {
       public SecurityImmutable? security => sec_md?.security;
       public double buy_quantity => (fill.Side == Side.Buy) ? fill.LastShares : 0;
       public double sell_quantity => (fill.Side == Side.Sell) ? fill.LastShares : 0;
-      public Fill.Types.FillKind fill_kind => fill.FillKind;
+      public SecurityType? security_type => security?.security_type();
 
       public override string ToString() {
          return $"Fill: ( {account} {exchange} {security.ToString()} buy: {buy_quantity} sell: {sell_quantity} ) Security: {security.ToString()}";

src/client/spark/view/Spark/PAndL/PnLSummaryRow.cs

@@ -13,7 +13,7 @@ namespace mbts.spark {
 
     internal class PnLSummaryRow : ViewModelBase, IEquatable<PnLSummaryRow> {
       DateTime _start_of_day;
-      (string account, Venue? exchange, string symbol, SecurityImmutable? security, SecurityMD? sec_md, Fill.Types.FillKind fill_kind) key;
+      (string account, Venue? exchange, string symbol, SecurityImmutable? security, SecurityMD? sec_md, SecurityType? sec_type) key;
       int _start_net_position, _total_net_position;
       uint _day_long_quantity, _day_short_quantity;
       double _total_pnl, _sod_pnl, _start_pnl, _pnl_price, _weighted_average_buy, _weighted_average_sell;
@@ -24,7 +24,7 @@ namespace mbts.spark {
 
       public PnLSummaryRow(pnl_calc_type calc_type,
          DateTime start_of_day,
-         (string, Venue?, string, SecurityImmutable?, SecurityMD?, Fill.Types.FillKind) key, 
+         (string, Venue?, string, SecurityImmutable?, SecurityMD?, SecurityType?) key, 
          uint day_buy_quant, uint day_sell_quant, int net_pos, int sod_pos, double wavg_buy, double wavg_sell, IEnumerable<FillImmutable> fills) {
          this._start_of_day = start_of_day;
          this.key = key;
@@ -41,19 +41,21 @@ namespace mbts.spark {
          update_pnl(calc_type);
       }
 
+      /// <summary>
+      /// Looks up the cash per point (cpp) for the given SecurityImmutable.
+      /// If the SecurityImmutable is a cross-product exchange-traded spread,
+      /// such as a crack spread, where the legs have differing cpps, then
+      /// we'll return Double.Nan;
+      /// </summary>
+      /// <param name="s">The SecurityImmutable for which we will look up a cpp</param>
+      /// <returns>The found cpp, else Double.NaN</returns>
       private double _get_cpp_for_security(SecurityImmutable s) {
          if (s.security_type() == SecurityType.Spread) {
             var distinct_cpps = s.Spread.Legs
                .Select(l => get_cpp_for_symbol(l.Security))
                .Distinct();
             bool all_legs_have_same_cpp = distinct_cpps.Count() < 2;
-            if (all_legs_have_same_cpp) {
-               return distinct_cpps.First();
-            } else {
-               Logger.error(bts.log.Subsystem.Ui,
-                  $"Security {s} contains legs with the following cash multipliers: { string.Join(",", distinct_cpps.Select(d => d.ToString())) } ");
-               return Double.NaN;
-            }
+            return all_legs_have_same_cpp ? distinct_cpps.First() : Double.NaN;
          } else {
             return get_cpp_for_symbol(s);
          }
@@ -62,8 +64,6 @@ namespace mbts.spark {
          double get_cpp_for_symbol(SecurityImmutable si) {
             if (cpp_dict.TryGetValue(si.Symbol, out double cpp_out)) {
                return cpp_out;
-            } else {
-               Logger.error(bts.log.Subsystem.Ui, $"No cash multiplier found for Security {si}, Symbol {si.Symbol}");
             }
             return double.NaN;
          }
@@ -170,14 +170,13 @@ namespace mbts.spark {
       }
 
       public SecurityImmutable? security => key.security;
+      public SecurityType security_type => key.sec_type.GetValueOrDefault();
 
       public string security_text {
          get => security.HasValue ? security.Value.ToString() : "Unknown Security";
          set => ignore();
       }
 
-      public Fill.Types.FillKind fill_kind => key.fill_kind;
-
 
       public int start_net_position {
          get => _start_net_position;

src/client/spark/view/Spark/Shell/CVOrderBook/CVOrderBookColumns.cs

@@ -54,6 +54,19 @@ namespace mbts.spark {
                   EqualityComparer.Create<SecurityImmutable, uint>(s => s.Id),
                   SecurityHelper.Comparer.InstanceImmutable
                )));
+            // symbol
+            Add(symbol_column = new SimpleTextColumn<string[]>(
+               Columns.Symbol,
+               DefaultColumnHeaders.Symbol,
+               nameof(CVOrderBookRowVM.symbol),
+               r => r.symbol.value,
+               DefaultColumnWidths.Symbol,
+               DiscreteValuesFilterVM.Create<CVOrderBookRowVM, string>(
+                  mbts.co2.UserSymbol.get_all().Select(u => u.symbol).Distinct().ToList(),
+                  r => r.symbol.value,
+                  value_path(nameof(CVOrderBookRowVM.symbol))
+               ),
+               DataGridFilterKnownTemplates.DiscreteValues));
             // time_created
             Add(time_created_column = new TimeCreatedColumn(
                new TimeFilterVM<CVOrderBookRowVM>(r => r.time_created.value, value_path(nameof(OrderBookRowVM.time_created)))));
@@ -223,6 +236,7 @@ namespace mbts.spark {
                status_column.filter.load_state(fs.statuses);
                last_message_column.filter.load_state(fs.exchange_message_regex);
                last_change_column.filter.load_state(fs.last_change);
+               symbol_column.filter.load_state(fs.symbols);
             }
             if (startup_data?.sort_criteria is List<SortingCriteria> lsc) {
                foreach (var sc in lsc)
@@ -238,6 +252,7 @@ namespace mbts.spark {
          public event Action filter_changed;
 
          public SecurityColumn security_column { get; }
+         public SimpleTextColumn<string[]> symbol_column { get; }
          public TimeCreatedColumn time_created_column { get; }
          public LastChangeTimeColumn last_change_column { get; }
          public SimpleTextColumn<string[]> user_column { get; }
@@ -279,7 +294,8 @@ namespace mbts.spark {
                statuses = status_column.filter.get_state(),
                exchange_message_regex = last_message_column.filter.get_state(),
                last_change = last_change_column.filter.get_state(),
-               security_criteria = security_column.filter.get_state()
+               security_criteria = security_column.filter.get_state(),
+               symbols = symbol_column.filter.get_state()
             };
 
          public void Dispose() {

src/client/spark/view/Spark/Shell/CVOrderBook/CVOrderBookRowVM.cs

@@ -31,6 +31,7 @@ namespace mbts.spark {
       IOrderBookCellStore<uint> _security_key_store;
       IOrderBookCellStore<SecurityImmutable?> _security_store;
       IOrderBookCellStore<Venue> _exchange_store;
+      IOrderBookCellStore<string> _symbol_store;
       // Def
       IOrderBookCellStore<CustomOrderDefinition.CustomOneofCase> _order_type_store;
       IOrderBookCellStore<Side> _side_store;
@@ -59,6 +60,7 @@ namespace mbts.spark {
          _security_key_store = CellFactory.create(this, _book_vm.columns.security_key_column, nameof(security_key), r => r._order.security_key(), r => r._security_key_store);
          _parent_order_key_store = CellFactory.create(this, _book_vm.columns.parent_key_column, nameof(parent_order_key), r => r._order.ParentKey, r => r._parent_order_key_store);
          _order_key_store = CellFactory.create(this, _book_vm.columns.order_key_column, nameof(order_key), r => r._order.Key, r => r._order_key_store);
+         _symbol_store = CellFactory.create(this, _book_vm.columns.symbol_column, nameof(symbol), r => r._security.HasValue ? _security.Value.Symbol : "unknown symbol", r => r._symbol_store);
          _time_created_store = CellFactory.create(this, _book_vm.columns.time_created_column, nameof(time_created), r => r._order.created(), r => r._time_created_store, format_date);
          _user_store = CellFactory.create(this, _book_vm.columns.user_column, nameof(user), r => _order.user(), r => r._user_store);
          _account_store = CellFactory.create(this, _book_vm.columns.account_column, nameof(account), r => r._order.account(), r => r._account_store);
@@ -88,7 +90,7 @@ namespace mbts.spark {
          }
          //
          _visible = backing(nameof(is_visible), true);
-         _expanded = backing(nameof(is_expanded), true);
+         _expanded = backing(nameof(is_expanded), false);
          _depth_level = backing(nameof(depth_level), () => {
             int i = 0;
             var curr = this;
@@ -120,6 +122,7 @@ namespace mbts.spark {
       public IOrderBookCell<uint> security_key => _security_key_store.current;
       public IOrderBookCell<SecurityImmutable?> security => _security_store.current;
       public IOrderBookCell<Venue> exchange => _exchange_store.current;
+      public IOrderBookCell<string> symbol => _symbol_store.current;
       // Def
       public IOrderBookCell<CustomOrderDefinition.CustomOneofCase> order_type => _order_type_store.current;
       public IOrderBookCell<Side> side => _side_store.current;
@@ -190,6 +193,7 @@ namespace mbts.spark {
             yield return _price_store;
             yield return _avg_price_store;
             yield return _last_price_store;
+            yield return _symbol_store;
          }
       }
 
@@ -282,6 +286,7 @@ namespace mbts.spark {
             case Columns.ParentKey: return row._parent_order_key_store;
             case Columns.SecurityKey: return row._security_key_store;
             case Columns.Security: return row._security_store;
+            case Columns.Symbol: return row._symbol_store;
             case Columns.Exchange: return row._exchange_store;
             case Columns.OrderType: return row._order_type_store;
             case Columns.Side: return row._side_store;

src/client/spark/view/Spark/Shell/CVOrderBook/CVOrderBookVM.cs

@@ -38,6 +38,8 @@ namespace mbts.spark {
       SetMap<uint, uint> _parent_to_children = new SetMap<uint, uint>();
       SetMap<uint, uint> _security_orders = new SetMap<uint, uint>();
 
+      bool _expand_collapse_all_status = false;
+
       public CVOrderBookVM(IBus bus, OrdersModel model, OrderBookStartupData state) {
          columns = new CVOrderBookColumnCollection(bus, model, state, _order_securities_subject);
          foreach (ColumnBase col in columns)
@@ -69,6 +71,15 @@ namespace mbts.spark {
          SetMap<uint, uint> missing_security_orders = new SetMap<uint, uint>();
          IReadOnlyDictionary<uint, SecurityMD> security_dict = new Dictionary<uint, SecurityMD>();
 
+         expand_collapse_all_rows_command = new DelegateCommand(
+            () => {
+               _order_rows.Where(r => r.is_visible && r.passes_filter).ForEach(r =>
+                  r.is_expanded = !_expand_collapse_all_status
+               );
+               _expand_collapse_all_status = !_expand_collapse_all_status;
+            }
+         );
+
          _dispose = Disposable.Create(
             bus.subscribe(SparkTags.SecuritiesTag).OnDispatcher().Subscribe(on_securities),
             model.on_orders.OnDispatcher().Subscribe(on_orders),
@@ -232,6 +243,8 @@ namespace mbts.spark {
       public CVOrderBookRowVM current_row { get => _current_row.Value; set => _current_row.Value = value ?? _selected_rows.FirstOrDefault(); }
       public OrderRecordImmutable? current_order => current_row?.order;
       public IEnumerable<OrderRecordImmutable> selected_orders => _selected_rows.Select(r => r.order);
+
+      public System.Windows.Input.ICommand expand_collapse_all_rows_command { get; }
       void IOrderBookVM.on_sorting(DIYGridSortingEventArgs e) { }
 
       public void row_selection_changed(SelectionChangedEventArgs e) {

src/client/spark/view/Spark/Shell/OrderBook/OrderBookVM.cs

@@ -18,6 +18,7 @@ using System.Linq;
 using System.Reactive.Linq;
 using System.Windows.Controls;
 using System.Windows.Data;
+using System.Windows.Input;
 using static bts.shell.OrderBookWindowFactory;
 using static mbts.spark.OrderBookView;
 using static mbts.ui.controls.DIYGrid;
@@ -427,6 +428,9 @@ namespace mbts.spark {
       public ObservableCollection<OrderBookRowVM> selected_rows => _selected_rows;
       public IEnumerable<OrderRecordImmutable> selected_orders => _selected_rows.Select(r => r.order);
 
+      public ICommand expand_collapse_all_rows_command => throw new NotImplementedException();
+
+
       #region Filters
       public IDataGridColumnPersistentFilter<OrderBookRowVM, DoubleMinMaxState> price_filter { get; } = new DoubleMinMax<OrderBookRowVM>(r => r.price, nameof(OrderBookRowVM.price));
       public IDataGridColumnPersistentFilter<OrderBookRowVM, UIntMinMaxState> qty_filter { get; } = new UIntMinMax<OrderBookRowVM>(r => r.quantity, nameof(OrderBookRowVM.quantity));
@@ -448,7 +452,6 @@ namespace mbts.spark {
       public IDataGridColumnPersistentFilter<OrderBookRowVM, string> exchange_message_filter { get; }
       public IDataGridColumnPersistentFilter<OrderBookRowVM, string> security_filter { get; } = new RegexFilterVM<OrderBookRowVM>(r => r.security_text, nameof(OrderBookRowVM.security_text));
 
-
       public FilterStates get_filter_states() =>
          new FilterStates {
             order_key_regex = order_key_filter.get_state(),

src/client/spark/view/Spark/Shell/OrderBook/OrderBookView.xaml.cs

@@ -17,10 +17,11 @@ namespace mbts.spark {
       void on_sorting(DIYGridSortingEventArgs e);
       List<SortingCriteria> get_sorting_states();
       OrderBookWindowFactory.FilterStates get_filter_states();
+      System.Windows.Input.ICommand expand_collapse_all_rows_command { get; }
    }
 
    internal partial class OrderBookView : UserControl {
-      public enum Columns { Security, Time, LastChange, User, Account, Exchange, TimeInForce, OrderType, SubType, Side, Size, Price, Filled, AvgFilled, LastPrice, Status, LastMessage, SecurityKey, OrderKey, ParentKey }
+      public enum Columns { Security, Time, LastChange, User, Account, Exchange, TimeInForce, OrderType, SubType, Side, Size, Price, Filled, AvgFilled, LastPrice, Status, LastMessage, SecurityKey, OrderKey, ParentKey, Symbol }
       public const string
          SubTypeFilterTemplateKey = "SubTypeFilterTemplate",
          ExpandCollapseToggleStyleKey = "ExpandCollapseToggleStyle",

src/client/spark/view/Spark/Shell/OrderBook/OrderBookWindow.cs

@@ -28,7 +28,8 @@ namespace mbts.spark {
          , _cancel_sell_orders_command
          , _cancel_all_orders_command
          , _cancel_selected_orders_command
-         , _recreate_selected_orders_command;
+         , _recreate_selected_orders_command
+         , _toggle_collapse_expand_all_command;
 
 
       public OrderBookWindow(OrdersModel model, OrderBookWindowFactory factory, IBus bus, OrderBookWindowFactory.OrderBookStartupData state, ImageSource icon) : base("Order Book", bus, factory, SparkCommands.ShowOrders.icon) {
@@ -115,6 +116,10 @@ namespace mbts.spark {
             can_execute: () => _vm.selected_orders.Any(x => _passes_recreate_search_criteria(x))
          );
 
+         _toggle_collapse_expand_all_command = new SparkCommandCommand("Expand / Collapse All", 
+            Icons.UpArrow,
+            _vm.expand_collapse_all_rows_command);
+
          _commands = new SparkCommand[] {
             _show_fills_command,
             _modify_order_command,
@@ -122,13 +127,14 @@ namespace mbts.spark {
             _cancel_buy_orders_command,
             _cancel_sell_orders_command,
             _cancel_all_orders_command,
-            _recreate_selected_orders_command
+            _recreate_selected_orders_command,
+            _toggle_collapse_expand_all_command
          };
          _vm.selected_order_changed += _vm_selected_order_changed;
       }
 
       private bool _passes_recreate_search_criteria(OrderRecordImmutable record) {
-         return record.status() == OrderStatus.Canceled 
+         return (record.status() == OrderStatus.Canceled || record.status() == OrderStatus.Error)
             && !record.has_parent() 
             && record.State.Quantity > record.State.FilledQuantity;
       }

src/client/spark_api/api/Update.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.Immutable;
 using System.Linq;
 using System.Reactive.Linq;
 
@@ -43,9 +44,9 @@ namespace bts.api.core {
 
    public class TableUpdate<T, TKey> where TKey : IEquatable<TKey> {
 
-      readonly Dictionary<TKey, T> _state;
+      ImmutableDictionary<TKey, T> _state;
 
-      internal TableUpdate(Dictionary<TKey, T> init, IReadOnlyList<T> added, IReadOnlyList<T> removed) {
+      internal TableUpdate(ImmutableDictionary<TKey, T> init, IReadOnlyList<T> added, IReadOnlyList<T> removed) {
          _state = init;
          this.added = added;
          this.removed = removed;
@@ -59,14 +60,10 @@ namespace bts.api.core {
          this.update(update.removed, update.added, update.key);
 
       internal TableUpdate<T, TKey> update(IReadOnlyList<T> removed, IReadOnlyList<T> added, Func<T, TKey> key) {
-         var table = new TableUpdate<T, TKey>(new Dictionary<TKey, T>(_state), added, removed);
-         foreach (var i in removed) {
-            table._state.Remove(key(i));
-         }
-         foreach (var item in added) {
-            table._state.Add(key(item), item);
-         }
-         return table;
+         var builder = _state.ToBuilder();
+         builder.RemoveRange(removed.Select(i => key(i)));
+         builder.AddRange(added.Select(i => new KeyValuePair<TKey, T>(key(i), i)));
+         return new TableUpdate<T, TKey>(builder.ToImmutableDictionary(), added, removed);
       }
 
       internal TableUpdate<T, TKey> as_snapshot() {
@@ -102,7 +99,7 @@ namespace bts.api.core {
 
       public static IObservable<TableUpdate<T,TKey>> Table<T,TKey>(this IObservable<Update<T,TKey>> source) where TKey : IEquatable<TKey> =>
          source.Scan(
-            new TableUpdate<T,TKey>(new Dictionary<TKey, T>(), new List<T>(), new List<T>()),
+            new TableUpdate<T,TKey>(ImmutableDictionary<TKey,T>.Empty, new List<T>(), new List<T>()),
             (table, update) => table.update(update));
 
       public static IObservable<TableUpdate<T,TKey>> Cache<T, TKey>(this IObservable<TableUpdate<T, TKey>> source) where TKey : IEquatable<TKey> {

src/client/spark_api/spark_api.csproj

@@ -1,34 +1,35 @@
-<Project Sdk="Microsoft.NET.Sdk">
-  <PropertyGroup>
-    <TargetFrameworks>netstandard2.0</TargetFrameworks>
-    <Authors>Blue Trading Systems</Authors>
-    <Company>Blue Trading Systems</Company>
-    <Product>Spark</Product>
-    <Description />
-    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <Platforms>x64</Platforms>
-    <PlatformTarget>AnyCPU</PlatformTarget>
-    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
-    <WarningsAsErrors />
-    <Version>1.0.0</Version>
-    <LangVersion>7.2</LangVersion>
-  </PropertyGroup>
-  
-  <ItemGroup>
-    <PackageReference Include="Google.Protobuf" Version="3.4.0" />
-    <PackageReference Include="System.Reactive" Version="3.1.1" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <Proto3 Include="**/*.proto" />
-    <Proto3 Include="../../lib/api/*.proto" />
-    <Proto3 Include="../../lib/api/services/*.proto" />
-    <!--Proto3 Remove="../../lib/api/services/MarketDataRelay.proto" /-->
-    <Proto3 Include="../../lib/md/protos/co2.proto" />
-    <Proto3 Include="../../lib/domain_model/Security.proto" />
-    <Proto3 Include="../../lib/sparkproto/SparkProtos.proto" />
-    <Proto3 Update="../../lib/api/services/OutOfProcessURLListener.proto">
-      <!-- <CompileOptions>internal_access:</CompileOptions> -->
-    </Proto3>
-  </ItemGroup>
-</Project>
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFrameworks>netstandard2.0</TargetFrameworks>
+    <Authors>Blue Trading Systems</Authors>
+    <Company>Blue Trading Systems</Company>
+    <Product>Spark</Product>
+    <Description />
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <Platforms>x64</Platforms>
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+    <WarningsAsErrors />
+    <Version>1.0.0</Version>
+    <LangVersion>7.2</LangVersion>
+  </PropertyGroup>
+  
+  <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.4.0" />
+    <PackageReference Include="System.Collections.Immutable" Version="1.5.0" />
+    <PackageReference Include="System.Reactive" Version="3.1.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Proto3 Include="**/*.proto" />
+    <Proto3 Include="../../lib/api/*.proto" />
+    <Proto3 Include="../../lib/api/services/*.proto" />
+    <!--Proto3 Remove="../../lib/api/services/MarketDataRelay.proto" /-->
+    <Proto3 Include="../../lib/md/protos/co2.proto" />
+    <Proto3 Include="../../lib/domain_model/Security.proto" />
+    <Proto3 Include="../../lib/sparkproto/SparkProtos.proto" />
+    <Proto3 Update="../../lib/api/services/OutOfProcessURLListener.proto">
+      <!-- <CompileOptions>internal_access:</CompileOptions> -->
+    </Proto3>
+  </ItemGroup>
+</Project>

src/client/spark_core/Co2/MDConsumerHandle.cs

@@ -124,8 +124,8 @@ namespace mbts.co2
       public double session_high { get { return _cold->session_high;} }
       public double session_low { get {  return _cold->session_low; } }
       public double open_interest { get {  return _cold->open_interest; } }
-      public double session_high_bid { get {  return _cold->session_high_bid; } }
-      public double session_low_offer { get {  return _cold->session_low_offer; } }
+      // public double session_high_bid { get {  return _cold->session_high_bid; } }
+      // public double session_low_offer { get {  return _cold->session_low_offer; } }
       public double fixing_price { get {  return _cold->fixing_price; } }
 
       //

src/lib/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cpp_config.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cpp_config.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/quickbus_cfg.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp)
+set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cpp_config.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cpp_config.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/quickbus_cfg.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp)

src/lib/config/protos/MarketData.proto

@@ -145,4 +145,12 @@ message CMETradingSchedule {
    // we can elaborate later, but for now it's very simple:
    // this is "hours after midnight, in Central Time"
    optional double gen_open_hour = 1 [default = 8.5]; // 8:30 am CT
+}
+
+message SimSource {
+    oneof source {
+        string replay_url = 1 [default = "http://mdreplay.bluetradesys.local/spark/futures.coconut"];
+        string replay_file = 2 [default = "futures.coconut"];
+        bool local_relay = 3;
+    }
 }

src/lib/md/Diag/ShmDumper.cpp

@@ -28,8 +28,10 @@ double dump(TimeStamp time) {
 string dump(PriceSize ps) {
    stringstream res;
    res << "{";
-   res << "'price': " << ps.price() << ", ";
-   res << "'size': " << ps.size();
+   res << "'price': " << std::setprecision(11) << ps.price() << ", ";
+   res << "'size': " << ps.size() << " ";
+   // res << "'mant': " << ps.mantissa() << ", ";
+   // res << "'exp': " << (int)ps.exponent() << " ";
    res << "}";
    return res.str();
 }

src/lib/md/MDProducerHandle.cpp

@@ -115,7 +115,7 @@ void MDProducerHandle::set_status(CoconutSecurityStatus status) {
    }
 }
 
-void MDProducerHandle::add_trade(TimeStamp time, const PriceSize price, LLDateTime tradetime, uint32_t volume) {
+void MDProducerHandle::add_trade(TimeStamp time, const PriceSize price, LLDateTime tradetime, co2::AggressorSide agside, uint32_t volume) {
    cdata().last_trade.store(price);
    cdata().last_trade_time = tradetime;
    cdata().total_volume = volume;
@@ -127,6 +127,7 @@ void MDProducerHandle::add_trade(TimeStamp time, const PriceSize price, LLDateTi
       trade.set_mantissa(price.mantissa());
       trade.set_size(price.size());
       trade.set_time(tradetime);
+      trade.set_aggressor_side(agside);
       if (volume > 0) trade.set_total_volume(volume);
       trade.set_key(key());
       _channel.add_trade(trade);

src/lib/md/MDProducerHandle.h

@@ -56,7 +56,7 @@ public:
    void set_status(co2::CoconutSecurityStatus status); // hot
 
    // Updates trade log and last trade
-   void add_trade(TimeStamp time, const PriceSize pricesize, datetime::LLDateTime date, uint32_t total_volume=0); // cold
+   void add_trade(TimeStamp time, const PriceSize pricesize, datetime::LLDateTime date, co2::AggressorSide agside, uint32_t total_volume=0); // cold
    // Update last_trade without updating trade log
    void set_last_trade(TimeStamp time, const PriceSize ps, datetime::LLDateTime date, uint32_t total_volume);
    void add_rfq(TimeStamp time, const co2::QuoteRequestMessage& rfq); // cold

src/lib/md/protos/co2.proto

@@ -119,6 +119,12 @@ message MiscMessage {
    double session_low_offer    = 12;
    double fixing_price		    = 13;
 }
+   
+enum AggressorSide {
+    AGGRESSOR_SIDE_NONE = 0;
+    AGGRESSOR_SIDE_BUY = 1;
+    AGGRESSOR_SIDE_SELL = 2;
+}
 
 message TradeMessage {
    uint32 key = 1;
@@ -130,6 +136,7 @@ message TradeMessage {
    uint32 size = 6; 
    uint32 total_volume = 7;
    bool gap = 8;
+   AggressorSide aggressor_side = 9;
 }
 message QuoteRequestMessage {
     enum Side {

src/lib/md/relay/RelayClient.cpp

@@ -283,15 +283,38 @@ struct RelayScheduler {
 
 void relay_everything() {
    api::ServiceContext ctx;
-   auto relay = ctx.locate<MarketDataRelay>();
    std::set<UserSymbol> system_symbols;
    get_system_md_symbols(system_symbols);
    RelayProducer producer(true);
-   RelayClient client(relay, nullptr, system_symbols, producer);
-   RelayScheduler scheduler(ctx, config::get<RelayOptions>().naggle_ms(), client);
-   ctx.post([&]() { scheduler.run(); });
+   relay_selected(ctx, system_symbols, producer);
 }
 
+void relay_selected(api::ServiceContext& ctx, const std::set<UserSymbol>& symbols, IRelayWriter& writer) {
+   auto relay = ctx.locate<MarketDataRelay>();
+   RelayClient client(relay, nullptr, symbols, writer);
+   RelayScheduler scheduler(ctx, config::get<RelayOptions>().naggle_ms(), client);
+   ctx.post([&]() { scheduler.run(); });
+
+   // block forever, I guess
+   utils::Dispatcher::current().run();
+}
+void local_relay(IRelayWriter& writer) {
+   api::ServiceContext ctx;
+   auto relay = ctx.locate<MarketDataRelay>();
+   std::set<UserSymbol> symbols;
+   {
+      std::set<UserSymbol> system_symbols;
+      get_system_md_symbols(system_symbols);
+
+      // Filter out VENUE_SIM
+      for (const auto& s : system_symbols) {
+         if (s.exchange() != api::VENUE_SIM) {
+            symbols.insert(s);
+         }
+      }
+   }
+   relay_selected(ctx, symbols, writer);
+}
 
 void record_everything(std::string path, std::string filename) {
    auto options = config::get<RecordOptions>();

src/lib/md/relay/RelayClient.h

@@ -14,6 +14,9 @@ namespace bts {
 namespace pubsub {
    class PubsubHub;
 }
+namespace api {
+   class ServiceContext;
+}
 namespace md {
 
 struct RelayControl;
@@ -26,6 +29,9 @@ struct IRelayWriter {
 
 utils::Dispatcher& scan_securities_thread();
 
+void relay_selected(api::ServiceContext& ctx, const std::set<UserSymbol>& symbols, IRelayWriter& writer);
+void local_relay(IRelayWriter& writer);
+
 extern "C" {
 BTS_EXPORT void relay_everything();
 

src/lib/md/relay/RelayRecorder.cpp

@@ -157,7 +157,7 @@ void replay_recording(istream& in, IRelayWriter& writer, int32_t naggle_ms) {
    }
 }
 
-void replay_file(const std::string& filename, IRelayWriter& writer, int32_t naggle_ms, bool loop = false) {
+void replay_file(const std::string& filename, IRelayWriter& writer, int32_t naggle_ms, bool loop) {
    do {
       ifstream in;
       in.open(filename, ios::binary | ios::in);

src/lib/md/relay/RelayRecorder.h

@@ -30,6 +30,7 @@ void unprint_recording(const std::string& src, const std::string& dest);
 void trace_recording(const std::string& file, security_key key);
 void replay_recording(const std::string&, int32_t delay_ms);
 void replay_url(const std::string& url, IRelayWriter& writer, int32_t naggle_ms);
+void replay_file(const std::string& filename, IRelayWriter& writer, int32_t naggle_ms, bool loop=false);
 
 }}//namespace
 

src/lib/md/test/Co2TestInstrument.cpp

@@ -1,4 +1,4 @@
-#ifndef NO_CPP0X
+#define _DISABLE_EXTENDED_ALIGNED_STORAGE
 #include "Co2TestInstrument.h"
 #include "lib/md/MemorySegmentUtils.h"
 
@@ -50,4 +50,3 @@ MDConsumerHandle Co2TestInstrument::consumer() {
 Co2TestInstrument::~Co2TestInstrument() { }
 
 }}//namespace
-#endif

src/server/sparkd/CMakeLists.txt

@@ -24,7 +24,7 @@ target_link_libraries( sparkd
   tbb
   double-conversion
   pugixml
-  onixs-cmemdh-all-4.5.2
+  onixs-cmemdh-all-4.8.0
   quickfix
   benchmark
   lttng-ust

src/server/sparkd/exchange_sim/ExchangeSim.cpp

@@ -104,11 +104,27 @@ int start_exchange_sim()
       acceptor.start();
       
       std::map<uint32_t,md::Security> known_securities;
+      const auto& conf = config::get<md::SimSource>(); 
       while ( true ) {
          MDPlayer player(application,known_securities);
-         md::replay_url("http://mdreplay.bluetradesys.local/spark/futures.coconut",
-                        player,
-                        config::get<md::RelayOptions>().naggle_ms());
+         switch(conf.source_case()) {
+            case md::SimSource::SOURCE_NOT_SET:
+            case md::SimSource::kReplayUrl:
+               md::replay_url(conf.replay_url(),
+                  player,
+                  config::get<md::RelayOptions>().naggle_ms());
+            break;
+            case md::SimSource::kReplayFile:
+               md::replay_file(conf.replay_file(),
+                  player,
+                  config::get<md::RelayOptions>().naggle_ms(), true);
+            break;
+            case md::SimSource::kLocalRelay: {
+               md::local_relay(player);
+            }
+            break;
+
+         }
       }
       acceptor.stop();
       return 0;

src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp

@@ -474,7 +474,8 @@ void ExchangeSimApplication::update_md(std::string const &sec_desc) {
              utils::TimeStamp::now(),
              md::PriceSize::from_double<7>(t.price.value() * price_factor,
                                            t.quantity.value()),
-             datetime::ll_now());
+             datetime::ll_now(),
+             co2::AGGRESSOR_SIDE_NONE);
    }
 }
 

src/server/sparkd/http/ApiGateway.cpp

@@ -99,7 +99,7 @@ struct ApiGatewayImpl {
          // buffer = [size][ResponseHeader][Payload].
          GatewayResponse response;
          response.set_call_result(buffer, length);
-         LOG(Api,Info, "Gateway Result: {}", api::toJSON(response));
+         LOG(Api,Diag, "Gateway Result: {}", api::toJSON(response));
          // in response send to client
          const Responder* r = static_cast<const Responder*>(extra);
          if (r != nullptr) {

src/server/sparkd/http/WebServer.cpp

@@ -77,6 +77,7 @@ namespace http {
          _server.set_http_handler(bind(&ServerState::on_http, this, ::_1));
          // Turn off logging
          _server.clear_access_channels(websocketpp::log::alevel::all);
+         _server.clear_error_channels(websocketpp::log::alevel::all);
          run();
       }
 

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp

@@ -3,6 +3,7 @@
 #include <vector>
 #include "lib/utils/helper_macros.h"
 #include "lib/config/protos/MarketData.pb.h"
+#include "lib/md/protos/co2.pb.h"
 
 using namespace std;
 using namespace bts::co2;
@@ -10,74 +11,6 @@ using namespace bts::co2;
 namespace bts {
 namespace md {
 
-void decode_ticks(int vtt, double mintick, FixedSecurity& dest) {
-   switch(vtt) {
-      default:
-      case 0:
-         dest.low_tick = mintick;
-         dest.high_tick = mintick;
-         dest.price_break = 0;
-         break;
-      case 1:
-         dest.price_break = 500;
-         dest.low_tick = 5;
-         dest.high_tick = 10;
-         break;
-      case 2:
-         dest.price_break = 5;
-         dest.low_tick = 0.5;
-         dest.high_tick = 1;
-         break;
-      case 3:
-         dest.price_break = 10;
-         dest.low_tick = 1;
-         dest.high_tick = 2;
-         break;
-      case 4:
-         dest.price_break = 500;
-         dest.low_tick = 5;
-         dest.high_tick = 25;
-         break;
-      case 10:
-         dest.price_break = 300;
-         dest.low_tick = 5;
-         dest.high_tick = 25;
-         break;
-      case 11:
-         dest.price_break = 300;
-         dest.low_tick = 5;
-         dest.high_tick = 10;
-         break;
-      case 12:
-         dest.price_break = 5;
-         dest.low_tick = 0.25;
-         dest.high_tick = 0.5;
-         break;
-   }
-}
-
-int floorlog10(int64_t m) {
-   bts_assert(m >= 0);
-   if (m < 10) return 0;
-   if (m < 100) return 1;
-   if (m < 1000) return 2;
-   if (m < 10000) return 3;
-   if (m < 100000) return 4;
-   if (m < 1000000) return 5;
-   if (m < 10000000) return 6;
-   if (m < 100000000) return 7;
-   if (m < 1000000000) return 8;
-   if (m < 10000000000) return 9;
-   if (m < 100000000000) return 10;
-   if (m < 1000000000000) return 11;
-   if (m < 10000000000000) return 12;
-   if (m < 100000000000000) return 13;
-   if (m < 1000000000000000) return 14;
-   if (m < 10000000000000000) return 15;
-   if (m < 100000000000000000) return 16;
-   if (m < 1000000000000000000) return 17;
-   return 19;
-}
 
 MDP3Listener::MDP3Listener(
       CMESecurityIndex& index,
@@ -181,159 +114,6 @@ void MDP3Listener::onEndOfPacket(onixs::Handler&, const onixs::PacketArgs&) {
 //
 // Security discovery
 
-template <class Price>
-double double_of_price(const Price& price, int8_t display_factor) {
-   // order of multiplication is important because of floating point error
-   return (price.mantissa() * pow10(price.exponent())) * pow10(display_factor);
-}
-
-inline datetime::LLDateTime ll_of_timestamp(const onixs::Timestamp& ts) {
-   return datetime::ll_from_parts(ts.year(), (int)ts.month(), ts.day(), ts.hour(), ts.minute(), ts.second(), ts.millisecond());
-}
-
-datetime::Date date_of_timestamp(const onixs::Timestamp& ts) {
-   return datetime::Date(ts.year(), (int)ts.month(), ts.day());
-}
-
-template <class Definition>
-datetime::Date date_of_maturity(const Definition& def) {
-   for (const auto& event : def.events()) {
-      if (event.eventType() == onixs::EventType::LastEligibleTradeDate) {
-         return date_of_timestamp(event.eventTime());
-      }
-   }
-   return datetime::Date();
-}
-
-
-template <class Definition>
-void set_security_common(const Definition& def, FixedSecurity& fsec) {
-   fsec.numeric_id = def.securityId();
-   fsec.symbol(def.asset().items(), def.asset().size());
-   fsec.cfi_code(def.cfiCode().items(), def.cfiCode().size());
-   fsec.alt_symbol(def.securityGroup().items(), def.securityGroup().size());
-   fsec.security_desc(def.symbol().items(), def.symbol().size());
-   fsec.exchange = api::Venue::VENUE_GLBX;
-   // tickRule not present in futures
-   fsec.cme_market_segment_id = def.marketSegmentId();
-   fsec.display_factor = def.displayFactor().exponent() + floorlog10(def.displayFactor().mantissa());
-
-   def.mainFraction(fsec.cme_main_fraction);
-   def.subFraction(fsec.cme_sub_fraction);
-   def.priceDisplayFormat(fsec.cme_price_display_format);
-}
-
-void MDP3Listener::onDefinition(onixs::Handler&, const onixs::Security& osec, const onixs::InstrumentDefinitionFutureArgs& def) {
-   tracepoint (bts_trace,trace_order,(char*) "onDefinition.future", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   const auto& sec = def.message();
-   auto exchange_id = sec.securityId();
-   // LOG(md, Info, "onDefinition future {} {}",  
-   //    std::string(sec.asset().items(), sec.asset().size()),
-   //    std::string(sec.symbol().items(), sec.symbol().size()));
-
-   if (filter_security(exchange_id, sec) != ADD) return;
-   bts_assert(exchange_id == osec.id());
-
-   FixedFutureSecurity fsec;
-
-   set_security_common(sec, fsec);
-   fsec.price_break = 0;
-   // low/high tick are stored non-premultiplied and multiplied in conversion to api::Secutity
-   double min_tick = double_of_price(sec.minPriceIncrement(), 0);
-   fsec.low_tick = fsec.high_tick = min_tick;
-   onixs::PRICE min_tick_amount;
-   sec.minPriceIncrementAmount(min_tick_amount);
-   double min_tick_dbl = double_of_price(min_tick_amount, 0);
-   fsec.cash_per_point = min_tick_dbl / min_tick / pow10(fsec.display_factor);
-   auto expiration = date_of_maturity(sec);
-   fsec.expiration_year = expiration.year;
-   fsec.expiration_month = expiration.month;
-   fsec.expiration_day = expiration.day;
-   add_security(fsec);
-}
-void MDP3Listener::onDefinition(onixs::Handler&, const onixs::Security&, const onixs::InstrumentDefinitionOptionArgs& def) {
-   tracepoint (bts_trace,trace_order,(char*) "onDefinition.option", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   const auto& sec = def.message();
-   auto exchange_id = sec.securityId();
-   // LOG(Info, MarketData) << "onDefinition option " << exchange_id;
-   if (filter_security(exchange_id, sec) != ADD) return;
-
-   FixedOptionSecurity fsec;
-   set_security_common(sec, fsec);
-   auto expiration = date_of_maturity(sec);
-   fsec.expiration_year = expiration.year;
-   fsec.expiration_month = expiration.month;
-   fsec.expiration_day = expiration.day;
-   
-   onixs::PRICE strike_price;
-   bts_assert(sec.strikePrice(strike_price));
-   fsec.strike = double_of_price(strike_price, fsec.display_factor);
-
-   onixs::PRICE min_tick, min_tick_amount;
-   sec.minPriceIncrement(min_tick);
-   sec.minPriceIncrementAmount(min_tick_amount);
-
-   int8_t tick_rule;
-   if (sec.tickRule(tick_rule)) {
-      decode_ticks(tick_rule, double_of_price(min_tick, 0), fsec);
-   }
-   fsec.cash_per_point = double_of_price(min_tick_amount, 0) / fsec.low_tick / pow10(fsec.display_factor);
-
-   fsec.call_put = sec.putOrCall() == onixs::PutOrCall::Call ? api::PUTORCALL_CALL : api::PUTORCALL_PUT;
-
-   add_security(fsec);
-}
-
-void MDP3Listener::onDefinition(onixs::Handler&, const onixs::Security&, const onixs::InstrumentDefinitionSpreadArgs& spread_args) {
-   tracepoint (bts_trace,trace_order,(char*) "onDefinition.spread", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   const auto& sec = spread_args.message();
-   auto exchange_id = sec.securityId();
-   // LOG(md, Info, "onDefinition spread {} {}",  
-   //    std::string(sec.asset().items(), sec.asset().size()),
-   //    std::string(sec.symbol().items(), sec.symbol().size()));
-   if (filter_security(exchange_id, sec) != ADD) return;
-   
-   // Filter out any already expired spreads
-   auto expiration = date_of_maturity(sec);
-   if (expiration < datetime::today()) return;
-
-   FixedSpreadSecurityL fsec;
-   set_security_common(sec, fsec);
-   vector<uint32_t> leg_exchange_ids;
-   leg_exchange_ids.reserve(sec.legs().size());
-   
-   onixs::PRICE min_tick = sec.minPriceIncrement();
-   
-   int8_t tick_rule;
-   if (sec.tickRule(tick_rule)) {
-      decode_ticks(tick_rule, double_of_price(min_tick, 0), fsec);
-   }
-   else {
-      double mintick = double_of_price(min_tick, 0);
-      fsec.low_tick = mintick;
-      fsec.high_tick = mintick;
-      fsec.price_break = 0;
-   }
-
-   fsec.set_leg_count(sec.legs().size());
-   for(size_t x = 0; x < fsec.leg_count(); ++x) {
-      const auto& leg = sec.legs()[x];
-      onixs::PRICE price;
-      if (leg.legPrice(price)) {
-         fsec.legs[x].leg_price = double_of_price(price, fsec.display_factor);
-      }
-      onixs::DecimalQty qty;
-      if (leg.legOptionDelta(qty)) {
-         fsec.legs[x].leg_delta = double_of_price(qty, 0);
-      }
-      fsec.legs[x].leg_quantity = leg.legRatioQty();
-      fsec.legs[x].leg_side = leg.legSide() == onixs::LegSide::BuySide ? api::SIDE_BUY : api::SIDE_SELL;
-      leg_exchange_ids.push_back(leg.legSecurityId());
-   }
-   add_security(fsec);
-   _index.add_spread_legs(fsec.numeric_id, leg_exchange_ids);
-}
-
 void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::DirectBook& book) {
    if (_booktype == CMEOptions::BOOKTYPE_DIRECT) {
       onBookUpdate(sec, book);
@@ -349,48 +129,28 @@ void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, con
 //
 // Trades
 
-void on_trade(SecurityData* data, const onixs::PRICE& price, int size, const onixs::Timestamp& onixs_date) {
+AggressorSide map_aggressor(onixs::AggressorSide::Enum agside) {
+   switch(agside) {
+      default:
+      case onixs::AggressorSide::NoAggressor:
+         return AGGRESSOR_SIDE_NONE;
+      case onixs::AggressorSide::Buy:
+         return AGGRESSOR_SIDE_BUY;
+      case onixs::AggressorSide::Sell:
+         return AGGRESSOR_SIDE_SELL;
+   }
+}
+
+void on_trade(SecurityData* data, const onixs::Decimal& price, int size, onixs::AggressorSide::Enum agside, const onixs::Timestamp& onixs_date) {
    auto ts = utils::TimeStamp::now();
    PriceSize ps = pricesize_of_price_size(price, data->handle.security().display_factor, size);
    auto date = ll_of_timestamp(onixs_date);
-   data->handle.add_trade(ts, ps, date, data->electronic_volume + size);
-}
-
-void MDP3Listener::onTrade(onixs::Handler&, const onixs::Security& sec, const onixs::RecoveryEntryArgs& arg) {
-   tracepoint (bts_trace,trace_order,(char*) "onTrade.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != NULL) {
-      int size = 0;
-      arg.entry().entrySize(size);
-      onixs::PRICE price;
-      arg.entry().entryPx(price);
-      on_trade(data, price, size, arg.message().transactTime());
-   }
-}
-void MDP3Listener::onTrade(onixs::Handler&, const onixs::Security& sec, const onixs::TradeSummaryArgs& arg) {
-   tracepoint (bts_trace,trace_order,(char*) "onTrade.summary", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != NULL) {
-      auto size = arg.entry().entrySize();
-      onixs::PRICE price = arg.entry().entryPx();
-      on_trade(data, price, size, arg.message().transactTime());
-   }
-}
-void MDP3Listener::onTrade(onixs::Handler&, const onixs::Security& sec, const onixs::TradeArgs& arg) {
-   tracepoint (bts_trace,trace_order,(char*) "onTrade.trade_args", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   if (arg.entry().updateAction() == onixs::UpdateAction::New) {
-      auto* data = _index.get_security(sec.id());
-      if (data != nullptr) {
-         int size = arg.entry().entrySize();
-         onixs::PRICE price = arg.entry().entryPx();
-         on_trade(data, price, size, arg.message().transactTime());
-      }
-   }
+   data->handle.add_trade(ts, ps, date, map_aggressor(agside), data->electronic_volume + size);
 }
 
 //
 // RFQ
-void MDP3Listener::onQuoteRequest(onixs::Handler&, const onixs::Security& sec, const onixs::QuoteRequestRelatedSymArgs& arg) {
+void MDP3Listener::onQuoteRequest(onixs::Handler&, const onixs::Security& sec, const onixs::QuoteRequest39RelatedSymArgs& arg) {
    tracepoint (bts_trace,trace_order,(char*) "onQuoteRequest", _latest_sequence, (uint16_t) _latest_sequence, 0);
    auto* data = _index.get_security(sec.id());
    if (data != nullptr) {
@@ -439,7 +199,7 @@ void MDP3Listener::onQuoteRequest(onixs::Handler&, const onixs::Security& sec, c
 
 //
 // Volume
-void MDP3Listener::onElectronicVolume(onixs::Handler&, const onixs::Security& sec, const onixs::VolumeArgs& args) {
+void MDP3Listener::onElectronicVolume(onixs::Handler&, const onixs::Security& sec, const onixs::Volume37Args& args) {
    tracepoint (bts_trace,trace_order,(char*) "onElectronicVolume.volume", _latest_sequence, (uint16_t) _latest_sequence, 0);
    auto* data = _index.get_security(sec.id());
    if (data != nullptr) {
@@ -449,162 +209,6 @@ void MDP3Listener::onElectronicVolume(onixs::Handler&, const onixs::Security& se
       data->electronic_volume = args.entry().entrySize();
    }
 }
-void MDP3Listener::onElectronicVolume(onixs::Handler&, const onixs::Security& sec, const onixs::RecoveryEntryArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onElectronicVolume.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data) {
-      int32_t size;
-      if (args.entry().entrySize(size)) {
-         data->electronic_volume = size;
-      }
-   }
-}
-
-//
-// Misc
-void MDP3Listener::onSettlementPrice(onixs::Handler&, const onixs::Security& sec, const onixs::DailyStatisticsArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onSettlementPrice.daily", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      onixs::PRICE price;
-      onixs::Timestamp time;
-      args.entry().entryPx(price);
-      args.entry().tradingReferenceDate(time);
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_settlement(
-         utils::TimeStamp::now(), 
-         double_of_price(price, display_factor),
-         date_of_timestamp(time));
-   }
-}
-void MDP3Listener::onSettlementPrice(onixs::Handler&, const onixs::Security& sec, const onixs::RecoveryEntryArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onSettlementPrice.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      onixs::PRICE price;
-      onixs::Timestamp time;
-      args.entry().entryPx(price);
-      args.entry().tradingReferenceDate(time);
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_settlement(
-         utils::TimeStamp::now(), 
-         double_of_price(price, display_factor),
-         date_of_timestamp(time));
-   }
-}
-void MDP3Listener::onOpenInterest(onixs::Handler&, const onixs::Security& sec, const onixs::DailyStatisticsArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onOpenInterest.daily", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      onixs::PRICE price;
-      args.entry().entryPx(price);
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_open_interest(utils::TimeStamp::now(), double_of_price(price, display_factor));
-   }
-}
-void MDP3Listener::onOpenInterest(onixs::Handler&, const onixs::Security& sec, const onixs::RecoveryEntryArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onOpenInterest.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      onixs::PRICE price;
-      args.entry().entryPx(price);
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_open_interest(utils::TimeStamp::now(), double_of_price(price, display_factor));
-   }
-}
-void MDP3Listener::onOpeningPrice(onixs::Handler&, const onixs::Security& sec, const onixs::SessionStatisticsArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onOpeningPrice.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      onixs::PRICE price = args.entry().entryPx();
-      onixs::Timestamp time = args.message().transactTime();
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_opening(
-         utils::TimeStamp::now(), 
-         double_of_price(price, display_factor),
-         date_of_timestamp(time));
-   }
-}
-void MDP3Listener::onOpeningPrice(onixs::Handler&, const onixs::Security& sec, const onixs::RecoveryEntryArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onOpeningPrice.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      onixs::PRICE price; 
-      args.entry().entryPx(price);
-      onixs::Timestamp time;
-      args.entry().tradingReferenceDate(time);
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_opening(
-         utils::TimeStamp::now(), 
-         double_of_price(price, display_factor),
-         date_of_timestamp(time));
-   }
-}
-void MDP3Listener::onHighTradePrice(onixs::Handler&, const onixs::Security& sec, const onixs::SessionStatisticsArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onHighTradePrice.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_session_high(
-         utils::TimeStamp::now(),
-         double_of_price(args.entry().entryPx(), display_factor));
-   }
-}
-void MDP3Listener::onHighTradePrice(onixs::Handler&, const onixs::Security& sec, const onixs::RecoveryEntryArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onHighTradePrice.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      auto display_factor = data->handle.security().display_factor;
-      onixs::PRICE price; 
-      args.entry().entryPx(price);
-      data->handle.update_session_high(
-         utils::TimeStamp::now(),
-         double_of_price(price, display_factor));
-   }
-}
-void MDP3Listener::onLowTradePrice(onixs::Handler&, const onixs::Security& sec, const onixs::SessionStatisticsArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onLowTradePrice.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_session_low(
-         utils::TimeStamp::now(),
-         double_of_price(args.entry().entryPx(), display_factor));
-   }
-}
-void MDP3Listener::onLowTradePrice(onixs::Handler&, const onixs::Security& sec, const onixs::RecoveryEntryArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onLowTradePrice.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      auto display_factor = data->handle.security().display_factor;
-      onixs::PRICE price; 
-      args.entry().entryPx(price);
-      data->handle.update_session_low(
-         utils::TimeStamp::now(),
-         double_of_price(price, display_factor));
-   }
-}
-void MDP3Listener::onHighestBid(onixs::Handler&, const onixs::Security& sec, const onixs::SessionStatisticsArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onHighestBid.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_session_high_bid(
-         utils::TimeStamp::now(),
-         double_of_price(args.entry().entryPx(), display_factor));
-   }
-}
-void MDP3Listener::onLowestOffer(onixs::Handler&, const onixs::Security& sec, const onixs::SessionStatisticsArgs& args) {
-   tracepoint (bts_trace,trace_order,(char*) "onLowestOffer.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
-   auto* data = _index.get_security(sec.id());
-   if (data != nullptr) {
-      auto display_factor = data->handle.security().display_factor;
-      data->handle.update_session_low_offer(
-         utils::TimeStamp::now(),
-         double_of_price(args.entry().entryPx(), display_factor));
-   }
-}
-
 
 //
 // Non-onixs calls

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h

@@ -5,7 +5,9 @@
 #include "CMESecurityIndex.h"
 #include "server/sparkd/tracing/bts_trace.h"
 
+#include "MDP3ListenerHelpers.h"
 #include "onixs.h"
+
 #include <tbb/concurrent_queue.h>
 #include <vector>
 #include <atomic>
@@ -14,11 +16,6 @@
 namespace bts {
 namespace md {
 
-inline PriceSize pricesize_of_price_size(const onixs::PRICE price, int8_t display_factor, Size size) {
-   PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
-   return result;
-}
-
 class MDP3Listener :
    public onixs::HandlerListener,
    public onixs::MarketDataListener,
@@ -54,9 +51,21 @@ public:
 
    //
    // Security discovery
-   void onDefinition(onixs::Handler&, const onixs::Security&, const onixs::InstrumentDefinitionFutureArgs&) override;
-   void onDefinition(onixs::Handler&, const onixs::Security&, const onixs::InstrumentDefinitionOptionArgs&) override;
-   void onDefinition(onixs::Handler&, const onixs::Security&, const onixs::InstrumentDefinitionSpreadArgs&) override;
+   template <typename InstrumentDefinitionFuture>
+   void onFuture(onixs::Handler&, const onixs::Security&, const InstrumentDefinitionFuture&);
+   template <typename InstrumentDefinition>
+   void onOption(onixs::Handler&, const onixs::Security&, const InstrumentDefinition&);
+   template <typename InstrumentDefinition>
+   void onSpread(onixs::Handler&, const onixs::Security&, const InstrumentDefinition&);
+
+   void onDefinition(onixs::Handler& h, const onixs::Security& s, const onixs::InstrumentDefinitionFuture27Args& def) override { onFuture(h, s, def); }
+   void onDefinition(onixs::Handler& h, const onixs::Security& s, const onixs::InstrumentDefinitionFuture54Args& def) override { onFuture(h, s, def); }
+
+   void onDefinition(onixs::Handler& h, const onixs::Security& s, const onixs::InstrumentDefinitionOption41Args& def) override { onOption(h, s, def); }
+   void onDefinition(onixs::Handler& h, const onixs::Security& s, const onixs::InstrumentDefinitionOption55Args& def) override { onOption(h, s, def); }
+
+   void onDefinition(onixs::Handler& h, const onixs::Security& s, const onixs::InstrumentDefinitionSpread29Args& def) override { onSpread(h, s, def); }
+   void onDefinition(onixs::Handler& h, const onixs::Security& s, const onixs::InstrumentDefinitionSpread56Args& def) override { onSpread(h, s, def); }
 
    //
    // Book
@@ -66,79 +75,74 @@ public:
    void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ConsolidatedBook&) override;
 
    template <class TBook>
-   void onBookUpdate(const onixs::Security& sec, const TBook& book) {
-      tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", _latest_sequence, (uint16_t) _latest_sequence, 0);
-
-      auto data = _index.get_security(sec.id());
-      if (data == nullptr) return;
-      bts_assert(data->id == (uint32_t)sec.id());
-      auto display_factor = data->handle.security().display_factor;
-
-      // Put the data into the cache. Flush at the end of the packet.
-      auto key = data->handle.key();
-      auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) {
-         return key == entry.handle.key();
-      });
-      bool found = find_iter != _book_updates.end();
-      if (!found) {
-         _book_updates.emplace_back(data->handle);
-      }
-      BookData& cache = found ? *find_iter : _book_updates.back();
-
-      // Overwrite all entries every time
-      size_t bids_size = book.bids().size();
-      size_t asks_size = book.offers().size();
-      if (!data->handle.has_depth()) {
-         bids_size = std::min(bids_size, size_t { 1 });
-         asks_size = std::min(asks_size, size_t { 1 });
-      }
-      for (size_t x = 0; x < bids_size; ++x) {
-         cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
-      }
-      for (size_t x = 0; x < asks_size; ++x) {
-         cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
-      }
-      // NaN out the remaining depth
-      for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
-         cache.bids[x] = PriceSize::NO_MARKET;
-      }
-      for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
-         cache.asks[x] = PriceSize::NO_MARKET;
-      }
-   }
+   void onBookUpdate(const onixs::Security& sec, const TBook& book);
 
    //
    // Trades
-   void onTrade(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
-   void onTrade(onixs::Handler&, const onixs::Security&, const onixs::TradeArgs&) override;
-   void onTrade(onixs::Handler&, const onixs::Security&, const onixs::TradeSummaryArgs&) override;
+   template <typename TradeRecovery>
+   void onTradeRecovery(const onixs::Security&, const TradeRecovery&);
+   void onTrade(onixs::Handler&, const onixs::Security& s, const onixs::Recovery52EntryArgs& arg) override { onTradeRecovery(s, arg); }
+   void onTrade(onixs::Handler&, const onixs::Security& s, const onixs::Recovery38EntryArgs& arg) override { onTradeRecovery(s, arg); }
+
+   template <typename Trade>
+   void onTradeSummary(const onixs::Security&, const Trade&);
+   void onTrade(onixs::Handler&, const onixs::Security& s, const onixs::TradeSummary48Args& arg) override { onTradeSummary(s, arg); }
+   void onTrade(onixs::Handler&, const onixs::Security& s, const onixs::TradeSummary42Args& arg) override { onTradeSummary(s, arg); }
 
    //
    // RFQ
-   void onQuoteRequest(onixs::Handler&, const onixs::Security&, const onixs::QuoteRequestRelatedSymArgs&) override;
+   void onQuoteRequest(onixs::Handler&, const onixs::Security&, const onixs::QuoteRequest39RelatedSymArgs&) override;
 
    //
    // Volume
-   void onElectronicVolume(onixs::Handler&, const onixs::Security&, const onixs::VolumeArgs&) override;
-   void onElectronicVolume(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
+   void onElectronicVolume(onixs::Handler&, const onixs::Security&, const onixs::Volume37Args&) override;
+   template <typename RecoveryEntry>
+   void onElectronicVolume(const onixs::Security&, const RecoveryEntry&);
+   void onElectronicVolume(onixs::Handler&, const onixs::Security& s, const onixs::Recovery52EntryArgs& arg) override { onElectronicVolume(s, arg); }
+   void onElectronicVolume(onixs::Handler&, const onixs::Security& s, const onixs::Recovery38EntryArgs& arg) override { onElectronicVolume(s, arg); }
 
    //
    // Misc
-   void onSettlementPrice(onixs::Handler&, const onixs::Security&, const onixs::DailyStatisticsArgs&) override;
-   void onSettlementPrice(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
-   void onOpenInterest(onixs::Handler&, const onixs::Security&, const onixs::DailyStatisticsArgs&) override;
-   void onOpenInterest(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
-   void onOpeningPrice(onixs::Handler&, const onixs::Security&, const onixs::SessionStatisticsArgs&) override;
-   void onOpeningPrice(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
-   void onHighTradePrice(onixs::Handler&, const onixs::Security&, const onixs::SessionStatisticsArgs&) override;
-   void onHighTradePrice(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
-   void onLowTradePrice(onixs::Handler&, const onixs::Security&, const onixs::SessionStatisticsArgs&) override;
-   void onLowTradePrice(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
+   template <typename StatisticsArgs>
+   void onSettlementPrice(const onixs::Security& sec, const StatisticsArgs& args);
+   void onSettlementPrice(onixs::Handler&, const onixs::Security& s, const onixs::DailyStatistics49Args& arg) override { onSettlementPrice(s, arg); }
+   void onSettlementPrice(onixs::Handler&, const onixs::Security& s, const onixs::DailyStatistics33Args& arg) override { onSettlementPrice(s, arg); }
+   void onSettlementPrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery52EntryArgs& arg) override { onSettlementPrice(s, arg); }
+   void onSettlementPrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery38EntryArgs& arg) override { onSettlementPrice(s, arg); }
 
-   void onHighestBid(onixs::Handler&, const onixs::Security&, const onixs::SessionStatisticsArgs&) override;
-   void onLowestOffer(onixs::Handler&, const onixs::Security&, const onixs::SessionStatisticsArgs&) override;
-   
+   template <typename InterestArgs>
+   void onOpenInterest(const onixs::Security& sec, const InterestArgs& args);
+   void onOpenInterest(onixs::Handler&, const onixs::Security& s, const onixs::DailyStatistics49Args& arg) override { onOpenInterest(s, arg); }
+   void onOpenInterest(onixs::Handler&, const onixs::Security& s, const onixs::DailyStatistics33Args& arg) override { onOpenInterest(s, arg); }
+   void onOpenInterest(onixs::Handler&, const onixs::Security& s, const onixs::Recovery52EntryArgs& arg) override { onOpenInterest(s, arg); }
+   void onOpenInterest(onixs::Handler&, const onixs::Security& s, const onixs::Recovery38EntryArgs& arg) override { onOpenInterest(s, arg); }
 
+   template <typename SessionArgs>
+   void onSessionOpeningPrice(const onixs::Security&, const SessionArgs&);
+   template <typename RecoveryArgs>
+   void onRecoveryOpeningPrice(const onixs::Security& sec, const RecoveryArgs& args);
+   void onOpeningPrice(onixs::Handler&, const onixs::Security& s, const onixs::SessionStatistics51Args& arg) override { onSessionOpeningPrice(s, arg); }
+   void onOpeningPrice(onixs::Handler&, const onixs::Security& s, const onixs::SessionStatistics35Args& arg) override { onSessionOpeningPrice(s, arg); }
+   void onOpeningPrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery52EntryArgs& arg) override { onRecoveryOpeningPrice(s, arg); }
+   void onOpeningPrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery38EntryArgs& arg) override { onRecoveryOpeningPrice(s, arg); }
+
+   template <typename SessionArgs>
+   void onSessionHighTradePrice(const onixs::Security&, const SessionArgs&);
+   template <typename RecoveryArgs>
+   void onRecoveryHighTradePrice(const onixs::Security&, const RecoveryArgs&);
+   void onHighTradePrice(onixs::Handler&, const onixs::Security& s, const onixs::SessionStatistics51Args& arg) override { onSessionHighTradePrice(s, arg); }
+   void onHighTradePrice(onixs::Handler&, const onixs::Security& s, const onixs::SessionStatistics35Args& arg) override { onSessionHighTradePrice(s, arg); }
+   void onHighTradePrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery52EntryArgs& arg) override { onRecoveryHighTradePrice(s, arg); }
+   void onHighTradePrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery38EntryArgs& arg) override { onRecoveryHighTradePrice(s, arg); }
+
+   template <typename SessionArgs>
+   void onSessionLowTradePrice(const onixs::Security&, const SessionArgs&);
+   template <typename RecoveryArgs>
+   void onRecoveryLowTradePrice(const onixs::Security&, const RecoveryArgs&);
+   void onLowTradePrice(onixs::Handler&, const onixs::Security& s, const onixs::SessionStatistics51Args& arg) override { onSessionLowTradePrice(s, arg); }
+   void onLowTradePrice(onixs::Handler&, const onixs::Security& s, const onixs::SessionStatistics35Args& arg) override { onSessionLowTradePrice(s, arg); }
+   void onLowTradePrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery52EntryArgs& arg) override { onRecoveryLowTradePrice(s, arg); }
+   void onLowTradePrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery38EntryArgs& arg) override { onRecoveryLowTradePrice(s, arg); }
 
    //
    // Discover securities
@@ -190,4 +194,345 @@ private:
 
 };
 
+//////////////////////////////////////////
+// Template Member Function Definitions
+//////////////////////////////////////////
+
+template <class Definition>
+void set_security_common(const Definition& def, FixedSecurity& fsec) {
+   fsec.numeric_id = def.securityId();
+   fsec.symbol(def.asset().items(), def.asset().size());
+   fsec.cfi_code(def.cfiCode().items(), def.cfiCode().size());
+   fsec.alt_symbol(def.securityGroup().items(), def.securityGroup().size());
+   fsec.security_desc(def.symbol().items(), def.symbol().size());
+   fsec.exchange = api::Venue::VENUE_GLBX;
+   // tickRule not present in futures
+   fsec.cme_market_segment_id = def.marketSegmentId();
+   fsec.display_factor = def.displayFactor().exponent() + floorlog10(def.displayFactor().mantissa());
+
+   def.mainFraction(fsec.cme_main_fraction);
+   def.subFraction(fsec.cme_sub_fraction);
+   def.priceDisplayFormat(fsec.cme_price_display_format);
+}
+
+
+template <typename InstrumentDefinition>
+void MDP3Listener::onFuture(onixs::Handler&, const onixs::Security& osec, const InstrumentDefinition& def) {
+   tracepoint (bts_trace,trace_order,(char*) "onDefinition.future", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   const auto& sec = def.message();
+   auto exchange_id = sec.securityId();
+   // LOG(md, Info, "onDefinition future {} {}",  
+   //    std::string(sec.asset().items(), sec.asset().size()),
+   //    std::string(sec.symbol().items(), sec.symbol().size()));
+
+   if (filter_security(exchange_id, sec) != ADD) return;
+   bts_assert(exchange_id == osec.id());
+
+   FixedFutureSecurity fsec;
+
+   set_security_common(sec, fsec);
+   fsec.price_break = 0;
+   // low/high tick are stored non-premultiplied and multiplied in conversion to api::Secutity
+   double min_tick = double_of_price(sec.minPriceIncrement(), 0);
+   fsec.low_tick = fsec.high_tick = min_tick;
+   onixs::Decimal min_tick_amount;
+   sec.minPriceIncrementAmount(min_tick_amount);
+   double min_tick_dbl = double_of_price(min_tick_amount, 0);
+   fsec.cash_per_point = min_tick_dbl / min_tick / pow10(fsec.display_factor);
+   auto expiration = date_of_maturity(sec);
+   fsec.expiration_year = expiration.year;
+   fsec.expiration_month = expiration.month;
+   fsec.expiration_day = expiration.day;
+   add_security(fsec);
+}
+
+template <typename InstrumentDefinition>
+void MDP3Listener::onOption(onixs::Handler&, const onixs::Security&, const InstrumentDefinition& def) {
+   tracepoint (bts_trace,trace_order,(char*) "onDefinition.option", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   const auto& sec = def.message();
+   auto exchange_id = sec.securityId();
+   // LOG(Info, MarketData) << "onDefinition option " << exchange_id;
+   if (filter_security(exchange_id, sec) != ADD) return;
+
+   FixedOptionSecurity fsec;
+   set_security_common(sec, fsec);
+   auto expiration = date_of_maturity(sec);
+   fsec.expiration_year = expiration.year;
+   fsec.expiration_month = expiration.month;
+   fsec.expiration_day = expiration.day;
+   
+   onixs::Decimal strike_price;
+   bts_assert(sec.strikePrice(strike_price));
+   fsec.strike = double_of_price(strike_price, fsec.display_factor);
+
+   onixs::Decimal min_tick, min_tick_amount;
+   sec.minPriceIncrement(min_tick);
+   sec.minPriceIncrementAmount(min_tick_amount);
+
+   int8_t tick_rule;
+   if (sec.tickRule(tick_rule)) {
+      decode_ticks(tick_rule, double_of_price(min_tick, 0), fsec);
+   }
+   fsec.cash_per_point = double_of_price(min_tick_amount, 0) / fsec.low_tick / pow10(fsec.display_factor);
+
+   fsec.call_put = sec.putOrCall() == onixs::PutOrCall::Call ? api::PUTORCALL_CALL : api::PUTORCALL_PUT;
+
+   add_security(fsec);
+}
+
+// min_tick specializations
+inline onixs::Decimal tick_rule(const onixs::InstrumentDefinitionSpread29& sec) {
+   return sec.minPriceIncrement();
+}
+inline onixs::Decimal tick_rule(const onixs::InstrumentDefinitionSpread56& sec) {
+   onixs::Decimal min_tick;
+   sec.minPriceIncrement(min_tick);
+   return min_tick;
+}
+
+template <typename InstrumentDefinition>
+void MDP3Listener::onSpread(onixs::Handler&, const onixs::Security&, const InstrumentDefinition& spread_args) {
+   tracepoint (bts_trace,trace_order,(char*) "onDefinition.spread", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   const auto& sec = spread_args.message();
+   auto exchange_id = sec.securityId();
+   // LOG(md, Info, "onDefinition spread {} {}",  
+   //    std::string(sec.asset().items(), sec.asset().size()),
+   //    std::string(sec.symbol().items(), sec.symbol().size()));
+   if (filter_security(exchange_id, sec) != ADD) return;
+   
+   // Filter out any already expired spreads
+   auto expiration = date_of_maturity(sec);
+   if (expiration < datetime::today()) return;
+
+   FixedSpreadSecurityL fsec;
+   set_security_common(sec, fsec);
+   std::vector<uint32_t> leg_exchange_ids;
+   leg_exchange_ids.reserve(sec.legs().size());
+
+   onixs::Decimal min_tick = tick_rule(sec);
+   
+   int8_t tick_rule;
+   if (sec.tickRule(tick_rule)) {
+      decode_ticks(tick_rule, double_of_price(min_tick, 0), fsec);
+   }
+   else {
+      double mintick = double_of_price(min_tick, 0);
+      fsec.low_tick = mintick;
+      fsec.high_tick = mintick;
+      fsec.price_break = 0;
+   }
+
+   fsec.set_leg_count(sec.legs().size());
+   for(size_t x = 0; x < fsec.leg_count(); ++x) {
+      const auto& leg = sec.legs()[x];
+      onixs::Decimal price;
+      if (leg.legPrice(price)) {
+         fsec.legs[x].leg_price = double_of_price(price, fsec.display_factor);
+      }
+      onixs::Decimal qty;
+      if (leg.legOptionDelta(qty)) {
+         fsec.legs[x].leg_delta = double_of_price(qty, 0);
+      }
+      fsec.legs[x].leg_quantity = leg.legRatioQty();
+      fsec.legs[x].leg_side = leg.legSide() == onixs::LegSide::BuySide ? api::SIDE_BUY : api::SIDE_SELL;
+      leg_exchange_ids.push_back(leg.legSecurityId());
+   }
+   add_security(fsec);
+   _index.add_spread_legs(fsec.numeric_id, leg_exchange_ids);
+}
+
+
+template <class TBook>
+void MDP3Listener::onBookUpdate(const onixs::Security& sec, const TBook& book) {
+   tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", _latest_sequence, (uint16_t) _latest_sequence, 0);
+
+   auto data = _index.get_security(sec.id());
+   if (data == nullptr) return;
+   bts_assert(data->id == (uint32_t)sec.id());
+   auto display_factor = data->handle.security().display_factor;
+
+   // Put the data into the cache. Flush at the end of the packet.
+   auto key = data->handle.key();
+   auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) {
+      return key == entry.handle.key();
+   });
+   bool found = find_iter != _book_updates.end();
+   if (!found) {
+      _book_updates.emplace_back(data->handle);
+   }
+   BookData& cache = found ? *find_iter : _book_updates.back();
+
+   // Overwrite all entries every time
+   size_t bids_size = book.bids().size();
+   size_t asks_size = book.offers().size();
+   if (!data->handle.has_depth()) {
+      bids_size = std::min(bids_size, size_t { 1 });
+      asks_size = std::min(asks_size, size_t { 1 });
+   }
+   for (size_t x = 0; x < bids_size; ++x) {
+      cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
+   }
+   for (size_t x = 0; x < asks_size; ++x) {
+      cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
+   }
+   // NaN out the remaining depth
+   for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
+      cache.bids[x] = PriceSize::NO_MARKET;
+   }
+   for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
+      cache.asks[x] = PriceSize::NO_MARKET;
+   }
+}
+
+void on_trade(SecurityData* data, const onixs::Decimal& price, int size, onixs::AggressorSide::Enum agside, const onixs::Timestamp& onixs_date);
+
+template <typename TradeRecovery>
+void MDP3Listener::onTradeRecovery(const onixs::Security& sec, const TradeRecovery& arg) {
+   tracepoint (bts_trace,trace_order,(char*) "onTrade.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != NULL) {
+      int size = 0;
+      arg.entry().entrySize(size);
+      onixs::Decimal price;
+      arg.entry().entryPx(price);
+      on_trade(data, price, size, onixs::AggressorSide::NoAggressor, arg.message().transactTime());
+   }
+}
+
+template <typename TradeSummary>
+void MDP3Listener::onTradeSummary(const onixs::Security& sec, const TradeSummary& arg) {
+   tracepoint (bts_trace,trace_order,(char*) "onTrade.summary", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != NULL) {
+      auto size = arg.entry().entrySize();
+      onixs::Decimal price = arg.entry().entryPx();
+      onixs::AggressorSide::Enum agside = onixs::AggressorSide::NoAggressor;
+      arg.entry().aggressorSide(agside);
+      on_trade(data, price, size, agside, arg.message().transactTime());
+   }
+}
+
+
+template <typename RecoveryEntryArgs>
+void MDP3Listener::onElectronicVolume(const onixs::Security& sec, const RecoveryEntryArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onElectronicVolume.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data) {
+      int32_t size;
+      if (args.entry().entrySize(size)) {
+         data->electronic_volume = size;
+      }
+   }
+}
+
+template <typename StatisticsArgs>
+void MDP3Listener::onSettlementPrice(const onixs::Security& sec, const StatisticsArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onSettlementPrice.daily", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != nullptr) {
+      onixs::Decimal price;
+      onixs::Timestamp time;
+      args.entry().entryPx(price);
+      args.entry().tradingReferenceDate(time);
+      auto display_factor = data->handle.security().display_factor;
+      data->handle.update_settlement(
+         utils::TimeStamp::now(), 
+         double_of_price(price, display_factor),
+         date_of_timestamp(time));
+   }
+}
+
+template <typename InterestArgs>
+void MDP3Listener::onOpenInterest(const onixs::Security& sec, const InterestArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onOpenInterest.daily", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != nullptr) {
+      onixs::Decimal price;
+      args.entry().entryPx(price);
+      auto display_factor = data->handle.security().display_factor;
+      data->handle.update_open_interest(utils::TimeStamp::now(), double_of_price(price, display_factor));
+   }
+}
+
+template <typename SessionArgs>
+void MDP3Listener::onSessionOpeningPrice(const onixs::Security& sec, const SessionArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onOpeningPrice.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != nullptr) {
+      onixs::Decimal price = args.entry().entryPx();
+      onixs::Timestamp time = args.message().transactTime();
+      auto display_factor = data->handle.security().display_factor;
+      data->handle.update_opening(
+         utils::TimeStamp::now(), 
+         double_of_price(price, display_factor),
+         date_of_timestamp(time));
+   }
+}
+template <typename RecoveryArgs>
+void MDP3Listener::onRecoveryOpeningPrice(const onixs::Security& sec, const RecoveryArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onOpeningPrice.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != nullptr) {
+      onixs::Decimal price; 
+      args.entry().entryPx(price);
+      onixs::Timestamp time;
+      args.entry().tradingReferenceDate(time);
+      auto display_factor = data->handle.security().display_factor;
+      data->handle.update_opening(
+         utils::TimeStamp::now(), 
+         double_of_price(price, display_factor),
+         date_of_timestamp(time));
+   }
+}
+
+template <typename SessionArgs>
+void MDP3Listener::onSessionHighTradePrice(const onixs::Security& sec, const SessionArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onHighTradePrice.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != nullptr) {
+      auto display_factor = data->handle.security().display_factor;
+      data->handle.update_session_high(
+         utils::TimeStamp::now(),
+         double_of_price(args.entry().entryPx(), display_factor));
+   }
+}
+template <typename RecoveryArgs>
+void MDP3Listener::onRecoveryHighTradePrice(const onixs::Security& sec, const RecoveryArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onHighTradePrice.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != nullptr) {
+      auto display_factor = data->handle.security().display_factor;
+      onixs::Decimal price; 
+      args.entry().entryPx(price);
+      data->handle.update_session_high(
+         utils::TimeStamp::now(),
+         double_of_price(price, display_factor));
+   }
+}
+template <typename SessionArgs>
+void MDP3Listener::onSessionLowTradePrice(const onixs::Security& sec, const SessionArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onLowTradePrice.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != nullptr) {
+      auto display_factor = data->handle.security().display_factor;
+      data->handle.update_session_low(
+         utils::TimeStamp::now(),
+         double_of_price(args.entry().entryPx(), display_factor));
+   }
+}
+template <typename RecoveryArgs>
+void MDP3Listener::onRecoveryLowTradePrice(const onixs::Security& sec, const RecoveryArgs& args) {
+   tracepoint (bts_trace,trace_order,(char*) "onLowTradePrice.recovery", _latest_sequence, (uint16_t) _latest_sequence, 0);
+   auto* data = _index.get_security(sec.id());
+   if (data != nullptr) {
+      auto display_factor = data->handle.security().display_factor;
+      onixs::Decimal price; 
+      args.entry().entryPx(price);
+      data->handle.update_session_low(
+         utils::TimeStamp::now(),
+         double_of_price(price, display_factor));
+   }
+}
+
+
 }}//namespace

src/server/sparkd/md/platforms/mdp3_onixs/MDP3ListenerHelpers.h

@@ -0,0 +1,112 @@
+#include "lib/utils/bts_assert.h"
+#include "lib/md/coconut2.h"
+
+#include "onixs.h"
+
+#include <vector>
+#include <atomic>
+#include <array>  
+
+namespace bts {
+namespace md {
+
+inline PriceSize pricesize_of_price_size(const onixs::Decimal price, int8_t display_factor, Size size) {
+   PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
+   return result;
+}
+
+template <class Price>
+double double_of_price(const Price& price, int8_t display_factor) {
+   // order of multiplication is important because of floating point error
+   return (price.mantissa() * pow10(price.exponent())) * pow10(display_factor);
+}
+
+inline void decode_ticks(int vtt, double mintick, FixedSecurity& dest) {
+   switch(vtt) {
+      default:
+      case 0:
+         dest.low_tick = mintick;
+         dest.high_tick = mintick;
+         dest.price_break = 0;
+         break;
+      case 1:
+         dest.price_break = 500;
+         dest.low_tick = 5;
+         dest.high_tick = 10;
+         break;
+      case 2:
+         dest.price_break = 5;
+         dest.low_tick = 0.5;
+         dest.high_tick = 1;
+         break;
+      case 3:
+         dest.price_break = 10;
+         dest.low_tick = 1;
+         dest.high_tick = 2;
+         break;
+      case 4:
+         dest.price_break = 500;
+         dest.low_tick = 5;
+         dest.high_tick = 25;
+         break;
+      case 10:
+         dest.price_break = 300;
+         dest.low_tick = 5;
+         dest.high_tick = 25;
+         break;
+      case 11:
+         dest.price_break = 300;
+         dest.low_tick = 5;
+         dest.high_tick = 10;
+         break;
+      case 12:
+         dest.price_break = 5;
+         dest.low_tick = 0.25;
+         dest.high_tick = 0.5;
+         break;
+   }
+}
+
+inline int floorlog10(int64_t m) {
+   bts_assert(m >= 0);
+   if (m < 10) return 0;
+   if (m < 100) return 1;
+   if (m < 1000) return 2;
+   if (m < 10000) return 3;
+   if (m < 100000) return 4;
+   if (m < 1000000) return 5;
+   if (m < 10000000) return 6;
+   if (m < 100000000) return 7;
+   if (m < 1000000000) return 8;
+   if (m < 10000000000) return 9;
+   if (m < 100000000000) return 10;
+   if (m < 1000000000000) return 11;
+   if (m < 10000000000000) return 12;
+   if (m < 100000000000000) return 13;
+   if (m < 1000000000000000) return 14;
+   if (m < 10000000000000000) return 15;
+   if (m < 100000000000000000) return 16;
+   if (m < 1000000000000000000) return 17;
+   return 19;
+}
+
+inline datetime::LLDateTime ll_of_timestamp(const onixs::Timestamp& ts) {
+   return datetime::ll_from_parts(ts.year(), (int)ts.month(), ts.day(), ts.hour(), ts.minute(), ts.second(), ts.millisecond());
+}
+
+inline datetime::Date date_of_timestamp(const onixs::Timestamp& ts) {
+   return datetime::Date(ts.year(), (int)ts.month(), ts.day());
+}
+
+template <class Definition>
+datetime::Date date_of_maturity(const Definition& def) {
+   for (const auto& event : def.events()) {
+      if (event.eventType() == onixs::EventType::LastEligibleTradeDate) {
+         return date_of_timestamp(event.eventTime());
+      }
+   }
+   return datetime::Date();
+}
+
+
+}}//namespace

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp

@@ -299,6 +299,12 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(const string& channel_id) {
       loggerSettings.filename(log_path() + "/log_" + channel_id);
       loggerSettings.severityLevel((onixs::LogSeverity::Enum)options.onixs_log_mode());
       logger = new onixs::FileLogger(loggerSettings);
+      if (options.onixs_log_mode() == CMEOptions::Debug) {
+         // trace all the things
+         settings.debugLogging().traceBooks(true);
+         settings.debugLogging().traceFeeds(true);
+         settings.debugLogging().traceMessages(onixs::MessageTracing::FixFormat);
+      }
    }
 
 

src/server/sparkd/orderman/actors/ExecutionGateway.cpp

@@ -203,11 +203,10 @@ void ExecutionGateway::on_exe_report(
       data->side = api::Side(msg.get<fix::Side>());
       if (msg.has<fix::TimeInForce>())
          data->time_in_force =
-            api::orderman::TimeInForce(msg.get<fix::TimeInForce>());
+             api::orderman::TimeInForce(msg.get<fix::TimeInForce>());
       else
          data->time_in_force = api::orderman::TIME_IN_FORCE_UNDEFINED;
-         
-         
+
       data->order_type =
           api::orderman::NativeOrderType(msg.get<fix::OrdType>());
       data->order_status =
@@ -234,18 +233,19 @@ void ExecutionGateway::on_exe_report(
          data->quantity = Quantity(msg.get<fix::OrderQty>());
       else
          data->quantity = Quantity(0);
-         
+
       data->filled_quantity = Quantity(msg.get<fix::CumQty>());
       data->last_shares = msg.has<fix::LastShares>()
                               ? Quantity(msg.get<fix::LastShares>())
                               : Quantity(0);
+      auto last_px_sec = data->leg_security? data->leg_security:sec;
       data->last_price = msg.has<fix::LastPx>()
                              ? securities::decode_exchange_price(
-                                   *sec, Price(msg.get<fix::LastPx>()))
+                                   *last_px_sec, Price(msg.get<fix::LastPx>()))
                              : Price(0);
       data->avg_price = msg.has<fix::AvgPx>()
                             ? securities::decode_exchange_price(
-                                  *sec, Price(msg.get<fix::AvgPx>()))
+                               *last_px_sec, Price(msg.get<fix::AvgPx>()))
                             : Price(0);
       if (msg.has<fix::ExecID>())
          str_to(msg.get<fix::ExecID>(), &data->exchange_fill_id);

src/server/sparkd/orderman/limiter/LimiterUtils.h

@@ -21,14 +21,14 @@ public:
          net_pos_new = net_pos + quantity.value();
       else if(side == api::Side::SIDE_SELL)
          net_pos_new = net_pos - quantity.value();
-      if (net_pos_new > long_pos_limit){
+      if (net_pos_new > long_pos_limit && side==api::Side::SIDE_BUY){
          out_res.ok = false;
          if (is_security_level)
             out_res.error_msg = fmt::format("Security Long Position (Max {} Current {})", long_pos_limit, net_pos);
          else
             out_res.error_msg = fmt::format("Product Long Position (Max {} Current {})", long_pos_limit, net_pos);
       }
-      else if (net_pos_new < short_pos_limit){
+      else if (net_pos_new < short_pos_limit && side==api::Side::SIDE_SELL){
          out_res.ok = false;
          if (is_security_level)
             out_res.error_msg = fmt::format("Security Short Position (Max {} Current {})", short_pos_limit, net_pos);

src/server/sparkd/orderman/orders/IcebergOrder.cpp

@@ -271,9 +271,17 @@ void IcebergOrder::on_child_order_update(OrderStateUpdateEvent const &ev) {
             break;
          }
 
+         case ORDER_STATUS_CANCELED:
+            LOG(OrderMan, Info,
+                "Unexpected cancel of child {} in Iceberg Order {}. Canceling "
+                "parent",
+                ev.key.value(),
+                key().value());
+            transition(ORDER_STATUS_CANCELED);
+            break;
+
          case ORDER_STATUS_DISCONNECTED:
          case ORDER_STATUS_REJECTED_BY_SPARK:
-         case ORDER_STATUS_CANCELED:
          case ORDER_STATUS_ERROR:
             me->child.clear();
             transition(ORDER_STATUS_ERROR, "Unexpected child order state");

src/server/sparkd/orderman/orders/NativeOrder.cpp

@@ -588,6 +588,12 @@ void NativeOrder::on_exe_report(const NewExecutionReportEvent &er) {
    auto &state = me->state;
    auto &native_state = get_native_state(state);
 
+   if (er.data->leg_security) {
+      LOG(OrderMan, Diag,
+          "Ignoring leg execution report for order {}", key().value());
+      return;
+   }
+
    if (native_state.last_msg_seq_num() > er.data->msg_seq_num) {
       LOG(OrderMan, Warn,
           "Ignoring already seen execution report {} in order {}",

src/server/sparkd/orderman/orders/TriggerOrder.cpp

@@ -251,6 +251,12 @@ bool TriggerOrder::_on_init_new(const api::orderman::OrderDefinition& order_def,
    const auto* sec_ptr = pre_os.sec_db.resolve(SecurityKey(order_def.security_key()));
    me->tick_size=std::min(sec_ptr->contract_spec().low_tick(),
                           sec_ptr->contract_spec().high_tick());
+   LOG(OrderMan, Info, "TriggerOrder::_on_init_new key:{} state:{} "
+                        "trigger_price_type:{} child_order_price:{}",   
+                        key(),_state,
+                        trigger_order_def.trigger_condition().price_type(),
+                        trigger_order_def.trigger_action().price());
+
    return true;
 }
 void TriggerOrder::_report_state(){
@@ -326,6 +332,12 @@ void TriggerOrder::_observe_market_once(){
       if(obs_md != nullptr && obs_md->size()>0){
          me->market_observed_once=true;
          me->trigger_above_market=_trigger_price()>=obs_md->price();
+         LOG(OrderMan, Info, "TriggerOrder::_observe_market_once "
+                             "key:{} state:{} "
+                             "md_p:{} md_q:{} trigger_above_md:{}",
+                              key(),_state,
+                              obs_md->price(), obs_md->size(),
+                              me->trigger_above_market);
       }
    }
 }

src/server/sparkd/orderman/orders/spreader_orders/SpreaderOrder.cpp

@@ -110,8 +110,8 @@ void SpreaderOrder::_on_child_order_update(const OrderStateUpdateEvent& ord_ev){
 #endif
    
    Quantity filled_qty = Quantity (ord_ev.state.filled_quantity());
-   Price last_price = Price(ord_ev.state.last_price());
-   LOG(OrderMan, Diag, "SpreaderOrder::_on_child_order_update key: {} sec_key: {} order_key: {} status: {} filled_qty: {} last_price: {} msg: {}", key(), sec_key, order_key, status, filled_qty, last_price, ord_ev.state.last_msg ());
+   Price last_price = Price(ord_ev.state.last_price()),
+         order_price=Price(ord_ev.state.price());
    if (!SpreaderUtils::order_acknowledged(status)){
       LOG(OrderMan, Diag, "SpreaderOrder::_on_child_order_update, unack'ed callback for key: {} sec_key: {} order_key: {} status: {} ignoring", key(), sec_key, order_key, status);
       bts_assert(filled_qty.value() == 0);
@@ -121,8 +121,14 @@ void SpreaderOrder::_on_child_order_update(const OrderStateUpdateEvent& ord_ev){
    if (leg_data_ptr != nullptr){
       SpreaderLegData& leg_data = *leg_data_ptr;
       ChildOrderError order_error = _get_order_error (ord_ev, leg_data.quoter().expecting_cancel_update());
-      LOG(OrderMan, Info, "SpreaderOrder::_on_child_order_update quoter order key: {} sec_key: {} order_key: {} status: {} filled_qty: {} last_price: {} msg: {} child_order_error: {}",
-          key(), sec_key, order_key, status, filled_qty, last_price, ord_ev.state.last_msg (), static_cast<int>(order_error));
+      LOG(OrderMan, Info, "SpreaderOrder::_on_child_order_update quoter order "
+          "key:{} sec_key:{} state:{} order_key:{} status:{} "
+          "filled_qty:{} price:{} last_price:{} "
+          "msg:{} child_order_error:{}",
+          key(), sec_key,
+          static_cast<int>(_state_worker->state()), order_key, status,
+          filled_qty, order_price, last_price,
+          ord_ev.state.last_msg(), static_cast<int>(order_error));
       leg_data.order_update_message().on_order_update(ord_ev);
       if (order_error == ChildOrderError::NONE){
          if (leg_data.order_update_message().last_update_filled_quantity().value() > 0)
@@ -138,8 +144,14 @@ void SpreaderOrder::_on_child_order_update(const OrderStateUpdateEvent& ord_ev){
       SingleFillHedger* hedger = _bookkeeping_data().find_hedger_from_orderkey(order_key);
       if (hedger != nullptr){
          ChildOrderError order_error = _get_order_error (ord_ev, hedger->expecting_cancel_update());
-         LOG(OrderMan, Info, "SpreaderOrder::_on_child_order_update hedger order: key: {} sec_key: {} order_key: {} status: {} filled_qty: {} last_price: {} msg: {} child_order_error: {}",
-             key(), sec_key, order_key, status, filled_qty, last_price, ord_ev.state.last_msg (), static_cast<int>(order_error));
+         LOG(OrderMan, Info, "SpreaderOrder::_on_child_order_update hedger "
+             "key:{} sec_key:{} state:{} order_key:{} status:{} "
+             "filled_qty:{} price:{} last_price:{} " 
+             "msg:{} child_order_error:{}",
+             key(), sec_key, 
+             static_cast<int>(_state_worker->state()), order_key, status,
+             filled_qty, order_price, last_price,
+             ord_ev.state.last_msg (), static_cast<int>(order_error));
          hedger->order_state().on_order_update(ord_ev);
          if (order_error == ChildOrderError::NONE)
             _state_worker->on_hedger_order_update (*hedger);
@@ -273,6 +285,7 @@ void SpreaderOrder::_send_hedge(size_t hedge_leg_index,
 void SpreaderOrder::_compute_hedge_price_quantity(size_t filled_leg_index,
                                                   Price fill_price,
                                                   double fill_quantity,
+                                                  Price quote_order_price,
                                                   size_t hedge_leg_index,
                                                   Price& hedge_price,
                                                   double& hedge_quantity){
@@ -288,7 +301,7 @@ void SpreaderOrder::_compute_hedge_price_quantity(size_t filled_leg_index,
                                                       computed_hedge_price,
                                                       hedge_quantity);
    else{
-      const QuoteLegDriver& driver_data = filled_leg_data.quoter().driver_data(fill_price, hedge_leg_index);
+      const QuoteLegDriver& driver_data = filled_leg_data.quoter().driver_data(quote_order_price, hedge_leg_index);
       SpreaderUtils::compute_hedge_price_size_n_legs (filled_leg_data.leg_property_data(),
                                                       fill_price,
                                                       fill_quantity,

src/server/sparkd/orderman/orders/spreader_orders/SpreaderOrder.h

@@ -173,6 +173,7 @@ private:
    void _compute_hedge_price_quantity(size_t filled_leg_index,
                                       Price filled_price,
                                       double filled_quantity,
+                                      Price order_price,
                                       size_t hedged_leg_index,
                                       Price& hedge_price,
                                       double& hedge_quantity);

src/server/sparkd/orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp

@@ -19,6 +19,7 @@ _status(src._status),
 _filled_price(src._filled_price),
 _filled_quantity(src._filled_quantity),
 _last_filled_price(src._last_filled_price),
+_last_reported_order_price(src._last_reported_order_price),
 _last_filled_quantity(src._last_filled_quantity)
 { }
 SpreaderChildOrderState::SpreaderChildOrderState (bts::orderman::ChildOrder& child_order,
@@ -45,6 +46,7 @@ void SpreaderChildOrderState::_init_running_values(){
    _filled_price = Price(0);
    _filled_quantity = Quantity (0);
    _last_filled_price = Price(0);
+   _last_reported_order_price=Price(0);
    _last_filled_quantity = Quantity(0);
    _out_price = Price(0);
    _out_quantity = Quantity(0);

src/server/sparkd/orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.h

@@ -29,6 +29,7 @@ private:
    Price _filled_price;
    Quantity _filled_quantity;
    Price _last_filled_price;
+   Price _last_reported_order_price;
    Quantity _last_filled_quantity;
    Price _out_price;
    Quantity _out_quantity;
@@ -54,9 +55,12 @@ public:
    }
    // if the last fill was a status change
    // this will be zero
-   Price last_update_filled_price () const {
+   Price last_update_filled_price() const {
       return _last_filled_price;
    }
+   Price last_reported_order_price() const{
+      return _last_reported_order_price;
+   }
    // if the last fill was a status change
    // this will be zero
    Quantity last_update_filled_quantity () const {
@@ -109,6 +113,7 @@ public:
       _status = e.state.status();
       _last_filled_quantity = Quantity(e.state.filled_quantity() - _filled_quantity.value());
       _last_filled_price = Price(e.state.last_price());
+      _last_reported_order_price=Price(e.state.price());
       SpreaderUtils::compute_average_price(_filled_price,
                                            _filled_quantity,
                                            _last_filled_price,

src/server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp

@@ -43,7 +43,12 @@ const QuoteDriverSet* QuoteDriverSetHistory::_find_driver_set(Price quote_price)
                                                _tick_size))
          if (rv==nullptr||d.quote_seq_no()>rv->quote_seq_no())
             rv=&d;
-   return rv!=nullptr ? rv:_latest();
+   if (rv==nullptr){
+      LOG(OrderMan, Warn, "QuoteDriverSetHistory::_find_driver_set "
+                          "could not find driver:{}",quote_price);
+      rv=_latest();
+   }
+   return rv;
 }
 const QuoteLegDriver* QuoteDriverSetHistory::find_driver(Price quote_price,
                                                           size_t leg_index) const{

src/server/sparkd/orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp

@@ -92,11 +92,7 @@ void SpreaderWorkingStateWorker::on_quoter_order_fill(SpreaderLegData& leg_data,
    // ii. not send a hedge on leg_2 since its already filled
    // however doing the above would mess up the leg ratio's so we are preffering
    // getting overfilled
-   SpreaderChildOrderState& order_update_message = leg_data.order_update_message();
-   LOG(OrderMan, Info, "SpreaderWorkingStateWorker::on_quoter_order_fill order: key: {} order_key: {} status: {} filled_qty: {} last_price: {}",
-       _spreader->key(), order_update_message.order_key(), order_update_message.status(),
-       order_update_message.last_update_filled_quantity(),
-       order_update_message.last_update_filled_price());
+   SpreaderChildOrderState& order_update_message = leg_data.order_update_message();   
    bts_assert (order_update_message.last_update_filled_quantity().value() > 0);
    for (size_t q_idx = 0; q_idx < _spreader->_bookkeeping_data().n_legs(); q_idx++){      
       if (q_idx != leg_data.leg_property_data().leg_index()){
@@ -106,6 +102,7 @@ void SpreaderWorkingStateWorker::on_quoter_order_fill(SpreaderLegData& leg_data,
          _spreader->_compute_hedge_price_quantity(leg_data.leg_property_data().leg_index(),
                                                   order_update_message.last_update_filled_price(),
                                                   order_update_message.last_update_filled_quantity().value(),
+                                                  order_update_message.last_reported_order_price(),
                                                   q_idx,
                                                   hedge_price,
                                                   hedge_qty);

src/server/sparkd/support/ErrorReportService.cpp

@@ -73,7 +73,7 @@ void ErrorReportServer::send_report(
       }
 
       // log error for visibility
-      LOG(Gen, Error, "Support request from {} - {} \n{}", request.user(), request.contact(), request.user_description());
+      LOG(Gen, Error, "Support request from {} - {} {}", request.user(), request.contact(), request.user_description());
 
       ErrorResponse r;
       reply(r);

src/web/ui/sass/main.scss

@@ -139,7 +139,7 @@ body {
 .is-main-control-bar {
   margin:2rem 3rem 0 0;
   overflow: hidden;
-  text-align: center;
+  //text-align: center;
   .button {
     margin-left: 1em;
   }

src/web/ui/src/Accounts/State.fs

@@ -24,8 +24,8 @@ let init (context:ServiceContext.ServiceContext) =
     mdRelayExpNameData = Array.empty<ExpirationNameList>
     scratch = emptyAccount
     previousScratch = emptyAccount
-    AllValue = ""
-    AllValueDisplay = ""
+    AllValue = "*"
+    AllValueDisplay = "All"
     OptionsMenuOpen = ""
     ConfirmModal = "", (Msg.SelectTab General)
     SelectedClipLimits = []
@@ -51,16 +51,21 @@ let applyUpdate (account:BTSAccount) = function
   | IsActive b -> { account with isActive = b }
 
 let applyCMEUpdate (data:CMEAccountData) = function
+ | CMEExchangeField.CTI v -> { data with cti = v }
+ | CMEExchangeField.Origin v -> { data with origin = v }
+ | CMEExchangeField.TakeupMember v -> { data with takeupMemberCode = v }
+ | CMEExchangeField.ClearingAccount v -> { data with clearingAccount = v }
+ | CMEExchangeField.DirectElectronicAccess v -> { data with directElectronicAccess = v }
+ | CMEExchangeField.TradingCapacity v -> { data with tradingCapacity = v }
+ | CMEExchangeField.LiquidityProvision v ->  { data with liquidityProvision = v }
+ | CMEExchangeField.CommodityDerivativeIndicator v -> { data with commodityDerivativeIndicator = v }
+ | CMEExchangeField.MiscTagMapping mp -> { data with miscTags = mp }
+
+let applySIMUpdate (data:SIMAccountData) = function
  | CTI v -> { data with cti = v }
  | Origin v -> { data with origin = v }
- | TakeupMember v -> { data with takeupMemberCode = v }
  | ClearingAccount v -> { data with clearingAccount = v }
- | DirectElectronicAccess v -> { data with directElectronicAccess = v }
- | TradingCapacity v -> { data with tradingCapacity = v }
- | LiquidityProvision v ->  { data with liquidityProvision = v }
- | CommodityDerivativeIndicator v -> { data with commodityDerivativeIndicator = v }
- | MiscTagMapping mp -> { data with miscTags = mp }
-
+ | _ -> data
 
 let saveAccount (admin:IAdminService) (account:BTSAccount) = promise {
   let! result = admin.update_account { completeData = account }
@@ -250,10 +255,14 @@ let update msg (model:Model) : Model * Cmd<Msg> =
     { model with scratch = { model.scratch with connections = { model.scratch.connections with exchangeConnectionNames = newCons }}; ConfirmModal = ("", Msg.SelectTab model.tab)}, Cmd.Empty
   //
   // Exchange data
-  | UpdateCME field ->
+  | UpdateCME (field:CMEExchangeField) ->
     let newCme = applyCMEUpdate model.scratch.exchangeAccountData.cmeAccountData field
     let scratch = { model.scratch with exchangeAccountData = { cmeAccountData = newCme; iceAccountData = model.scratch.exchangeAccountData.iceAccountData; simAccountData = model.scratch.exchangeAccountData.simAccountData } }
     { model with scratch = scratch }, Cmd.Empty
+  | UpdateSIM field ->
+    let newSim = applySIMUpdate model.scratch.exchangeAccountData.simAccountData field
+    let scratch = { model.scratch with exchangeAccountData = { cmeAccountData = model.scratch.exchangeAccountData.cmeAccountData ; iceAccountData = model.scratch.exchangeAccountData.iceAccountData; simAccountData = newSim } }
+    { model with scratch = scratch }, Cmd.Empty
   | RequestProducts ->
       model, Cmd.ofExistingPromise (requestProducts model.mdRelay) id (box>>ErrorOccured)
   | UpdateProducts data ->

src/web/ui/src/Accounts/Types.fs

@@ -35,6 +35,11 @@ type CMEExchangeField =
   | CommodityDerivativeIndicator of string
   | MiscTagMapping of JSMap<string>
 
+type SIMExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type Msg =
   | UpdateAdminData of AdminData
   | NoAdminData of string
@@ -64,6 +69,7 @@ type Msg =
   | DeleteSecurityLimit of int
   // ExchangeData
   | UpdateCME of CMEExchangeField
+  | UpdateSIM of SIMExchangeField
   | RequestProducts
   | UpdateProducts of Product[]
   | RequestExpirationNames

src/web/ui/src/Accounts/View.fs

@@ -69,10 +69,12 @@ let GetSymbolArray (model:Model) =
   |> Array.distinct
 
 let GetFilteredSymbolArray (model:Model) (remove:string []) =
-  Array.map<bts.api.Product, string> (fun a -> a.symbol ) model.mdRelayProductData
-  |> Array.filter (fun elm -> (not <| Array.exists (fun e -> elm = e) remove))
-  |> Array.distinct
-
+  let a = (Array.map<bts.api.Product, string> (fun a -> a.symbol ) model.mdRelayProductData
+          |> Array.filter (fun elm -> (not <| Array.exists (fun e -> elm = e) remove))
+          |> Array.distinct)
+  match a.Length with
+  | 0 -> a
+  | _ -> [ [|model.AllValueDisplay|]; a ] |> Array.concat
 
 let limitFilter (positionLimit:AccountSecurityPositionLimitUnit option) (clipLimit:AccountSecurityClipLimitUnit option) allValueDisplay (array:'a []) =
   match positionLimit, clipLimit with
@@ -193,7 +195,11 @@ let clipLimitsView model dispatch =
                                                               && (elm.security.securityType = limit.security.securityType)
                                                             )
                                                             model.scratch.securityClipLimits.limit
-    |> Array.map (fun (elm:AccountSecurityClipLimitUnit) -> elm.security.securitySymbol)
+    |> Array.map (fun (elm:AccountSecurityClipLimitUnit) ->
+              match (elm.security.securitySymbol = model.AllValue) with
+              | true -> model.AllValueDisplay
+              | false -> elm.security.securitySymbol
+            )
 
 
 
@@ -223,7 +229,7 @@ let clipLimitsView model dispatch =
             )
             TableControl.Cell (Fields.enumSelect<string>
                           ""
-                          limit.security.securitySymbol
+                          (match (limit.security.securitySymbol = model.AllValue) with | true -> model.AllValueDisplay | false -> limit.security.securitySymbol)
                           (GetFilteredSymbolArray model <| (getUsedSymbolNames limit))
                           []
                           (fun value -> ((limit, (changeSymbol limit value)) |> ClipLimits |> dispatch))
@@ -262,7 +268,7 @@ let clipLimitsView model dispatch =
             div [ ClassName "column is-size-7 has-text-grey-mid is-help-text";  ]
               [str "Clip limits can be set for one security, for all securities in a product (by selecting \"All\" for securities), or for all products within a security type (by selecting \"All\" for both symbol and security)"]
       ]
-      TableControl.createDataTable (Some headerRow) rws (Button.button "Add" "" "plus" Button.Enabled addRow)
+      TableControl.createDataTable (Some headerRow) rws (Button.button "Add" "" "plus-circle" Button.Enabled addRow)
     ]
 
 let productLimits model dispatch =
@@ -362,7 +368,11 @@ let productLimits model dispatch =
                                                                 && (elm.product.securityType = limit.product.securityType)
                                                             )
                                                             model.scratch.productPositionLimits
-    |> Array.map (fun (elm:AccountProductPositionLimitUnit) -> elm.product.securitySymbol)
+    |> Array.map (fun (elm:AccountProductPositionLimitUnit) ->
+            match (elm.product.securitySymbol = model.AllValue) with
+            | true -> model.AllValueDisplay
+            | false -> elm.product.securitySymbol
+            )
 
 
   let rws:TableControl.row list =
@@ -390,7 +400,7 @@ let productLimits model dispatch =
             )
             TableControl.Cell (Fields.enumSelect<string>
                           ""
-                          limit.product.securitySymbol
+                          (match (limit.product.securitySymbol = model.AllValue) with | true -> model.AllValueDisplay | false -> limit.product.securitySymbol)
                           (GetFilteredSymbolArray model (getUsedSymbolNames limit))
                           []
                           (fun value -> ((limit, (changeSymbol limit value)) |> ProductLimits |> dispatch))
@@ -431,9 +441,9 @@ let productLimits model dispatch =
       div [ ClassName "columns" ][
             h1 [ ClassName "is-size-3 column is-narrow" ][str "Position Limits" ]
             div [ ClassName "column is-size-7 has-text-grey-mid is-help-text" ]
-              [str "This limits the net position for a product (i.e. accumulated position over all securities belonging to a product). This can be set for one product, or for all products within a security type by selecting \"All\" for symbol"]
+              [str "This limits the net position for a product (i.e. accumulated position over all securities belonging to a product). This can be set for one product, or for all products within a security type by selecting \"All\" for symbol. (Note that \"All\" applies the same limits to each symbol, not the combiled position of all symbols)"]
       ]
-      TableControl.createDataTable (Some headerRow) rws (Button.button "Add" "" "plus" Button.Enabled addRow)
+      TableControl.createDataTable (Some headerRow) rws (Button.button "Add" "" "plus-circle" Button.Enabled addRow)
       br []
     ]
 
@@ -551,7 +561,7 @@ let securityLimitsView model dispatch =
             div [ ClassName "column is-size-7 has-text-grey-mid is-help-text" ]
               [str "Optionally, the net position for one or more securities can also be limited. For example, the Product Limits can be set to restrict the Max Net Long Position across all ES futures to 10,000 contracts, but Security Limits can be used to restrict the ES Jun future to 500 only"]
       ]
-      TableControl.createDataTable (Some headerRow) rws (Button.button "Add" "" "plus" Button.Enabled addRow)
+      TableControl.createDataTable (Some headerRow) rws (Button.button "Add" "" "plus-circle" Button.Enabled addRow)
     ]
 
 let LimitsView model dispatch =
@@ -623,17 +633,24 @@ let creditLimitsView model dispatch = div [] [str "lorem ipsum"]
 
 let exchangeSettings model dispatch =
   let cme = model.scratch.exchangeAccountData.cmeAccountData
+  let sim = model.scratch.exchangeAccountData.simAccountData
   let send = UpdateCME>>dispatch
+  let sendSim = UpdateSIM>>dispatch
   div [] [
     h2 [Class "is-size-3"] [ str "CME" ]
-    Fields.enumSelect "CTI" (toHuman false CTICode.values cme.cti) (allToHuman false CTICode.values) [] (fromHuman CTICode.values>>CTI>>send)
-    Fields.enumSelect "Origin" (toHuman false CustomerOrFirm.values cme.origin) (allToHuman false CustomerOrFirm.values) [] (fromHuman CustomerOrFirm.values>>Origin>>send)
+    Fields.enumSelect "CTI" (toHuman false CTICode.values cme.cti) (allToHuman false CTICode.values) [] (fromHuman CTICode.values>>CMEExchangeField.CTI>>send)
+    Fields.enumSelect "Origin" (toHuman false CustomerOrFirm.values cme.origin) (allToHuman false CustomerOrFirm.values) [] (fromHuman CustomerOrFirm.values>>CMEExchangeField.Origin>>send)
     //Fields.text "Takeup Member Code" cme.takeupMemberCode [] (TakeupMember>>send)
-    Fields.text "Clearing Account" cme.clearingAccount [] (ClearingAccount>>send)
+    Fields.text "Clearing Account" cme.clearingAccount [] (CMEExchangeField.ClearingAccount>>send)
     //Fields.text "Direct Electronic Access" cme.directElectronicAccess [] (DirectElectronicAccess>>send)
     //Fields.text "Trading Capacity" cme.tradingCapacity [] (TradingCapacity>>send)
     //Fields.text "Liquidity Provision" cme.liquidityProvision [] (LiquidityProvision>>send)
     //Fields.map "Misc Tag Mapping" cme.miscTags [] (MiscTagMapping>>send)
+    h2 [Class "is-size-3"] [ str "SIM" ]
+    Fields.enumSelect "CTI" (toHuman false CTICode.values sim.cti) (allToHuman false CTICode.values) [] (fromHuman CTICode.values>>SIMExchangeField.CTI>>sendSim)
+    Fields.enumSelect "Origin" (toHuman false CustomerOrFirm.values sim.origin) (allToHuman false CustomerOrFirm.values) [] (fromHuman CustomerOrFirm.values>>SIMExchangeField.Origin>>sendSim)
+    Fields.text "Clearing Account" sim.clearingAccount [] (SIMExchangeField.ClearingAccount>>sendSim)
+
   ]
 
 let accountControls model (dispatch:Msg->unit) =

tools/cpp_helper/Generate-Cpps-reqs.ps1

@@ -0,0 +1,23 @@
+$known_good = @{
+	"IC" = 50
+	"IH" = 50
+	"IP" = 50
+	"IS" = 50
+	"IW" = 50
+	"KE" = 50
+	"KET" = 50
+	"MCX" = 50
+	"MKC" = 10
+	"MWE" = 50
+	"SBT" = 50
+	"WMK" = 50
+	"XC" = 10
+	"XK" = 10
+	"XW" = 10
+	"ZC" = 50
+	"ZO" = 50
+	"ZS" = 50
+	"ZW" = 50
+	"ZCT" = 50
+	"ZWT" = 50
+}

tools/cpp_helper/Generate-Cpps.ps1

@@ -1,43 +1,9 @@
-$known_good = @{
-	"6A" = 1000
-	"6B" = 625
-	"6D" = 1000
-	"6E" = 1250
-	"6J" = 1250
-	"M6E" = 125
-	"NQ" = 20
-	"ES" = 50
-	"YM" = 5
-	"RTY" = 50
-	"VX" = 1000
-	"EMD" = 100
-	"GE" = 2500
-	"ZB" = 1000
-	"ZN" = 1000
-	"ZF" = 1000
-	"ZT" = 2000
-	"ZQ" = 4167
-	"SI" = 50
-	"YI" = 10
-	"GC" = 100
-	"HG" = 250
-	"CL" = 1000
-	"NG" = 100
-	"HO" = 420
-	"QM" = 500
-	"RB" = 420
-	"ZC" = 50
-	"ZS" = 50
-	"ZW" = 50
-	"LE" = 400
-	"HE" = 400
-	"ZM" = 100
-	"ZL" = 600
-}
+# Dot sourcing the reqs file will make 
+. .\Generate-Cpps-reqs.ps1
 
 $cme_file_url = "ftp://ftp.cmegroup.com/pub/fprf/csv/cmeg.fut.prf.csv"
 $csv_save_path = ".\cmeg.fut.prf.csv"
-$output_file = ".\cpp_out.txt"
+$output_file = ".\cpp_config.conf"
 
 # turn off console update progress while downloading
 $CurrentProgressPref = $ProgressPreference;
@@ -47,11 +13,23 @@ $ProgressPreference = $CurrentProgressPref
 
 $a = Import-Csv $csv_save_path
 $b = $a | select Sym, Mult | Group Sym | Sort Name
+
+@"
+# Generated by automated script.  DO NOT EDIT!
+# source: tools/cpp_helper/generate-cpps.ps1
+"@ | out-file -Encoding ascii $output_file 
+"CppConfig`n{`n`tvalue`n`t{" | out-file -Encoding ascii $output_file -Append
 $b | % { 
 	if ($known_good.ContainsKey($_.Group[0].Sym)) { 
 		$cpp = $known_good[$_.Group[0].Sym] 
 	} else { 
 		$cpp = $_.Group[0].Mult 
 	}
-	"cpp_entry {{ `n`tkey: `"{0}`"`n`tvalue: {1}`n}}" -f $_.Group[0].Sym, $cpp } `
-	| out-file -Encoding ascii $output_file
+	"`t`tcpp_entry {{ key: `"{0}`" value: {1} }}" -f $_.Group[0].Sym, $cpp } `
+	| out-file -Encoding ascii $output_file -Append
+"`t}`n}" | out-file -Encoding ascii $output_file -Append
+Remove-Item $csv_save_path -Force
+
+# You can now copy the output file to defaults assuming all looks good
+# Copy-Item .\cpp_config.conf ..\..\src\lib\config\defaults\ -Force
+# Remove-Item cpp_config.conf