SPARK-1446 Try to add securities to security DB even if they already have an assigned security key

We were asserting that completed orders must have non zero fills, but
its easy for an algo to break that so we handle it.

src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs

@@ -173,7 +173,8 @@ namespace mbts.spark {
                }
                _is_setting_price_mults = false;
             };
-            _can_add_leg = backing(nameof(can_add_leg), leg_tuples.ObserveSelf().Select(_ => leg_count < 2), leg_count < 2);
+            //_can_add_leg = backing(nameof(can_add_leg), leg_tuples.ObserveSelf().Select(_ => leg_count < 2), leg_count < 2);
+            _can_add_leg = backing(nameof(can_add_leg), leg_tuples.ObserveSelf().Select(_ => true), true);
          } else {
             _can_add_leg = backing(nameof(can_add_leg), false);
          }

src/client/spark/Modules/Trading/Fills/FillsVM.cs

@@ -130,7 +130,7 @@ namespace mbts.spark {
       protected override IEnumerable<SortingCriteria> get_default_sorting() { yield return new SortingCriteria(nameof(Row.fill_time), false); }
 
       static IObservable<IIncrementalUpdate<FillTuple, FillTupleKey>> _get_observable(IBus bus, OrdersModel model, StartupData state) {
-         if (state.orders != null && state.orders.Length == 0) {
+         if (!state.orders.is_null_or_empty()) {
             var orders_hash = new HashSet<uint>(state.orders);
             return model.on_fill_tuples.filter(f => orders_hash.Contains(f.fill.OrderKey));
          } else return model.on_fill_tuples;

src/client/spark/Modules/Trading/LadderWindowFactory.cs

@@ -30,8 +30,7 @@ namespace bts.shell {
          _wm = wm;
          _settings_hub = settings_hub;
          _orders_model = orders_model;
-         var window_props = Configuration.get<bts.config.IShellWindowProps>();
-         _max_ladders_open = (int)(window_props?.ladder?.max_total_open ?? 75);
+         _max_ladders_open = (int)(Configuration.get<bts.config.LadderKnobs>().max_open_ladders);
          _dispose = Disposable.Create(
             bus.subscribe(SparkTags.SecuritiesTag).Subscribe(s => _securities = s),
             bus.subscribe(SparkTags.ShowLadderTag).OnDispatcher().Subscribe(s => show_windows(new[] { s })),

src/client/spark/Orders/OrderActivitySummarizer.cs

@@ -114,7 +114,7 @@ namespace bts.spark {
             (var security, var spec) = get_security(neworder.Def.SecurityKey);
             //
             // description
-            var mod_desc = modified_description(oldorder, neworder, spec.Value);
+            var mod_desc = modified_description(oldorder, neworder, spec);
             if (!string.IsNullOrEmpty(mod_desc)) {
                log_status(ActivityLogCategory.modify, security, mod_desc);
             }

src/client/spark/core/Utility/AnonymousEqualityComparer.cs

@@ -3,6 +3,8 @@ using System.Collections.Generic;
 
 namespace mbts.utils {
    public static class EqualityComparer {
+      const int _hash_seed = 63607, _hash_multiplier = 6451;
+
       public static IEqualityComparer<T> Create<T>(Func<T, T, bool> equals) =>
          new AnonymousEqualityComparer<T>(equals);
 
@@ -29,7 +31,7 @@ namespace mbts.utils {
          non_null_equals(x, y);
 
       public static int aggregate_hash(int item1_hash, int item2_hash, int item3_hash) {
-         int rv = 63607;
+         int rv = _hash_seed;
          unchecked {
             _accumulate_hash(ref rv, ref item1_hash);
             _accumulate_hash(ref rv, ref item2_hash);
@@ -38,9 +40,20 @@ namespace mbts.utils {
          return rv;
       }
 
+      // Note: be wary of the performance cost this method may have, as frequently creating large arrays can be expensive.
+      // If you wish to avoid this cost, consider making an overload like the method above, with the appropriate number of item-hash arguments
+      public static int aggregate_hash(params int[] item_hashes) {
+         int hash = _hash_seed;
+         unchecked {
+            foreach (int item_hash in item_hashes)
+               hash = hash * _hash_multiplier + item_hash;
+         }
+         return hash;
+      }
+
       // WARNING: be sure to call this within in an unchecked scope
       static void _accumulate_hash(ref int aggregate_hash, ref int item_hash) =>
-         aggregate_hash = aggregate_hash * 6451 + item_hash;
+         aggregate_hash = aggregate_hash * _hash_multiplier + item_hash;
 
       private class AnonymousEqualityComparer<T> : IEqualityComparer<T> {
          readonly Func<T, T, bool> _equals;

src/client/spark/view/Controls/DataGridEx/Filters/Views/TimeFilter.cs

@@ -54,8 +54,9 @@ namespace mbts.ui {
       public static TimeTextTuple FromTicks(long ticks) =>
          FromDateTime(new DateTime(ticks));
 
-      public static TimeTextTuple FromDateTime(DateTime date_time) =>
-         new TimeTextTuple(DateTimeFormatInfo.CurrentInfo.AbbreviatedMonthNames[date_time.Month - 1], date_time.Day.ToString(), date_time.Year.ToString(), date_time.ToString("t"));
+      public static TimeTextTuple FromDateTime(DateTime date_time) => FromDateTime(date_time, "t");
+      public static TimeTextTuple FromDateTime(DateTime date_time, string time_format) =>
+         new TimeTextTuple(DateTimeFormatInfo.CurrentInfo.AbbreviatedMonthNames[date_time.Month - 1], date_time.Day.ToString(), date_time.Year.ToString(), date_time.ToString(time_format));
 
       public TimeTextTuple(string month_text, string day_text, string year_text, string time_text) {
          _month_text = month_text;
@@ -107,14 +108,23 @@ namespace mbts.ui {
 
 
       static DateTime? _parse(string month_text, string day_text, string year_text, string time_text) {
-         DateTime? t = _parse(time_text);
+         TimeSpan? t = _parse_time(time_text);
          if (t.HasValue) {
             DateTime? d = _parse($"{month_text}/{day_text}/{year_text}");
-            if (d.HasValue) return d.Value.Date.AddDays(t.Value.TimeOfDay.TotalDays);
+            if (d.HasValue) return d.Value.Date.Add(t.Value);
          }
          return null;
       }
 
+      static TimeSpan? _parse_time(string text) {
+         DateTime? dt = _parse(text);
+         if (dt.HasValue) {
+            return dt.Value.TimeOfDay;
+         } else if (TimeSpan.TryParse(text, out TimeSpan result) && result < TimeSpan.FromDays(1)) {
+            return result;
+         } else return null;
+      }
+
       static DateTime? _parse(string text) {
          if (!string.IsNullOrEmpty(text) && DateTime.TryParse(text, out DateTime result))
             return result;

src/client/spark/view/Controls/PriceEntryControl.xaml

@@ -28,7 +28,14 @@
         BorderBrush="{TemplateBinding BorderBrush}"
         BorderThickness="{TemplateBinding BorderThickness}"
         Text="{Binding Value, RelativeSource={RelativeSource TemplatedParent}, Converter={StaticResource TickDisplayTwoWayConverter}, Mode=TwoWay}"
-      />
+      >
+        <TextBox.CommandBindings>
+          <CommandBinding Command="{x:Static u:TextBoxBehavior.SelectAllCommand}" Executed="{x:Static u:TextBoxBehavior.SelectAllCommandExecutedHandler}"/>
+        </TextBox.CommandBindings>
+        <TextBox.InputBindings>
+          <MouseBinding MouseAction="LeftDoubleClick" Command="{x:Static u:TextBoxBehavior.SelectAllCommand}"/>
+        </TextBox.InputBindings>
+      </TextBox>
     </c:TickUpDown>
   </ControlTemplate>
   <Style TargetType="usp:PriceEntryControl">

src/client/spark/view/Controls/PriceEntryControl.xaml.cs

@@ -47,24 +47,30 @@ namespace mbts.ui.spark {
       #endregion
 
       TextBox _text;
-      IDisposable _text_input_hook;
 
       public PriceEntryControl() {
          TickUpCommand = new DelegateCommand(_tick_up);
          TickDownCommand = new DelegateCommand(_tick_down);
       }
 
+      IDisposable _keyboard_focus_sub;
+      bool _has_keyboard_focus;
+
       public override void OnApplyTemplate() {
+         Disposable.dispose(ref _keyboard_focus_sub);
          _text = this.assert_template_child<TextBox>("_text");
-         Disposable.dispose(ref _text_input_hook);
-         _text_input_hook = _text.SubscribePreviewTextInput((s, e) => e.Handled = !RegexFilters.PriceInput.IsMatch(_text.PeekText(e)));
+         _keyboard_focus_sub = _text.SubscribeKeyboardFocus(val => _has_keyboard_focus = val);
          _update_text_binding();
       }
 
       void _tick_up() {
+         if (_has_keyboard_focus) {
+            _text.GetBindingExpression(TextBox.TextProperty)?.UpdateSource();
+         }
          double v = Value;
-         if (double.IsNaN(v)) SetCurrentValue(ValueProperty, TickDefaultValue);
-         else {
+         if (double.IsNaN(v)) {
+            SetCurrentValue(ValueProperty, TickDefaultValue);
+         } else {
             var spec = ContractSpec;
             if (spec.HasValue) SetCurrentValue(ValueProperty, spec.Value.tickup(v));
             else SetCurrentValue(ValueProperty, TickDefaultValue);
@@ -72,9 +78,13 @@ namespace mbts.ui.spark {
       }
 
       void _tick_down() {
+         if (_has_keyboard_focus) {
+            _text.GetBindingExpression(TextBox.TextProperty)?.UpdateSource();
+         }
          double v = Value;
-         if (double.IsNaN(v)) SetCurrentValue(ValueProperty, TickDefaultValue);
-         else {
+         if (double.IsNaN(v)) {
+            SetCurrentValue(ValueProperty, TickDefaultValue);
+         } else {
             var spec = ContractSpec;
             if (spec.HasValue) SetCurrentValue(ValueProperty, spec.Value.tickdown(v));
             else SetCurrentValue(ValueProperty, TickDefaultValue);

src/client/spark/view/Controls/TickUpDown/TickUpDown.xaml.cs

@@ -23,38 +23,42 @@ namespace mbts.ui.controls {
       }
 
       protected override void OnPreviewKeyDown(KeyEventArgs e) {
-         switch (Keyboard.Modifiers) {
-            case ModifierKeys.None: {
-                  switch (e.Key) {
-                     case Key.Up:
-                        TickUpCommand.TryExecuteIfCan(TickUpParameter);
-                        e.Handled = true;
-                        break;
-                     case Key.Down:
-                        TickDownCommand.TryExecuteIfCan(TickDownParameter);
-                        e.Handled = true;
-                        break;
+         if (EnableArrowKeyTick) {
+            switch (Keyboard.Modifiers) {
+               case ModifierKeys.None: {
+                     switch (e.Key) {
+                        case Key.Up:
+                           TickUpCommand.TryExecuteIfCan(TickUpParameter);
+                           e.Handled = true;
+                           break;
+                        case Key.Down:
+                           TickDownCommand.TryExecuteIfCan(TickDownParameter);
+                           e.Handled = true;
+                           break;
+                     }
                   }
-               }
-               break;
-            case ModifierKeys.Alt: {
-                  // see: https://stackoverflow.com/questions/3099472/previewkeydown-is-not-seeing-alt-modifiers
-                  switch (e.SystemKey) {
-                     case Key.Up:
-                        TickUpCommand.TryExecuteIfCan(TickUpAltParameter);
-                        e.Handled = true;
-                        break;
-                     case Key.Down:
-                        TickDownCommand.TryExecuteIfCan(TickDownAltParameter);
-                        e.Handled = true;
-                        break;
+                  break;
+               case ModifierKeys.Alt: {
+                     // see: https://stackoverflow.com/questions/3099472/previewkeydown-is-not-seeing-alt-modifiers
+                     switch (e.SystemKey) {
+                        case Key.Up:
+                           TickUpCommand.TryExecuteIfCan(TickUpAltParameter);
+                           e.Handled = true;
+                           break;
+                        case Key.Down:
+                           TickDownCommand.TryExecuteIfCan(TickDownAltParameter);
+                           e.Handled = true;
+                           break;
+                     }
                   }
-               }
-               break;
+                  break;
+            }
          }
          base.OnPreviewKeyDown(e);
       }
 
+      public bool EnableArrowKeyTick { get; set; } = true;
+
       #region TickCommand
       public ICommand TickCommand {
          get { return (ICommand)GetValue(TickCommandProperty); }

src/client/spark/view/Controls/Time/DateTimePicker.xaml

@@ -31,8 +31,14 @@
               <ColumnDefinition Width="Auto"/>
               <ColumnDefinition/>
             </Grid.ColumnDefinitions>
-            <c:TickUpDown x:Name="PART_time_tick" Grid.Column="0" MinWidth="85" HorizontalAlignment="Stretch">
-              <TextBox x:Name="PART_time_text" Height="20" u:TextBoxBehavior.EnableArrowKeyTraverse="True" u:Behavior.DisablePaste="True"/>
+            <c:TickUpDown x:Name="PART_time_tick" Grid.Column="0" MinWidth="85" HorizontalAlignment="Stretch" EnableArrowKeyTick="False">
+              <TextBox x:Name="PART_time_text" Height="20" u:TextBoxBehavior.EnableArrowKeyTraverse="True" u:Behavior.DisablePaste="True">
+                <TextBox.InputBindings>
+                  <KeyBinding Key="Up" Command="{x:Static c:DateTimePicker.TickTimeComponentCommand}" CommandParameter="{u:Int 1}"/>
+                  <KeyBinding Key="Down" Command="{x:Static c:DateTimePicker.TickTimeComponentCommand}" CommandParameter="{u:Int -1}"/>
+                  <u:CustomInputBinding Gesture="{x:Static u:InputGestures.LeftTripleClick}" Command="{x:Static u:TextBoxBehavior.SelectAllCommand}"/>
+                </TextBox.InputBindings>
+              </TextBox>
             </c:TickUpDown>
             <c:TickUpDown x:Name="PART_month_tick" Grid.Column="2" Width="50">
               <TextBox x:Name="PART_month_text" Height="20" u:TextBoxBehavior.EnableArrowKeyTraverse="True" u:Behavior.DisablePaste="True"/>

src/client/spark/view/Controls/Time/DateTimePicker.xaml.cs

@@ -3,6 +3,7 @@ using mbts.ui.shell;
 using System;
 using System.Collections.Generic;
 using System.ComponentModel;
+using System.Linq;
 using System.Windows;
 using System.Windows.Controls;
 using System.Windows.Data;
@@ -14,7 +15,9 @@ namespace mbts.ui.controls {
    /// Interaction logic for DateTimePicker.xaml
    /// </summary>
    public class DateTimePicker : Control {
-      TimeTupleVM _vm;
+      public static readonly RoutedCommand TickTimeComponentCommand = new RoutedCommand();
+
+      DTPController _controller;
       bool _suspend_notify_vm;
 
       static DateTimePicker() =>
@@ -24,8 +27,8 @@ namespace mbts.ui.controls {
          new FrameworkPropertyMetadata(default(TimeTextTuple), FrameworkPropertyMetadataOptions.BindsTwoWayByDefault, OnTimeTupleChanged));
       static void OnTimeTupleChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
          var dtp = (DateTimePicker)d;
-         if (!dtp._suspend_notify_vm && dtp._vm != null)
-            dtp._vm.result = (TimeTextTuple)e.NewValue;
+         if (!dtp._suspend_notify_vm && dtp._controller != null)
+            dtp._controller.result = (TimeTextTuple)e.NewValue;
       }
       public TimeTextTuple TimeTuple { get => (TimeTextTuple)GetValue(TimeTupleProperty); set => SetValue(TimeTupleProperty, value); }
 
@@ -41,59 +44,74 @@ namespace mbts.ui.controls {
          var time = this.assert_template_child<TickUpDown>("PART_time_tick");
          var time_text = this.assert_template_child<TextBox>("PART_time_text");
 
-         if (_vm != null) _vm.PropertyChanged -= _vm_PropertyChanged;
-         _vm = new TimeTupleVM();
-         _vm.result = TimeTuple;
-         _vm.PropertyChanged += _vm_PropertyChanged;
+         if (_controller != null) _controller.PropertyChanged -= _vm_PropertyChanged;
+         _controller = new DTPController(time_text);
+         _controller.result = TimeTuple;
+         _controller.PropertyChanged += _vm_PropertyChanged;
+         time.CommandBindings.Add(new CommandBinding(TickTimeComponentCommand, (s, e) => {
+            _controller.tick_time_component((int)e.Parameter);
+         }));
+         time.CommandBindings.Add(new CommandBinding(TextBoxBehavior.SelectAllCommand, TextBoxBehavior.SelectAllCommandExecutedHandler));
 
          const int delay = 500;
-         time.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_time") { Source = _vm });
+         time.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_time") { Source = _controller });
          time.TickUpParameter = 1;
          time.TickDownParameter = -1;
          time.TickUpAltParameter = 60;
          time.TickDownAltParameter = -60;
-         time_text.SetBinding(TextBox.TextProperty, new Binding("time_text") { Source = _vm, Delay = delay, ValidatesOnExceptions = true });
-         _hooks.Add(time_text.SubscribePreviewTextInput((s, e) =>
-            e.Handled = !MatchAny(time_text.PeekText(e), RegexFilters.Time.StandardTime, RegexFilters.Time.MilitaryTime)
-         ));
+         time_text.SetBinding(TextBox.TextProperty, new Binding("time_text") { Source = _controller, Delay = delay, ValidatesOnExceptions = true });
+         _hooks.Add(time_text.SubscribePreviewTextInput((s, e) => {
+            string preview = time_text.PeekText(e);
+            if (RegexFilters.Time.StandardTimeOptionalSeconds.IsMatch(preview)) {
+               _controller.use_military_time = false;
+            } else if (RegexFilters.Time.MilitaryTimeOptionalSeconds.IsMatch(preview)) {
+               _controller.use_military_time = true;
+            } else {
+               e.Handled = true;
+            }
+         }));
 
-         month.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_month") { Source = _vm });
+         month.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_month") { Source = _controller });
          month.TickUpParameter = 1;
          month.TickDownParameter = -1;
-         month_text.SetBinding(TextBox.TextProperty, new Binding("month_text") { Source = _vm, Delay = delay });
+         month_text.SetBinding(TextBox.TextProperty, new Binding("month_text") { Source = _controller, Delay = delay });
          _hooks.Add(month_text.SubscribePreviewTextInput((s, e) =>
             e.Handled = !RegexFilters.Time.MonthAbbrev.IsMatch(month_text.PeekText(e))
          ));
 
-         day.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_day") { Source = _vm });
+         day.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_day") { Source = _controller });
          day.TickUpParameter = 1;
          day.TickDownParameter = -1;
-         day_text.SetBinding(TextBox.TextProperty, new Binding("day_text") { Source = _vm, Delay = delay });
+         day_text.SetBinding(TextBox.TextProperty, new Binding("day_text") { Source = _controller, Delay = delay });
          _hooks.Add(day_text.SubscribePreviewTextInput((s, e) =>
             e.Handled = !RegexFilters.Time.DaysInMonth.IsMatch(day_text.PeekText(e))
          ));
 
-         year.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_year") { Source = _vm });
+         year.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_year") { Source = _controller });
          year.TickUpParameter = 1;
          year.TickDownParameter = -1;
-         year_text.SetBinding(TextBox.TextProperty, new Binding("year_text") { Source = _vm, Delay = delay });
+         year_text.SetBinding(TextBox.TextProperty, new Binding("year_text") { Source = _controller, Delay = delay });
          _hooks.Add(year_text.SubscribePreviewTextInput((s, e) =>
             e.Handled = !RegexFilters.Time.Year.IsMatch(year_text.PeekText(e))
          ));
       }
 
       void _vm_PropertyChanged(object sender, PropertyChangedEventArgs e) {
-         if (e.PropertyName == nameof(TimeTupleVM.result)) {
+         if (e.PropertyName == nameof(DTPController.result)) {
             _suspend_notify_vm = true;
-            TimeTuple = _vm.result;
+            TimeTuple = _controller.result;
             _suspend_notify_vm = false;
          }
       }
 
-      private class TimeTupleVM : ViewModelBase {
+      // This is not exactly a view-model, bc it accesses the time TextBox directly so it can get/set the CaretIndex,
+      // which is used to determine which time-componenet - hh:mm:ss AM/PM - should be incremented/decremented via the up/down arrow-keys.
+      class DTPController : ViewModelBase {
+         readonly TextBox _time_text_box;
          TimeTextTuple _result;
 
-         public TimeTupleVM() {
+         public DTPController(TextBox time_text_box) {
+            _time_text_box = time_text_box;
             tick_time = new DelegateCommand<int>(_tick_time);
             tick_month = new DelegateCommand<int>(_tick_month);
             tick_day = new DelegateCommand<int>(_tick_day);
@@ -135,12 +153,52 @@ namespace mbts.ui.controls {
                RaisePropertyChanged(nameof(month_text), nameof(day_text), nameof(year_text), nameof(time_text));
             }
          }
+
+         public bool use_military_time { get; set; }
          public ICommand tick_time { get; }
          public ICommand tick_month { get; }
          public ICommand tick_day { get; }
          public ICommand tick_year { get; }
+         string _time_format => use_military_time ? StringFormats.Time.MilitaryWithSeconds : StringFormats.Time.StandardWithSeconds;
 
+         public void tick_time_component(int increment) {
+            _ensure_time_text_up_to_date();
+            var r = _result.result;
+            if (r.HasValue) {
+               int index = _time_text_box.CaretIndex;
+               using (detect_property_changes()) {
+                  DateTime dt;
+                  if (index <= 0) {
+                     dt = r.Value.AddHours(increment);
+                  } else {
+                     int segment = _time_text_box.Text.Take(index).Count(c => c == ':');
+                     switch (segment) {
+                        case 0: dt = r.Value.AddHours(increment); break;
+                        case 1: dt = r.Value.AddMinutes(increment); break;
+                        case 2:
+                           if (_time_text_box.Text.Take(index).Contains(' ')) {
+                              // toggle AM/PM
+                              dt = r.Value.AddHours(r.Value.Hour < 12 ? 12 : -12);
+                           } else {
+                              dt = r.Value.AddSeconds(increment);
+                           }
+                           break;
+                        default:
+                           dt = r.Value.AddSeconds(increment);
+                           break;
+                     }
+                  }
+                  _result = TimeTextTuple.FromDateTime(dt, _time_format);
+               }
+               _time_text_box.CaretIndex = index;
+            } else {
+               using (detect_property_changes())
+                  _force_valid_inputs();
+            }
+         }
+ 
          void _tick_time(int increment) {
+            _ensure_time_text_up_to_date();
             using (detect_property_changes()) {
                var r = _result.result;
                if (r.HasValue) {
@@ -187,6 +245,11 @@ namespace mbts.ui.controls {
                }
             }
          }
+ 
+         void _ensure_time_text_up_to_date() {
+            if (_time_text_box.IsKeyboardFocused)
+               _time_text_box.GetBindingExpression(TextBox.TextProperty)?.UpdateSource();
+         }
 
          void _force_valid_inputs() {
             var dt = DateTime.Today.Date;

src/client/spark/view/Extensions/InputGestures.cs

@@ -0,0 +1,16 @@
+using System.Windows.Input;
+
+namespace mbts.ui {
+   class CustomInputBinding : InputBinding { }
+
+   static class InputGestures {
+      public static readonly InputGesture LeftTripleClick = new LeftTripleClickGesture();
+
+      class LeftTripleClickGesture : InputGesture {
+         public override bool Matches(object targetElement, InputEventArgs inputEventArgs) =>
+            inputEventArgs is MouseButtonEventArgs mouse_args &&
+            mouse_args.ChangedButton == MouseButton.Left &&
+            mouse_args.ClickCount == 3;
+      }
+   }
+}

src/client/spark/view/Extensions/TextElement/TextBoxBehavior.cs

@@ -7,7 +7,14 @@ using System.Windows.Input;
 
 namespace mbts.ui {
    static class TextBoxBehavior {
-      public static readonly RoutedCommand ReleaseCaptureCommand = new RoutedCommand();
+      #region RoutedCommands/Handlers
+      public static readonly RoutedCommand
+         ReleaseCaptureCommand = new RoutedCommand(),
+         SelectAllCommand = new RoutedCommand();
+
+      public static readonly ExecutedRoutedEventHandler SelectAllCommandExecutedHandler =
+         new ExecutedRoutedEventHandler((s, e) => (e.OriginalSource as TextBox ?? e.Source as TextBox ?? s as TextBox)?.SelectAll());
+      #endregion
 
       #region SelectAllOnFocus
 
@@ -168,7 +175,7 @@ namespace mbts.ui {
       public static void SetInputFilter(DependencyObject d, TextBoxInputFilter value) => d.SetValue(InputFilterProperty, value);
       #endregion
 
-      #region
+      #region IncrementDecrementOnMouseClick
       public static readonly DependencyProperty IncrementDecrementOnMouseClickProperty = DependencyProperty.RegisterAttached(
          "IncrementDecrementOnMouseClick", typeof(bool), typeof(TextBoxBehavior), new PropertyMetadata(false, IncrementDecrementOnMouseClickPropertyChanged)
          );

src/client/spark/view/Extensions/TextElement/TextElementHelper.cs

@@ -2,11 +2,28 @@
 using System.Windows;
 using System.Windows.Controls;
 using System.Windows.Data;
+using System.Windows.Input;
+
+namespace mbts.ui {
+   static class TextElementHelper {
+      public static IDisposable SubscribeKeyboardFocus(this TextBox text_box, Action<bool> callback) {
+         text_box.GotKeyboardFocus += on_got_keyboard_focus;
+         text_box.LostKeyboardFocus += on_lost_keyboard_focus;
+         return Disposable.Create(() => {
+            text_box.GotKeyboardFocus -= on_got_keyboard_focus;
+            text_box.LostKeyboardFocus -= on_lost_keyboard_focus;
+         });
+         void on_got_keyboard_focus(object sender, KeyboardFocusChangedEventArgs e) {
+            if (e.NewFocus == text_box)
+               callback(true);
+
+         }
+         void on_lost_keyboard_focus(object sender, KeyboardFocusChangedEventArgs e) {
+            if (e.OldFocus == text_box)
+               callback(false);
+         }
+      }
 
-namespace mbts.ui
-{
-   public class TextElementHelper
-   {
       public static void GetTargetProperty(DependencyObject d, out FrameworkElement text_element, out DependencyProperty text_property) {
          text_element = (FrameworkElement)d;
          if (text_element is TextBox) text_property = TextBox.TextProperty;

src/client/spark/view/Spark/ContractPicker/ContractPickerModel.cs

@@ -197,10 +197,6 @@ namespace mbts.spark
          selected_securities = secs.Select(s => s.Id).ToList();
          ok_button_command.TryRaiseCanExecuteChanged();
          _compose_criteria_clause_name();
-
-         // _security_in_criteria is our fun to be returned to our callers
-         // test on click -- verify our func works
-         //var results = all_securities.Select(all => all.Value.security).Where(x => _security_in_criteria(x)).ToList();
       }
 
       private void on_securities_changed(IReadOnlyDictionary<uint, SecurityMD> changed_securities) {
@@ -212,10 +208,6 @@ namespace mbts.spark
             all_securities = changed_securities;
          }
 
-         securities.sync_with(all_securities
-            .Select(entry => entry.Value.security)
-         );
-
          _update_exchanges();
          _update_security_types();
          _update_symbols();
@@ -327,7 +319,6 @@ namespace mbts.spark
             .Select(s => s.Value.security.Symbol)
             .Distinct()
          );
-            
       }
 
       private void _update_securities() {

src/client/spark/view/Spark/Ladder/LadderGridHost.cs

@@ -533,6 +533,7 @@ namespace mbts.spark {
          _visualHost.AddVisualChild(_mid_mkt_break_vis);
          _visualHost.AddVisualChild(_ask_mkt_break_vis);
          _visualHost.AddVisualChild(_bid_mkt_break_vis);
+         _render_grid_background();
          _init_tactile_adorner();
          // this effectively sets the background of the "Last" column
          if (_scrollViewer != null) {

src/client/spark/view/Spark/Ladder/LadderSettings.cs

@@ -1,5 +1,6 @@
 using bts.api.orderman;
 using mbts.ui;
+using mbts.utils;
 using ProtoBuf;
 using System;
 using System.Linq;
@@ -124,8 +125,6 @@ namespace mbts.spark {
                last_column = last_column,
                calculator_buttons = new[] { 1, 5, 10, 25, 50, 100 },
                scroll_bar_visible = false,
-               ticks_to_recenter = 10,
-               seconds_to_recenter = 300,
                default_order_type = CustomOrderDefinition.CustomOneofCase.Limit,
                default_time_in_force = TimeInForce.Day,
                animate_recenter = true,
@@ -163,10 +162,8 @@ namespace mbts.spark {
       public int[] calculator_buttons { get; set; }
       [ProtoMember(10)]
       public bool scroll_bar_visible { get; set; }
-      [ProtoMember(11)]
-      public int ticks_to_recenter { get; set; }
-      [ProtoMember(12)]
-      public int seconds_to_recenter { get; set; }
+      // USED [ProtoMember(11)]
+      // USED [ProtoMember(12)]
       [ProtoMember(13)]
       public CustomOrderDefinition.CustomOneofCase default_order_type { get; set; }
       [ProtoMember(14)]
@@ -197,11 +194,8 @@ namespace mbts.spark {
          other.price_column.safe_equals(price_column) &&
          other.calculator_buttons.safe_equals(calculator_buttons, (a, b) => a.SequenceEqual(b)) &&
          other.scroll_bar_visible == scroll_bar_visible &&
-         other.ticks_to_recenter == ticks_to_recenter &&
-         other.seconds_to_recenter == seconds_to_recenter &&
          other.default_order_type == default_order_type &&
          other.default_time_in_force == default_time_in_force &&
-         other.seconds_to_recenter == seconds_to_recenter &&
          other.size == size &&
          other.animate_recenter == animate_recenter &&
          other.badge_style == badge_style &&
@@ -213,26 +207,24 @@ namespace mbts.spark {
       public override bool Equals(object obj) => Equals(obj as LadderSettings);
 
       public override int GetHashCode() =>
-         row_height.GetHashCode() +
-         grid_line_color.GetHashCode() +
-         orders_column.GetHashCode() +
-         bid_column.GetHashCode() +
-         ask_column.GetHashCode() +
-         price_column.GetHashCode() +
-         last_column.GetHashCode() +
-         calculator_buttons.GetHashCode() +
-         scroll_bar_visible.GetHashCode() +
-         ticks_to_recenter.GetHashCode() +
-         seconds_to_recenter.GetHashCode() +
-         default_order_type.GetHashCode() +
-         default_time_in_force.GetHashCode() +
-         seconds_to_recenter.GetHashCode() +
-         size.GetHashCode() +
-         animate_recenter.GetHashCode() +
-         badge_style.GetHashCode() +
-         bid_break_line.GetHashCode() +
-         ask_break_line.GetHashCode() +
-         mid_break_line.GetHashCode() +
-         alternate_order_size.GetHashCode();
+         EqualityComparer.aggregate_hash(
+            row_height.GetHashCode(),
+            grid_line_color.GetHashCode(),
+            orders_column.GetHashCode(),
+            bid_column.GetHashCode(),
+            ask_column.GetHashCode(),
+            price_column.GetHashCode(),
+            last_column.GetHashCode(),
+            calculator_buttons.GetHashCode(),
+            scroll_bar_visible.GetHashCode(),
+            default_order_type.GetHashCode(),
+            default_time_in_force.GetHashCode(),
+            size.GetHashCode(),
+            animate_recenter.GetHashCode(),
+            badge_style.GetHashCode(),
+            bid_break_line.GetHashCode(),
+            ask_break_line.GetHashCode(),
+            mid_break_line.GetHashCode(),
+            alternate_order_size.GetHashCode());
    }
 }

src/client/spark/view/Spark/Ladder/LadderVM.cs

@@ -16,8 +16,6 @@ using System.Reactive.Linq;
 using System.Threading.Tasks;
 using System.Windows.Input;
 using System.Windows.Media;
-using mbts.utils;
-using mbts.spark.utilities;
 
 namespace mbts.spark {
    class LadderVM : DisposableViewModelBase, IKeyListener {
@@ -25,7 +23,7 @@ namespace mbts.spark {
       readonly LadderModel _model;
       readonly Property<double> _high, _low, _change, _open, _settlement;
       readonly Property<ulong> _volume;
-      readonly Property<bool> _revert;
+      readonly Property<bool> _auto_clear;
       readonly Property<Brush> _bid_background, _bid_foreground, _ask_background, _ask_foreground;
       IWindowSettingsManager<LadderSettings> _settings_manager;
       ulong _misc_time = 0;
@@ -44,7 +42,7 @@ namespace mbts.spark {
          _open = backing(nameof(open), double.NaN);
          _settlement = backing(nameof(settlement), double.NaN);
          _volume = backing(nameof(volume), 0ul);
-         _revert = backing(nameof(revert), false);
+         _auto_clear = backing(nameof(auto_clear), false);
          _status = backing(nameof(status_message), "");
          _status_brush = backing(nameof(status_background), (Brush)BTSColors.UI.spark_menu_background);
          // brush props
@@ -176,7 +174,7 @@ namespace mbts.spark {
       public double open => _open.Value;
       public double settlement => _settlement.Value;
       public ulong volume => _volume.Value;
-      public bool revert { get => _revert.Value; set => _revert.set(value); }
+      public bool auto_clear { get => _auto_clear.Value; set => _auto_clear.set(value); }
       public CalculatorLogic size_calculator { get; }
       public OrderPropertiesEditorVM order_properties_vm { get; }
       public ObservableCollection<LadderOrderVM> orders { get; }
@@ -232,16 +230,8 @@ namespace mbts.spark {
                
             uint size = args.size_from_alternate ? s.alternate_order_size : (uint)Math.Abs(size_calculator.value.Value);
             await _model.order_gateway.send_order(order, args.price, size);
-            if (revert) {
-               size_calculator.value = s.size;
-               if (order_properties_vm.selected_time_in_force != s.default_time_in_force && order_properties_vm.time_in_forces.Contains(s.default_time_in_force))
-                  order_properties_vm.selected_time_in_force = s.default_time_in_force;
-               foreach (var ot in order_properties_vm.order_types) {
-                  ot.load_from(_model.order_gateway.factory.custom_order(ot.order_type));
-                  if (ot.order_type == s.default_order_type)
-                     order_properties_vm.selected_order_type = ot;
-               }
-            }
+            if (auto_clear)
+               size_calculator.value = 0;
          }
       }
 

src/client/spark/view/Spark/Ladder/LadderView.xaml

@@ -394,7 +394,7 @@
         </ItemsControl>
         <Button Margin="2" HorizontalAlignment="Stretch" Command="{Binding size_calculator.clear}">Clear</Button>
         <!-- reset to defaults after order -->
-        <CheckBox Margin="2" IsChecked="{Binding revert}">Auto Clear</CheckBox>
+        <CheckBox Margin="2" IsChecked="{Binding auto_clear}">Auto Clear</CheckBox>
       </StackPanel>
       <Grid Grid.Row="1">
         <Grid.RowDefinitions>

src/client/spark/view/Spark/Ladder/RecenterCountdown.cs

@@ -17,16 +17,17 @@ namespace mbts.spark {
       };
       SecurityImmutable? _security;
       MDConsumerHandle _handle;
-      LadderSettings _settings;
       DateTime _last_recenter = DateTime.MaxValue;
       double _last_mid = 0.0;
       IDisposable _dispose;
+      readonly TimeSpan _seconds_to_center;
 
       public RecenterCountdown() {
          DataContextChanged += RecenterCountdown_DataContextChanged;
          Unloaded += (o,e) => on_unload(false);
          _timer.Tick += _timer_Tick;
          _timer.Start();
+         _seconds_to_center = TimeSpan.FromSeconds(config.Configuration.get<bts.config.LadderKnobs>().seconds_to_recenter);
       }
 
       static (double,double) tick_range(double price, int tick_count, ContractSpecificationImmutable spec) {
@@ -34,11 +35,10 @@ namespace mbts.spark {
       }
 
       private void _timer_Tick(object sender, EventArgs e) {
-         if (_settings != null && _security != null) {
+         if (_security != null) {
             var mid = _handle.mid();
-            (double low, double high) = tick_range(_last_mid, _settings.ticks_to_recenter, _security.Value.ContractSpec);
             if (//mid < low || mid > high ||
-               (_last_recenter + TimeSpan.FromSeconds(_settings.seconds_to_recenter) < DateTime.UtcNow)) {
+               (_last_recenter + _seconds_to_center < DateTime.UtcNow)) {
                trigger_recenter(mid);
             }
          }
@@ -59,7 +59,6 @@ namespace mbts.spark {
             vm.OnDisposed += on_unload;
             _dispose =
                Disposable.Create(
-                  vm.settings.Subscribe(s => _settings = s),
                   Disposable.Create(() => vm.OnDisposed -= on_unload));
          }
       }

src/client/spark/view/Spark/Utilities/WorkLoop.cs

@@ -10,7 +10,7 @@ namespace mbts.spark {
       static readonly TimeSpan default_interval;
 
       static WorkLoop() {
-         var timer_params = config.Configuration.get<bts.config.LadderTimerParams>();
+         var timer_params = config.Configuration.get<bts.config.LadderKnobs>();
          default_priority = Enum.TryParse(timer_params.priority, true, out DispatcherPriority p) ? p : DispatcherPriority.ApplicationIdle;
          default_interval = TimeSpan.FromMilliseconds(timer_params.interval_ms);
       }

src/client/spark/view/Utility/TextUtils.cs

@@ -130,12 +130,22 @@ namespace mbts.ui.utility.text {
          public static class Time {
             public static readonly Regex
                StandardTime = new Regex(@"^((0?[0-9])|(1?[0-2]?))(:?|:([0-5]?|[0-5][0-9]?)) ?(?i)((a|p)m?)?$"),
+               StandardTimeOptionalSeconds = new Regex(@"^((\d?|0\d|1[0-2])|((\d|0\d|1[0-2])(:?|(:([0-5]?|[0-5]\d(( ?|( ([aApP]?|[aApP][mM])))(:?|(: ?)|(: [aApP]?[mM]?)|(:[0-5]?\d? ?[aApP]?[mM]?))|:([0-5]?|([0-5]\d( ?|( ([aApP]?|([aApP][mM]))))))))))))$"),
                MilitaryTime = new Regex(@"^(([0-1]?[0-9])|(2?[0-3]?))(:?|:([0-5]?|[0-5][0-9]?))$"),
+               MilitaryTimeOptionalSeconds = new Regex(@"^((\d?|[0-1]\d|2[0-3])|((\d|[0-1]\d|2[0-3])(:?|(:([0-5]?|[0-5]\d(:?|:([0-5]?|[0-5]\d)))))))$"),
                DaysInMonth = new Regex(@"^([0-9]?|[0-2][0-9]|3[0-1])?$"),
                MonthAbbrev = new Regex(@"^(?i)(a(ug?|pr?)?|d(e|ec)?|f(e?b)?|j(an?|u[nl]?)?|m(a[ry]?)?|(o(ct?)?)|s(ep?)?|n(ov?)?)?$"),
                Year = new Regex(@"^\d{0,4}$"),
                TimeSpan = new Regex(@"^(\d+(:?|:([0-5]?|[0-5]\d(:?|:([0-5]?|[0-5]\d)))))?$");
          }
       }
+
+      public static class StringFormats {
+         public static class Time {
+            public const string
+               MilitaryWithSeconds = "HH:mm:ss",
+               StandardWithSeconds = "hh:mm:ss tt";
+         }
+      }
    }
 }

src/client/spark_api/security/SecurityFormatter.cs

@@ -1,18 +1,18 @@
-using System;
-using System.Collections.Concurrent;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using static bts.api.Security;
 
 namespace bts.api {
    struct FutureLeg {
       public readonly Venue exchange;
-      public readonly string symbol;
+      public readonly string symbol, exchange_desc;
       public readonly FutureContractImmutable future;
       public readonly int quantity;
 
-      public FutureLeg(Venue exchange, string symbol, FutureContractImmutable future, int quantity) {
+      public FutureLeg(Venue exchange, string symbol, string exchange_desc, FutureContractImmutable future, int quantity) {
          this.exchange = exchange;
          this.symbol = symbol;
+         this.exchange_desc = exchange_desc;
          this.future = future;
          this.quantity = quantity;
       }
@@ -32,12 +32,14 @@ namespace bts.api {
          return string.Join("", _format_future_legs(future_legs));
       }
 
-      public static string format_future(string symbol, FutureContractImmutable future) =>
-         $"{symbol}{DateFormatter.Format(future.ExpirationYear, future.ExpirationMonth, future.ExpirationDay, DateFormat.MonthCodeLastDigitYear)}";
+      public static string format_future(string symbol, string exch_desc, FutureContractImmutable future) =>
+         string.IsNullOrEmpty(exch_desc) ?
+            $"{symbol}{DateFormatter.Format(future.ExpirationYear, future.ExpirationMonth, future.ExpirationDay, DateFormat.MonthCodeLastDigitYear)}" :
+            exch_desc;
 
       static string _format(SecurityImmutable security) {
          switch (security.ContractCase) {
-            case ContractOneofCase.Future: return format_future(security.Symbol, security.Future);
+            case ContractOneofCase.Future: return format_future(security.Symbol, security.glbx_desc(), security.Future);
             case ContractOneofCase.Spread:
                if (security.HasGlbx) {
                   // In case we ever bring it back, pretty printed spread names can be found here:
@@ -61,7 +63,7 @@ namespace bts.api {
       }
 
       static string _get_first_leg_name(FutureLeg fl) {
-         string sym_exp_name = format_future(fl.symbol, fl.future);
+         string sym_exp_name = format_future(fl.symbol, fl.exchange_desc, fl.future);
          switch (fl.quantity) {
             case 1: return sym_exp_name;
             case -1: return $"-{sym_exp_name}";
@@ -70,7 +72,7 @@ namespace bts.api {
       }
 
       static string _get_subsequent_leg_name(FutureLeg fl) {
-         string sym_exp_name = format_future(fl.symbol, fl.future);
+         string sym_exp_name = format_future(fl.symbol, fl.exchange_desc, fl.future);
          switch (fl.quantity) {
             case 1: return $"+{sym_exp_name}";
             case -1: return $"-{sym_exp_name}";

src/client/spark_api/security/SecurityHelper.cs

@@ -19,7 +19,7 @@ namespace bts.api {
          foreach (var l in legs) {
             switch (l.Security.ContractCase) {
                case ContractOneofCase.Future:
-                  yield return new FutureLeg(l.Security.Exchange, l.Security.Symbol, l.Security.Future, qty * l.SignedQuantity());
+                  yield return new FutureLeg(l.Security.Exchange, l.Security.Symbol, l.Security.glbx_desc(), l.Security.Future, qty * l.SignedQuantity());
                   break;
                case ContractOneofCase.Spread:
                   foreach (var fl in _get_flat_future_legs(l.Security.Spread.Legs, qty * l.SignedQuantity()))
@@ -56,6 +56,9 @@ namespace bts.api {
 
       public static string DisplayExchange(this SecurityImmutable security) =>
          security.ContractCase == ContractOneofCase.Synthetic ? $"[{string.Join(",", get_flat_future_legs(security).Select(flt => flt.exchange.ToString()).Distinct().OrderBy(e => e))}]" : security.Exchange.ToString();
+
+      public static string glbx_desc(this SecurityImmutable security) =>
+         security.HasGlbx ? security.Glbx.SecurityDesc ?? string.Empty : string.Empty;
       #endregion
 
       public static class Comparer {

src/lib/api/service_helpers/AdminServiceUtils.cpp

@@ -820,19 +820,49 @@ void AdminServiceUtils::delete_user (bts::api::admin::AdminData& admin_data,
 ////////////////////////////////////////////
 bool AdminServiceUtils::_sort_security_scopes (const bts::api::admin::SecurityScope& a,
                                                const bts::api::admin::SecurityScope& b){
-   if (has_id(a.security_id()))
-      return true;
+   // for both the sec and product scope sorting, the more
+   // qualified arg is sent to the top (vs wildcard). the
+   // order of qualification is sec_id, symbol, type, exchange.
+   // if both a and b are equally qualified, it is a
+   // a dont-care case (in terms of sec/prod res) and 
+   // ascending orderring is used
+   if (has_id(a.security_id())){
+      if (has_id(b.security_id()))
+         return a.security_id() < b.security_id();
+      else
+         return true;
+   }
    if (has_id(b.security_id()))
      return false;
-   if (a.security_symbol() != "*")
-      return true; 
-   return false;
+   // symbol
+   if (is_symbol_set(a.security_symbol())){
+      if (is_symbol_set(b.security_symbol()))
+         return a.security_symbol() < b.security_symbol();
+      else
+         return true;
+   }
+   if (is_symbol_set(b.security_symbol()))
+      return false;
+   // sec type
+   if (a.security_type() != b.security_type())
+      return a.security_type() < b.security_type();
+   return a.exchange() < b.exchange();
 }
 bool AdminServiceUtils::_sort_product_scopes (const bts::api::admin::ProductScope& a,
-                                              const bts::api::admin::ProductScope&){
-   if (a.security_symbol() != "*")
-      return true;
-   return false;
+                                              const bts::api::admin::ProductScope& b){
+   // symbol
+   if (is_symbol_set(a.security_symbol())){
+      if (is_symbol_set(b.security_symbol()))
+         return a.security_symbol() < b.security_symbol();
+      else
+         return true;
+   }
+   if (is_symbol_set(b.security_symbol()))
+      return false;
+   // sec type
+   if (a.security_type() != b.security_type())
+      return a.security_type() < b.security_type();
+   return a.exchange() < b.exchange();
 }
 void AdminServiceUtils::sort_limits (bts::api::admin::BTSAccount& account){
    std::sort (account.mutable_security_position_limits()->begin(),
@@ -901,7 +931,7 @@ bool AdminServiceUtils::match_security_scope (const bts::api::admin::SecuritySco
 bool AdminServiceUtils::match_product_scope (const bts::api::admin::ProductScope& scope,
                                              const bts::api::Product& product){
    return product.exchange() == scope.exchange() &&
-          product.type() && scope.security_type() &&
+          product.type() == scope.security_type() &&
           _match_limit_rule_symbol(product.symbol(), scope.security_symbol());
 }
 bool AdminServiceUtils::match_product(const api::Security& security,
@@ -1025,4 +1055,7 @@ void AdminServiceUtils::bootstrap_data (bts::api::admin::AdminData& data){
    acct_a1_clip_size->set_clip_size(100);
 }
 bool AdminServiceUtils::has_id (uint32_t id){return id>0;}
+bool AdminServiceUtils::is_symbol_set (const std::string& symbol){
+   return symbol !="*";
+}
 }}

src/lib/api/service_helpers/AdminServiceUtils.h

@@ -240,6 +240,7 @@ public:
    // misc//////// ////////////////////////////
    ////////////////////////////////////////////
    static bool has_id (uint32_t id);
+   static bool is_symbol_set (const std::string& symbol);
    static void bootstrap_data (bts::api::admin::AdminData& data);
 };
 }}

src/lib/api/services/OrderService.proto

@@ -233,7 +233,7 @@ enum CTICode {
   CTI_CODE_CTI_CODE1 = 49;
   CTI_CODE_CTI_CODE2 = 50;
   CTI_CODE_CTI_CODE3 = 51;
-  CTI_CODE_CTI4 = 52;
+  CTI_CODE_CTI_CODE4 = 52;
 }
 
 enum NativeOrderStatus 

src/lib/config/protos/MarketData.proto

@@ -76,6 +76,10 @@ message CMEOptions
       repeated uint32 affinity = 10;
       optional uint32 udp_socket_buffer_size = 11 [default = 33554432]; //32mb
    }
+   enum BookType {
+       BOOKTYPE_CONSOLIDATED = 0;
+       BOOKTYPE_DIRECT = 1;
+   }
    message CMEChannel {
       repeated string id = 2;
       optional string feed_engine = 3 [default = "default"];
@@ -98,6 +102,7 @@ message CMEOptions
       optional bool join_snapshot_on_gap = 32 [default = false];
 
       optional bool load_security_db = 33 [default = true];
+      optional BookType book_type = 34 [default = BOOKTYPE_CONSOLIDATED];
 
    }
    optional string onixs_license = 1 [default = "{default}"];

src/lib/config/protos/UI.proto

@@ -28,20 +28,6 @@ message IShellModuleProps
    repeated ModuleProps module = 1;
 }
 
-message IShellWindowProps {
-   // TODO: Maybe IWindowManager should be responsible for total-windows-open limits?
-   // To achieve that we would need to use a sort of type-windowprops map.
-   // However, that makes the problem more complicated, and at this point we are just aiming to limit n_ladders open.
-   // So for now, just let the LadderWindowFactory manage this.
-   message WindowProps {
-      // NOTE: 50 will be a dangerously high default for most other windows (than Ladder),
-      // so be careful if you add other WindowProps members.
-      optional uint32 max_total_open = 1 [default = 50];
-   }
-
-   optional WindowProps ladder = 1;
-}
-
 message GridSettings {
     optional string font_family = 1 [default = "Calibri"];
     optional float font_size = 2 [default = 10];
@@ -62,9 +48,11 @@ message ShellLimits {
    optional uint64 limit_group_memory_mb = 3;
 }
 
-// priorities (ordered): SystemIdle, ApplicationIdle, ContextIdle, Background, Input, Loaded, Render, DataBind, Normal, Send
-// It seems like all priorities higher than ApplicationIdle choke the main thread.
-message LadderTimerParams {
+message LadderKnobs {
+  // priorities (ordered): SystemIdle, ApplicationIdle, ContextIdle, Background, Input, Loaded, Render, DataBind, Normal, Send
+  // It seems like all priorities higher than ApplicationIdle choke the main thread.
   optional string priority = 1 [default = "ApplicationIdle"];
   optional double interval_ms = 2 [default = 100];
+  optional uint32 seconds_to_recenter = 3 [default = 300];
+  optional uint32 max_open_ladders = 4 [default = 75];
 }

src/lib/md/SecurityDB.cpp

@@ -89,7 +89,7 @@ size_t load_securities(
 
 void store_if_new(const std::vector<Security*>& securities) {
    auto dbptr = open_db();
-   auto insert = utils::prepare_query(dbptr, "insert into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)");
+   auto insert = utils::prepare_query(dbptr, "insert or ignore into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)");
    sqlite3_exec_nofail(dbptr, "BEGIN TRANSACTION");
    std::string buffer;
    for (const api::Security* psecurity : securities) {
@@ -100,10 +100,7 @@ void store_if_new(const std::vector<Security*>& securities) {
          SQLITE_CHK(sqlite3_bind_int(insert.get(), 4, securitytype_of(*psecurity)), dbptr);
          SQLITE_CHK(sqlite3_bind_blob(insert.get(), 5, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr);
          SQLITE_CHK(sqlite3_bind_int(insert.get(), 6, expiration(*psecurity).to_short()), dbptr);
-         auto result = sqlite3_step(insert.get());
-         if (result != SQLITE_DONE) {
-            LOG(md, Warn, "Failed to insert security into security db. key={}", psecurity->id());
-         }
+         sqlite3_step(insert.get());
          sqlite3_reset(insert.get());
       }
    }

src/lib/md/SecurityIDRecorder.cpp

@@ -133,6 +133,8 @@ void SecurityIDRecorder::work_db() {
    // Once a security has a key it is public so record it in the security database
    std::shared_ptr<SecurityIDEntry> temp;
    vector<std::shared_ptr<SecurityIDEntry>> new_entries;
+   // SPARK-1446 Need to try to add existing securities to the db in case we deleted them by hand
+   vector<std::shared_ptr<SecurityIDEntry>> existing_securities;
    vector<std::shared_ptr<SecurityIDEntry>> defer;
    vector<string> new_descriptors;
    size_t count = 0;
@@ -150,6 +152,7 @@ void SecurityIDRecorder::work_db() {
          if (key != invalid_security_key) {
             temp->security.set_id(key);
             temp->callback(temp->security);
+            existing_securities.push_back(move(temp));
          }
          else {
             // always push in parallel
@@ -168,7 +171,7 @@ void SecurityIDRecorder::work_db() {
    }
    
 
-   if (new_entries.size() > 0) {
+   if (!new_entries.empty()) {
       sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
 
       for (size_t x = 0; x < new_entries.size(); ++x) {
@@ -179,15 +182,22 @@ void SecurityIDRecorder::work_db() {
       }
 
       sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
+   }
 
-      // record the securities
+   // record the securities
+   if (!new_entries.empty() || !existing_securities.empty()) {
       std::vector<Security*> save_these;
-      save_these.reserve(new_entries.size());
+      save_these.reserve(new_entries.size() + existing_securities.size());
       for (const auto& e : new_entries) {
          save_these.push_back(&e->security);
       }
+      for (const auto& e : existing_securities) {
+         save_these.push_back(&e->security);
+      }
       db::store_if_new(save_these);
+   }
 
+   if (!new_entries.empty()) {
       for (size_t x = 0; x < new_entries.size(); ++x) {
          Security& security = new_entries[x]->security;
          new_entries[x]->callback(security);
@@ -199,9 +209,18 @@ void SecurityIDRecorder::bulk_insert(const std::vector<const api::Security*>& da
    if (data.size() == 0) return;
    sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
    std::string descriptor;
-   for (const api::Security* sec : data) {
-      descriptor = security_descriptor(*sec);
-      insert(prefix + descriptor, securitytype_of(*sec));
+   {
+      auto insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)");
+      for (const api::Security* sec : data) {
+         descriptor = prefix + security_descriptor(*sec);
+         auto type = securitytype_of(*sec);
+
+         sqlite3_bind_int(insert_stmt.get(), 1, sec->id());
+         sqlite3_bind_text(insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(), SQLITE_STATIC);
+         sqlite3_bind_int(insert_stmt.get(), 3, (int)type);
+         sqlite3_step(insert_stmt.get());
+         sqlite3_reset(insert_stmt.get());
+      }
    }
    sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
 }

src/lib/md/relay/QueryServer.cpp

@@ -15,6 +15,9 @@
 #include "lib/api/apicore.pb.h"
 #include "lib/api/services/MarketDataRelay.server.pb.h"
 #include "lib/api/public/ServiceContext.h"
+#ifndef _WIN32
+#include "server/sparkd/metrics/metrics.h"
+#endif
 
 #include <vector>
 #include <list>
@@ -61,13 +64,29 @@ struct DatedHandle {
 
 struct MDConsumerGroup {
    // probably just have to send the consumer in from outside
-   MDConsumerGroup(const UserSymbol& symbol, const MDConsumer& consumer) :
+   MDConsumerGroup(const UserSymbol& symbol, const MDConsumer& consumer, metrics::MetricsContainer& stats) :
       symbol(symbol),
       platform(platform_for_exchange(symbol.exchange())),
       channel(consumer.get_channel(symbol)),
       handles()
    {
       set_consumer(consumer);
+      
+      // create and publish a stat for this group
+#ifndef _WIN32
+      if (symbol.type() != SecurityType::SECURITYTYPE_SYNTHETIC) {
+         std::vector<metrics::Label> labels;
+         labels.push_back({ "symbol", symbol.symbol() });
+         labels.push_back({ "type", SecurityType_Name(symbol.type()).substr(13) /*SECURITYTYPE_*/ });
+         labels.push_back({ "exch", Venue_Name(symbol.exchange()).substr(6) /*VENUE_*/ });
+         stats.publish(
+            std::make_shared<metrics::LazyCounterMetric>("spark_md_book_updates", [=]() { return std::max(channel.bbo_count(), channel.depth_count());  }, labels));
+         stats.publish(
+            std::make_shared<metrics::LazyCounterMetric>("spark_md_security_count", [=]() { return channel.security_count();  }, labels));
+         stats.publish(
+            std::make_shared<metrics::LazyCounterMetric>("spark_md_trade_count", [=]() { return channel.trade_count();  }, labels));
+      }
+#endif
    }
    MDConsumerGroup(const MDConsumerGroup& other) = delete;
 
@@ -109,6 +128,7 @@ void set_price(PriceSize ps, co2::TradeMessage* ptrade) {
    ptrade->set_exponent(ps.exponent());
    ptrade->set_size(ps.size());
 }
+
 void copy_misc(MiscData misc, co2::MiscMessage* pmisc) {
    pmisc->set_settlement_price(misc.settlement_price);
    pmisc->set_settlement_date(misc.settlement_date.to_short());
@@ -134,14 +154,14 @@ class QueryServerImpl : public IQueryServer {
    boost::container::flat_map<UserSymbol,shared_ptr<MDConsumerGroup>> _groups;
 
 public:
-   QueryServerImpl(const std::set<UserSymbol>& symbols, std::shared_ptr<api::ISyntheticSecurityService>& ss) :
+   QueryServerImpl(const std::set<UserSymbol>& symbols, std::shared_ptr<api::ISyntheticSecurityService>& ss, metrics::MetricsContainer& stats) :
       _consumer(symbols, ss),
       _symbols(symbols),
       _ss(ss),
       _groups()
    {
       for (const auto& sym : symbols) {
-         _groups.emplace(std::make_pair(sym, std::make_unique<MDConsumerGroup>(sym, _consumer)));
+         _groups.emplace(std::make_pair(sym, std::make_unique<MDConsumerGroup>(sym, _consumer, stats)));
       }
       _groups.shrink_to_fit();
 
@@ -336,24 +356,36 @@ public:
 struct MDRelayServer : bts::co2::MarketDataRelayBase {
    const QueryServerImpl& _db;
    utils::Dispatcher& _threadpool;
-    MDRelayServer(const QueryServerImpl& db, utils::Dispatcher& threadpool, bts::api::ServiceContext ctx) : 
+#ifndef _WIN32
+   std::shared_ptr<metrics::HistogramMetric<5>> _time,_size;
+#endif
+
+    MDRelayServer(const QueryServerImpl& db, utils::Dispatcher& threadpool, bts::api::ServiceContext ctx, metrics::MetricsContainer& stats) : 
        bts::co2::MarketDataRelayBase(ctx),
        _db(db), _threadpool(threadpool)
-    { }
+#ifndef _WIN32
+       , _time(std::make_shared<metrics::HistogramMetric<5>>("spark_qserver_response_time_ms", std::array<double,5>({ 10.0, 50.0, 100.0, 500.0, 1000.0 }))),
+       _size(std::make_shared<metrics::HistogramMetric<5>>("spark_qserver_size_bytes", std::array<double,5>({ 1024 /*kb*/, 1024*1024 /*mb*/, (1024*1024)*4, (1024*1024)*16, (1024*1024)*32 })))
+    {
+       stats.publish(_time);
+       stats.publish(_size);
+    }
+#else
+   {}
+#endif
 
     virtual void requestmd(const MDRequest & request, const std::function<void(const MDReply&)>& reply) override {
+        TimeStamp start = TimeStamp::now();
        _threadpool.post([=]() {
           google::protobuf::Arena arena;
           MDReply* preply = google::protobuf::Arena::CreateMessage<MDReply>(&arena);
-          TimeStamp start = TimeStamp::now();
           _db.query(request, preply);
-          auto elapsed = (TimeStamp::now() - start);
-          LOG(md, Diag,
-             "Request for: {} took {}us and returned {} bytes before compression",
-             request.host(),
-             elapsed.microseconds(),
-             preply->ByteSize());
           reply(*preply);
+          auto elapsed = (TimeStamp::now() - start);
+#ifndef _WIN32
+          _time->observe(elapsed.milliseconds());
+          _size->observe(preply->ByteSize());
+#endif
        });
     }
     
@@ -382,11 +414,15 @@ struct MDRelayServer : bts::co2::MarketDataRelayBase {
 };
 
 struct QueryServer {
-    QueryServer(std::set<UserSymbol> symbols, utils::Dispatcher& threadpool, const bts::api::ServiceContext& ctx, std::shared_ptr<api::ISyntheticSecurityService>& ss):
-    db(symbols, ss),
-    relay(db, threadpool, ctx),
-    security_timer(),
-    health_timer()
+    QueryServer(std::set<UserSymbol> symbols, 
+                utils::Dispatcher& threadpool, 
+                const bts::api::ServiceContext& ctx, 
+                std::shared_ptr<api::ISyntheticSecurityService>& ss,
+                metrics::MetricsContainer& stats):
+      db(symbols, ss, stats),
+      relay(db, threadpool, ctx, stats),
+      security_timer(),
+      health_timer()
     { }
     QueryServerImpl db;
     MDRelayServer relay;
@@ -394,7 +430,7 @@ struct QueryServer {
     std::unique_ptr<utils::DispatcherTimer> health_timer;
 };
    
-QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared_ptr<api::ISyntheticSecurityService>& ss) {
+QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared_ptr<api::ISyntheticSecurityService>& ss, metrics::MetricsContainer& stats) {
    auto& threadpool = utils::Dispatcher::get_named("MDQuery");
 
    std::set<UserSymbol> all_symbols;
@@ -403,7 +439,8 @@ QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared
    SecurityType type;
    bts_assert(api::SecurityType_Parse(config::get<PlatformConfig>().type(), &type));
    get_system_md_symbols(venue, type, all_symbols);
-   QueryServerPtr container = std::make_shared<QueryServer>(all_symbols, threadpool, ctx, ss);
+
+   QueryServerPtr container = std::make_shared<QueryServer>(all_symbols, threadpool, ctx, ss, stats);
    auto& db = container->db;
 
     // Scan for new securities every half second
@@ -426,8 +463,8 @@ QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared
    return container;
 }
 
-std::unique_ptr<IQueryServer> create_query_server(const std::set<UserSymbol>& symbols, std::shared_ptr<api::ISyntheticSecurityService>& ss) {
-   return std::make_unique<QueryServerImpl>(symbols, ss);
+std::unique_ptr<IQueryServer> create_query_server(const std::set<UserSymbol>& symbols, std::shared_ptr<api::ISyntheticSecurityService>& ss, metrics::MetricsContainer& stats) {
+   return std::make_unique<QueryServerImpl>(symbols, ss, stats);
 }
 
 }}//namespace

src/lib/md/relay/QueryServer.h

@@ -11,6 +11,7 @@
 
 namespace bts {
     namespace api { class ServiceContext; }
+    namespace metrics { class MetricsContainer; }
 namespace md {
 
 struct IQueryServer {
@@ -18,11 +19,11 @@ struct IQueryServer {
    virtual void scan_securities() = 0;
    virtual void query(const co2::MDRequest& request, co2::MDReply* reply) const = 0;
 };
-std::unique_ptr<IQueryServer> create_query_server(const std::set<UserSymbol>&, std::shared_ptr<api::ISyntheticSecurityService>&);
+std::unique_ptr<IQueryServer> create_query_server(const std::set<UserSymbol>&, std::shared_ptr<api::ISyntheticSecurityService>&, metrics::MetricsContainer&);
 
 struct QueryServer;
 typedef std::shared_ptr<QueryServer> QueryServerPtr;
-QueryServerPtr run_query_server(const bts::api::ServiceContext&, std::shared_ptr<api::ISyntheticSecurityService>&);
+QueryServerPtr run_query_server(const bts::api::ServiceContext&, std::shared_ptr<api::ISyntheticSecurityService>&, metrics::MetricsContainer&);
 
 }}//namespace
 

src/lib/md/relay/RelayRecorder.cpp

@@ -121,9 +121,6 @@ void record_security_keys(md::SecurityIDRecorder& idrec, const co2::MDReply& rep
          }
       }
    }
-   // Sort by id only actually works if the recording was done on a fresh machine
-   std::sort(data.begin(), data.end(), [](const Security* a, const Security* b) { return a->id() < b->id(); });
-   // We could insert garbage until data[0]->id()
    idrec.bulk_insert(data, "REPLAY: ");
 }
 #endif

src/lib/md/test/md2_tests.cpp

@@ -10,6 +10,7 @@
 #include "lib/utils/defer.h"
 #include "lib/utils/warn_ctrl.h"
 #include "lib/utils/bts_chrono.h"
+#include "server/sparkd/metrics/metrics.h"
 
 #include "lib/config/protos/MarketData.pb.h"
 
@@ -30,6 +31,15 @@
 using namespace std;
 
 namespace bts {
+
+#ifdef _WIN32
+namespace metrics {
+struct MetricCollection{};
+MetricsContainer::MetricsContainer() {}
+MetricsContainer::~MetricsContainer() {}
+}
+#endif
+
 int md_tests = 0;
 namespace md {
 // import this from RelayClient.cpp
@@ -1275,7 +1285,8 @@ struct Co2RelayTests {
       set<UserSymbol> symbols;
       symbols.insert(symbol);
       shared_ptr<api::ISyntheticSecurityService> empty;
-      auto server = create_query_server(symbols, empty);
+      metrics::MetricsContainer stats;
+      auto server = create_query_server(symbols, empty, stats);
       REQUIRE(server);
       server->scan_securities();
 

src/lib/quickbus/network/QBConnection.cpp

@@ -193,10 +193,18 @@ void QBConnection::handle_msg(msgs::QBMsg *msgs)
             me->intro=op.intro();
             me->my_id = NodeID(me->intro.node_id());
 
-            if (!me->router.add_neighbour(me->my_id,shared_from_this())) {
-               close_connection("Node is already connected");
+            if (me->router.is_neighbour(me->my_id)) {
+               LOG(QBus,Warn, "Refusing connection from {} to {} reason: {} is already connected" ,
+                   me->my_desc , me->remote_desc , me->my_id.value());
+               if (me->socket.is_open()) 
+                  me->socket.close();
                return;
             }
+            else {
+               LOG(QBus,Warn, "Accepted connection from {} to {}. {} is now connected." ,
+                   me->my_desc , me->remote_desc , me->my_id.value());
+               me->router.add_neighbour(me->my_id,shared_from_this());
+            }
          }
          break;
       }

src/lib/quickbus/routing/Router.cpp

@@ -184,6 +184,11 @@ void Router::remove_neighbour(NodeID const&removed_neighbour_id)
    me->neighbours.erase(removed_neighbour_id);
 }
 
+bool Router::is_neighbour(NodeID const&neighbour_id) const
+{
+   return me->neighbours.find(neighbour_id) != me->neighbours.end();
+}
+
 void Router::advertise(NodeID const& sender,TableID const&tid,Advertisement const&ad) 
 {
    LOG(QBus,Diag, "{} received advertisement from nid: {}  tid: {} ad topic: {}",

src/lib/quickbus/routing/Router.h

@@ -15,7 +15,8 @@ public:
    NodeID id() const ;
    bool add_neighbour (NodeID const& neighbour_id, INodePtr const& neighbour);
    void remove_neighbour (NodeID const& neighbour_id);
-
+   bool is_neighbour(NodeID const&neighbour_id) const;
+   
    // from INode 
    void advertise(NodeID const& sender,TableID const&id,Advertisement const&adv) override;
    void unadvertise(NodeID const& sender,TableID const&tid) override;

src/server/sparkd/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/Md.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;dspark/mirage.cpp;dspark/mirage/routing/Router.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;main.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/MDP3Utils.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;mercury/actors/Actor.cpp;mercury/broker/AsyncEventBroker.cpp;mercury/broker/CoreEventBroker.cpp;mercury/broker/IEventBroker.cpp;mercury/events/TimerEvent.cpp;mercury/mercury_utils/Executor.cpp;mercury/mercury_utils/IPollableActor.cpp;mercury/mercury_utils/mercury_common.cpp;mercury/synopsis/SynopsisBuilder.cpp;mercury/topics/ITopic.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/TickOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/Line truncated
+set(sources cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/Md.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage.cpp;dspark/mirage/routing/Router.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;main.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/MDP3Utils.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;mercury/actors/Actor.cpp;mercury/broker/AsyncEventBroker.cpp;mercury/broker/CoreEventBroker.cpp;mercury/broker/IEventBroker.cpp;mercury/events/TimerEvent.cpp;mercury/mercury_utils/Executor.cpp;mercury/mercury_utils/IPollableActor.cpp;mercury/mercury_utils/mercury_common.cpp;mercury/synopsis/SynopsisBuilder.cpp;mercury/topics/ITopic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/TickOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreaderLine truncated

src/server/sparkd/cmds/Run.cpp

@@ -8,6 +8,7 @@
 #include "lib/logging/logging.h"
 #include "lib/os/os.h"
 #include "lib/quickbus/quickbus.h"
+#include "lib/utils/Dispatcher.h"
 
 #include "lib/md/SyntheticSecurityService.h"
 #include "server/sparkd/exchange_sim/ExchangeSim.h"
@@ -16,6 +17,8 @@
 #include "server/sparkd/persistence/PersistenceServiceServer.h"
 #include "server/sparkd/support/ErrorReportService.h"
 #include "server/sparkd/http/WebServer.h"
+#include "server/sparkd/metrics/metrics.h"
+#include "server/sparkd/metrics/system_metrics.h"
 #include "server/sparkd/user/AdminServiceServer.h"
 #include "server/sparkd/user/UserServiceServer.h"
 
@@ -37,7 +40,9 @@ int Run::execute(std::vector<std::string> const &) {
    qbus::Quickbus quickbus(qb_cfg);
    api::ServiceContext sc(&quickbus);
 
-   http::SparkWebServer webserver(sc, ports.config_port());
+   metrics::MetricsContainer stats;
+   metrics::publish_system_metrics(stats);
+   http::SparkWebServer webserver(sc, stats, ports.config_port());
 
    persistence::PersistenceServiceServer settings_saver(sc, "settings");
    user::UserServiceServer login_server(sc);
@@ -45,10 +50,15 @@ int Run::execute(std::vector<std::string> const &) {
    std::shared_ptr<api::ISyntheticSecurityService> synthetic_service =
        std::make_shared<api::synthetic::SyntheticSecurityService>(sc);
    md::QueryServerPtr md_query_server(
-       md::run_query_server(sc, synthetic_service));
+       md::run_query_server(sc, synthetic_service, stats));
    orderman::OrderServiceServer order_service_server(sc, synthetic_service);
    support::ErrorReportServer error_report_service(sc);
    admin::AdminServiceServer admin_service(sc);
+
+   auto bg_health = std::make_shared<metrics::IntCounterMetric>("spark_background_thread");
+   utils::DispatcherTimer timer(utils::Dispatcher::background(), 5000, [=]() { bg_health->increment(); });
+   timer.start();
+   stats.publish(bg_health);
    
    std::thread([]() {
       os::set_thread_name("ExchangeSim");

src/server/sparkd/cmds/Web.cpp

@@ -0,0 +1,56 @@
+#include "Web.h"
+
+#include "lib/config/config.h"
+#include "lib/config/protos/quickbus_cfg.pb.h"
+#include "lib/config/protos/Ports.pb.h"
+#include "lib/quickbus/quickbus.h"
+#include "lib/api/public/ServiceContext.h"
+
+#include "server/sparkd/metrics/metrics.h"
+#include "server/sparkd/metrics/system_metrics.h"
+#include "server/sparkd/http/WebServer.h"
+#include "lib/md/relay/QueryServer.h"
+#include "lib/md/SyntheticSecurityService.h"
+#include "server/sparkd/user/AdminServiceServer.h"
+#include "server/sparkd/user/UserServiceServer.h"
+#include "server/sparkd/md/run_md.h"
+
+#include <vector>
+#include <string>
+
+namespace bts {
+namespace cmd {
+
+int Web::execute(const std::vector<std::string>& args) {
+   config::QuickbusNodeCfg qb_cfg;
+   auto& ports = config::get<config::Ports>();
+   qb_cfg.set_my_port(ports.qbus_port());
+   qbus::Quickbus quickbus(qb_cfg);
+   api::ServiceContext sc(&quickbus);
+
+   metrics::MetricsContainer stats;
+   metrics::publish_system_metrics(stats);
+   http::SparkWebServer webserver(sc, stats, ports.config_port());
+
+   user::UserServiceServer login_server(sc);
+   if (std::find(args.begin(), args.end(), "-md") != args.end()) {
+      md::run_market_datapublishers();
+   }
+
+   std::shared_ptr<api::ISyntheticSecurityService> synthetic_service =
+       std::make_shared<api::synthetic::SyntheticSecurityService>(sc);
+   md::QueryServerPtr md_query_server(
+       md::run_query_server(sc, synthetic_service, stats));
+
+   admin::AdminServiceServer admin_service(sc);
+
+
+   
+   std::this_thread::sleep_until(
+       std::chrono::time_point<std::chrono::system_clock>::max());
+   return 0;
+}
+
+std::string Web::help() { return "\t run only the services required for the web admin.  This cannot be run concurrently with 'sparkd run'\n\tAdd -md to start market data too."; }
+
+}}//namespace

src/server/sparkd/cmds/Web.h

@@ -0,0 +1,17 @@
+
+#pragma once
+
+#include "ICmd.h"
+
+namespace bts {
+namespace cmd {
+
+class Web : public ICmd 
+{
+public:
+   int execute(std::vector<std::string> const &args);
+   std::string help ();
+};
+
+}}
+

src/server/sparkd/http/WebServer.cpp

@@ -5,6 +5,7 @@
 #include "lib/api/apigateway.pb.h"
 #include "lib/logging/logging.h"
 #include "lib/os/os.h"
+#include "lib/utils/TimeStamp.h"
 
 #include <websocketpp/config/asio_no_tls.hpp>
 #include <websocketpp/server.hpp>
@@ -49,24 +50,33 @@ namespace http {
       std::string _www;
       std::string _conf;
       const api::ServiceContext& _ctx;
+      metrics::MetricsContainer& _metrics;
+      std::shared_ptr<metrics::HistogramMetric<4>> _mtimer;
 
-      ServerState(const api::ServiceContext& ctx, int port) :
+      ServerState(const api::ServiceContext& ctx, metrics::MetricsContainer& metrics, int port) :
          _m(),
          _dispatcher(utils::Dispatcher::get_named("Webserver")), 
          _session(), 
          _server(),
          _port(port),
          _www(),
-         _ctx(ctx)
+         _ctx(ctx),
+         _metrics(metrics),
+         _mtimer(std::make_shared<metrics::HistogramMetric<4>>("spark_metrics_query_duration_ms", 
+                                                               std::array<double,4>({ 0.1, 0.5, 1.0, 5.0 })))
       {
          _www = os::sparkdata_path("www");
          _conf = os::sparkdata_path("conf");
 
+         _metrics.publish(_mtimer);
+
          _server.init_asio(_dispatcher.io_service());
          _server.set_open_handler(bind(&ServerState::on_open, this, ::_1));
          _server.set_close_handler(bind(&ServerState::on_close, this, ::_1));
          _server.set_message_handler(bind(&ServerState::on_message, this, ::_1, ::_2));
          _server.set_http_handler(bind(&ServerState::on_http, this, ::_1));
+         // Turn off logging
+         _server.clear_access_channels(websocketpp::log::alevel::all);
          run();
       }
 
@@ -137,6 +147,15 @@ namespace http {
          con->set_status(websocketpp::http::status_code::ok);
       }
 
+      void return_metrics(connection_ptr& con) {
+         utils::TimeStamp start = utils::TimeStamp::now();
+         std::stringstream ss;
+         _metrics.print(ss);
+         con->set_body(ss.str());
+         con->set_status(websocketpp::http::status_code::ok);
+         _mtimer->observe((utils::TimeStamp::now() - start).microseconds() / 1000.0);
+      }
+
       void directory_index(connection_ptr& con, const std::string& resource, const boost::filesystem::path& path) {
          // directory listing
          std::string body = "<!doctype html>\n";
@@ -198,11 +217,22 @@ namespace http {
          con->set_body(read_file(path.string()));
          con->set_status(websocketpp::http::status_code::ok);
       }
+
       void return_bts_installer(connection_ptr& con) {
          std::string installer_bits = read_file(_www+"/bts_installer.exe");
          const std::string installer_config = _www+"install_config.json";
          if (boost::filesystem::is_regular_file(installer_config)) {
-            installer_bits += read_file(installer_config);
+            auto conf = read_file(installer_config);
+            std::string needle = "$HOST";
+            std::size_t pos;
+            do {
+               pos = conf.find(needle,0);
+               if (pos != std::string::npos) {
+                  auto host = con->get_uri()->get_host();
+                  conf.replace(pos, needle.size(), host);
+               }
+            } while(pos != std::string::npos);
+            installer_bits += conf;
          }
          con->set_body(installer_bits);
          con->set_status(websocketpp::http::status_code::ok);
@@ -219,10 +249,13 @@ namespace http {
       void on_http(connection_hdl hdl) {
          auto con = _server.get_con_from_hdl(hdl);
          auto resource = strip_query_str(con->get_resource());
-         LOG(Api,Info, "REQUEST: {}", resource);
+         LOG(Api,Diag, "HTTP GET: {}", resource);
          if (boost::iequals(resource, "/config/.all")) {
             return_dotall(con);
          }
+         else if (boost::iequals(resource, "/metrics")) {
+            return_metrics(con);
+         }
          else if (boost::iequals(resource, "/bts_installer.exe") && boost::filesystem::is_regular_file(_www+"/bts_installer.exe")) {
             return_bts_installer(con);
          }
@@ -256,8 +289,8 @@ namespace http {
 
 
 
-SparkWebServer::SparkWebServer(const api::ServiceContext& ctx, int port) :
-   _me(std::make_shared<ServerState>(ctx, port))
+SparkWebServer::SparkWebServer(const api::ServiceContext& ctx, metrics::MetricsContainer& metrics, int port) :
+   _me(std::make_shared<ServerState>(ctx, metrics, port))
 { 
 }
 

src/server/sparkd/http/WebServer.h

@@ -1,5 +1,6 @@
 #pragma once
 #include "lib/api/public/ServiceContext.h"
+#include "server/sparkd/metrics/metrics.h"
 #include <memory>
 
 namespace bts {
@@ -10,7 +11,7 @@ namespace http {
    class SparkWebServer {
       std::shared_ptr<ServerState> _me;
       public:
-      SparkWebServer(const api::ServiceContext& ctx, int port);
+      SparkWebServer(const api::ServiceContext& ctx, metrics::MetricsContainer& metrics, int port);
    };
 
 }}

src/server/sparkd/jetfix/gateway/CoreGateway.cpp

@@ -449,7 +449,7 @@ void CoreGateway::sync_time(EventData const &ev) {
                me->session_info.reset();
                me->start_of_pause = datetime::now();
                LOG(Jetfix, Error,
-                   "Failed to connect to exchange gateway {} too many"
+                   "Failed to connect to exchange gateway {} too many "
                    "times. Giving up permanently. You must bounce "
                    "the server to recover.",
                    me->cfg.name());

src/server/sparkd/jetfix/store/FIXFileStore.cpp

@@ -127,17 +127,6 @@ std::string store_path (std::string base_path,
    return result.string ();
 }
 
-void create_current_simlink (std::string const &path)
-{
-   auto sym_link_path = boost::filesystem::path (path);
-   sym_link_path /= "..";
-   sym_link_path /= "current";
-   if (boost::filesystem::exists (sym_link_path)) 
-      boost::filesystem::remove (sym_link_path);
-   
-   boost::filesystem::create_symlink (boost::filesystem::path (path),sym_link_path);
-}
-
 std::map<uint32_t,InMsgBuffer>
 load_replay (config::JetFIX::Connection const &cfg,std::string const&path, uint32_t max_sn)
 {
@@ -227,8 +216,6 @@ FIXFileStore::FIXFileStore (config::JetFIX::Connection const&cfg,std::string bas
          }
       }
       me->fix_store.open (fix_log_p,std::ofstream::app);
-
-      create_current_simlink (path);
       me->writer_thread = std::thread (std::bind (&FIXFileStore::write,this));
    }
 }

src/server/sparkd/main.cpp

@@ -15,6 +15,7 @@
 #include "server/sparkd/cmds/AutoCert.h"
 #include "server/sparkd/cmds/QuickTest.h"
 #include "server/sparkd/cmds/Span.h"
+#include "server/sparkd/cmds/Web.h"
 
 int main(int argc, char **argv) {
    using namespace bts;
@@ -33,6 +34,7 @@ int main(int argc, char **argv) {
    REGISTER(QuickTest);
    REGISTER(AutoCert);
    REGISTER(Span);
+   REGISTER(Web);
    END_CMD;
 
    utils::finit_spark_lib();

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp

@@ -1,8 +1,8 @@
 #include "MDP3Listener.h"
 #include "lib/utils/datetime.h"
 #include <vector>
-#include "server/sparkd/tracing/bts_trace.h"
 #include "lib/utils/helper_macros.h"
+#include "lib/config/protos/MarketData.pb.h"
 
 using namespace std;
 using namespace bts::co2;
@@ -84,7 +84,8 @@ MDP3Listener::MDP3Listener(
       MDProducer& producer,
       SecurityIDRecorder& recorder,
       co2::SettlementRecorder& ,
-      std::vector<MDProducerChannel> channels) :
+      const std::vector<MDProducerChannel> channels,
+      CMEOptions::BookType bt) :
    _index(index),
    _producer(producer),
    _packet_arrived(),
@@ -94,6 +95,7 @@ MDP3Listener::MDP3Listener(
    _symbol_filter_storage(),
    _symbol_filter(),
    _channels(channels),
+   _booktype(bt),
    _concurrency_monitor(0),
    _book_updates()
 {
@@ -188,11 +190,6 @@ double double_of_price(const Price& price, int8_t display_factor) {
    return (price.mantissa() * pow10(price.exponent())) * pow10(display_factor);
 }
 
-inline PriceSize pricesize_of_price_size(const onixs::PRICE price, int8_t display_factor, Size size) {
-   PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
-   return result;
-}
-
 inline datetime::LLDateTime ll_of_timestamp(const onixs::Timestamp& ts) {
    return datetime::ll_from_parts(ts.year(), (int)ts.month(), ts.day(), ts.hour(), ts.minute(), ts.second(), ts.millisecond());
 }
@@ -335,49 +332,18 @@ void MDP3Listener::onDefinition(onixs::Handler&, const onixs::Security&, const o
    _index.add_spread_legs(fsec.numeric_id, leg_exchange_ids);
 }
 
-
-//
-// Book
-void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::ConsolidatedBook& book) {
-   tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", 0,_latest_sequence);
-
-   auto data = _index.get_security(sec.id());
-   if (data == nullptr) return;
-   bts_assert(data->id == (uint32_t)sec.id());
-   auto display_factor = data->handle.security().display_factor;
-
-   // Put the data into the cache. Flush at the end of the packet.
-   auto key = data->handle.key();
-   auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) {
-      return key == entry.handle.key();
-   });
-   bool found = find_iter != _book_updates.end();
-   if (!found) {
-      _book_updates.emplace_back(data->handle);
-   }
-   BookData& cache = found ? *find_iter : _book_updates.back();
-
-   // Overwrite all entries every time
-   size_t bids_size = book.bids().size();
-   size_t asks_size = book.offers().size();
-   if (!data->handle.has_depth()) {
-      bids_size = min(bids_size, size_t { 1 });
-      asks_size = min(asks_size, size_t { 1 });
-   }
-   for (size_t x = 0; x < bids_size; ++x) {
-      cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
-   }
-   for (size_t x = 0; x < asks_size; ++x) {
-      cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
-   }
-   // NaN out the remaining depth
-   for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
-      cache.bids[x] = PriceSize::NO_MARKET;
-   }
-   for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
-      cache.asks[x] = PriceSize::NO_MARKET;
+void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::DirectBook& book) {
+   if (_booktype == CMEOptions::BOOKTYPE_DIRECT) {
+      onBookUpdate(sec, book);
    }
 }
+
+void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::ConsolidatedBook& book) {
+   if (_booktype == CMEOptions::BOOKTYPE_CONSOLIDATED) {
+      onBookUpdate(sec, book);
+   }
+}
+
 //
 // Trades
 

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h

@@ -3,6 +3,7 @@
 #include "lib/utils/bts_no_copy.h"
 #include "lib/md/coconut2.h"
 #include "CMESecurityIndex.h"
+#include "server/sparkd/tracing/bts_trace.h"
 
 #include "onixs.h"
 #include <tbb/concurrent_queue.h>
@@ -13,13 +14,23 @@
 namespace bts {
 namespace md {
 
+inline PriceSize pricesize_of_price_size(const onixs::PRICE price, int8_t display_factor, Size size) {
+   PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
+   return result;
+}
+
 class MDP3Listener :
    public onixs::HandlerListener,
    public onixs::MarketDataListener,
    public onixs::SecurityListener
 {
 public:
-   MDP3Listener(CMESecurityIndex& index, MDProducer&, SecurityIDRecorder&, co2::SettlementRecorder&, std::vector<MDProducerChannel>);
+   MDP3Listener(CMESecurityIndex& index, 
+                MDProducer&, 
+                SecurityIDRecorder&, 
+                co2::SettlementRecorder&, 
+                const std::vector<MDProducerChannel>,
+                CMEOptions::BookType);
    virtual ~MDP3Listener();
 
    //
@@ -51,10 +62,52 @@ public:
    //
    // Book
    // void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook&) override;
-   // void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override;
+   void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override;
    // void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ImpliedBook&) override;
    void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ConsolidatedBook&) override;
 
+   template <class TBook>
+   void onBookUpdate(const onixs::Security& sec, const TBook& book) {
+      tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", 0,_latest_sequence);
+
+      auto data = _index.get_security(sec.id());
+      if (data == nullptr) return;
+      bts_assert(data->id == (uint32_t)sec.id());
+      auto display_factor = data->handle.security().display_factor;
+
+      // Put the data into the cache. Flush at the end of the packet.
+      auto key = data->handle.key();
+      auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) {
+         return key == entry.handle.key();
+      });
+      bool found = find_iter != _book_updates.end();
+      if (!found) {
+         _book_updates.emplace_back(data->handle);
+      }
+      BookData& cache = found ? *find_iter : _book_updates.back();
+
+      // Overwrite all entries every time
+      size_t bids_size = book.bids().size();
+      size_t asks_size = book.offers().size();
+      if (!data->handle.has_depth()) {
+         bids_size = std::min(bids_size, size_t { 1 });
+         asks_size = std::min(asks_size, size_t { 1 });
+      }
+      for (size_t x = 0; x < bids_size; ++x) {
+         cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
+      }
+      for (size_t x = 0; x < asks_size; ++x) {
+         cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
+      }
+      // NaN out the remaining depth
+      for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
+         cache.bids[x] = PriceSize::NO_MARKET;
+      }
+      for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
+         cache.asks[x] = PriceSize::NO_MARKET;
+      }
+   }
+
    //
    // Trades
    void onTrade(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
@@ -125,6 +178,7 @@ private:
    std::vector<std::string> _symbol_filter_storage;
    std::set<onixs::StrRef> _symbol_filter;
    std::vector<MDProducerChannel> _channels;
+   CMEOptions::BookType _booktype;
    std::atomic<int> _concurrency_monitor;
 
    struct BookData {

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp

@@ -213,7 +213,12 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(const string& channel_id) {
    settings.feeds().connectivityConfigurationFile(config_path());
 
    // Market data options
-   settings.bookManagement().consolidatedBooks().maintain(true);
+   if (options.book_type() == CMEOptions::BOOKTYPE_CONSOLIDATED) {
+      settings.bookManagement().consolidatedBooks().maintain(true);
+   }
+   else {
+      settings.bookManagement().directBooks().maintain(true); 
+   }
 
    // 10k default is *far* too low
    settings.recoveryQueueMaxSize(options.recovery_queue_size());
@@ -334,12 +339,13 @@ unique_ptr<MDP3Listener> MDP3Main::prepare_listener(onixs::Handler& handler, con
       producer_channels.push_back(_producer.get_channel(symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
   }
 
-   unique_ptr<MDP3Listener> listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels));
+   const auto& options = channel_settings_for(channel.id);
+
+   unique_ptr<MDP3Listener> listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels, options.book_type()));
    for (auto& symbol : channel.symbols) {
       listener->add_filter_symbol(symbol.symbol());
    }
 
-   const auto& options = channel_settings_for(channel.id);
    if (options.load_security_db()) {
       // Load the list of known securities and add them to the index for fast startup
       db::load_securities(channel.symbols, [&](const api::Security& security) {

src/server/sparkd/metrics/metrics.cpp

@@ -0,0 +1,77 @@
+#include "metrics.h"
+#include <sstream>
+#include <tbb/concurrent_vector.h>
+
+using std::vector;
+using std::shared_ptr;
+
+namespace bts {
+namespace metrics {
+
+Metric::Metric(const std::string& cname, const std::vector<Label>& clabels):
+   name(cname),
+   labels(clabels) {
+}
+
+std::string Metric::str() const {
+   std::stringstream ss;
+   print(ss);
+   return ss.str();
+}
+
+std::ostream& operator<<(std::ostream& os, const std::vector<Label>& labels) {
+   if (!labels.empty()) {
+      os << "{";
+      for (auto iter = labels.begin(); iter != labels.end(); ++iter) {
+         if (iter != labels.begin()) os << ",";
+         os << iter->name << "=\"" << iter->value << "\"";
+      }
+      os << "}";
+   }
+   return os;
+}
+
+const std::string double_to_string(double d) {
+   std::stringstream ss;
+   ss << d;
+   return ss.str();
+}
+
+const std::vector<Label> vector_concat(const std::vector<Label>& orig, const std::vector<Label>& extra) {
+   if (orig.empty()) return extra; 
+   std::vector<Label> result;
+   result.reserve(orig.size() + extra.size());
+   result.insert(result.end(), orig.begin(), orig.end());
+   result.insert(result.end(), extra.begin(), extra.end());
+   return result;
+}
+
+
+void LazyCounterMetric::print(std::ostream& os) const {
+   os << name 
+       << labels
+       << " " 
+       << _get()
+       << std::endl;
+}
+
+//
+// Histogram
+// -- collection of CounterMetrics?
+
+struct MetricCollection {
+   tbb::concurrent_vector<shared_ptr<Metric>> metrics;
+};
+
+MetricsContainer::MetricsContainer() : _me(std::make_unique<MetricCollection>()) {}
+MetricsContainer::~MetricsContainer() {}
+void MetricsContainer::print(std::ostream& out) const {
+   for (const auto& m : _me->metrics) {
+      m->print(out);
+   }
+}
+void MetricsContainer::publish(std::shared_ptr<Metric> m) {
+   _me->metrics.push_back(m);
+}
+
+}}//namespace

src/server/sparkd/metrics/metrics.h

@@ -0,0 +1,150 @@
+#pragma once
+
+#include <iostream>
+#include <string>
+#include <memory>
+#include <vector>
+#include <functional>
+#include <atomic>
+#include <sstream>
+
+namespace bts {
+namespace metrics {
+class MetricsContainer;
+
+struct Label {
+   std::string name;
+   std::string value;
+};
+std::ostream& operator<<(std::ostream& os, const std::vector<Label>& l);
+
+struct Metric {
+   const std::string name;
+   const std::vector<Label> labels;
+   Metric(const std::string&, const std::vector<Label>&);
+   virtual void print(std::ostream&) const = 0;
+   std::string str() const;
+   virtual ~Metric() {}
+};
+
+template <class T>
+struct CounterMetric : public Metric {
+   T _val;
+   CounterMetric(const std::string& name) : Metric(name, {}), _val() {}
+   CounterMetric(const std::string& name, const std::vector<Label>& labels) : Metric(name, labels), _val() {}
+   ~CounterMetric() {};
+   void print(std::ostream& out) const override {
+      out << name << labels << " " << _val << std::endl;
+   }
+   void increment() { _val++; }
+   void increment(T val) { _val += val; }
+   void decrement() { _val--; }
+   void decrement(T val) { _val -= val; }
+   void set(T val) { _val = val; }
+};
+typedef CounterMetric<int> IntCounterMetric;
+
+template <class T>
+class AtomicCounterMetric : public Metric {
+   std::atomic<T> _val;
+   public:
+   AtomicCounterMetric(const std::string& name) : Metric(name, {}), _val() {}
+   AtomicCounterMetric(const std::string& name, const std::vector<Label>& labels) : Metric(name, labels), _val() {}
+   AtomicCounterMetric(const AtomicCounterMetric& o) : Metric(o.name, o.labels), _val(o._val.load()) { }
+   ~AtomicCounterMetric() {};
+   void print(std::ostream& out) const override {
+      out << name << labels << " " << _val << std::endl;
+   }
+   void increment() { _val++; }
+   void increment(T val) { _val += val; }
+   void decrement() { _val--; }
+   void decrement(T val) { _val -= val; }
+   void set(T val) { _val = val; }
+};
+
+class LazyCounterMetric : public Metric {
+   const std::function<long()> _get;
+   public:
+   LazyCounterMetric(
+      const std::string& name, 
+      const std::function<long()>& fn) : Metric(name, {}), _get(fn) {}
+   LazyCounterMetric(
+      const std::string& name, 
+      const std::function<long()>& fn,
+      const std::vector<Label>& labels) : Metric(name, labels), _get(fn) {}
+   void print(std::ostream&) const override;
+};
+
+const std::string double_to_string(double d);
+const std::vector<Label> vector_concat(const std::vector<Label>& orig, const std::vector<Label>& extra);
+
+template <size_t N, size_t... Is>
+std::array<CounterMetric<long>,N> mk_counters(const std::string& name,
+                                              const std::vector<Label>& labels,
+                                              const std::array<double,N>& in,
+                                              std::index_sequence<Is...>) {
+   return {{ CounterMetric<long>(name+"_bucket", vector_concat(labels, {{ "le", double_to_string(in[Is]) }}))... }};
+}
+template <size_t N, size_t... Is> 
+std::array<CounterMetric<long>,N> mk_counters(const std::string& name,
+                                              const std::vector<Label>& labels,
+                                              const std::array<double,N>& in) {
+   return mk_counters(name, labels, in, std::make_index_sequence<N>());
+}
+
+
+// Calculate the 90th percentile of response times
+// histogram_quantile(0.9, sum(rate(spark_metrics_query_duration_us_bucket[5m])) by (le))
+template <size_t N>
+class HistogramMetric : public Metric {
+   std::array<CounterMetric<long>,N> _buckets;
+   std::array<double,N> _bucket_values;
+   CounterMetric<double> _sum;
+   CounterMetric<long> _count;
+   public:
+   HistogramMetric(
+      const std::string& name,
+      const std::array<double,N>& buckets) 
+         : Metric(name,{}), _buckets(mk_counters(name, {}, buckets)), _bucket_values(buckets), _sum(name+"_sum"), _count(name+"_count") {}
+   HistogramMetric(
+      const std::string& name,
+      const std::array<double,N>& buckets,
+      const std::vector<Label>& labels) 
+         : Metric(name,labels), _buckets(mk_counters(name, labels, buckets)), _bucket_values(buckets), _sum(name+"_sum"), _count(name+"_count") {}
+   void print(std::ostream& os) const override {
+      for (size_t x = 0; x < N; ++x) {
+         _buckets[x].print(os);
+      }
+      os << name << "_bucket{le=\"+Inf\"";
+      for (auto iter = labels.begin(); iter != labels.end(); ++iter) {
+         os << "," << iter->name << "=\"" << iter->value << "\"";
+      }
+      os << "} " << _count._val << std::endl;
+      _sum.print(os);
+      _count.print(os);
+   }
+   void observe(double v) {
+      for (size_t x = 0; x < N; ++x) {
+         if (v <= _bucket_values[x]) {
+            _buckets[x].increment();
+         }
+      }
+      _sum.increment(v);
+      _count.increment();
+   }
+};
+
+struct MetricCollection;
+class MetricsContainer {
+   std::unique_ptr<MetricCollection> _me;
+   public:
+   MetricsContainer();
+   ~MetricsContainer();
+
+   void print(std::ostream&) const;
+   void publish(std::shared_ptr<Metric>);
+};
+
+
+}}//namespace
+

src/server/sparkd/metrics/system_metrics.cpp

@@ -0,0 +1,86 @@
+#include "system_metrics.h"
+
+#include "lib/config/config.h"
+#include "lib/config/protos/system.pb.h"
+
+#include <memory>
+#include <functional>
+#include <unistd.h>
+#include <fstream>
+
+namespace bts {
+namespace metrics {
+
+struct ProcessStats {
+   int pid;
+   std::string comm, state, ppid, pgrp, session, tty_nr;
+   std::string tpgid, flags, minflt, cminflt, majflt, cmajflt;
+   // utime - user time
+   // stime - kernel time
+   // cutime - cumulative user time
+   // cstime - cumulative system time
+   long utime, stime, cutime, cstime, priority, nice;
+   std::string O, itrealvalue, starttime;
+   // virtual size
+   uint64_t vsize;
+   // resident set
+   int64_t  rss;
+};
+
+void read_stats(ProcessStats& stats) {
+   std::ifstream stat_stream("/proc/self/stat", std::ios_base::in);
+   stat_stream 
+            >> stats.pid >> stats.comm >> stats.state >> stats.ppid >> stats.pgrp >> stats.session >> stats.tty_nr
+            >> stats.tpgid >> stats.flags >> stats.minflt >> stats.cminflt >> stats.majflt >> stats.cmajflt
+            >> stats.utime >> stats.stime >> stats.cutime >> stats.cstime >> stats.priority >> stats.nice
+            >> stats.O >> stats.itrealvalue >> stats.starttime >> stats.vsize >> stats.rss;
+   stat_stream.close();
+}
+
+void process_mem_usage(double& vm, double& resident) {
+   using std::ios_base;
+   using std::ifstream;
+   vm = resident = 0.0;
+
+   ProcessStats stats;
+   read_stats(stats);
+
+   long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024;
+   vm = stats.vsize / 1024.0;
+   resident = stats.rss * page_size_kb;
+}
+
+
+void publish_system_metrics(MetricsContainer& stats) {
+   auto version = std::make_shared<CounterMetric<int>>("spark_version", std::vector<Label> ({{"version", config::get<config::System>().version()}}));
+   version->set(1);
+   stats.publish(version);
+
+   // publish system cpu use
+
+   // publish sparkd cpu use and thread use (/proc/self/task/*)
+
+
+   //
+   // sparkd memory use
+   // Note:  It would be nice to have 2 metrics work on the same 'read' call
+   stats.publish(
+      std::make_shared<LazyCounterMetric>("spark_memory_res_kb", []() {
+         double vm,res;
+         process_mem_usage(vm,res);
+         return res;
+      }));
+
+   stats.publish(
+      std::make_shared<LazyCounterMetric>("spark_memory_vm_kb", []() {
+         double vm,res;
+         process_mem_usage(vm,res);
+         return vm;
+      }));
+
+
+}
+
+
+
+}}

src/server/sparkd/metrics/system_metrics.h

@@ -0,0 +1,9 @@
+#pragma once
+#include "metrics.h"
+
+namespace bts {
+namespace metrics {
+
+void publish_system_metrics(MetricsContainer&);
+
+}}

src/server/sparkd/orderman/actors/OrderStore.cpp

@@ -1,10 +1,10 @@
 #include "OrderStore.h"
 
-#include "lib/utils/PBUtils.h"
-#include "server/sparkd/persistence/SqliteStore.h"
 #include "lib/api/services/OrderService.pb.h"
-#include "server/sparkd/orderman/events/StoredOrderEvent.h"
+#include "lib/utils/PBUtils.h"
 #include "server/sparkd/orderman/events/OrderStateUpdateEvent.h"
+#include "server/sparkd/orderman/events/StoredOrderEvent.h"
+#include "server/sparkd/persistence/SqliteStore.h"
 
 #include <unordered_set>
 
@@ -13,74 +13,74 @@ namespace orderman {
 
 static auto const orders_ns = "orders";
 
-struct OrderStore::PImpl 
-{
-   PImpl() :
-      known_orders(),
-      store(new persistence::SqliteStore("orders")),
-      order_stream()
-   {}
+struct OrderStore::PImpl {
+   PImpl()
+       : known_orders(), store(new persistence::SqliteStore("orders")),
+         order_stream() {}
 
    const GatewayName gw_name;
-   std::unordered_map<OrderKey,std::unique_ptr<api::orderman::OrderRecord>> known_orders;
+   std::unordered_map<OrderKey, google::protobuf::Timestamp> known_orders;
    std::unique_ptr<persistence::IStore> store;
    mercury::Stream<StoredOrderEvent> order_stream;
 };
 
-OrderStore::OrderStore () : me (new PImpl()) {}
-OrderStore::~OrderStore () {}
+OrderStore::OrderStore() : me(new PImpl()) {}
+OrderStore::~OrderStore() {}
 
-void OrderStore::init()
-{
+void OrderStore::init() {
    api::orderman::OrderRecord rec;
    std::vector<StoredOrderEvent> stored_batch;
 
    auto all = me->store->retrieve_all()[orders_ns];
-   for (auto const &kvp : all)  
-      if (utils::pb::from_string<api::orderman::OrderRecord> (kvp.second,&rec)) {
+   for (auto const &kvp : all)
+      if (utils::pb::from_string<api::orderman::OrderRecord>(kvp.second,
+                                                             &rec)) {
          auto ok = OrderKey(std::stoi(kvp.first));
-         auto order_record = std::make_unique<api::orderman::OrderRecord> (rec);
-         stored_batch.emplace_back(*order_record.get());
-         me->known_orders[ok] = std::move(order_record);
-      }
-      else
-         LOG(OrderMan,Error,
+         stored_batch.emplace_back(rec);
+         me->known_orders[ok] = rec.state().created();
+      } else
+         LOG(OrderMan, Error,
              "Could not recover order {} from storage. "
-             "This should be investigated", kvp.first);
+             "This should be investigated",
+             kvp.first);
 
-   me->order_stream = make_stream<StoredOrderEvent> ();
-   LOG(OrderMan,Info,
-       "Order Store published {} orders recoved from disk", all.size());
-   publish_batch(me->order_stream,stored_batch);
-   subscribe<OrderStateUpdateEvent> ([this] (auto const&ev) {this->on_order_state_update(ev);});
+   me->order_stream = make_stream<StoredOrderEvent>();
+   LOG(OrderMan, Info, "Order Store published {} orders recoved from disk",
+       all.size());
+   publish_batch(me->order_stream, stored_batch);
+   subscribe<OrderStateUpdateEvent>(
+       [this](auto const &ev) { this->on_order_state_update(ev); });
 }
 
-void OrderStore::on_order_state_update(mercury::Update<OrderStateUpdateEvent> const&u)
-{
+void OrderStore::on_order_state_update(
+    mercury::Update<OrderStateUpdateEvent> const &u) {
    if (u.events)
       for (auto const &ev : *u.events) {
-         auto &ptr = me->known_orders[ev.key];
-         auto new_order = !ptr; 
-         if (new_order) {
-            ptr.reset (new api::orderman::OrderRecord());
-            *ptr->mutable_def() = *ev.definition;
-            ptr->set_key(ev.key.value());
-            ptr->set_parent_key(ev.parent_key.value());
-            *ptr->mutable_state()->mutable_created() = utils::pb::pb_now();
-         }
 
-         auto created = ptr->state().created(); 
-         *ptr->mutable_state() = ev.state;
-         *ptr->mutable_state()->mutable_created() = created;
-         *ptr->mutable_state()->mutable_last_change() = utils::pb::pb_now();
-         auto k = std::to_string(ptr->key());
-         auto v = utils::pb::to_string(*ptr);
-         if (new_order)
-            me->store->insert(orders_ns,k,v);
-         else
-            me->store->update(orders_ns,k,v);
-         publish(me->order_stream,*ptr.get());
+         api::orderman::OrderRecord order;
+         order.set_key(ev.key.value());
+         order.set_parent_key(ev.parent_key.value());
+         *order.mutable_def() = *ev.definition;
+         *order.mutable_state() = ev.state;
+         *order.mutable_state()->mutable_last_change() = utils::pb::pb_now();
+
+         auto it = me->known_orders.find(ev.key);
+         if (it == me->known_orders.end()) {
+            // new order
+            auto now = utils::pb::pb_now();
+            me->known_orders[ev.key] = now;
+            *order.mutable_state()->mutable_created() = now;
+            me->store->insert(orders_ns, std::to_string(order.key()),
+                              utils::pb::to_string(order));
+         } else {
+            *order.mutable_state()->mutable_created() = it->second;
+            me->store->update(orders_ns, std::to_string(order.key()),
+                              utils::pb::to_string(order));
+         }
+         LOG(OrderMan, Diag, "Storing order {}", utils::pb::to_string(order));
+         publish(me->order_stream, order);
       }
 }
 
-}}
+} // namespace orderman
+} // namespace bts

src/server/sparkd/orderman/actors/ServiceBridge.cpp

@@ -92,10 +92,10 @@ void ServiceBridge::on(mercury::Update<StoredOrderEvent> const &update) {
       bool was_ready = is_ready();
       me->have_orders = true;
 
-      std::vector<api::orderman::OrderRecord const *> orders;
+      std::vector<api::orderman::OrderRecord> orders;
       orders.reserve(update.events->size());
       for (auto ev : *(update.events)) {
-         me->known_orders.insert(OrderKey(ev.order_record->key()));
+         me->known_orders.insert(OrderKey(ev.order_record.key()));
          orders.push_back(ev.order_record);
       }
       me->on_orders(orders);

src/server/sparkd/orderman/actors/ServiceBridge.h

@@ -39,8 +39,8 @@ public:
    template <class EV> using OnEv = std::function<void(EV const&)>;
 
    using OnReady = std::function<void()>;
-   using OnOrders = std::function<void (std::vector<api::orderman::OrderRecord const*> )>;
-   using OnFills = std::function<void (std::vector<api::orderman::Fill const*> )>;
+   using OnOrders = std::function<void (std::vector<api::orderman::OrderRecord> const& )>;
+   using OnFills = std::function<void (std::vector<api::orderman::Fill const*> const&)>;
    
    ServiceBridge(OnReady const&, OnOrders const&, OnFills const&);
    void init() override;

src/server/sparkd/orderman/actors/order_manager/OrderManager.cpp

@@ -6,6 +6,7 @@
 #include "server/sparkd/orderman/actors/order_manager/OrderServicesData.h"
 #include "server/sparkd/orderman/events/NewAdminDataEvent.h"
 #include "server/sparkd/orderman/events/OrderManagerReadyEvent.h"
+#include "server/sparkd/orderman/events/StoredOrderEvent.h"
 #include "server/sparkd/orderman/order_factory/OrderFactory.h"
 #include "server/sparkd/orderman/orders/Order.h"
 #include "server/sparkd/securities/ISecurityDB.h"
@@ -96,8 +97,8 @@ void OrderManager::on_order_recovery(
          // recover orders from disk
          for (auto const &ev : *(update.events)) {
             auto &order =
-                me->services_data.order_factory.recover(*ev.order_record);
-            auto const &ostate = ev.order_record->state();
+                me->services_data.order_factory.recover(ev.order_record);
+            auto const &ostate = ev.order_record.state();
             OrderInfo oi{order, me->services_data};
             auto serv_ptr = oi.order_services.get();
             me->orders[order.key()] = std::move(oi);

src/server/sparkd/orderman/actors/order_manager/OrderServicesData.h

@@ -24,7 +24,6 @@
 #include "server/sparkd/orderman/events/NewSecuritiesEvent.h"
 #include "server/sparkd/orderman/events/OrderStateUpdateEvent.h"
 #include "server/sparkd/orderman/events/OutFIXMsgEvent.h"
-#include "server/sparkd/orderman/events/StoredOrderEvent.h"
 #include "server/sparkd/orderman/events/UpdateOrderEvent.h"
 #include "server/sparkd/orderman/limiter/Limiter.h"
 #include "server/sparkd/orderman/order_router/OrderRouter.h"

src/server/sparkd/orderman/events/StoredOrderEvent.h

@@ -5,6 +5,7 @@
 #include "lib/api/services/OrderService.pb.h"
 #include "server/sparkd/mercury/synopsis/MergeByKey.h"
 #include "lib/api/services/OrderService.pb.h"
+#include "server/sparkd/mercury/events/Event.h"
 
 namespace bts {
 namespace orderman {
@@ -14,10 +15,10 @@ class StoredOrderEvent : public mercury::Event <StoredOrderEvent>
 public:
    using SynopsisPolicy = mercury::MergeByKey<StoredOrderEvent>;
 
-   StoredOrderEvent(api::orderman::OrderRecord &record) : order_record(&record) {}
+   StoredOrderEvent(api::orderman::OrderRecord const &record) : order_record(record) {}
    
-   auto key() const {return order_record->key();}
-   api::orderman::OrderRecord *order_record;
+   auto key() const {return order_record.key();}
+   api::orderman::OrderRecord order_record;
 };
 
 }}

src/server/sparkd/orderman/orderman_utils/fix_order_codec.h

@@ -14,7 +14,7 @@ namespace api {class Security;}
 namespace orderman {
 
 constexpr int clordid_sz() {return 20;}
-constexpr int price_sz() {return 15;}
+constexpr int price_sz() {return 17;}
 constexpr int quantity_sz() {return 6;}
 
 inline void encode_clorid(OrderKey ok, SecurityKey sk,std::string *out) {

src/server/sparkd/orderman/orders/IcebergOrder.cpp

@@ -88,7 +88,8 @@ void IcebergOrder::on_init_recovered(api::orderman::OrderRecord const &rec,
    }
 }
 
-void IcebergOrder::on_activate(Price p, Quantity q, OrderServices &services, TickToTradeCorID) {
+void IcebergOrder::on_activate(Price p, Quantity q, OrderServices &services,
+                               TickToTradeCorID) {
    bts_assert(!me->services);
    me->services = &services;
    me->child_builder.activate(services);
@@ -184,6 +185,7 @@ void IcebergOrder::transition(api::orderman::OrderStatus new_status,
    auto const &to = OrderStatus_Name(new_status);
    LOG(OrderMan, Info, "Transitioning iceberg order {} from {} to {}. Msg: {}",
        key().value(), from, to, msg);
+   me->state.set_last_msg(msg);
    me->state.set_status(new_status);
 }
 
@@ -229,8 +231,9 @@ void IcebergOrder::send_next_child_order() {
    auto remaining = me->state.quantity() - me->state.filled_quantity();
    auto to_send = std::min(remaining, me->show_quantity.value());
    me->child.order = me->child_builder.build();
-   LOG(OrderMan, Info, "Iceberg order {} creating new child order {} with size "
-                       "{} price {}. Filled {} so far out of {}",
+   LOG(OrderMan, Info,
+       "Iceberg order {} creating new child order {} with size "
+       "{} price {}. Filled {} so far out of {}",
        key().value(), me->child.order.key().value(), to_send, me->state.price(),
        me->state.filled_quantity(), me->state.quantity());
 
@@ -251,14 +254,20 @@ void IcebergOrder::on_child_order_update(OrderStateUpdateEvent const &ev) {
             break;
 
          case ORDER_STATUS_COMPLETED: {
-            api::orderman::OrderState merged_state;
-            merge_states(me->state, me->child.state, &merged_state);
-            me->state = merged_state;
-            me->child.clear();
-            if (me->state.quantity() > me->state.filled_quantity())
-               send_next_child_order();
-            else
-               transition(ORDER_STATUS_COMPLETED);
+            if (me->child.state.filled_quantity() > 0) {
+               api::orderman::OrderState merged_state;
+               merge_states(me->state, me->child.state, &merged_state);
+               me->state = merged_state;
+               me->child.clear();
+               if (me->state.quantity() > me->state.filled_quantity())
+                  send_next_child_order();
+               else
+                  transition(ORDER_STATUS_COMPLETED);
+            } else {
+               me->child.clear();
+               transition(ORDER_STATUS_ERROR,
+                          "Child order completed without fills");
+            }
             break;
          }
 
@@ -302,5 +311,5 @@ void IcebergOrder::on_child_order_update(OrderStateUpdateEvent const &ev) {
    }
    report_state();
 }
-}
-}
+} // namespace orderman
+} // namespace bts

src/server/sparkd/orderman/orders/NativeOrder.cpp

@@ -126,9 +126,10 @@ void NativeOrder::on_init_recovered(OrderRecord const &rec,
 
       // bail if we can't find the security for the order
       if (!me->sec) {
-         LOG(OrderMan, Error, "Unknown security {} referenced by recovered "
-                              "order {}. This is a serious "
-                              "problem which has to be corrected manually. ",
+         LOG(OrderMan, Error,
+             "Unknown security {} referenced by recovered "
+             "order {}. This is a serious "
+             "problem which has to be corrected manually. ",
              def.security_key(), rec.key());
          transition_to_error(
              NativeOrderState::SUB_ERROR_UNKNOWN_SECURITY,
@@ -282,7 +283,7 @@ void NativeOrder::on_activate(Price p, Quantity q,
       me->state.set_filled_quantity(0);
       me->state.set_last_price(0);
       me->state.set_avg_price(0);
-      
+
       me->session_services = &me->services->get_session_services(
           get_native_state(me->state).routing_record().session());
       me->price_encoding_precision =
@@ -290,8 +291,9 @@ void NativeOrder::on_activate(Price p, Quantity q,
 
       // is our gateway connected?
       if (!me->session_services->is_gateway_connected()) {
-         LOG(OrderMan, Warn, "Attempt to send order to a disconnected gateway. "
-                             "Dropping order {}",
+         LOG(OrderMan, Warn,
+             "Attempt to send order to a disconnected gateway. "
+             "Dropping order {}",
              key().value());
          transition(ORDER_STATUS_REJECTED_BY_SPARK, "Exchange gateway is down");
          break;
@@ -302,9 +304,9 @@ void NativeOrder::on_activate(Price p, Quantity q,
           UserID(me->definition.user_id()),
           AccountID(me->definition.account_id()));
       if (!lqr.ok) {
-         LOG(OrderMan, Warn, "Unauthorized order attempted. User: {} on "
-                             "Account: {}. Msg: {} Order: {} "
-                             "Dropping order {}",
+         LOG(OrderMan, Warn,
+             "Unauthorized order attempted. User: {} on "
+             "Account: {}. Msg: {} Order: {}. Dropping order",
              me->definition.user(), me->definition.account(), lqr.error_msg,
              key().value());
 
@@ -319,8 +321,9 @@ void NativeOrder::on_activate(Price p, Quantity q,
           me->definition.side(), q);
 
       if (!lqr.ok) {
-         LOG(OrderMan, Warn, "New order would breach limit. User: {} Account: "
-                             "{} Msg: {} Order: {}. Dropping order",
+         LOG(OrderMan, Warn,
+             "New order would breach limit. User: {} Account: "
+             "{} Msg: {} Order: {}. Dropping order",
              me->definition.user(), me->definition.account(), lqr.error_msg,
              key().value());
 
@@ -331,8 +334,9 @@ void NativeOrder::on_activate(Price p, Quantity q,
 
       // are we under the messaging threshold?
       if (!me->session_services->governor().can_send()) {
-         LOG(OrderMan, Warn, "New order would cause breach of messaging "
-                             "limits. Dropping order {}",
+         LOG(OrderMan, Warn,
+             "New order would cause breach of messaging "
+             "limits. Dropping order {}",
              key().value());
          transition(ORDER_STATUS_REJECTED_BY_SPARK,
                     "Messaging threshold breached");
@@ -357,8 +361,8 @@ void NativeOrder::on_activate(Price p, Quantity q,
       case CustomOrderDefinition::kStopLimit:
          me->create_price_bm = encoding.overwrite(
              me->create_price_bm,
-             securities::encode_exchange_price(*me->sec, p).value(), 7,
-             price_sz());
+             securities::encode_exchange_price(*me->sec, p).value(),
+             me->price_encoding_precision, price_sz());
          break;
       case CustomOrderDefinition::kStop:
       case CustomOrderDefinition::kMarket:
@@ -484,11 +488,12 @@ void NativeOrder::on_update(Price p, Quantity q, TickToTradeCorID cid) {
 
       // are we under the messaging threshold?
       if (!me->session_services->governor().can_send()) {
-         LOG(OrderMan, Warn, "Cannot update order {} because of messaging limits. ",
+         LOG(OrderMan, Warn,
+             "Cannot update order {} because of messaging limits. ",
              key().value());
          auto msg = "Messaging threshold breached";
          if (parent_key() == null_order_key())
-            transition(me->state.status(),msg);
+            transition(me->state.status(), msg);
          else {
             me->state.set_last_msg(msg);
             on_cancel(cid);
@@ -500,8 +505,8 @@ void NativeOrder::on_update(Price p, Quantity q, TickToTradeCorID cid) {
       auto &encoding = me->update_encoding;
       me->update_price_bm = encoding.overwrite(
           me->update_price_bm,
-          securities::encode_exchange_price(*me->sec, p).value(), 7,
-          price_sz());
+          securities::encode_exchange_price(*me->sec, p).value(),
+          me->price_encoding_precision, price_sz());
       me->update_quantity_bm =
           encoding.overwrite(me->update_quantity_bm, q.value(), quantity_sz());
 
@@ -747,8 +752,9 @@ void NativeOrder::report_state() {
              me->definition.side(),
              Quantity(previous_position.value() - current_position.value()));
    } else
-      LOG(OrderMan, Error, "Order {} ignored for purposes of limit enforcing "
-                           "because it references unknown security {} ",
+      LOG(OrderMan, Error,
+          "Order {} ignored for purposes of limit enforcing "
+          "because it references unknown security {} ",
           key().value(), me->definition.security_key());
 
    me->services->report_state(me->state);
@@ -829,8 +835,10 @@ void NativeOrder::init_fix() {
       cancel_encoding.add<fix::OrdType>(NATIVE_ORDER_TYPE_LIMIT);
 
       // price
-      me->create_price_bm = create_encoding.add<fix::Price>(0, 7, price_sz());
-      me->update_price_bm = update_encoding.add<fix::Price>(0, 7, price_sz());
+      me->create_price_bm = create_encoding.add<fix::Price>(
+          0, me->price_encoding_precision, price_sz());
+      me->update_price_bm = update_encoding.add<fix::Price>(
+          0, me->price_encoding_precision, price_sz());
       break;
 
    case CustomOrderDefinition::kStop: {
@@ -843,8 +851,10 @@ void NativeOrder::init_fix() {
       auto stop_px = def.custom_def().stop().stop_price();
       auto encoded_stop_px =
           securities::encode_exchange_price(*me->sec, Price(stop_px)).value();
-      create_encoding.add<fix::StopPx>(encoded_stop_px, -1, -1);
-      update_encoding.add<fix::StopPx>(encoded_stop_px, -1, -1);
+      create_encoding.add<fix::StopPx>(encoded_stop_px,
+                                       me->price_encoding_precision, -1);
+      update_encoding.add<fix::StopPx>(encoded_stop_px,
+                                       me->price_encoding_precision, -1);
       break;
    }
 
@@ -858,12 +868,16 @@ void NativeOrder::init_fix() {
       auto stop_px = def.custom_def().stop_limit().stop_price();
       auto encoded_stop_px =
           securities::encode_exchange_price(*me->sec, Price(stop_px)).value();
-      create_encoding.add<fix::StopPx>(encoded_stop_px, -1, -1);
-      update_encoding.add<fix::StopPx>(encoded_stop_px, -1, -1);
+      create_encoding.add<fix::StopPx>(encoded_stop_px,
+                                       me->price_encoding_precision, -1);
+      update_encoding.add<fix::StopPx>(encoded_stop_px,
+                                       me->price_encoding_precision, -1);
 
       // price
-      me->create_price_bm = create_encoding.add<fix::Price>(0, 7, price_sz());
-      me->update_price_bm = update_encoding.add<fix::Price>(0, 7, price_sz());
+      me->create_price_bm = create_encoding.add<fix::Price>(
+          0, me->price_encoding_precision, price_sz());
+      me->update_price_bm = update_encoding.add<fix::Price>(
+          0, me->price_encoding_precision, price_sz());
       break;
    }
    default:
@@ -1017,5 +1031,5 @@ NativeOrder::get_native_state(api::orderman::OrderState &st) {
    bts_assert(false);
    return *st.mutable_stop_limit()->mutable_native_state();
 }
-}
-}
+} // namespace orderman
+} // namespace bts

src/server/sparkd/orderman/orders/spreader_orders/SpreaderOrder.cpp

@@ -53,7 +53,7 @@ void SpreaderOrder::on_activate(Price price, Quantity quantity,
 }
 void SpreaderOrder::on_update(Price price, Quantity quantity,TickToTradeCorID){
    if (!_active) return;
-   LOG(OrderMan, Info, "SpreaderOrder.on_update key: {} price: {} quantity: {}", key().value(), price.value(), quantity.value());   
+   LOG(OrderMan, Info, "SpreaderOrder.on_update key: {} price: {} quantity: {}", key().value(), price.value(), quantity.value());
    _state_worker->on_update(price, quantity);
    _report_state();
 }
@@ -242,45 +242,50 @@ bool SpreaderOrder::_quoters_can_accept_modification (){
    }
    return true;
 }
-void SpreaderOrder::_send_hedge(size_t filled_leg_index,
-                                Price filled_price,
-                                Quantity filled_quantity, 
-                                size_t hedge_leg_index){
-   LOG(OrderMan, Info, "SpreaderOrder::_send_hedge key: {}, filled_leg_index: {} filled_price: {} filled_quantity: {} hedge_leg_index: {}",
-                        key().value(), filled_leg_index, filled_price.value(), filled_quantity.value(), hedge_leg_index);
-   Price hedge_price;
-   Quantity hedge_quantity;
-   _compute_hedge_price_quantity (filled_leg_index, filled_price, 
-                                  filled_quantity, hedge_leg_index,
-                                  hedge_price, hedge_quantity);
-   SingleFillHedger& hedger = _bookkeeping_data().add_new_hedger (*this, 
+void SpreaderOrder::_send_hedge(size_t hedge_leg_index,
+                                Price hedge_price,
+                                Quantity hedge_qty){
+   LOG(OrderMan, Info, "SpreaderOrder::_send_hedge key: {}, hedge_leg_index: {} hedge_price: {} hedge_qty: {}",
+                        key().value(), hedge_leg_index, hedge_price, hedge_qty);
+   SingleFillHedger& hedger = _bookkeeping_data().add_new_hedger (*this,
                                                                   hedge_leg_index,
                                                                   hedge_price,
-                                                                  hedge_quantity);
+                                                                  hedge_qty);
    hedger.init ();
 }
 void SpreaderOrder::_compute_hedge_price_quantity(size_t filled_leg_index,
-                                                  Price fill_price, Quantity fill_quantity,
+                                                  Price fill_price,
+                                                  double fill_quantity,
                                                   size_t hedge_leg_index,
-                                                  Price& hedge_price, Quantity& hedge_quantity){
+                                                  Price& hedge_price,
+                                                  double& hedge_quantity){
    Price computed_hedge_price;
    SpreaderLegData& filled_leg_data = _bookkeeping_data().get_leg_data(filled_leg_index);
    SpreaderLegData& hedged_leg_data = _bookkeeping_data().get_leg_data(hedge_leg_index);
    if (_bookkeeping_data().n_legs() == 2)
       SpreaderUtils::compute_hedge_price_size_2_legs (_bookkeeping_data().price(),
-                                                      filled_leg_data.leg_property_data(), fill_price, fill_quantity,
-                                                      hedged_leg_data.leg_property_data(), computed_hedge_price, hedge_quantity);
+                                                      filled_leg_data.leg_property_data(),
+                                                      fill_price,
+                                                      fill_quantity,
+                                                      hedged_leg_data.leg_property_data(),
+                                                      computed_hedge_price,
+                                                      hedge_quantity);
    else{
-      const QuoterLegDriverData& driver_data = filled_leg_data.quoter().driver_data(hedge_leg_index);
-      SpreaderUtils::compute_hedge_price_size_n_legs (filled_leg_data.leg_property_data(), fill_price, fill_quantity,
-                                                      hedged_leg_data.leg_property_data(), driver_data,
-                                                      computed_hedge_price, hedge_quantity); 
+      const QuoteLegDriver& driver_data = filled_leg_data.quoter().driver_data(fill_price, hedge_leg_index);
+      SpreaderUtils::compute_hedge_price_size_n_legs (filled_leg_data.leg_property_data(),
+                                                      fill_price,
+                                                      fill_quantity,
+                                                      hedged_leg_data.leg_property_data(),
+                                                      driver_data,
+                                                      computed_hedge_price,
+                                                      hedge_quantity); 
    }
    hedge_price = SpreaderUtils::apply_pre_hedge_parameters (hedged_leg_data.leg_property_data().leg_params().pre_hedge_params(), 
                                                             hedged_leg_data.book(),
                                                             hedged_leg_data.leg_property_data().trading_side(),
                                                             hedged_leg_data.leg_property_data().tick_size(),
-                                                            computed_hedge_price, hedge_quantity);
+                                                            Price(computed_hedge_price),
+                                                            Quantity(hedge_quantity));
 }
 void SpreaderOrder::_report_state (){ 
    if (_activated()){ 

src/server/sparkd/orderman/orders/spreader_orders/SpreaderOrder.h

@@ -133,10 +133,11 @@ private:
    void _transform_to_END(std::string* m = nullptr);
    // swtches to the state passed in the arg
    void _switch_state (ISpreaderStateWorker* new_state);
-   // this means on_init_new has been called and we have the
-   // the structure built
    // misc ////////////////////
    ////////////////////////////
+   // this means on_init_new or on_init_recovered
+   // has been called and we have the bookkeeping
+   // data built
    bool _inited () const {
       return _bookkeeping_data_ptr != nullptr;
    }
@@ -165,17 +166,15 @@ private:
    // on leg 'leg_to_hedge_index' in response to a fill
    // on leg 'filled_leg_index' with price 'filled_price'
    // quantity 'filled_quantity'
-   void _send_hedge(size_t filled_leg_index, Price filled_price,
-                    Quantity filled_quantity,
-                    size_t leg_to_hedge_index);
-   // purely a computating function that computes the
-   // hedge price and quantity (last 2 args) for leg
-   // 'hedged_leg_index' for a fill on leg 'filled_leg_index'
-   // with price 'filled_price' and quantity 'filled_quantity'
+   void _send_hedge(size_t hedge_leg_index,
+                    Price hedge_price,
+                    Quantity hedge_qty);
    void _compute_hedge_price_quantity(size_t filled_leg_index,
-                                      Price filled_price, Quantity filled_quantity,
+                                      Price filled_price,
+                                      double filled_quantity,
                                       size_t hedged_leg_index,
-                                      Price& hedge_price, Quantity& hedge_quantity);
+                                      Price& hedge_price,
+                                      double& hedge_quantity);
    // sends the report out
    void _report_state ();      
    SpreaderBookkeepingData& _bookkeeping_data(){

src/server/sparkd/orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp

@@ -20,6 +20,7 @@ _spreader(spreader),
 _this_leg_data (this_leg_property),
 _n_spreader_legs (n_spreader_legs),
 _child_order_state (child_order_state),
+_driver_cache(n_spreader_legs,n_spreader_legs>2?10:0,this_leg_property.tick_size()),
 _price_quantity_state(Price(0), Quantity(0), child_order_state),
 _is_permenantly_cancelled (false),
 _worker(nullptr),
@@ -33,8 +34,7 @@ _end_state_worker(this){
    if (_active_quoting){
       if (!_two_legged){
          for (size_t q_idx = 0; q_idx < _n_spreader_legs; q_idx++){
-            _drivers_commited.push_back (QuoterLegDriverData());
-            _drivers_temp_usage.push_back (QuoterLegDriverData());          
+            _drivers_temp_usage.push_back (QuoteLegDriver());
          }
       }
    }
@@ -49,7 +49,7 @@ _this_leg_data (other._this_leg_data),
 _n_spreader_legs (other._n_spreader_legs),
 _child_order_state (other._child_order_state),
 _drivers_temp_usage(other._drivers_temp_usage),
-_drivers_commited (other._drivers_commited),   
+_driver_cache(other._driver_cache), 
 _price_quantity_state(other._price_quantity_state.target_spread_price(), 
                       other._price_quantity_state.target_leg_quantity(),
                       other._price_quantity_state.order_state()), 
@@ -131,13 +131,8 @@ bool SingleLegQuoter::is_terminal_state () const {
 bool SingleLegQuoter::accept_modifications() const {
     return _worker->state() == QuoterState::WAITING_FOR_SIZE || _worker->state() == QuoterState::WORKING;
 }
-Price SingleLegQuoter::driver_price (size_t leg_idx) const{
-   bts_assert (leg_idx < _n_spreader_legs);
-   return _drivers_commited[leg_idx].driver_price;
-}
-void SingleLegQuoter::_incorporate_other_leg_fill (size_t leg_index, Quantity filled_quantity){
-   Quantity this_leg_reduction = SpreaderUtils::get_leg_quantity_from_another_leg_quantity (_spreader.get_leg_data(leg_index), filled_quantity, _this_leg_data);
-   _price_quantity_state.add_hedged_reduction (this_leg_reduction);
+void SingleLegQuoter::_incorporate_other_leg_fill (size_t, Quantity hedge_reduction){
+   _price_quantity_state.add_hedged_reduction (hedge_reduction);
 }
 void SingleLegQuoter::_compute_price_quantity(Price& price, Quantity& quantity){
    Quantity remaining_quantity = _price_quantity_state.remaining_to_fill();
@@ -152,11 +147,9 @@ void SingleLegQuoter::_compute_price_quantity(Price& price, Quantity& quantity){
        SpreaderUtils::compute_quoting_price_size_n_legs (_price_quantity_state.target_spread_price(), remaining_quantity,
                                                          _this_leg_data.leg_index(), _spreader, _drivers_temp_usage, price, quantity);
 }
-void SingleLegQuoter::_commit_drivers(){
-    if (!_two_legged)
-    // this could be done with a pointer swap too
-       for (size_t idx = 0; idx < _n_spreader_legs; idx++)
-         _drivers_commited[idx] = _drivers_temp_usage[idx];
+void SingleLegQuoter::_commit_drivers(Price quote_price){
+   if (!_two_legged)
+      _driver_cache.add(_drivers_temp_usage,quote_price);
 }
 /*********************************************************************************/
 /*******************************state transformations*****************************/

src/server/sparkd/orderman/orders/spreader_orders/spreader/SingleLegQuoter.h

@@ -14,9 +14,10 @@
 #include "server/sparkd/orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.h"
 #include "server/sparkd/orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.h"
 #include "server/sparkd/orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.h"
-#include "server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoterLegDriverData.h"
+#include "server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.h"
 #include "server/sparkd/orderman/actors/order_manager/ChildOrder.h"
 #include "server/sparkd/orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.h"
+#include "server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.h"
 #include <sstream>
 
 
@@ -24,7 +25,6 @@ namespace bts {
 namespace orderman {
 
 class ISpreaderManager;
-//class SpreaderChildOrderState;
 class SingleLegQuoter
 {
 friend QuoterStartStateWorker;
@@ -45,8 +45,8 @@ private:
    // legs requires auxiliary storage for drivers, which
    // _drivers_temp_usage provides. once we are ready
    // to send the orders, the drivers are committed to
-   // _drivers_commited
-   std::vector<QuoterLegDriverData> _drivers_temp_usage, _drivers_commited;
+   std::vector<QuoteLegDriver> _drivers_temp_usage;
+   QuoteDriverSetHistory _driver_cache;
    QuoteLegPriceQuantityState _price_quantity_state;
    bool _is_permenantly_cancelled;
    // state workers
@@ -108,13 +108,10 @@ public:
    // the quoter returns true if it is in a state where orders
    // can be modified
    bool accept_modifications() const;
-   // for a given leg, returns the driver price that was used
-   // to price the current order
-   Price driver_price (size_t leg_idx) const;
-   const QuoterLegDriverData& driver_data (size_t leg_idx) const
-   {
-      bts_assert (leg_idx < _drivers_commited.size());
-      return _drivers_commited[leg_idx];
+   const QuoteLegDriver& driver_data (Price quote_price, size_t leg_idx) const{
+      const QuoteLegDriver* leg_driver = _driver_cache.find_driver(quote_price,leg_idx);
+      bts_assert(leg_driver!=nullptr);
+      return *leg_driver;
    }
    // tells if it is expecting to get a cancel (this helps catch
    // unsolicited cancels). returns true is its expecting a cancel
@@ -152,12 +149,12 @@ private:
    }
    void _cancel_order () {_child_order_state.cancel_order();}
    // this updates the hedge reduction in _price_quantity_state
-   void _incorporate_other_leg_fill (size_t leg_index, Quantity filled_quantity);
+   void _incorporate_other_leg_fill (size_t leg_index, Quantity hedge_reduction);
    // this is the function that computes the quoting price and the   
    // quantity for this leg.
    void _compute_price_quantity(Price& price, Quantity& quantity);
    // copies the computed driver to committed dri
-   void _commit_drivers ();
+   void _commit_drivers (Price quote_price);
    void _transform_to_START ();
    void _transform_to_WAITING_FOR_SIZE ();
    void _transform_to_WAITING_FOR_FIRST_ACK ();

src/server/sparkd/orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp

@@ -23,6 +23,7 @@ _reported_state(){
    _quoter_child_order_states.reserve(_n_legs);
    _quoters.reserve (_n_legs);
    _hedgers.reserve (_n_legs);
+   _unhedged_deltas.reserve(_n_legs);
    _leg_data.reserve (_n_legs);
    for (const auto& lpd_iter : legs){
       _leg_prop_data.push_back(lpd_iter);
@@ -34,9 +35,15 @@ _reported_state(){
       _quoter_child_order_states.emplace_back (empty_order,leg_property_data.leg_security_key(),_is_live,true);
       _quoters.emplace_back(spreader_manager, leg_property_data, _n_legs, _quoter_child_order_states.back());
       _hedgers.emplace_back ();
-      _leg_data.emplace_back(leg_property_data, _quoters.back(), _quoter_order_builders.back(), 
-                             _hedger_order_builders.back(), _hedgers.back (), _quoter_child_order_states.back(),
-                             _books.back());
+      _unhedged_deltas.emplace_back();
+      _leg_data.emplace_back(leg_property_data,
+                             _quoters.back(),
+                             _quoter_order_builders.back(), 
+                             _hedger_order_builders.back(),
+                             _hedgers.back (),
+                             _quoter_child_order_states.back(),
+                             _books.back(),
+                             _unhedged_deltas.back());
       bts_assert (_sec_key_to_index.find(leg_property_data.security_key()) == _sec_key_to_index.end());
       _sec_key_to_index.insert(std::pair<uint32_t, size_t>(leg_property_data.security_key(), leg_property_data.leg_index()));
    }

src/server/sparkd/orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.h

@@ -8,6 +8,7 @@
 #include "server/sparkd/orderman/actors/order_manager/OrderBuilder.h"
 #include "server/sparkd/orderman/orders/PreActivationOrderServices.h"
 #include "server/sparkd/orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.h"
+#include "server/sparkd/orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.h"
 
 namespace bts{
 namespace orderman { 
@@ -32,7 +33,8 @@ private:
    std::vector<SpreaderChildOrderState> _quoter_child_order_states;
    std::vector<SingleLegQuoter> _quoters;
    std::vector<std::vector<SingleFillHedger>> _hedgers;
-   std::vector<SpreaderLegData> _leg_data;
+   std::vector<UnhedgedDelta> _unhedged_deltas;
+   std::vector<SpreaderLegData> _leg_data;   
    boost::container::flat_map<uint32_t, size_t> _sec_key_to_index;
    // state thats sent out in report (its kept here amd modified
    // vs creating a new one everytime)

src/server/sparkd/orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp

@@ -9,7 +9,8 @@ SpreaderLegData::SpreaderLegData (const LegPropertyData& leg_property_data,
                                   SpreaderOrderBuilder& hedger_order_builder,
                                   std::vector<SingleFillHedger>& hedger_data_vector,
                                   SpreaderChildOrderState& quoter_child_order_state,
-                                  Book& book)
+                                  Book& book,
+                                  UnhedgedDelta& unhedged_delta)
 :
 _leg_property_data (leg_property_data),
 _quoter(quoter),
@@ -17,7 +18,8 @@ _quoter_order_builder (quoter_order_builder),
 _hedger_order_builder (hedger_order_builder),
 _quoter_child_order_state  (quoter_child_order_state),
 _hedger_data_vector (hedger_data_vector),     
-_book(book)
+_book(book),
+_unhedged_delta(unhedged_delta)
 {}
 
 SpreaderLegData::SpreaderLegData(SpreaderLegData&& other)
@@ -28,7 +30,8 @@ _quoter_order_builder(other._quoter_order_builder),
 _hedger_order_builder(other._hedger_order_builder),
 _quoter_child_order_state (other._quoter_child_order_state),
 _hedger_data_vector (other._hedger_data_vector),
-_book(other._book){
+_book(other._book),
+_unhedged_delta(other._unhedged_delta){
    bts_assert (false);
 }
 

src/server/sparkd/orderman/orders/spreader_orders/spreader/SpreaderLegData.h

@@ -5,6 +5,7 @@
 #include "SpreaderChildOrderState.h"
 #include "server/sparkd/orderman/actors/order_manager/OrderBuilder.h"
 #include "server/sparkd/orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.h"
+#include "server/sparkd/orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.h"
 
 namespace bts{
 namespace orderman{
@@ -28,6 +29,7 @@ private:
    SpreaderChildOrderState& _quoter_child_order_state;
    std::vector<SingleFillHedger>& _hedger_data_vector;   
    Book& _book;
+   UnhedgedDelta& _unhedged_delta;
 public:  
    SpreaderLegData (const LegPropertyData& leg_property_data,
                     SingleLegQuoter& quoter,                    
@@ -35,7 +37,8 @@ public:
                     SpreaderOrderBuilder& hedger_order_builder,
                     std::vector<SingleFillHedger>& hedger_data_vector,
                     SpreaderChildOrderState& quoter_child_order_state,
-                    Book& book);
+                    Book& book,
+                    UnhedgedDelta& unhedged_delta);
    SpreaderLegData() = delete;
    SpreaderLegData(const SpreaderLegData&) = delete;
    // this cons is not deleted since for each leg,
@@ -51,12 +54,13 @@ public:
    /////////////////////////////////
    const LegPropertyData& leg_property_data() const {return _leg_property_data;}
    SingleLegQuoter& quoter() {return _quoter;}
-   const bts::orderman::Book& book () const {return _book;}   
+   const bts::orderman::Book& book () const {return _book;}
    SpreaderChildOrderState& order_update_message () {return _quoter_child_order_state;}
    bts::orderman::OrderKey key() {return _quoter_child_order_state.order_key();}
    bts::orderman::OrderKey order_key() {return _quoter_child_order_state.order_key();}
    SpreaderOrderBuilder& quoter_order_builder() {return _quoter_order_builder;}
    SpreaderOrderBuilder& hedger_order_builder() {return _hedger_order_builder;}
+   UnhedgedDelta& unhedged_delta() {return _unhedged_delta;}
    // returns the number of active hedgers
    size_t n_hedgers() const { return _hedger_data_vector.size(); }
    // returns the hedger at a given index

src/server/sparkd/orderman/orders/spreader_orders/spreader/SpreaderUtils.h

@@ -6,7 +6,7 @@
 #include "server/sparkd/orderman/orderman_utils/Book.h"
 #include "server/sparkd/orderman/orders/spreader_orders/spreader/LegPropertyData.h"
 #include "server/sparkd/orderman/orders/spreader_orders/spreader/ISpreaderManager.h"
-#include "server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoterLegDriverData.h"
+#include "server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.h"
 #include "lib/api/services/OrderService.pb.h"
 #include "server/sparkd/orderman/actors/order_manager/OrderBuilder.h"
 
@@ -423,17 +423,28 @@ public:
    // so if we get an N lot fill in l2, we send a hedge on l1
    // sized s1 * N / s2
    ////////////////////////general functions
+   static double get_leg_quantity_from_another_leg_quantity (const LegPropertyData& source_leg,
+                                                             double source_leg_quantity,
+                                                             const LegPropertyData& target_leg){
+      return std::abs(target_leg.size_multiplier () * source_leg_quantity / source_leg.size_multiplier ());
+   }
    static Quantity get_leg_quantity_from_another_leg_quantity (const LegPropertyData& source_leg,
                                                                Quantity source_leg_quantity,
                                                                const LegPropertyData& target_leg){
-      double rv = std::abs(target_leg.size_multiplier () * source_leg_quantity.value () / source_leg.size_multiplier ());
+      double rv = get_leg_quantity_from_another_leg_quantity(source_leg,
+                                                             static_cast<double>(source_leg_quantity.value()),
+                                                             target_leg);            
       return Quantity(_round_leg_qty_computed_from_another_leg_qty(rv));
-   }
+   }   
    // given the quantity of the spreader order, determines the quantity
    // of the leg
+   static double get_leg_quantity_from_spreader_quantity(const LegPropertyData& leg,
+                                                         double spreader_order_quantity){
+      return std::abs (leg.size_multiplier () * spreader_order_quantity);
+   }
    static Quantity get_leg_quantity_from_spreader_quantity(const LegPropertyData& leg,
                                                            Quantity spreader_order_quantity){
-      double rv = std::abs (leg.size_multiplier () * spreader_order_quantity.value());
+      double rv = get_leg_quantity_from_spreader_quantity(leg,static_cast<double>(spreader_order_quantity.value()));
       return Quantity (_round_leg_qty_computed_from_spreader_qty (rv));
    }
    // finds the average weighted price and total quantity
@@ -450,53 +461,71 @@ public:
          out_avg_price = Price((base_price.value() * base_qty.value() + inc_price.value() * inc_qty.value())/out_total_qty.value());
    }
    ////////////////////////2 legged spreads
+   // given a quoting target qty, finds the "driver" price
+   // and qty, thats needed to hedge the quote. It also
+   // finds the ammended quoting target (since the hedge 
+   // book might not have enough to support the entire
+   // quoting target qty)
+   static void compute_driver(Quantity quote_target,
+                              const LegPropertyData& quote_leg_data,
+                              const LegPropertyData& hedge_leg_data,
+                              const bts::orderman::Book& hedge_leg_book,
+                              Price& out_hedge_price,
+                              Quantity& out_hedge_quantity,
+                              Quantity& out_quote_target){
+      // effective_mult is the number of contracts needed to
+      // hedge one quoting contract
+      double effective_mult = std::abs(hedge_leg_data.leg_params().quote_params().lean_quantity_ratio() * hedge_leg_data.size_multiplier() / quote_leg_data.size_multiplier());
+      uint32_t hedge_requirement = _round_hedge_target (effective_mult * quote_target.value());       // rounding hedging up
+      SpreaderUtils::traverse_depth_for_quantity(hedge_leg_book,
+                                                 SpreaderUtils::invert_side (hedge_leg_data.trading_side()),
+                                                 Quantity(hedge_requirement),
+                                                 out_hedge_price,
+                                                 out_hedge_quantity);
+      // when we floor the quoting target (ie _round_quoting_qty)
+      // we are essentially demanding less in terms of hedging, ex :
+      // consider the hedge book depth with qty's 5, 10, 5, 20, 10 (5 is the inside)
+      // and hedge ratio 20 and target qty = 5, so effectively target = 100,
+      // we get 50 at the last level, out_qty = _round_quoting_qty (50/20) = 2,
+      // which means we need 2 * 20 and should re-evaluate
+      uint32_t out_quote_target_uint = _round_quoting_qty (out_hedge_quantity.value() / effective_mult);    // rounding quoting down
+      out_quote_target_uint = std::min(out_quote_target_uint,quote_target.value());
+      out_quote_target = Quantity (out_quote_target_uint);
+   }
    // calculates the computed price and quantity for a quoting
    // leg given the data for the other leg
    static void compute_price_quantity_2_legs(Price target_spread_price,
-                                             Quantity this_leg_target_quantity,
-                                             const LegPropertyData& this_leg_data,
-                                             const LegPropertyData& other_leg_data,
-                                             const bts::orderman::Book& other_leg_book,
-                                             Price& price, Quantity& quantity){
-      // other_leg_price and other_leg_quantity are the price and quantity calculated
-      // from traversing the other book, for the requires size   
-      Price other_leg_price; Quantity other_leg_quantity;
-      // this is the size ratio of the other leg vs this leg
-      double effective_mult = std::abs(other_leg_data.leg_params().quote_params().lean_quantity_ratio() * other_leg_data.size_multiplier() / this_leg_data.size_multiplier());
-      // to be conservative round the required size up
-      uint32_t other_leg_target_quantity = _round_hedge_target (effective_mult * this_leg_target_quantity.value());
-      SpreaderUtils::traverse_depth_for_quantity(other_leg_book, SpreaderUtils::invert_side (other_leg_data.trading_side()), Quantity(other_leg_target_quantity), other_leg_price, other_leg_quantity);
-      // when we floor we essentially go back up the hedging ladder
-      // so we should re-evaluate 
-      // ex, consider the hedge book depth with qty's 5, 10, 5, 20, 10 (5 is the inside)
-      // and hedge ratio 20 and target qty = 5, so effectively target = 100,
-      // we get 50 at the last level, out_qty = floor (50/20) = 2, so the
-      // price should not be the last level but the second last level
-      uint32_t out_quantity = _round_quoting_qty (other_leg_quantity.value() / effective_mult);
-      quantity = Quantity (out_quantity);
-      double out_price = (target_spread_price.value() - other_leg_price.value() * other_leg_data.price_multiplier())/this_leg_data.price_multiplier();
-      out_price = bts::securities::raw_round_to_quote_price (out_price, this_leg_data.tick_size(), this_leg_data.trading_side());
-      price = Price(out_price);
+                                             Quantity quote_target,
+                                             const LegPropertyData& quote_leg_data,
+                                             const LegPropertyData& hedge_leg_data,
+                                             const bts::orderman::Book& hedge_leg_book,
+                                             Price& quote_price,
+                                             Quantity& quote_qty){
+      Price hedge_price;
+      Quantity hedge_qty;
+      compute_driver(quote_target,quote_leg_data,hedge_leg_data,hedge_leg_book,
+                     hedge_price,hedge_qty,quote_qty);
+      double out_price = (target_spread_price.value() - hedge_price.value() * hedge_leg_data.price_multiplier())/quote_leg_data.price_multiplier();
+      out_price = bts::securities::raw_round_to_quote_price (out_price, quote_leg_data.tick_size(), quote_leg_data.trading_side());
+      quote_price = Price(out_price);      
    }
    static void compute_hedge_price_size_2_legs (Price spread_price,
                                                 const LegPropertyData& quote_filled_leg,
                                                 Price quote_fill_price,
-                                                Quantity quote_fill_quantity,
+                                                double quote_fill_quantity,
                                                 const LegPropertyData& hedge_leg,
                                                 Price& out_hedge_price,
-                                                Quantity& out_hedge_quantity){
-      // for ratio spreads hedge sizes are simply rounded off (the fraction can be a setting)
+                                                double& out_hedge_quantity){
       out_hedge_quantity = get_leg_quantity_from_another_leg_quantity (quote_filled_leg, quote_fill_quantity, hedge_leg);
       double out_hedge_price_d = (spread_price.value() - quote_fill_price.value() * quote_filled_leg.price_multiplier())/hedge_leg.price_multiplier();
-      out_hedge_price_d = bts::securities::raw_round_to_quote_price (out_hedge_price_d, hedge_leg.tick_size(), hedge_leg.trading_side());
-      out_hedge_price = Price(out_hedge_price_d);
+      out_hedge_price = Price(bts::securities::raw_round_to_quote_price (out_hedge_price_d, hedge_leg.tick_size(), hedge_leg.trading_side()));
    }
    ////////////////////////N legged case
    static void compute_quoting_price_size_n_legs (Price target_spread_price,
                                                   Quantity quoting_leg_target_quantity,
                                                   size_t quoting_leg_index,
                                                   const ISpreaderManager& spreader,
-                                                  std::vector<QuoterLegDriverData>& driver_vector,
+                                                  std::vector<QuoteLegDriver>& driver_vector,
                                                   Price& out_price,
                                                   Quantity& out_quantity){
       compute_drivers (quoting_leg_target_quantity, quoting_leg_index, spreader, driver_vector);
@@ -514,19 +543,23 @@ public:
    static void compute_drivers(Quantity quoting_leg_target_quantity,
                                size_t quoting_leg_index,
                                const ISpreaderManager& spreader,
-                               std::vector<QuoterLegDriverData>& out_driver_vector){
-      const LegPropertyData& quoting_leg_data = spreader.get_leg_data (quoting_leg_index);
+                               std::vector<QuoteLegDriver>& out_driver_vector){
+      const LegPropertyData& quote_leg_data = spreader.get_leg_data (quoting_leg_index);
       size_t n_legs = out_driver_vector.size();
       for (size_t idx = 0; idx < n_legs; idx++)
          if (idx != quoting_leg_index){
-            const bts::orderman::Book& driver_leg_book = spreader.get_book(idx);
-            const LegPropertyData& driver_leg_data = spreader.get_leg_data(idx);            
-            double driver_effective_mult = std::abs(driver_leg_data.leg_params().quote_params().lean_quantity_ratio() * driver_leg_data.size_multiplier() / quoting_leg_data.size_multiplier());
-            uint32_t driver_leg_target_quantity = _round_hedge_target (driver_effective_mult * quoting_leg_target_quantity.value());
-            Price driver_price;  Quantity driver_quantity;
-            SpreaderUtils::traverse_depth_for_quantity(driver_leg_book, SpreaderUtils::invert_side (driver_leg_data.trading_side()), Quantity(driver_leg_target_quantity), driver_price, driver_quantity);
-            Quantity quoting_leg_calc_quantity = Quantity (_round_quoting_qty (driver_quantity.value() / driver_effective_mult));
-            out_driver_vector[idx].set (driver_price, driver_quantity, quoting_leg_calc_quantity);
+            const bts::orderman::Book& hedge_leg_book = spreader.get_book(idx);
+            const LegPropertyData& hedge_leg_data = spreader.get_leg_data(idx);
+            Price hedge_price;
+            Quantity hedge_qty, quote_qty;
+            compute_driver(quoting_leg_target_quantity,
+                           quote_leg_data,
+                           hedge_leg_data,
+                           hedge_leg_book,
+                           hedge_price,
+                           hedge_qty,
+                           quote_qty);
+            out_driver_vector[idx].set (hedge_price, hedge_qty, quote_qty);
          }
    }
    // computes the quoting price from the leg drivers
@@ -534,13 +567,13 @@ public:
    static Price compute_quoting_price_from_drivers(Price target_spread_price,
                                                    size_t quoting_leg_index,
                                                    const ISpreaderManager& spreader,
-                                                   const std::vector<QuoterLegDriverData>& driver_vector){
+                                                   const std::vector<QuoteLegDriver>& driver_vector){
       const LegPropertyData& quoting_leg_data = spreader.get_leg_data (quoting_leg_index);
       size_t n_legs = driver_vector.size();
       double sum = 0;
       for (size_t idx = 0; idx < n_legs; idx++)
          if (idx != quoting_leg_index)
-            sum += driver_vector[idx].driver_price.value() * spreader.get_leg_data (idx).price_multiplier();
+            sum += driver_vector[idx].driver_price().value() * spreader.get_leg_data (idx).price_multiplier();
       double rv = (target_spread_price.value() - sum)/quoting_leg_data.price_multiplier();
       rv = bts::securities::raw_round_to_quote_price (rv, quoting_leg_data.tick_size(), quoting_leg_data.trading_side());
       return Price(rv);
@@ -549,26 +582,40 @@ public:
    // what each hedging leg can support. this function
    // simply finds that minimum    
    static Quantity compute_quoting_quantity_from_drivers (size_t quoting_leg_index,
-                                                          const std::vector<QuoterLegDriverData>& driver_vector){
+                                                          const std::vector<QuoteLegDriver>& driver_vector){
       uint32_t rv = std::numeric_limits<uint32_t>::max();
       for (size_t idx = 0; idx < driver_vector.size(); idx++)
          if (idx != quoting_leg_index)
-            if (driver_vector[idx].quoter_leg_quantity.value() < rv)
-               rv = driver_vector[idx].quoter_leg_quantity.value();
+            if (driver_vector[idx].quoter_leg_quantity().value() < rv)
+               rv = driver_vector[idx].quoter_leg_quantity().value();
       return Quantity(rv);
    }
    static void compute_hedge_price_size_n_legs (const LegPropertyData& quote_filled_leg,
                                                 Price,
-                                                Quantity quote_fill_quantity,
+                                                double quote_fill_quantity,
                                                 const LegPropertyData& hedge_leg,
-                                                const QuoterLegDriverData& leg_driver_data,
-                                                Price& out_hedge_price, Quantity& out_hedge_quantity){
-      out_hedge_price = leg_driver_data.driver_price;
-      out_hedge_quantity = get_leg_quantity_from_another_leg_quantity (quote_filled_leg, quote_fill_quantity, hedge_leg);
+                                                const QuoteLegDriver& leg_driver_data,
+                                                Price& out_hedge_price,
+                                                double& out_hedge_quantity){
+      out_hedge_price = Price(leg_driver_data.driver_price().value());
+      out_hedge_quantity = get_leg_quantity_from_another_leg_quantity (quote_filled_leg, 
+                                                                       quote_fill_quantity,
+                                                                       hedge_leg);
    }
    ////////////////////////////////////////////////////
    // Rounding functions //////////////////////////////
    ////////////////////////////////////////////////////
+   // rounds a double to a whole number if its too close
+   // to it (this is ued in the case of rounding errors
+   // when 0.9999999 or 1.0000001 is really 1)
+   static double round_if_near_whole(double qty,
+                                     double vicinity=0.000000001){
+      double round_qty = std::round(qty);
+      if (std::abs(round_qty - qty) <= vicinity)
+         return round_qty;
+      else
+         return qty;
+   }
    static uint32_t _round_quantity(double qty, bool is_up){
       double round_err = 0.000000001;
       double round_qty = std::round(qty);

src/server/sparkd/orderman/orders/spreader_orders/spreader/hedger/IHedgerStateWorker.h

@@ -10,8 +10,7 @@ namespace orderman {
 class SpreaderChildOrderState;
 class Book;
 class SingleFillHedger;
-class IHedgerStateWorker
-{   
+class IHedgerStateWorker{
 public:  
    // called once when transition to this state happens   
    virtual void on_transition () = 0;

src/server/sparkd/orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp

@@ -0,0 +1,29 @@
+#include "UnhedgedDelta.h"
+#include <cmath>
+#include "lib/utils/bts_assert.h"
+#include "server/sparkd/orderman/orders/spreader_orders/spreader/SpreaderUtils.h"
+
+namespace bts{
+namespace orderman{
+
+UnhedgedDelta::UnhedgedDelta()
+:
+_qty(0),
+_avg_price(0){
+}
+void UnhedgedDelta::add_unhedged(double price,double qty){
+   bts_assert(qty >= 0);
+   auto total_qty=_qty+qty;
+   _avg_price = (_qty*_avg_price + qty*price)/total_qty;
+   _qty=total_qty;
+}
+uint32_t UnhedgedDelta::subtract_hedge_if_available(){
+   uint32_t rv = 0;
+   double qty = SpreaderUtils::round_if_near_whole(_qty);
+   if (qty >= 1){
+      rv = std::floor(qty);
+      _qty = qty-rv;
+   }
+   return rv;
+}
+}}

src/server/sparkd/orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <cstdint>
+
+namespace bts{
+namespace orderman{
+// in the case when the hedge quantity is not a full contract
+// (ex if we get a 1 lot fill on a 1 X 3 spreader, where 3 is 
+// a quoting leg), this class is used to accumulate the unhedged
+// delta
+class UnhedgedDelta{
+private:
+   double _qty;
+   double _avg_price;
+public:
+   UnhedgedDelta();
+   // adds to the unhedged qty and updates the
+   // running average price
+   void add_unhedged (double price, double qty);
+   // if >= 1 unhedged delta is available, it
+   // subtracts the integer part and returns it.
+   // else returns 0
+   uint32_t subtract_hedge_if_available();
+};
+}}

src/server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp

@@ -0,0 +1,29 @@
+#include "QuoteDriverSet.h"
+
+namespace bts {
+namespace orderman {
+
+QuoteDriverSet::QuoteDriverSet(size_t n_legs)
+:
+_quote_seq_no(0),
+_quote_price(0.0),
+_drivers(){
+   _drivers.reserve(n_legs);
+   for (size_t i=0; i<n_legs; i++)
+      _drivers.emplace_back();
+}
+void QuoteDriverSet::update(uint16_t quote_seq_no,
+                             Price quote_price,
+                             const std::vector<QuoteLegDriver>& drivers){
+   _quote_seq_no=quote_seq_no;
+   _quote_price=quote_price;
+   bts_assert(_drivers.size()==drivers.size());
+   int i=0;
+   for(const auto& d:drivers)
+      _drivers[i++]=d;
+}
+const QuoteLegDriver& QuoteDriverSet::leg_driver(size_t leg_index) const{
+   bts_assert(leg_index<_drivers.size());
+   return _drivers[leg_index];   
+}
+}}

src/server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "QuoteLegDriver.h"
+
+namespace bts {
+namespace orderman {
+
+// this is the list of all drivers for a 
+// single quoting leg
+class QuoteDriverSet{
+private:
+   uint16_t _quote_seq_no;
+   Price _quote_price;
+   std::vector<QuoteLegDriver> _drivers;
+public:   
+   QuoteDriverSet (size_t n_legs);
+   uint16_t quote_seq_no() const{
+      return _quote_seq_no; 
+   }
+   Price quote_price() const{
+      return _quote_price;
+   }
+   void update(uint16_t quote_seq_no,
+               Price quote_price,
+               const std::vector<QuoteLegDriver>& drivers);
+   const QuoteLegDriver& leg_driver (size_t leg_index) const;
+};
+}}

src/server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp

@@ -0,0 +1,54 @@
+#include "QuoteDriverSetHistory.h"
+#include "lib/domain_model/Security.h"
+
+namespace bts {
+namespace orderman {
+
+QuoteDriverSetHistory::QuoteDriverSetHistory(size_t n_legs,
+                                               size_t cache_size,
+                                               double tick_size)
+:
+_n_legs(n_legs),
+_cache_size(cache_size),
+_tick_size(tick_size),
+_quote_seq_no(0),
+_driver_sets(),
+_h_ptr(0){
+   _driver_sets.reserve(cache_size);
+   for(size_t i=0;i<cache_size;i++)
+      _driver_sets.emplace_back(n_legs);
+}
+void QuoteDriverSetHistory::add(const std::vector<QuoteLegDriver>& drivers,
+                                 Price quote_price){
+   bts_assert(_h_ptr<_cache_size);
+   bts_assert(_cache_size>0);
+   _quote_seq_no++;
+   _driver_sets[_h_ptr].update(_quote_seq_no,quote_price,drivers);
+   // move it to the next writeable place
+   if(_h_ptr == _cache_size-1)
+      _h_ptr = 0;
+   else
+      _h_ptr++;
+}
+const QuoteDriverSet* QuoteDriverSetHistory::_latest() const{
+   if (_quote_seq_no>0)
+      return &(_driver_sets[_h_ptr==0 ? _cache_size-1 : _h_ptr-1]);
+   return
+      nullptr;
+}
+const QuoteDriverSet* QuoteDriverSetHistory::_find_driver_set(Price quote_price) const{
+   const QuoteDriverSet* rv=nullptr;
+   for(const auto& d : _driver_sets)
+      if (bts::securities::test_price_equality(d.quote_price().value(),
+                                               quote_price.value(),
+                                               _tick_size))
+         if (rv==nullptr||d.quote_seq_no()>rv->quote_seq_no())
+            rv=&d;
+   return rv!=nullptr ? rv:_latest();
+}
+const QuoteLegDriver* QuoteDriverSetHistory::find_driver(Price quote_price,
+                                                          size_t leg_index) const{
+   const QuoteDriverSet* driver_set_ptr=_find_driver_set(quote_price);
+   return driver_set_ptr==nullptr ? nullptr : &(driver_set_ptr->leg_driver(leg_index));
+}
+}}

src/server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "QuoteDriverSet.h"
+
+namespace bts {
+namespace orderman {
+
+class QuoteDriverSetHistory{
+private:   
+   const size_t _n_legs, _cache_size;
+   const double _tick_size;
+   uint16_t _quote_seq_no;
+   std::vector<QuoteDriverSet> _driver_sets;
+   uint16_t _h_ptr;
+   const QuoteDriverSet* _latest() const;
+   const QuoteDriverSet* _find_driver_set(Price quote_price) const;
+public:
+   QuoteDriverSetHistory(size_t n_legs,
+                          size_t cache_size,
+                          double tick_size);
+   void add(const std::vector<QuoteLegDriver>& drivers,
+            Price quote_price);      
+   const QuoteLegDriver* find_driver(Price quote_price, size_t leg_index) const;
+};
+
+}}

src/server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp

@@ -0,0 +1,12 @@
+#include "QuoteLegDriver.h"
+
+namespace bts {
+namespace orderman {
+
+QuoteLegDriver::QuoteLegDriver()
+: 
+_driver_price(0.0),
+_driver_quantity(0),
+_quoter_leg_quantity(0){   
+}
+}}

src/server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include "server/sparkd/orderman/orderman_utils/orderman_common.h"
+
+namespace bts {
+namespace orderman {
+// Should be read as Quote_LegDriver, this is the
+// price and qty in a hedging leg that can sustain
+// a "_quoter_leg_quantity" sized quote in the
+// quoting leg
+class QuoteLegDriver
+{
+private:
+   Price _driver_price;
+   Quantity _driver_quantity;
+   Quantity _quoter_leg_quantity;
+public:
+   // driver_price and driver_quantity are the price and quantity
+   // in the hedging (or driver) leg that can drive a quoter_leg_quantity
+   // sized order in the quoting leg      
+   QuoteLegDriver();   
+   QuoteLegDriver& operator=(const QuoteLegDriver& s){
+      set(s.driver_price(),s.driver_quantity(), s.quoter_leg_quantity());
+      return *this;
+   }
+   void set(Price driver_price,
+            Quantity driver_quantity,
+            Quantity quoter_leg_quantity){
+      _driver_price = driver_price;
+      _driver_quantity = driver_quantity;
+      _quoter_leg_quantity = quoter_leg_quantity;      
+   }
+   Price driver_price() const{
+      return _driver_price;
+   }
+   Quantity driver_quantity() const{
+      return _driver_quantity;
+   }
+   Quantity quoter_leg_quantity() const{
+      return _quoter_leg_quantity;
+   }
+};
+}}

src/server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoterLegDriverData.h

@@ -1,32 +0,0 @@
-#pragma once
-
-#include "server/sparkd/orderman/orderman_utils/orderman_common.h"
-
-namespace bts {
-namespace orderman {
-// the concept of a "driver" is the size and price of a leg that 
-// can sustain a quote in the quoted leg
-class QuoterLegDriverData
-{
-public:
-   // driver_price and driver_quantity are the price and quantity
-   // in the hedging (or driver) leg that can drive a quoter_leg_quantity
-   // sized order in the quoting leg
-   Price driver_price;
-   Quantity driver_quantity;
-   Quantity quoter_leg_quantity;
-   
-   QuoterLegDriverData()
-   : 
-   driver_price(0.0),
-   driver_quantity(0),
-   quoter_leg_quantity(0)
-   {}
-   void set(Price new_driver_price, Quantity new_driver_quantity, Quantity new_quoter_leg_quantity)
-   {
-      driver_price = new_driver_price;
-      driver_quantity = new_driver_quantity;
-      quoter_leg_quantity = new_quoter_leg_quantity;
-   }
-};
-}}

src/server/sparkd/orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp

@@ -51,7 +51,7 @@ void QuoterWaitingForSizeStateWorker::_longpath_analyze_and_act (orderman::TickT
       Price out_price; Quantity out_quantity;
       _quoter->_compute_price_quantity (out_price, out_quantity);
       if (out_quantity.value() > 0){
-         _quoter->_commit_drivers ();
+         _quoter->_commit_drivers (out_price);
          LOG(OrderMan, Info, "QuoterWaitingForSizeStateWorker sending order key: {} price: {} quantity: {}", _quoter->_key(), out_price, out_quantity);
         // this is the only place where a quoter order is entered
          _quoter->_place_order(out_price, out_quantity, cid);

src/server/sparkd/orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp

@@ -94,7 +94,7 @@ void QuoterWorkingStateWorker::_modify_order (orderman::TickToTradeCorID cid, Pr
                          bts::securities::test_price_equality(_quoter->_price_quantity_state.out_price().value(), 
                                                               price.value(), _quoter->_this_leg_data.tick_size());
    if (!same_price_qty){
-      _quoter->_commit_drivers ();
+      _quoter->_commit_drivers (price);
       _quoter->_modify_order(price, quantity, cid);
    }
 }

src/server/sparkd/orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp

@@ -40,13 +40,19 @@ void SpreaderStartStateWorker::on_init_recovered(const bts::api::orderman::Order
                                                  const SpreaderPreActivationOrderServices& spreader_pao_services)
 {
    bts_assert(!_spreader->_inited());
-   // we are not handling this yet, so lets send it directly to END
+   // we are not handling recovery, so if order is not in a terminal state,
+   // instead of carrying on from where we left, we go directly to the end state
+   // and mark the order cancelled. if its already in a terminal state we 
+   // just go to end and not do anything
+   bts::api::orderman::OrderRecord order_record_copy = order_record;
+   if (!SpreaderUtils::order_in_terminal_state(order_record_copy.mutable_state()->status()))
+      order_record_copy.mutable_state()->set_status(bts::api::orderman::OrderStatus::ORDER_STATUS_CANCELED);
    _spreader->_bookkeeping_data_ptr = std::make_unique<SpreaderBookkeepingData>(*_spreader,
-                                                                                order_record,
+                                                                                order_record_copy,
                                                                                 spreader_pao_services,
                                                                                 std::vector<LegPropertyData>());
    LOG(OrderMan, Warn, "SpreaderStartStateWorker.on_init_recovered key: {} , going to END", _spreader->key().value());
-   std::string msg = "Recovered spreader orders and not supported";
+   std::string msg = "Recovered spreader orders (from last session) are cancelled";
    _spreader->_transform_to_END (&msg);
 }
 void SpreaderStartStateWorker::on_activate(Price price, Quantity quantity, SpreaderOrderServices& order_services){

src/server/sparkd/orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp

@@ -97,19 +97,31 @@ void SpreaderWorkingStateWorker::on_quoter_order_fill(SpreaderLegData& leg_data)
        order_update_message.last_update_filled_quantity(),
        order_update_message.last_update_filled_price());
    bts_assert (order_update_message.last_update_filled_quantity().value() > 0);
-   for (size_t q_idx = 0; q_idx < _spreader->_bookkeeping_data().n_legs(); q_idx++){
-      SpreaderLegData& other_leg_data = _spreader->_bookkeeping_data().get_leg_data (q_idx);
+   for (size_t q_idx = 0; q_idx < _spreader->_bookkeeping_data().n_legs(); q_idx++){      
       if (q_idx != leg_data.leg_property_data().leg_index()){
-         // if it is an actively quoting leg, inform the quoter about it
-         // so it can decrease the proportional size
-         if (other_leg_data.leg_property_data().is_active_quoting())
-            other_leg_data.quoter().update_other_leg_fills(leg_data.leg_property_data().leg_index(),
-                                                           order_update_message.last_update_filled_quantity());
-         // a hedge is sent regardless
-         _spreader->_send_hedge (leg_data.leg_property_data().leg_index(),
-                                 order_update_message.last_update_filled_price(),
-                                 order_update_message.last_update_filled_quantity(),
-                                 q_idx);
+         SpreaderLegData& other_leg_data = _spreader->_bookkeeping_data().get_leg_data (q_idx);   
+         Price hedge_price;
+         double hedge_qty;
+         _spreader->_compute_hedge_price_quantity(leg_data.leg_property_data().leg_index(),
+                                                  order_update_message.last_update_filled_price(),
+                                                  order_update_message.last_update_filled_quantity().value(),
+                                                  q_idx,
+                                                  hedge_price,
+                                                  hedge_qty);
+         UnhedgedDelta& leg_unhedged_delta = other_leg_data.unhedged_delta();
+         leg_unhedged_delta.add_unhedged(hedge_price.value(), hedge_qty);
+         uint32_t total_unhedged_qty=leg_unhedged_delta.subtract_hedge_if_available();
+         if (total_unhedged_qty != 0){
+            // if it is an actively quoting leg, inform the quoter about it
+            // so it can decrease the proportional size
+            if (other_leg_data.leg_property_data().is_active_quoting())
+                  other_leg_data.quoter().update_other_leg_fills(leg_data.leg_property_data().leg_index(),
+                                                                 Quantity(total_unhedged_qty));
+            // a hedge is sent regardless
+            _spreader->_send_hedge(other_leg_data.leg_property_data().leg_index(),
+                                   hedge_price,
+                                   Quantity(total_unhedged_qty)); 
+         }
       }
    }
    // this is done at the end (i dont like ordering though) because

src/server/sparkd/orderman/orders/spreader_orders/tests/spreader_tests.cpp

@@ -483,7 +483,7 @@ TEST_CASE ("compute quoting price quantity 3 legs", "[spreader]")
    std::vector<LegSpreaderParams> params; params.push_back(leg_params_1);params.push_back(leg_params_2);params.push_back(leg_params_3);
 
    SpreaderManagerForTest* sm =  create_spreader_manager_n_legs (OrderKey(), legs, params);
-   std::vector<QuoterLegDriverData> driver_vector;
+   std::vector<QuoteLegDriver> driver_vector;
    for(size_t i = 0; i < legs.size(); i++) driver_vector.emplace_back();
 
    SECTION("basic joining")
@@ -570,8 +570,9 @@ TEST_CASE ("compute quoting price quantity 3 legs", "[spreader]")
 TEST_CASE ("compute hedge price quantity 2 legs", "[spreader]")
 {
    ContractLeg leg_1, leg_2;
-   Price out_hedge_price; Quantity out_hedge_quantity;
-   Price spreader_price = Price (0.35);
+   Price out_hedge_price;
+   double out_hedge_quantity;
+   Price spreader_price = Price(0.35);
    double size_mult_1 = 0.25, size_mult_2 = -1.1, // same used for price and qty
              price_mult_1 = 0.35, price_mult_2 = -1.5;
    double lean_mult = 1.0;
@@ -580,23 +581,23 @@ TEST_CASE ("compute hedge price quantity 2 legs", "[spreader]")
    LegPropertyData leg_prop_data_1 = create_leg_property_data (leg_1, 0, price_mult_1, true, lean_mult);   
    LegPropertyData leg_prop_data_2 = create_leg_property_data (leg_2, 1, price_mult_2, true, lean_mult);
       // md
-   SpreaderUtils::compute_hedge_price_size_2_legs (spreader_price, leg_prop_data_2, Price (2500.5), Quantity (3),
+   SpreaderUtils::compute_hedge_price_size_2_legs (spreader_price, leg_prop_data_2, Price(2500.5), 3,
                                                                    leg_prop_data_1, out_hedge_price, out_hedge_quantity);
-   REQUIRE (out_hedge_quantity.value() == 1);
+   REQUIRE (std::abs(out_hedge_quantity - 0.6818181818181818181) < 0.000000001);
    REQUIRE (out_hedge_price.value() == 10717.25);
 
-   SpreaderUtils::compute_hedge_price_size_2_legs (spreader_price, leg_prop_data_2, Price (2500.5), Quantity (2),
+   SpreaderUtils::compute_hedge_price_size_2_legs (spreader_price, leg_prop_data_2, Price(2500.5), 2,
                                                                    leg_prop_data_1, out_hedge_price, out_hedge_quantity);
-   REQUIRE (out_hedge_quantity.value() == 0);
+   REQUIRE (std::abs(out_hedge_quantity - 0.45454545454545454545) < 0.000000001);
    REQUIRE (out_hedge_price.value() == 10717.25);
    // reversing the side of the quote fill
-   SpreaderUtils::compute_hedge_price_size_2_legs (spreader_price, leg_prop_data_1, Price (10000.5), Quantity (2),
+   SpreaderUtils::compute_hedge_price_size_2_legs (spreader_price, leg_prop_data_1, Price(10000.5), 2,
                                                                    leg_prop_data_2, out_hedge_price, out_hedge_quantity);
-   REQUIRE (out_hedge_quantity.value() == 9);
+   REQUIRE (std::abs(out_hedge_quantity - 8.8)<0.000000001);
    REQUIRE (out_hedge_price.value() == 2333.25);
-   SpreaderUtils::compute_hedge_price_size_2_legs (spreader_price, leg_prop_data_1, Price (9997), Quantity (6),
+   SpreaderUtils::compute_hedge_price_size_2_legs (spreader_price, leg_prop_data_1, Price(9997), 6,
                                                                    leg_prop_data_2, out_hedge_price, out_hedge_quantity);
-   REQUIRE (out_hedge_quantity.value() == 26);
+   REQUIRE (std::abs(out_hedge_quantity - 26.40) < 0.000000001);
    REQUIRE (out_hedge_price.value() == 2332.5);
 }
 /************************************************************************************************************/
@@ -2341,6 +2342,201 @@ TEST_CASE ("prehedge_payup", "[spreader]"){
       REQUIRE(bts::securities::test_price_equality(out_price.value(),21.10,0.05));
    }
 }
+// n legged spreader price evaluation /////////////////////
+///////////////////////////////////////////////////////////
+TEST_CASE ("nlegs_compute_driver", "[spreader]"){
+   bts::orderman::OrderKey order_key(1);
+   ContractLeg leg_1, leg_2, leg_3, leg_4;
+   fill_future_contract_leg (leg_1, 1, "CL", bts::api::Venue::VENUE_GLBX, 2018, 1, 15, 1.0, 1.0, api::Side::SIDE_BUY, 1.0, 0.01);
+   fill_future_contract_leg (leg_2, 2, "CL", bts::api::Venue::VENUE_GLBX, 2018, 2, 15, 1.0, 1.0, api::Side::SIDE_BUY, 1.0, 0.01);
+   fill_future_contract_leg (leg_3, 3, "CL", bts::api::Venue::VENUE_GLBX, 2018, 3, 15, 1.0, 1.0, api::Side::SIDE_BUY, 1.0, 0.01);
+   fill_future_contract_leg (leg_4, 4, "CL", bts::api::Venue::VENUE_GLBX, 2018, 4, 15, -3.0, -3.0, api::Side::SIDE_SELL, 1.0, 0.01);
+   std::vector<ContractLeg> legs;
+   legs.push_back(leg_1);legs.push_back(leg_2);legs.push_back(leg_3);legs.push_back(leg_4);
+   Security sec;
+   fill_synthetic_security (sec, 2000, "4legs", legs);
+   // execution params/order def
+   LegSpreaderParams lsp_1, lsp_2, lsp_3, lsp_4;
+   fill_leg_spreader_params(lsp_1, 1.0, false, 1.0);fill_leg_spreader_params(lsp_2, 1.0, false, 1.0);
+   fill_leg_spreader_params(lsp_3, 1.0, false, 1.0);fill_leg_spreader_params(lsp_4, -4.0, true, 1.0);
+  //  // post hedge time out here
+   std::vector<LegSpreaderParams> params;
+   params.push_back(lsp_1);params.push_back(lsp_2);
+   params.push_back(lsp_3);params.push_back(lsp_4);
+   bts::api::orderman::OrderDefinition order_def;
+   fill_spreader_order_definition(order_def, Side::SIDE_BUY, sec, params);
+   // putting it together in SpreaderBookkeepingData
+   SpreaderPreActivationOrderServices::TestOnlySpreaderPreActivationOrderServices
+   testonly_spreader_pao_services = _create_spreader_pao_services(sec);
+   SpreaderPreActivationOrderServices pao_services (&testonly_spreader_pao_services);
+   std::vector<LegPropertyData> leg_props;
+   SpreaderUtils::resolve_synthetic_spread(pao_services, order_def, leg_props);   
+   SpreaderManagerForTest spreader_manager (order_key, order_def, pao_services, leg_props);
+   SpreaderBookkeepingData& bookkeeping_data = spreader_manager.bookkeeping_data();
+   SECTION("compute_1"){
+      Quantity quoting_leg_target (300);
+      Price spread_price(0.25);
+      fill_book(spreader_manager, 0, new double[5]{65.31,65.32,65.34,65.36, 65.40},
+                                     new uint16_t[5]{12,18,20,19,23},bts::api::Side::SIDE_SELL,true);      
+      fill_book(spreader_manager, 1, new double[5]{64.40,64.41,64.43,64.45,64.50},
+                                     new uint16_t[5]{1,5,20,12,15},bts::api::Side::SIDE_SELL,true);
+      fill_book(spreader_manager, 2, new double[5]{64.10,64.18,64.19,64.20,64.21},
+                                     new uint16_t[5]{10,43,17,50,15},bts::api::Side::SIDE_SELL,true);
+      std::vector<QuoteLegDriver> out_driver_vector;
+      for (int i=0; i<4; i++)
+         out_driver_vector.emplace_back();
+      SpreaderUtils::compute_drivers(quoting_leg_target, 3, spreader_manager, out_driver_vector);
+      _verify_driver (out_driver_vector[0],92,65.40,276);
+      _verify_driver (out_driver_vector[1],53,64.50,159);
+      _verify_driver (out_driver_vector[2],100,64.20,300);
+      // size of the quoting leg
+      Quantity computed_quoted_qty=SpreaderUtils::compute_quoting_quantity_from_drivers(3,out_driver_vector);
+      REQUIRE(computed_quoted_qty.value()==159);
+      // 2nd pass drivers
+      SpreaderUtils::compute_drivers(computed_quoted_qty, 3, spreader_manager, out_driver_vector);
+      _verify_driver (out_driver_vector[0],53,65.36,159);
+      _verify_driver (out_driver_vector[1],53,64.50,159);
+      _verify_driver (out_driver_vector[2],53,64.18,159);
+      // price of the quoting leg
+      Price computed_price=SpreaderUtils::compute_quoting_price_from_drivers(spread_price,3,spreader_manager,out_driver_vector);
+      REQUIRE(computed_price.value()==64.60);
+      //hedging
+      Price hedge_price; double hedge_quantity;
+      SpreaderUtils::compute_hedge_price_size_n_legs(spreader_manager.get_leg_data(3),
+                                                     Price(0),
+                                                     18,
+                                                     spreader_manager.get_leg_data(0),
+                                                     out_driver_vector[0],
+                                                     hedge_price,
+                                                     hedge_quantity);
+      REQUIRE(hedge_price.value()==65.36);
+      REQUIRE(hedge_quantity==6);
+   }
+   SECTION("compute_2_fractional"){
+      Quantity quoting_leg_target (100);
+      Price spread_price(0.25);
+      fill_book(spreader_manager, 0, new double[5]{65.31,65.32,65.34,65.36, 65.40},
+                                     new uint16_t[5]{12,18,20,19,23},bts::api::Side::SIDE_SELL,true);      
+      fill_book(spreader_manager, 1, new double[5]{64.40,64.41,64.43,64.45,64.50},
+                                     new uint16_t[5]{1,5,20,12,15},bts::api::Side::SIDE_SELL,true);
+      fill_book(spreader_manager, 2, new double[5]{64.10,64.18,64.19,64.20,64.21},
+                                     new uint16_t[5]{11,1,1,5,1},bts::api::Side::SIDE_SELL,true);
+      std::vector<QuoteLegDriver> out_driver_vector;
+      for (int i=0; i<4; i++)
+         out_driver_vector.emplace_back();
+      SpreaderUtils::compute_drivers(quoting_leg_target, 3, spreader_manager, out_driver_vector);
+      _verify_driver (out_driver_vector[0],34,65.34,100);
+      _verify_driver (out_driver_vector[1],34,64.45,100);
+      _verify_driver (out_driver_vector[2],19,64.21,57);
+      // size of the quoting leg
+      Quantity computed_quoted_qty=SpreaderUtils::compute_quoting_quantity_from_drivers(3,out_driver_vector);
+      REQUIRE(computed_quoted_qty.value()==57);
+      // // 2nd pass drivers
+       SpreaderUtils::compute_drivers(computed_quoted_qty, 3, spreader_manager, out_driver_vector);
+      _verify_driver (out_driver_vector[0],19,65.32,57);
+      _verify_driver (out_driver_vector[1],19,64.43,57);
+      _verify_driver (out_driver_vector[2],19,64.21,57);
+      // // price of the quoting leg
+      Price computed_price=SpreaderUtils::compute_quoting_price_from_drivers(spread_price,3,spreader_manager,out_driver_vector);
+      REQUIRE(computed_price.value()==64.57);
+      // //hedging
+      Price hedge_price; double hedge_quantity;
+      SpreaderUtils::compute_hedge_price_size_n_legs(spreader_manager.get_leg_data(3),
+                                                     Price(0),
+                                                     10.0,
+                                                     spreader_manager.get_leg_data(0),
+                                                     out_driver_vector[0],
+                                                     hedge_price,
+                                                     hedge_quantity);
+      REQUIRE(hedge_price.value()==65.32);
+      REQUIRE(std::abs(hedge_quantity - 3.333333) < 0.000001);
+   }   
+}
+TEST_CASE ("quote driver caching", "[spreader]"){   
+   SECTION("zero_search"){
+      QuoteDriverSetHistory driver_cache(5,10,0.05);
+      const QuoteLegDriver* driver_ptr=driver_cache.find_driver(Price(100),3);
+      REQUIRE(driver_ptr==nullptr);
+   }
+   SECTION("simple_search"){
+      size_t n_legs=5, cache_sz=10;   
+      QuoteDriverSetHistory driver_cache(n_legs,cache_sz,0.05);
+      Price p1(100); std::vector<QuoteLegDriver> d1;
+      generate_driver_set(d1,n_legs);
+      driver_cache.add(d1,p1);
+      Price p2(100.05); std::vector<QuoteLegDriver> d2;
+      generate_driver_set(d2,n_legs);
+      driver_cache.add(d2,p2);
+      Price p3(100.10); std::vector<QuoteLegDriver> d3;
+      generate_driver_set(d3,n_legs);
+      driver_cache.add(d3,p3);
+      for(size_t i=0; i<n_legs; i++){
+         const auto d=driver_cache.find_driver(p1,i);
+         verify_driver(*d,d1[i]);
+      }
+      for(size_t i=0; i<n_legs; i++){
+         const auto d=driver_cache.find_driver(p2,i);
+         verify_driver(*d,d2[i]);
+      }
+      for(size_t i=0; i<n_legs; i++){
+         const auto d=driver_cache.find_driver(p3,i);
+         verify_driver(*d,d3[i]);
+      }
+      // if not found, returns the latest
+      for(size_t i=0; i<n_legs; i++){
+         const auto d=driver_cache.find_driver(Price(100.01),i);
+         verify_driver(*d,d3[i]);
+      }
+   }
+   SECTION("collision_search and continous add"){
+     // adding multiple updates to test circular nature
+      size_t n_runs=10003, n_legs=4, cache_sz=10;
+      QuoteDriverSetHistory driver_cache(n_legs,cache_sz,0.05);
+      for(size_t i=0; i<n_runs; i++){
+         Price p(gen_rand(1,100));
+         std::vector<QuoteLegDriver> d;
+         generate_driver_set(d,n_legs);
+         driver_cache.add(d,p);
+      }
+      Price p1(100); std::vector<QuoteLegDriver> d1;
+      generate_driver_set(d1,n_legs);
+      driver_cache.add(d1,p1);
+      Price p2(100.05); std::vector<QuoteLegDriver> d2;
+      generate_driver_set(d2,n_legs);
+      driver_cache.add(d2,p2);
+      Price p3(100.10); std::vector<QuoteLegDriver> d3;
+      generate_driver_set(d3,n_legs);
+      driver_cache.add(d3,p3);
+      Price p4(100); std::vector<QuoteLegDriver> d4;
+      generate_driver_set(d4,n_legs);
+      driver_cache.add(d4,p4);
+      Price p5(100.15); std::vector<QuoteLegDriver> d5;
+      generate_driver_set(d5,n_legs);
+      driver_cache.add(d5,p5);
+      for(size_t i=0; i<n_legs; i++){
+         const auto d=driver_cache.find_driver(p1,i);
+         verify_driver(*d,d4[i]);
+      }
+      for(size_t i=0; i<n_legs; i++){
+         const auto d=driver_cache.find_driver(p2,i);
+         verify_driver(*d,d2[i]);
+      }
+      for(size_t i=0; i<n_legs; i++){
+         const auto d=driver_cache.find_driver(p3,i);
+         verify_driver(*d,d3[i]);
+      }
+      for(size_t i=0; i<n_legs; i++){
+         const auto d=driver_cache.find_driver(p4,i);
+         verify_driver(*d,d4[i]);
+      }
+      // 100 is there twice, should returns the
+      // most recently added one
+      for(size_t i=0; i<n_legs; i++){
+         const auto d=driver_cache.find_driver(p5,i);
+         verify_driver(*d,d5[i]);
+      }
+   }
+}
 class RV
 {
 public:

src/server/sparkd/orderman/orders/spreader_orders/tests/tests_common.h

@@ -292,6 +292,20 @@ bts::orderman::Book get_book(SecurityKey key,
 {
    return get_book(key, n_levels, bid_prices, bid_sizes, n_levels, ask_prices, ask_sizes, delete_arrays);         
 }
+void fill_book (SpreaderManagerForTest& sm,
+                size_t index,
+                double* prices,
+                uint16_t* sizes,
+                bts::api::Side side,
+                bool delete_arrays){
+   Book book;
+   fill_book (book, prices, sizes, side);
+   sm.set_book(book, index);
+   if (delete_arrays){
+      delete[] prices;
+      delete[] sizes;      
+   }
+}
 /**********************************************************************/
 /******************************hedger**********************************/
 /**********************************************************************/
@@ -464,14 +478,38 @@ void set_spreader_orders (SpreaderOrder& spreader, double* leg_1_bid_prices, uin
 /**********************************************************************/
 /******************************misc************************************/
 /**********************************************************************/
+void _verify_driver(const QuoteLegDriver& driver_data,
+                    uint16_t driver_qty,
+                    double driver_price,
+                    uint16_t quoter_leg_quantity){
+   double err = 1e-9;
+   REQUIRE(std::abs(driver_data.driver_price().value() - driver_price)< err);
+   REQUIRE(driver_data.driver_quantity().value() == driver_qty);
+   REQUIRE(driver_data.quoter_leg_quantity().value() == quoter_leg_quantity);
+}
+void verify_driver(const QuoteLegDriver& d1,
+                    const QuoteLegDriver& d2){
+   _verify_driver(d1,
+                  d2.driver_quantity().value(),
+                  d2.driver_price().value(),
+                  d2.quoter_leg_quantity().value());
+}
 double gen_rand (double min, double max)
 {
    double f = (double)rand() / RAND_MAX;
    return min + f * (max - min);
 }
-// SpreaderChildOrderState create_order_update(const LegPropertyData& leg_data)
-// {
-//    SpreaderChildOrderState rv(false, leg_data, true);
-//    return rv;
-// }
-}}
+void fill_random_driver(QuoteLegDriver& driver){
+   driver.set(Price(gen_rand(0,10)),
+              Quantity(gen_rand(1,100)),
+              Quantity(gen_rand(1,300)));
+}
+void generate_driver_set(std::vector<QuoteLegDriver>& drivers, size_t n_legs){
+   drivers.clear();
+   for(size_t i=0; i < n_legs; i++){
+      QuoteLegDriver d;
+      fill_random_driver(d);
+      drivers.push_back(d);      
+   }      
+}
+}}

src/server/sparkd/orderman/perf_analysis/load.sh

@@ -4,7 +4,12 @@ sqlite3 trace_data/trace.sqlite <<EOF
 
 .separator ","
 
-create table trace (seq integer, ts integer, hts text, msg text, cpu integer, tid integer,arg1 integer);
-create index trace_seq_idx on trace(seq);
-.import trace_data/parsed_trace.txt trace
+create table raw_trace (seq integer, ts integer, hts text, msg text, cpu integer, tid integer,arg1 integer);
+.import trace_data/parsed_trace.txt raw_trace
+create table trace as
+select
+   (seq % 65535) seq16, seq, ts, hts, msg, cpu, tid, arg1
+from
+   raw_trace;
+create index trace_seq_idx on trace(seq16);
 EOF

src/server/sparkd/orderman/server/OrderServiceServer.cpp

@@ -110,7 +110,7 @@ OrderServiceServer::~OrderServiceServer() {
 }
 
 void OrderServiceServer::on_orders(
-    std::vector<api::orderman::OrderRecord const *> const &orders) {
+    std::vector<api::orderman::OrderRecord> const &orders) {
    LOG(OrderMan, Info, "Received order update for publishing");
    std::vector<api::orderman::OrderRecord> created;
    created.reserve(orders.size());
@@ -118,12 +118,12 @@ void OrderServiceServer::on_orders(
    updated.reserve(orders.size());
 
    for (auto const &rec : orders) {
-      auto const &k = OrderKey(rec->key());
+      auto const &k = OrderKey(rec.key());
       if (me->known_orders.count(k) == 0) {
          me->known_orders.insert(k);
-         created.push_back(*rec);
+         created.push_back(rec);
       } else
-         updated.push_back(*rec);
+         updated.push_back(rec);
    }
 
    LOG(OrderMan, Info,

src/server/sparkd/orderman/server/OrderServiceServer.h

@@ -27,7 +27,7 @@ public:
 private:
    struct PImpl;
    std::unique_ptr<PImpl> me;
-   void on_orders(std::vector<api::orderman::OrderRecord const*> const&);
+   void on_orders(std::vector<api::orderman::OrderRecord> const&);
    void on_fills (std::vector<api::orderman::Fill const*> const&);
 };
 

src/server/sparkd/persistence/PersistenceServiceServer.cpp

@@ -88,7 +88,7 @@ void PersistenceServiceServer::retrieve(const RetrieveRequest& request,
 
    auto it = me->data.find(request.ns());
    if (it == me->data.end()) {
-      LOG(persist,Error, "Client tried to retrieve specific rows from unknown namespace {}/{}",
+      LOG(persist,Warn, "Client tried to retrieve specific rows from unknown namespace {}/{}",
           me->db,request.ns());
       response.set_result ( RetrieveResponse::NO_SUCH_NS);
       reply(response);
@@ -108,7 +108,7 @@ void PersistenceServiceServer::retrieve(const RetrieveRequest& request,
       }
       auto it = known_rows.find(k);
       if (it == known_rows.end()) {
-         LOG(persist,Error, "Client tried to retrieve unknown row with key {} in {}/{}", k,
+         LOG(persist,Warn, "Client tried to retrieve unknown row with key {} in {}/{}", k,
              me->db,request.ns());
          response.set_result ( RetrieveResponse::DUPLICATE_KEY);
          reply(response);
@@ -132,7 +132,7 @@ void PersistenceServiceServer::retrieve_all(const RetrieveAllRequest& request,
    RetrieveResponse response;
    
    if (request.ns().empty()) {
-      LOG(persist,Error, "Client used empty namespace while trying to retrieve rows in database {}",
+      LOG(persist,Warn, "Client used empty namespace while trying to retrieve rows in database {}",
           me->db);
       response.set_result ( RetrieveResponse::EMPTY_NS);
       reply(response);
@@ -171,7 +171,7 @@ void PersistenceServiceServer::remove(const RemoveRequest& request,
    
    auto it = me->data.find(request.ns());
    if (it == me->data.end()) {
-      LOG(persist,Error, "Client used unknown namespace {} while trying to remove rows in database {}",
+      LOG(persist,Warn, "Client used unknown namespace {} while trying to remove rows in database {}",
           request.ns(),
           me->db);
       response.set_result ( RemoveResponse::NO_SUCH_NS);
@@ -184,7 +184,7 @@ void PersistenceServiceServer::remove(const RemoveRequest& request,
    for (auto &k : request.key()) {
 
       if (keys.count(k)) {
-         LOG(persist,Error, "Client used duplicate key {} while trying to remove rows in {}/{}",
+         LOG(persist,Warn, "Client used duplicate key {} while trying to remove rows in {}/{}",
              k,me->db,request.ns());
          response.set_result ( RemoveResponse::DUPLICATE_KEY);
          reply(response);
@@ -194,7 +194,7 @@ void PersistenceServiceServer::remove(const RemoveRequest& request,
       keys.insert(k);
 
       if (k.empty()) {
-         LOG(persist,Error, "Client used empty key while trying to remove rows in {}/{}",
+         LOG(persist,Warn, "Client used empty key while trying to remove rows in {}/{}",
              me->db,request.ns());
          response.set_result ( RemoveResponse::EMPTY_KEY);
          reply(response);
@@ -202,7 +202,7 @@ void PersistenceServiceServer::remove(const RemoveRequest& request,
       }
 
       if (ns_rec.data.find(k) == ns_rec.data.end()) {
-         LOG(persist,Error, "Client tried to remove unknown row with key {} in {}/{}",
+         LOG(persist,Info, "Client tried to remove unknown row with key {} in {}/{}",
              k, me->db, request.ns());
          response.set_result ( RemoveResponse::NO_SUCH_KEY);
          reply(response);

src/server/sparkd/user/tests/admin_service_tests.cpp

@@ -268,4 +268,71 @@ TEST_CASE ("Bootstrapping", "[admin_service]")
       REQUIRE(account.connections().exchange_connection_names_size()==2);
    }
 }
+TEST_CASE("Sorting Limit","[admin_service]"){
+   bts::api::admin::AdminData admin_data;
+   bts::api::admin::AdminServiceCreateAccountResult account_result;
+   auto account = AdminServiceUtils::create_account(admin_data,
+                                                    0, 
+                                                    std::string("A"), 
+                                                    std::string("description"), 
+                                                    true,
+                                                    account_result);
+   auto l1 = account->mutable_security_clip_limits()->add_limit();
+   l1->mutable_security()->set_exchange(api::Venue::VENUE_GLBX);
+   l1->mutable_security()->set_security_type(api::SecurityType::SECURITYTYPE_FUTURE);
+   l1->mutable_security()->set_security_symbol("*");
+   l1->mutable_security()->set_security_id(0);
+   l1->set_clip_size(100);
+   auto l2 = account->mutable_security_clip_limits()->add_limit();
+   l2->mutable_security()->set_exchange(api::Venue::VENUE_GLBX);
+   l2->mutable_security()->set_security_type(api::SecurityType::SECURITYTYPE_FUTURE);
+   l2->mutable_security()->set_security_symbol("SI");
+   l2->mutable_security()->set_security_id(0);
+   l2->set_clip_size(101);
+   auto l3 = account->mutable_security_clip_limits()->add_limit();
+   l3->mutable_security()->set_exchange(api::Venue::VENUE_GLBX);
+   l3->mutable_security()->set_security_type(api::SecurityType::SECURITYTYPE_FUTURE);
+   l3->mutable_security()->set_security_symbol("CL");
+   l3->mutable_security()->set_security_id(0);
+   l3->set_clip_size(102);
+   auto l4 = account->mutable_security_clip_limits()->add_limit();
+   l4->mutable_security()->set_exchange(api::Venue::VENUE_GLBX);
+   l4->mutable_security()->set_security_type(api::SecurityType::SECURITYTYPE_FUTURE);
+   l4->mutable_security()->set_security_symbol("ZC");
+   l4->mutable_security()->set_security_id(10);
+   l4->set_clip_size(103);
+   auto l5 = account->mutable_security_clip_limits()->add_limit();
+   l5->mutable_security()->set_exchange(api::Venue::VENUE_GLBX);
+   l5->mutable_security()->set_security_type(api::SecurityType::SECURITYTYPE_FUTURE);
+   l5->mutable_security()->set_security_symbol("ZN");
+   l5->mutable_security()->set_security_id(5);
+   l5->set_clip_size(104);
+   auto l6 = account->mutable_security_clip_limits()->add_limit();
+   l6->mutable_security()->set_exchange(api::Venue::VENUE_GLBX);
+   l6->mutable_security()->set_security_type(api::SecurityType::SECURITYTYPE_SPREAD);
+   l6->mutable_security()->set_security_symbol("SI");
+   l6->mutable_security()->set_security_id(0);
+   l6->set_clip_size(105);
+   // tests
+   auto sorted_admin_data = AdminServiceUtils::sort_limits(admin_data);
+   const auto& r_account = sorted_admin_data.accounts(0);
+   const auto& r_l1 = r_account.security_clip_limits().limit(0);   
+   REQUIRE(r_l1.security().security_id()==5);
+   REQUIRE(r_l1.clip_size()==104);
+   const auto& r_l2 = r_account.security_clip_limits().limit(1);
+   REQUIRE(r_l2.security().security_id()==10);
+   REQUIRE(r_l2.clip_size()==103);
+   const auto& r_l3 = r_account.security_clip_limits().limit(2);
+   REQUIRE(r_l3.security().security_symbol()=="CL");
+   REQUIRE(r_l3.clip_size()==102);
+   const auto& r_l4 = r_account.security_clip_limits().limit(3);
+   REQUIRE(r_l4.security().security_symbol()=="SI");
+   REQUIRE(r_l4.clip_size()==101);
+   const auto& r_l5 = r_account.security_clip_limits().limit(4);
+   REQUIRE(r_l5.security().security_symbol()=="SI");
+   REQUIRE(r_l5.clip_size()==105);
+   const auto& r_l6 = r_account.security_clip_limits().limit(5);
+   REQUIRE(r_l6.security().security_symbol()=="*");
+   REQUIRE(r_l6.clip_size()==100);
+}
 }}

src/web/ui/src/Accounts/View.fs

@@ -207,7 +207,11 @@ let clipLimitsView model dispatch =
   let addRow _ = Array.append model.scratch.securityClipLimits.limit [| emptyLimit |] |> (ClipLimits >> dispatch)
   div []
     [
-      p [ClassName "is-size-7 has-text-grey-mid"][ str "Clip limits can be set for one security, for all securities in a product (by selecting \"All\" for securities), or for all products within a security type (by selecting \"All\" for both symbol and security)"]
+      div [ ClassName "columns" ][     
+            div [ ClassName "column is-size-7 has-text-grey-mid" ]
+              [str "Clip limits can be set for one security, for all securities in a product (by selecting \"All\" for securities), or for all products within a security type (by selecting \"All\" for both symbol and security)"]
+            div [ ClassName "column" ][]
+      ]
       TableControl.createTableWithStyle (Some headerRow) rws "table-bordered" [Width "100%"] 
       span [ClassName "is-pulled-right"] [ Button.button "Add" "contrast-button" "plus" Button.Enabled addRow ]
       span [ClassName "is-clearfix"][]
@@ -311,7 +315,7 @@ let productLimits model dispatch =
     product = {
                 exchange = bts.api.Venue.GLBX
                 securityType = bts.api.SecurityType.FUTURE
-                securitySymbol = model.AllValueDisplay
+                securitySymbol = model.AllValue
               }
     longPositionLimit = 0u
     shortPositionLimit = 0u

src/web/ui/src/Login/Login.fs

@@ -43,7 +43,10 @@ let update msg model =
         model, Cmd.Empty
     | Submit -> 
         printfn "Login with %s %s" model.user model.password
-        model, loginMsg (model.user.ToLower()) model.password
+        let isAdmin = ((model.user.ToLower().Contains "bts") || (model.user.ToLower().Contains "admin"))
+        match isAdmin with
+        | true -> model, loginMsg (model.user.ToLower()) model.password
+        | false -> { model with error = "Only administrators are allowed to login" }, Cmd.Empty
     | LoginFailure msg ->
         { model with error = msg }, Cmd.Empty
     | LoginSuccess _ -> failwith "Handled externally. Should not get here."

testing/cycle_info.txt

@@ -4,10 +4,10 @@ testers=(
 #'pedro.pinto'
 # 'eric.harding'
  'jake.norbut'
-# 'michael.harding'
+ 'michael.harding'
  'karl.ellefsen'
  'pete.fisher'
-# 'taha.afzal'
+ 'taha.afzal'
  'john.colvin'
  'kevin.darby'
  'brian.muhr'

testing/docs/spreader_common_setup_1.txt

@@ -2,7 +2,7 @@ This is not a test, but common setup instructions that will be used for a few te
 
 0. This needs sim market data **not** to be merged with replay (or cert), so we can create our own markets.
 
-Add or update this line in your spark.conf (probably in ~/www/config)
+Add or update this line in your spark.conf (probably in ~/sparkdata/config)
 ```
 ExchangeSim {value {merge_with_md:false}}
 ```

testing/manual/admin/clip_limits.txt

@@ -1,24 +1,24 @@
 This test verifies that clip limits are followed, and rule precedence works correctly
 
-1. Pick a future, lets say ESU8.
+1. Pick a future, lets say SIZ1.
 
 2. In the Spark Admin, go to accounts and select the default account for your user (let say ACC A1).
-Go to Position Limits and enter large values (for both security and product level), so that
-position limits are bypassed.
+Go to Position Limits and enter large values for the Product Limits and remove all rows for the Security Limits
 
 4. Now go to Clip Limits and delete all entries.
 
-5. Enter an order and it should fail with a message that should tell exactly why.
+5. Enter an order and it should fail with a message that should tell exactly why. Make sure the orderbook and the
+ladder show the message.
 
-6. Now enter a rule for ESU8 (in the first version we can not specify securities, so go as far as the symbol (ES)).
+6. Now enter a Clip Limit for SIZ1.
 Enter a Clip Size of 10. Make sure orders > 10 are rejected and < 10 are allowed.
 
-7. Now enter a non ES future order, lets say SI. Enter a 1 lot and it should be rejected.
+7. Now enter an order for another future, lets say SIN1. Enter a 1 lot and it should be rejected.
 
-8. Enter another rule, GLBX, FUTURES, * and enter a clip limit of 100.
+8. Enter another rule, GLBX, FUTURES, All, All and enter a clip limit of 100.
 
-9. Now try the SI order again. Verify it does the right thing for the clip limit you entered in 8.
+9. Now try the SIN1 order again. Verify it does the right thing for the clip limit you entered in 8.
 
-10. Verify that ESU8 is still controlled by a clip size of 10.
+10. Verify that SIZ1 is still controlled by a clip size of 10.
 
 11. Enter a future spread order. It should be rejected since we don't have any clip limits for futures.

testing/manual/admin/security_position_limits.txt

@@ -1,5 +1,5 @@
 This test verifies that security position limits are enforced correctly. (Position limits are
-enforced at both the security and account level)
+enforced (per account) at the product level and optionally at the security level)
 
 0. Clear up your orders and fills so we don't have any lingering positions
 (run rm -r orders and rm -r fills from /sparkdata/data/databases). Restart your sparkd
@@ -14,7 +14,7 @@ clip limits don't interfere with this test.
 4. Enter an ESU8 order. It should be rejected. Make sure the rejection message makes sense
 
 5. Now enter a product level rule for GLBX, Future, ES, 1,000,000, 1,000,000 (to bypass it),
-and a security level rule GLBX, Future, ES, 50, 100
+and a security level rule GLBX, Future, ES, ESU8, 50, 100.
 
 6. Enter a 10 lot. This brings our 'position' to 10 (where 'positions' = fills + working orders).
 Enter more orders to get it to 50. Now try to get past 50 and you should be stopped with a clear message.
@@ -24,12 +24,13 @@ don't cancel these yet.
 the way to 150 (net 100 short), but not beyond that. Verify that.
 
 8. Cancel all your buy orders. Cancel some of your sell orders to get the net short position to 50.
-Start selling again, and make sure you are stopped at 100.
+Start selling again, and make sure you are stopped at 100. Cancel all ESU8 orders.
 
 9. Try another future, lets say SIU8. Since we don't have a position rule for it, it should fail.
 
 10. Enter a new rule for the product (and use All for the symbol) and enter 10 for the long and short limits.
 
-11. Now enter a 1 lot SIU8 again. Keep buying more and make sure you are stopped at the long limit of 10.
+11. Now enter a 1 lot SIU8 again. Keep buying more and make sure you are stopped at the long limit of 10. Keep
+these working
 
-12. Start selling and make sure you are stopped at the short limit.
+12. Start selling and make sure you are stopped at the short limit (remember to account the buys from 11)

testing/manual/admin/spread_position_limits.txt

@@ -1,29 +1,36 @@
 This test verifies the following 2 things
-i. spread legs fills (and working orders) are applied toward enforcing
+i. future and spread product limits can co-exist and work correctly
+ii. spread legs fills (and working orders) are applied toward enforcing
 the position limits for the legs
-ii. leg limits are checked when entering spread orders
+iii. leg limits are checked when entering spread orders
 
 i:
 
-1. Pick a 2 legged spread (lets say SIU8-SIZ8). Set the clip limits high so that they don't interfere.
-Set the product position limits high as well so they don't interfere either.
+1. Pick a 2 legged spread (lets say SIU8-SIZ8). Set the clip limits high for both spreads and futures
+so that they don't interfere. Remove all Security Position limits.
 
-2. For the security limits (which is what we will be testing), we want Max Long and Max Short of 20 and 30
-for the SI futures and 100 and 110 for the SI spreads respectively.
+2. For the Product Limits, set Max Long and Max Short of 100 and 110
+for the SI futures and 20 and 30 for the SI spreads respectively.
 
+3. Verify that the limits set in 2 work for futures(remember working orders count toward limit enforcement so you dont need
+fills). Cancel all the orders.
 
-3. Enter a 15 lot buy order for the spread. This 15 should count toward the leg limits. The leg long limit
-is 20. So enter a 5 lot buy for SIU8 which should be allowed, but a subsequent 1 lot will violate the 20 long
-and should be rejected.
+4. Verify that the limits set in 2. work for spreads. Cancel all the orders.
 
-4. Now enter a 15 lot sell for SIZ8 which should be allowed but a subsequent one lot should not be allowed, as it
-will violate the short limit of 30.
+5. Make these product limits 1,000,000 to make them irrelevant
 
 ii:
 
-5. Cancel all orders
+6. For security limits enter long 10 and short 20 for SIU8 and SIZ8 futures respectively and 50 and 60 for
+the SIU8-SIZ8 spread.
 
-6. Enter a 15 lot buy for SIU8. Any spread buy order of > 5 should be rejected, since this will violate the 20 long limit.
+7. Enter a 5 lot buy for SIU8-SIZ8. This 5 should count toward SIU8 long limit (which is 10). Enter
+a 5 lot SIU8 buy, which should be allowed but a subsequent 1 lot buy should be rejected. Similiary, the
+spreader order should count toward the SIZ8 short limit of 20. Enter a 15 lot sell on SIZ8, and a subsequent
+1 lot should be rejected.
 
-8. Enter a 20 sell for SIZ8. Any spread buy order of > 10 should be rejected since it will
-violate the 30 short limit.
+8. Cancel all orders
+
+iii:
+
+Enter a 11 lot spread order. It should be rejected because of the 10 lot limit on SIU8.