SPARK-1409 - slow picker fixed?

src/client/spark/Modules/Trading/Fills/FillsVM.cs

@@ -130,7 +130,7 @@ namespace mbts.spark {
       protected override IEnumerable<SortingCriteria> get_default_sorting() { yield return new SortingCriteria(nameof(Row.fill_time), false); }
 
       static IObservable<IIncrementalUpdate<FillTuple, FillTupleKey>> _get_observable(IBus bus, OrdersModel model, StartupData state) {
-         if (state.orders != null && state.orders.Length == 0) {
+         if (!state.orders.is_null_or_empty()) {
             var orders_hash = new HashSet<uint>(state.orders);
             return model.on_fill_tuples.filter(f => orders_hash.Contains(f.fill.OrderKey));
          } else return model.on_fill_tuples;

src/client/spark/Modules/Trading/LadderWindowFactory.cs

@@ -30,8 +30,7 @@ namespace bts.shell {
          _wm = wm;
          _settings_hub = settings_hub;
          _orders_model = orders_model;
-         var window_props = Configuration.get<bts.config.IShellWindowProps>();
-         _max_ladders_open = (int)(window_props?.ladder?.max_total_open ?? 75);
+         _max_ladders_open = (int)(Configuration.get<bts.config.LadderKnobs>().max_open_ladders);
          _dispose = Disposable.Create(
             bus.subscribe(SparkTags.SecuritiesTag).Subscribe(s => _securities = s),
             bus.subscribe(SparkTags.ShowLadderTag).OnDispatcher().Subscribe(s => show_windows(new[] { s })),

src/client/spark/Orders/OrderActivitySummarizer.cs

@@ -114,7 +114,7 @@ namespace bts.spark {
             (var security, var spec) = get_security(neworder.Def.SecurityKey);
             //
             // description
-            var mod_desc = modified_description(oldorder, neworder, spec.Value);
+            var mod_desc = modified_description(oldorder, neworder, spec);
             if (!string.IsNullOrEmpty(mod_desc)) {
                log_status(ActivityLogCategory.modify, security, mod_desc);
             }

src/client/spark/core/Utility/AnonymousEqualityComparer.cs

@@ -3,6 +3,8 @@ using System.Collections.Generic;
 
 namespace mbts.utils {
    public static class EqualityComparer {
+      const int _hash_seed = 63607, _hash_multiplier = 6451;
+
       public static IEqualityComparer<T> Create<T>(Func<T, T, bool> equals) =>
          new AnonymousEqualityComparer<T>(equals);
 
@@ -29,7 +31,7 @@ namespace mbts.utils {
          non_null_equals(x, y);
 
       public static int aggregate_hash(int item1_hash, int item2_hash, int item3_hash) {
-         int rv = 63607;
+         int rv = _hash_seed;
          unchecked {
             _accumulate_hash(ref rv, ref item1_hash);
             _accumulate_hash(ref rv, ref item2_hash);
@@ -38,9 +40,20 @@ namespace mbts.utils {
          return rv;
       }
 
+      // Note: be wary of the performance cost this method may have, as frequently creating large arrays can be expensive.
+      // If you wish to avoid this cost, consider making an overload like the method above, with the appropriate number of item-hash arguments
+      public static int aggregate_hash(params int[] item_hashes) {
+         int hash = _hash_seed;
+         unchecked {
+            foreach (int item_hash in item_hashes)
+               hash = hash * _hash_multiplier + item_hash;
+         }
+         return hash;
+      }
+
       // WARNING: be sure to call this within in an unchecked scope
       static void _accumulate_hash(ref int aggregate_hash, ref int item_hash) =>
-         aggregate_hash = aggregate_hash * 6451 + item_hash;
+         aggregate_hash = aggregate_hash * _hash_multiplier + item_hash;
 
       private class AnonymousEqualityComparer<T> : IEqualityComparer<T> {
          readonly Func<T, T, bool> _equals;

src/client/spark/view/Controls/DataGridEx/Filters/Views/TimeFilter.cs

@@ -54,8 +54,9 @@ namespace mbts.ui {
       public static TimeTextTuple FromTicks(long ticks) =>
          FromDateTime(new DateTime(ticks));
 
-      public static TimeTextTuple FromDateTime(DateTime date_time) =>
-         new TimeTextTuple(DateTimeFormatInfo.CurrentInfo.AbbreviatedMonthNames[date_time.Month - 1], date_time.Day.ToString(), date_time.Year.ToString(), date_time.ToString("t"));
+      public static TimeTextTuple FromDateTime(DateTime date_time) => FromDateTime(date_time, "t");
+      public static TimeTextTuple FromDateTime(DateTime date_time, string time_format) =>
+         new TimeTextTuple(DateTimeFormatInfo.CurrentInfo.AbbreviatedMonthNames[date_time.Month - 1], date_time.Day.ToString(), date_time.Year.ToString(), date_time.ToString(time_format));
 
       public TimeTextTuple(string month_text, string day_text, string year_text, string time_text) {
          _month_text = month_text;
@@ -107,14 +108,23 @@ namespace mbts.ui {
 
 
       static DateTime? _parse(string month_text, string day_text, string year_text, string time_text) {
-         DateTime? t = _parse(time_text);
+         TimeSpan? t = _parse_time(time_text);
          if (t.HasValue) {
             DateTime? d = _parse($"{month_text}/{day_text}/{year_text}");
-            if (d.HasValue) return d.Value.Date.AddDays(t.Value.TimeOfDay.TotalDays);
+            if (d.HasValue) return d.Value.Date.Add(t.Value);
          }
          return null;
       }
 
+      static TimeSpan? _parse_time(string text) {
+         DateTime? dt = _parse(text);
+         if (dt.HasValue) {
+            return dt.Value.TimeOfDay;
+         } else if (TimeSpan.TryParse(text, out TimeSpan result) && result < TimeSpan.FromDays(1)) {
+            return result;
+         } else return null;
+      }
+
       static DateTime? _parse(string text) {
          if (!string.IsNullOrEmpty(text) && DateTime.TryParse(text, out DateTime result))
             return result;

src/client/spark/view/Controls/PriceEntryControl.xaml

@@ -28,7 +28,14 @@
         BorderBrush="{TemplateBinding BorderBrush}"
         BorderThickness="{TemplateBinding BorderThickness}"
         Text="{Binding Value, RelativeSource={RelativeSource TemplatedParent}, Converter={StaticResource TickDisplayTwoWayConverter}, Mode=TwoWay}"
-      />
+      >
+        <TextBox.CommandBindings>
+          <CommandBinding Command="{x:Static u:TextBoxBehavior.SelectAllCommand}" Executed="{x:Static u:TextBoxBehavior.SelectAllCommandExecutedHandler}"/>
+        </TextBox.CommandBindings>
+        <TextBox.InputBindings>
+          <MouseBinding MouseAction="LeftDoubleClick" Command="{x:Static u:TextBoxBehavior.SelectAllCommand}"/>
+        </TextBox.InputBindings>
+      </TextBox>
     </c:TickUpDown>
   </ControlTemplate>
   <Style TargetType="usp:PriceEntryControl">

src/client/spark/view/Controls/PriceEntryControl.xaml.cs

@@ -47,24 +47,30 @@ namespace mbts.ui.spark {
       #endregion
 
       TextBox _text;
-      IDisposable _text_input_hook;
 
       public PriceEntryControl() {
          TickUpCommand = new DelegateCommand(_tick_up);
          TickDownCommand = new DelegateCommand(_tick_down);
       }
 
+      IDisposable _keyboard_focus_sub;
+      bool _has_keyboard_focus;
+
       public override void OnApplyTemplate() {
+         Disposable.dispose(ref _keyboard_focus_sub);
          _text = this.assert_template_child<TextBox>("_text");
-         Disposable.dispose(ref _text_input_hook);
-         _text_input_hook = _text.SubscribePreviewTextInput((s, e) => e.Handled = !RegexFilters.PriceInput.IsMatch(_text.PeekText(e)));
+         _keyboard_focus_sub = _text.SubscribeKeyboardFocus(val => _has_keyboard_focus = val);
          _update_text_binding();
       }
 
       void _tick_up() {
+         if (_has_keyboard_focus) {
+            _text.GetBindingExpression(TextBox.TextProperty)?.UpdateSource();
+         }
          double v = Value;
-         if (double.IsNaN(v)) SetCurrentValue(ValueProperty, TickDefaultValue);
-         else {
+         if (double.IsNaN(v)) {
+            SetCurrentValue(ValueProperty, TickDefaultValue);
+         } else {
             var spec = ContractSpec;
             if (spec.HasValue) SetCurrentValue(ValueProperty, spec.Value.tickup(v));
             else SetCurrentValue(ValueProperty, TickDefaultValue);
@@ -72,9 +78,13 @@ namespace mbts.ui.spark {
       }
 
       void _tick_down() {
+         if (_has_keyboard_focus) {
+            _text.GetBindingExpression(TextBox.TextProperty)?.UpdateSource();
+         }
          double v = Value;
-         if (double.IsNaN(v)) SetCurrentValue(ValueProperty, TickDefaultValue);
-         else {
+         if (double.IsNaN(v)) {
+            SetCurrentValue(ValueProperty, TickDefaultValue);
+         } else {
             var spec = ContractSpec;
             if (spec.HasValue) SetCurrentValue(ValueProperty, spec.Value.tickdown(v));
             else SetCurrentValue(ValueProperty, TickDefaultValue);

src/client/spark/view/Controls/TickUpDown/TickUpDown.xaml.cs

@@ -23,38 +23,42 @@ namespace mbts.ui.controls {
       }
 
       protected override void OnPreviewKeyDown(KeyEventArgs e) {
-         switch (Keyboard.Modifiers) {
-            case ModifierKeys.None: {
-                  switch (e.Key) {
-                     case Key.Up:
-                        TickUpCommand.TryExecuteIfCan(TickUpParameter);
-                        e.Handled = true;
-                        break;
-                     case Key.Down:
-                        TickDownCommand.TryExecuteIfCan(TickDownParameter);
-                        e.Handled = true;
-                        break;
+         if (EnableArrowKeyTick) {
+            switch (Keyboard.Modifiers) {
+               case ModifierKeys.None: {
+                     switch (e.Key) {
+                        case Key.Up:
+                           TickUpCommand.TryExecuteIfCan(TickUpParameter);
+                           e.Handled = true;
+                           break;
+                        case Key.Down:
+                           TickDownCommand.TryExecuteIfCan(TickDownParameter);
+                           e.Handled = true;
+                           break;
+                     }
                   }
-               }
-               break;
-            case ModifierKeys.Alt: {
-                  // see: https://stackoverflow.com/questions/3099472/previewkeydown-is-not-seeing-alt-modifiers
-                  switch (e.SystemKey) {
-                     case Key.Up:
-                        TickUpCommand.TryExecuteIfCan(TickUpAltParameter);
-                        e.Handled = true;
-                        break;
-                     case Key.Down:
-                        TickDownCommand.TryExecuteIfCan(TickDownAltParameter);
-                        e.Handled = true;
-                        break;
+                  break;
+               case ModifierKeys.Alt: {
+                     // see: https://stackoverflow.com/questions/3099472/previewkeydown-is-not-seeing-alt-modifiers
+                     switch (e.SystemKey) {
+                        case Key.Up:
+                           TickUpCommand.TryExecuteIfCan(TickUpAltParameter);
+                           e.Handled = true;
+                           break;
+                        case Key.Down:
+                           TickDownCommand.TryExecuteIfCan(TickDownAltParameter);
+                           e.Handled = true;
+                           break;
+                     }
                   }
-               }
-               break;
+                  break;
+            }
          }
          base.OnPreviewKeyDown(e);
       }
 
+      public bool EnableArrowKeyTick { get; set; } = true;
+
       #region TickCommand
       public ICommand TickCommand {
          get { return (ICommand)GetValue(TickCommandProperty); }

src/client/spark/view/Controls/Time/DateTimePicker.xaml

@@ -31,8 +31,14 @@
               <ColumnDefinition Width="Auto"/>
               <ColumnDefinition/>
             </Grid.ColumnDefinitions>
-            <c:TickUpDown x:Name="PART_time_tick" Grid.Column="0" MinWidth="85" HorizontalAlignment="Stretch">
-              <TextBox x:Name="PART_time_text" Height="20" u:TextBoxBehavior.EnableArrowKeyTraverse="True" u:Behavior.DisablePaste="True"/>
+            <c:TickUpDown x:Name="PART_time_tick" Grid.Column="0" MinWidth="85" HorizontalAlignment="Stretch" EnableArrowKeyTick="False">
+              <TextBox x:Name="PART_time_text" Height="20" u:TextBoxBehavior.EnableArrowKeyTraverse="True" u:Behavior.DisablePaste="True">
+                <TextBox.InputBindings>
+                  <KeyBinding Key="Up" Command="{x:Static c:DateTimePicker.TickTimeComponentCommand}" CommandParameter="{u:Int 1}"/>
+                  <KeyBinding Key="Down" Command="{x:Static c:DateTimePicker.TickTimeComponentCommand}" CommandParameter="{u:Int -1}"/>
+                  <u:CustomInputBinding Gesture="{x:Static u:InputGestures.LeftTripleClick}" Command="{x:Static u:TextBoxBehavior.SelectAllCommand}"/>
+                </TextBox.InputBindings>
+              </TextBox>
             </c:TickUpDown>
             <c:TickUpDown x:Name="PART_month_tick" Grid.Column="2" Width="50">
               <TextBox x:Name="PART_month_text" Height="20" u:TextBoxBehavior.EnableArrowKeyTraverse="True" u:Behavior.DisablePaste="True"/>

src/client/spark/view/Controls/Time/DateTimePicker.xaml.cs

@@ -3,6 +3,7 @@ using mbts.ui.shell;
 using System;
 using System.Collections.Generic;
 using System.ComponentModel;
+using System.Linq;
 using System.Windows;
 using System.Windows.Controls;
 using System.Windows.Data;
@@ -14,7 +15,9 @@ namespace mbts.ui.controls {
    /// Interaction logic for DateTimePicker.xaml
    /// </summary>
    public class DateTimePicker : Control {
-      TimeTupleVM _vm;
+      public static readonly RoutedCommand TickTimeComponentCommand = new RoutedCommand();
+
+      DTPController _controller;
       bool _suspend_notify_vm;
 
       static DateTimePicker() =>
@@ -24,8 +27,8 @@ namespace mbts.ui.controls {
          new FrameworkPropertyMetadata(default(TimeTextTuple), FrameworkPropertyMetadataOptions.BindsTwoWayByDefault, OnTimeTupleChanged));
       static void OnTimeTupleChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
          var dtp = (DateTimePicker)d;
-         if (!dtp._suspend_notify_vm && dtp._vm != null)
-            dtp._vm.result = (TimeTextTuple)e.NewValue;
+         if (!dtp._suspend_notify_vm && dtp._controller != null)
+            dtp._controller.result = (TimeTextTuple)e.NewValue;
       }
       public TimeTextTuple TimeTuple { get => (TimeTextTuple)GetValue(TimeTupleProperty); set => SetValue(TimeTupleProperty, value); }
 
@@ -41,59 +44,74 @@ namespace mbts.ui.controls {
          var time = this.assert_template_child<TickUpDown>("PART_time_tick");
          var time_text = this.assert_template_child<TextBox>("PART_time_text");
 
-         if (_vm != null) _vm.PropertyChanged -= _vm_PropertyChanged;
-         _vm = new TimeTupleVM();
-         _vm.result = TimeTuple;
-         _vm.PropertyChanged += _vm_PropertyChanged;
+         if (_controller != null) _controller.PropertyChanged -= _vm_PropertyChanged;
+         _controller = new DTPController(time_text);
+         _controller.result = TimeTuple;
+         _controller.PropertyChanged += _vm_PropertyChanged;
+         time.CommandBindings.Add(new CommandBinding(TickTimeComponentCommand, (s, e) => {
+            _controller.tick_time_component((int)e.Parameter);
+         }));
+         time.CommandBindings.Add(new CommandBinding(TextBoxBehavior.SelectAllCommand, TextBoxBehavior.SelectAllCommandExecutedHandler));
 
          const int delay = 500;
-         time.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_time") { Source = _vm });
+         time.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_time") { Source = _controller });
          time.TickUpParameter = 1;
          time.TickDownParameter = -1;
          time.TickUpAltParameter = 60;
          time.TickDownAltParameter = -60;
-         time_text.SetBinding(TextBox.TextProperty, new Binding("time_text") { Source = _vm, Delay = delay, ValidatesOnExceptions = true });
-         _hooks.Add(time_text.SubscribePreviewTextInput((s, e) =>
-            e.Handled = !MatchAny(time_text.PeekText(e), RegexFilters.Time.StandardTime, RegexFilters.Time.MilitaryTime)
-         ));
+         time_text.SetBinding(TextBox.TextProperty, new Binding("time_text") { Source = _controller, Delay = delay, ValidatesOnExceptions = true });
+         _hooks.Add(time_text.SubscribePreviewTextInput((s, e) => {
+            string preview = time_text.PeekText(e);
+            if (RegexFilters.Time.StandardTimeOptionalSeconds.IsMatch(preview)) {
+               _controller.use_military_time = false;
+            } else if (RegexFilters.Time.MilitaryTimeOptionalSeconds.IsMatch(preview)) {
+               _controller.use_military_time = true;
+            } else {
+               e.Handled = true;
+            }
+         }));
 
-         month.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_month") { Source = _vm });
+         month.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_month") { Source = _controller });
          month.TickUpParameter = 1;
          month.TickDownParameter = -1;
-         month_text.SetBinding(TextBox.TextProperty, new Binding("month_text") { Source = _vm, Delay = delay });
+         month_text.SetBinding(TextBox.TextProperty, new Binding("month_text") { Source = _controller, Delay = delay });
          _hooks.Add(month_text.SubscribePreviewTextInput((s, e) =>
             e.Handled = !RegexFilters.Time.MonthAbbrev.IsMatch(month_text.PeekText(e))
          ));
 
-         day.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_day") { Source = _vm });
+         day.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_day") { Source = _controller });
          day.TickUpParameter = 1;
          day.TickDownParameter = -1;
-         day_text.SetBinding(TextBox.TextProperty, new Binding("day_text") { Source = _vm, Delay = delay });
+         day_text.SetBinding(TextBox.TextProperty, new Binding("day_text") { Source = _controller, Delay = delay });
          _hooks.Add(day_text.SubscribePreviewTextInput((s, e) =>
             e.Handled = !RegexFilters.Time.DaysInMonth.IsMatch(day_text.PeekText(e))
          ));
 
-         year.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_year") { Source = _vm });
+         year.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_year") { Source = _controller });
          year.TickUpParameter = 1;
          year.TickDownParameter = -1;
-         year_text.SetBinding(TextBox.TextProperty, new Binding("year_text") { Source = _vm, Delay = delay });
+         year_text.SetBinding(TextBox.TextProperty, new Binding("year_text") { Source = _controller, Delay = delay });
          _hooks.Add(year_text.SubscribePreviewTextInput((s, e) =>
             e.Handled = !RegexFilters.Time.Year.IsMatch(year_text.PeekText(e))
          ));
       }
 
       void _vm_PropertyChanged(object sender, PropertyChangedEventArgs e) {
-         if (e.PropertyName == nameof(TimeTupleVM.result)) {
+         if (e.PropertyName == nameof(DTPController.result)) {
             _suspend_notify_vm = true;
-            TimeTuple = _vm.result;
+            TimeTuple = _controller.result;
             _suspend_notify_vm = false;
          }
       }
 
-      private class TimeTupleVM : ViewModelBase {
+      // This is not exactly a view-model, bc it accesses the time TextBox directly so it can get/set the CaretIndex,
+      // which is used to determine which time-componenet - hh:mm:ss AM/PM - should be incremented/decremented via the up/down arrow-keys.
+      class DTPController : ViewModelBase {
+         readonly TextBox _time_text_box;
          TimeTextTuple _result;
 
-         public TimeTupleVM() {
+         public DTPController(TextBox time_text_box) {
+            _time_text_box = time_text_box;
             tick_time = new DelegateCommand<int>(_tick_time);
             tick_month = new DelegateCommand<int>(_tick_month);
             tick_day = new DelegateCommand<int>(_tick_day);
@@ -135,12 +153,52 @@ namespace mbts.ui.controls {
                RaisePropertyChanged(nameof(month_text), nameof(day_text), nameof(year_text), nameof(time_text));
             }
          }
+
+         public bool use_military_time { get; set; }
          public ICommand tick_time { get; }
          public ICommand tick_month { get; }
          public ICommand tick_day { get; }
          public ICommand tick_year { get; }
+         string _time_format => use_military_time ? StringFormats.Time.MilitaryWithSeconds : StringFormats.Time.StandardWithSeconds;
 
+         public void tick_time_component(int increment) {
+            _ensure_time_text_up_to_date();
+            var r = _result.result;
+            if (r.HasValue) {
+               int index = _time_text_box.CaretIndex;
+               using (detect_property_changes()) {
+                  DateTime dt;
+                  if (index <= 0) {
+                     dt = r.Value.AddHours(increment);
+                  } else {
+                     int segment = _time_text_box.Text.Take(index).Count(c => c == ':');
+                     switch (segment) {
+                        case 0: dt = r.Value.AddHours(increment); break;
+                        case 1: dt = r.Value.AddMinutes(increment); break;
+                        case 2:
+                           if (_time_text_box.Text.Take(index).Contains(' ')) {
+                              // toggle AM/PM
+                              dt = r.Value.AddHours(r.Value.Hour < 12 ? 12 : -12);
+                           } else {
+                              dt = r.Value.AddSeconds(increment);
+                           }
+                           break;
+                        default:
+                           dt = r.Value.AddSeconds(increment);
+                           break;
+                     }
+                  }
+                  _result = TimeTextTuple.FromDateTime(dt, _time_format);
+               }
+               _time_text_box.CaretIndex = index;
+            } else {
+               using (detect_property_changes())
+                  _force_valid_inputs();
+            }
+         }
+ 
          void _tick_time(int increment) {
+            _ensure_time_text_up_to_date();
             using (detect_property_changes()) {
                var r = _result.result;
                if (r.HasValue) {
@@ -187,6 +245,11 @@ namespace mbts.ui.controls {
                }
             }
          }
+ 
+         void _ensure_time_text_up_to_date() {
+            if (_time_text_box.IsKeyboardFocused)
+               _time_text_box.GetBindingExpression(TextBox.TextProperty)?.UpdateSource();
+         }
 
          void _force_valid_inputs() {
             var dt = DateTime.Today.Date;

src/client/spark/view/Extensions/InputGestures.cs

@@ -0,0 +1,16 @@
+using System.Windows.Input;
+
+namespace mbts.ui {
+   class CustomInputBinding : InputBinding { }
+
+   static class InputGestures {
+      public static readonly InputGesture LeftTripleClick = new LeftTripleClickGesture();
+
+      class LeftTripleClickGesture : InputGesture {
+         public override bool Matches(object targetElement, InputEventArgs inputEventArgs) =>
+            inputEventArgs is MouseButtonEventArgs mouse_args &&
+            mouse_args.ChangedButton == MouseButton.Left &&
+            mouse_args.ClickCount == 3;
+      }
+   }
+}

src/client/spark/view/Extensions/TextElement/TextBoxBehavior.cs

@@ -7,7 +7,14 @@ using System.Windows.Input;
 
 namespace mbts.ui {
    static class TextBoxBehavior {
-      public static readonly RoutedCommand ReleaseCaptureCommand = new RoutedCommand();
+      #region RoutedCommands/Handlers
+      public static readonly RoutedCommand
+         ReleaseCaptureCommand = new RoutedCommand(),
+         SelectAllCommand = new RoutedCommand();
+
+      public static readonly ExecutedRoutedEventHandler SelectAllCommandExecutedHandler =
+         new ExecutedRoutedEventHandler((s, e) => (e.OriginalSource as TextBox ?? e.Source as TextBox ?? s as TextBox)?.SelectAll());
+      #endregion
 
       #region SelectAllOnFocus
 
@@ -168,7 +175,7 @@ namespace mbts.ui {
       public static void SetInputFilter(DependencyObject d, TextBoxInputFilter value) => d.SetValue(InputFilterProperty, value);
       #endregion
 
-      #region
+      #region IncrementDecrementOnMouseClick
       public static readonly DependencyProperty IncrementDecrementOnMouseClickProperty = DependencyProperty.RegisterAttached(
          "IncrementDecrementOnMouseClick", typeof(bool), typeof(TextBoxBehavior), new PropertyMetadata(false, IncrementDecrementOnMouseClickPropertyChanged)
          );

src/client/spark/view/Extensions/TextElement/TextElementHelper.cs

@@ -2,11 +2,28 @@
 using System.Windows;
 using System.Windows.Controls;
 using System.Windows.Data;
+using System.Windows.Input;
+
+namespace mbts.ui {
+   static class TextElementHelper {
+      public static IDisposable SubscribeKeyboardFocus(this TextBox text_box, Action<bool> callback) {
+         text_box.GotKeyboardFocus += on_got_keyboard_focus;
+         text_box.LostKeyboardFocus += on_lost_keyboard_focus;
+         return Disposable.Create(() => {
+            text_box.GotKeyboardFocus -= on_got_keyboard_focus;
+            text_box.LostKeyboardFocus -= on_lost_keyboard_focus;
+         });
+         void on_got_keyboard_focus(object sender, KeyboardFocusChangedEventArgs e) {
+            if (e.NewFocus == text_box)
+               callback(true);
+
+         }
+         void on_lost_keyboard_focus(object sender, KeyboardFocusChangedEventArgs e) {
+            if (e.OldFocus == text_box)
+               callback(false);
+         }
+      }
 
-namespace mbts.ui
-{
-   public class TextElementHelper
-   {
       public static void GetTargetProperty(DependencyObject d, out FrameworkElement text_element, out DependencyProperty text_property) {
          text_element = (FrameworkElement)d;
          if (text_element is TextBox) text_property = TextBox.TextProperty;

src/client/spark/view/Spark/ContractPicker/ContractPickerModel.cs

@@ -197,10 +197,6 @@ namespace mbts.spark
          selected_securities = secs.Select(s => s.Id).ToList();
          ok_button_command.TryRaiseCanExecuteChanged();
          _compose_criteria_clause_name();
-
-         // _security_in_criteria is our fun to be returned to our callers
-         // test on click -- verify our func works
-         //var results = all_securities.Select(all => all.Value.security).Where(x => _security_in_criteria(x)).ToList();
       }
 
       private void on_securities_changed(IReadOnlyDictionary<uint, SecurityMD> changed_securities) {
@@ -212,10 +208,6 @@ namespace mbts.spark
             all_securities = changed_securities;
          }
 
-         securities.sync_with(all_securities
-            .Select(entry => entry.Value.security)
-         );
-
          _update_exchanges();
          _update_security_types();
          _update_symbols();
@@ -327,7 +319,6 @@ namespace mbts.spark
             .Select(s => s.Value.security.Symbol)
             .Distinct()
          );
-            
       }
 
       private void _update_securities() {

src/client/spark/view/Spark/Ladder/LadderGridHost.cs

@@ -533,6 +533,7 @@ namespace mbts.spark {
          _visualHost.AddVisualChild(_mid_mkt_break_vis);
          _visualHost.AddVisualChild(_ask_mkt_break_vis);
          _visualHost.AddVisualChild(_bid_mkt_break_vis);
+         _render_grid_background();
          _init_tactile_adorner();
          // this effectively sets the background of the "Last" column
          if (_scrollViewer != null) {

src/client/spark/view/Spark/Ladder/LadderSettings.cs

@@ -1,5 +1,6 @@
 using bts.api.orderman;
 using mbts.ui;
+using mbts.utils;
 using ProtoBuf;
 using System;
 using System.Linq;
@@ -124,8 +125,6 @@ namespace mbts.spark {
                last_column = last_column,
                calculator_buttons = new[] { 1, 5, 10, 25, 50, 100 },
                scroll_bar_visible = false,
-               ticks_to_recenter = 10,
-               seconds_to_recenter = 300,
                default_order_type = CustomOrderDefinition.CustomOneofCase.Limit,
                default_time_in_force = TimeInForce.Day,
                animate_recenter = true,
@@ -163,10 +162,8 @@ namespace mbts.spark {
       public int[] calculator_buttons { get; set; }
       [ProtoMember(10)]
       public bool scroll_bar_visible { get; set; }
-      [ProtoMember(11)]
-      public int ticks_to_recenter { get; set; }
-      [ProtoMember(12)]
-      public int seconds_to_recenter { get; set; }
+      // USED [ProtoMember(11)]
+      // USED [ProtoMember(12)]
       [ProtoMember(13)]
       public CustomOrderDefinition.CustomOneofCase default_order_type { get; set; }
       [ProtoMember(14)]
@@ -197,11 +194,8 @@ namespace mbts.spark {
          other.price_column.safe_equals(price_column) &&
          other.calculator_buttons.safe_equals(calculator_buttons, (a, b) => a.SequenceEqual(b)) &&
          other.scroll_bar_visible == scroll_bar_visible &&
-         other.ticks_to_recenter == ticks_to_recenter &&
-         other.seconds_to_recenter == seconds_to_recenter &&
          other.default_order_type == default_order_type &&
          other.default_time_in_force == default_time_in_force &&
-         other.seconds_to_recenter == seconds_to_recenter &&
          other.size == size &&
          other.animate_recenter == animate_recenter &&
          other.badge_style == badge_style &&
@@ -213,26 +207,24 @@ namespace mbts.spark {
       public override bool Equals(object obj) => Equals(obj as LadderSettings);
 
       public override int GetHashCode() =>
-         row_height.GetHashCode() +
-         grid_line_color.GetHashCode() +
-         orders_column.GetHashCode() +
-         bid_column.GetHashCode() +
-         ask_column.GetHashCode() +
-         price_column.GetHashCode() +
-         last_column.GetHashCode() +
-         calculator_buttons.GetHashCode() +
-         scroll_bar_visible.GetHashCode() +
-         ticks_to_recenter.GetHashCode() +
-         seconds_to_recenter.GetHashCode() +
-         default_order_type.GetHashCode() +
-         default_time_in_force.GetHashCode() +
-         seconds_to_recenter.GetHashCode() +
-         size.GetHashCode() +
-         animate_recenter.GetHashCode() +
-         badge_style.GetHashCode() +
-         bid_break_line.GetHashCode() +
-         ask_break_line.GetHashCode() +
-         mid_break_line.GetHashCode() +
-         alternate_order_size.GetHashCode();
+         EqualityComparer.aggregate_hash(
+            row_height.GetHashCode(),
+            grid_line_color.GetHashCode(),
+            orders_column.GetHashCode(),
+            bid_column.GetHashCode(),
+            ask_column.GetHashCode(),
+            price_column.GetHashCode(),
+            last_column.GetHashCode(),
+            calculator_buttons.GetHashCode(),
+            scroll_bar_visible.GetHashCode(),
+            default_order_type.GetHashCode(),
+            default_time_in_force.GetHashCode(),
+            size.GetHashCode(),
+            animate_recenter.GetHashCode(),
+            badge_style.GetHashCode(),
+            bid_break_line.GetHashCode(),
+            ask_break_line.GetHashCode(),
+            mid_break_line.GetHashCode(),
+            alternate_order_size.GetHashCode());
    }
 }

src/client/spark/view/Spark/Ladder/LadderVM.cs

@@ -16,8 +16,6 @@ using System.Reactive.Linq;
 using System.Threading.Tasks;
 using System.Windows.Input;
 using System.Windows.Media;
-using mbts.utils;
-using mbts.spark.utilities;
 
 namespace mbts.spark {
    class LadderVM : DisposableViewModelBase, IKeyListener {
@@ -25,7 +23,7 @@ namespace mbts.spark {
       readonly LadderModel _model;
       readonly Property<double> _high, _low, _change, _open, _settlement;
       readonly Property<ulong> _volume;
-      readonly Property<bool> _revert;
+      readonly Property<bool> _auto_clear;
       readonly Property<Brush> _bid_background, _bid_foreground, _ask_background, _ask_foreground;
       IWindowSettingsManager<LadderSettings> _settings_manager;
       ulong _misc_time = 0;
@@ -44,7 +42,7 @@ namespace mbts.spark {
          _open = backing(nameof(open), double.NaN);
          _settlement = backing(nameof(settlement), double.NaN);
          _volume = backing(nameof(volume), 0ul);
-         _revert = backing(nameof(revert), false);
+         _auto_clear = backing(nameof(auto_clear), false);
          _status = backing(nameof(status_message), "");
          _status_brush = backing(nameof(status_background), (Brush)BTSColors.UI.spark_menu_background);
          // brush props
@@ -176,7 +174,7 @@ namespace mbts.spark {
       public double open => _open.Value;
       public double settlement => _settlement.Value;
       public ulong volume => _volume.Value;
-      public bool revert { get => _revert.Value; set => _revert.set(value); }
+      public bool auto_clear { get => _auto_clear.Value; set => _auto_clear.set(value); }
       public CalculatorLogic size_calculator { get; }
       public OrderPropertiesEditorVM order_properties_vm { get; }
       public ObservableCollection<LadderOrderVM> orders { get; }
@@ -232,16 +230,8 @@ namespace mbts.spark {
                
             uint size = args.size_from_alternate ? s.alternate_order_size : (uint)Math.Abs(size_calculator.value.Value);
             await _model.order_gateway.send_order(order, args.price, size);
-            if (revert) {
-               size_calculator.value = s.size;
-               if (order_properties_vm.selected_time_in_force != s.default_time_in_force && order_properties_vm.time_in_forces.Contains(s.default_time_in_force))
-                  order_properties_vm.selected_time_in_force = s.default_time_in_force;
-               foreach (var ot in order_properties_vm.order_types) {
-                  ot.load_from(_model.order_gateway.factory.custom_order(ot.order_type));
-                  if (ot.order_type == s.default_order_type)
-                     order_properties_vm.selected_order_type = ot;
-               }
-            }
+            if (auto_clear)
+               size_calculator.value = 0;
          }
       }
 

src/client/spark/view/Spark/Ladder/LadderView.xaml

@@ -394,7 +394,7 @@
         </ItemsControl>
         <Button Margin="2" HorizontalAlignment="Stretch" Command="{Binding size_calculator.clear}">Clear</Button>
         <!-- reset to defaults after order -->
-        <CheckBox Margin="2" IsChecked="{Binding revert}">Auto Clear</CheckBox>
+        <CheckBox Margin="2" IsChecked="{Binding auto_clear}">Auto Clear</CheckBox>
       </StackPanel>
       <Grid Grid.Row="1">
         <Grid.RowDefinitions>

src/client/spark/view/Spark/Ladder/RecenterCountdown.cs

@@ -17,16 +17,17 @@ namespace mbts.spark {
       };
       SecurityImmutable? _security;
       MDConsumerHandle _handle;
-      LadderSettings _settings;
       DateTime _last_recenter = DateTime.MaxValue;
       double _last_mid = 0.0;
       IDisposable _dispose;
+      readonly TimeSpan _seconds_to_center;
 
       public RecenterCountdown() {
          DataContextChanged += RecenterCountdown_DataContextChanged;
          Unloaded += (o,e) => on_unload(false);
          _timer.Tick += _timer_Tick;
          _timer.Start();
+         _seconds_to_center = TimeSpan.FromSeconds(config.Configuration.get<bts.config.LadderKnobs>().seconds_to_recenter);
       }
 
       static (double,double) tick_range(double price, int tick_count, ContractSpecificationImmutable spec) {
@@ -34,11 +35,10 @@ namespace mbts.spark {
       }
 
       private void _timer_Tick(object sender, EventArgs e) {
-         if (_settings != null && _security != null) {
+         if (_security != null) {
             var mid = _handle.mid();
-            (double low, double high) = tick_range(_last_mid, _settings.ticks_to_recenter, _security.Value.ContractSpec);
             if (//mid < low || mid > high ||
-               (_last_recenter + TimeSpan.FromSeconds(_settings.seconds_to_recenter) < DateTime.UtcNow)) {
+               (_last_recenter + _seconds_to_center < DateTime.UtcNow)) {
                trigger_recenter(mid);
             }
          }
@@ -59,7 +59,6 @@ namespace mbts.spark {
             vm.OnDisposed += on_unload;
             _dispose =
                Disposable.Create(
-                  vm.settings.Subscribe(s => _settings = s),
                   Disposable.Create(() => vm.OnDisposed -= on_unload));
          }
       }

src/client/spark/view/Spark/Utilities/WorkLoop.cs

@@ -10,7 +10,7 @@ namespace mbts.spark {
       static readonly TimeSpan default_interval;
 
       static WorkLoop() {
-         var timer_params = config.Configuration.get<bts.config.LadderTimerParams>();
+         var timer_params = config.Configuration.get<bts.config.LadderKnobs>();
          default_priority = Enum.TryParse(timer_params.priority, true, out DispatcherPriority p) ? p : DispatcherPriority.ApplicationIdle;
          default_interval = TimeSpan.FromMilliseconds(timer_params.interval_ms);
       }

src/client/spark/view/Utility/TextUtils.cs

@@ -130,12 +130,22 @@ namespace mbts.ui.utility.text {
          public static class Time {
             public static readonly Regex
                StandardTime = new Regex(@"^((0?[0-9])|(1?[0-2]?))(:?|:([0-5]?|[0-5][0-9]?)) ?(?i)((a|p)m?)?$"),
+               StandardTimeOptionalSeconds = new Regex(@"^((\d?|0\d|1[0-2])|((\d|0\d|1[0-2])(:?|(:([0-5]?|[0-5]\d(( ?|( ([aApP]?|[aApP][mM])))(:?|(: ?)|(: [aApP]?[mM]?)|(:[0-5]?\d? ?[aApP]?[mM]?))|:([0-5]?|([0-5]\d( ?|( ([aApP]?|([aApP][mM]))))))))))))$"),
                MilitaryTime = new Regex(@"^(([0-1]?[0-9])|(2?[0-3]?))(:?|:([0-5]?|[0-5][0-9]?))$"),
+               MilitaryTimeOptionalSeconds = new Regex(@"^((\d?|[0-1]\d|2[0-3])|((\d|[0-1]\d|2[0-3])(:?|(:([0-5]?|[0-5]\d(:?|:([0-5]?|[0-5]\d)))))))$"),
                DaysInMonth = new Regex(@"^([0-9]?|[0-2][0-9]|3[0-1])?$"),
                MonthAbbrev = new Regex(@"^(?i)(a(ug?|pr?)?|d(e|ec)?|f(e?b)?|j(an?|u[nl]?)?|m(a[ry]?)?|(o(ct?)?)|s(ep?)?|n(ov?)?)?$"),
                Year = new Regex(@"^\d{0,4}$"),
                TimeSpan = new Regex(@"^(\d+(:?|:([0-5]?|[0-5]\d(:?|:([0-5]?|[0-5]\d)))))?$");
          }
       }
+
+      public static class StringFormats {
+         public static class Time {
+            public const string
+               MilitaryWithSeconds = "HH:mm:ss",
+               StandardWithSeconds = "hh:mm:ss tt";
+         }
+      }
    }
 }

src/client/spark_api/security/SecurityFormatter.cs

@@ -1,18 +1,18 @@
-using System;
-using System.Collections.Concurrent;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using static bts.api.Security;
 
 namespace bts.api {
    struct FutureLeg {
       public readonly Venue exchange;
-      public readonly string symbol;
+      public readonly string symbol, exchange_desc;
       public readonly FutureContractImmutable future;
       public readonly int quantity;
 
-      public FutureLeg(Venue exchange, string symbol, FutureContractImmutable future, int quantity) {
+      public FutureLeg(Venue exchange, string symbol, string exchange_desc, FutureContractImmutable future, int quantity) {
          this.exchange = exchange;
          this.symbol = symbol;
+         this.exchange_desc = exchange_desc;
          this.future = future;
          this.quantity = quantity;
       }
@@ -32,12 +32,14 @@ namespace bts.api {
          return string.Join("", _format_future_legs(future_legs));
       }
 
-      public static string format_future(string symbol, FutureContractImmutable future) =>
-         $"{symbol}{DateFormatter.Format(future.ExpirationYear, future.ExpirationMonth, future.ExpirationDay, DateFormat.MonthCodeLastDigitYear)}";
+      public static string format_future(string symbol, string exch_desc, FutureContractImmutable future) =>
+         string.IsNullOrEmpty(exch_desc) ?
+            $"{symbol}{DateFormatter.Format(future.ExpirationYear, future.ExpirationMonth, future.ExpirationDay, DateFormat.MonthCodeLastDigitYear)}" :
+            exch_desc;
 
       static string _format(SecurityImmutable security) {
          switch (security.ContractCase) {
-            case ContractOneofCase.Future: return format_future(security.Symbol, security.Future);
+            case ContractOneofCase.Future: return format_future(security.Symbol, security.glbx_desc(), security.Future);
             case ContractOneofCase.Spread:
                if (security.HasGlbx) {
                   // In case we ever bring it back, pretty printed spread names can be found here:
@@ -61,7 +63,7 @@ namespace bts.api {
       }
 
       static string _get_first_leg_name(FutureLeg fl) {
-         string sym_exp_name = format_future(fl.symbol, fl.future);
+         string sym_exp_name = format_future(fl.symbol, fl.exchange_desc, fl.future);
          switch (fl.quantity) {
             case 1: return sym_exp_name;
             case -1: return $"-{sym_exp_name}";
@@ -70,7 +72,7 @@ namespace bts.api {
       }
 
       static string _get_subsequent_leg_name(FutureLeg fl) {
-         string sym_exp_name = format_future(fl.symbol, fl.future);
+         string sym_exp_name = format_future(fl.symbol, fl.exchange_desc, fl.future);
          switch (fl.quantity) {
             case 1: return $"+{sym_exp_name}";
             case -1: return $"-{sym_exp_name}";

src/client/spark_api/security/SecurityHelper.cs

@@ -19,7 +19,7 @@ namespace bts.api {
          foreach (var l in legs) {
             switch (l.Security.ContractCase) {
                case ContractOneofCase.Future:
-                  yield return new FutureLeg(l.Security.Exchange, l.Security.Symbol, l.Security.Future, qty * l.SignedQuantity());
+                  yield return new FutureLeg(l.Security.Exchange, l.Security.Symbol, l.Security.glbx_desc(), l.Security.Future, qty * l.SignedQuantity());
                   break;
                case ContractOneofCase.Spread:
                   foreach (var fl in _get_flat_future_legs(l.Security.Spread.Legs, qty * l.SignedQuantity()))
@@ -56,6 +56,9 @@ namespace bts.api {
 
       public static string DisplayExchange(this SecurityImmutable security) =>
          security.ContractCase == ContractOneofCase.Synthetic ? $"[{string.Join(",", get_flat_future_legs(security).Select(flt => flt.exchange.ToString()).Distinct().OrderBy(e => e))}]" : security.Exchange.ToString();
+
+      public static string glbx_desc(this SecurityImmutable security) =>
+         security.HasGlbx ? security.Glbx.SecurityDesc ?? string.Empty : string.Empty;
       #endregion
 
       public static class Comparer {

src/lib/api/service_helpers/AdminServiceUtils.cpp

@@ -820,19 +820,49 @@ void AdminServiceUtils::delete_user (bts::api::admin::AdminData& admin_data,
 ////////////////////////////////////////////
 bool AdminServiceUtils::_sort_security_scopes (const bts::api::admin::SecurityScope& a,
                                                const bts::api::admin::SecurityScope& b){
-   if (has_id(a.security_id()))
-      return true;
+   // for both the sec and product scope sorting, the more
+   // qualified arg is sent to the top (vs wildcard). the
+   // order of qualification is sec_id, symbol, type, exchange.
+   // if both a and b are equally qualified, it is a
+   // a dont-care case (in terms of sec/prod res) and 
+   // ascending orderring is used
+   if (has_id(a.security_id())){
+      if (has_id(b.security_id()))
+         return a.security_id() < b.security_id();
+      else
+         return true;
+   }
    if (has_id(b.security_id()))
      return false;
-   if (a.security_symbol() != "*")
-      return true; 
-   return false;
+   // symbol
+   if (is_symbol_set(a.security_symbol())){
+      if (is_symbol_set(b.security_symbol()))
+         return a.security_symbol() < b.security_symbol();
+      else
+         return true;
+   }
+   if (is_symbol_set(b.security_symbol()))
+      return false;
+   // sec type
+   if (a.security_type() != b.security_type())
+      return a.security_type() < b.security_type();
+   return a.exchange() < b.exchange();
 }
 bool AdminServiceUtils::_sort_product_scopes (const bts::api::admin::ProductScope& a,
-                                              const bts::api::admin::ProductScope&){
-   if (a.security_symbol() != "*")
-      return true;
-   return false;
+                                              const bts::api::admin::ProductScope& b){
+   // symbol
+   if (is_symbol_set(a.security_symbol())){
+      if (is_symbol_set(b.security_symbol()))
+         return a.security_symbol() < b.security_symbol();
+      else
+         return true;
+   }
+   if (is_symbol_set(b.security_symbol()))
+      return false;
+   // sec type
+   if (a.security_type() != b.security_type())
+      return a.security_type() < b.security_type();
+   return a.exchange() < b.exchange();
 }
 void AdminServiceUtils::sort_limits (bts::api::admin::BTSAccount& account){
    std::sort (account.mutable_security_position_limits()->begin(),
@@ -901,7 +931,7 @@ bool AdminServiceUtils::match_security_scope (const bts::api::admin::SecuritySco
 bool AdminServiceUtils::match_product_scope (const bts::api::admin::ProductScope& scope,
                                              const bts::api::Product& product){
    return product.exchange() == scope.exchange() &&
-          product.type() && scope.security_type() &&
+          product.type() == scope.security_type() &&
           _match_limit_rule_symbol(product.symbol(), scope.security_symbol());
 }
 bool AdminServiceUtils::match_product(const api::Security& security,
@@ -1025,4 +1055,7 @@ void AdminServiceUtils::bootstrap_data (bts::api::admin::AdminData& data){
    acct_a1_clip_size->set_clip_size(100);
 }
 bool AdminServiceUtils::has_id (uint32_t id){return id>0;}
+bool AdminServiceUtils::is_symbol_set (const std::string& symbol){
+   return symbol !="*";
+}
 }}

src/lib/api/service_helpers/AdminServiceUtils.h

@@ -240,6 +240,7 @@ public:
    // misc//////// ////////////////////////////
    ////////////////////////////////////////////
    static bool has_id (uint32_t id);
+   static bool is_symbol_set (const std::string& symbol);
    static void bootstrap_data (bts::api::admin::AdminData& data);
 };
 }}

src/lib/api/services/OrderService.proto

@@ -233,7 +233,7 @@ enum CTICode {
   CTI_CODE_CTI_CODE1 = 49;
   CTI_CODE_CTI_CODE2 = 50;
   CTI_CODE_CTI_CODE3 = 51;
-  CTI_CODE_CTI4 = 52;
+  CTI_CODE_CTI_CODE4 = 52;
 }
 
 enum NativeOrderStatus 

src/lib/config/protos/UI.proto

@@ -28,20 +28,6 @@ message IShellModuleProps
    repeated ModuleProps module = 1;
 }
 
-message IShellWindowProps {
-   // TODO: Maybe IWindowManager should be responsible for total-windows-open limits?
-   // To achieve that we would need to use a sort of type-windowprops map.
-   // However, that makes the problem more complicated, and at this point we are just aiming to limit n_ladders open.
-   // So for now, just let the LadderWindowFactory manage this.
-   message WindowProps {
-      // NOTE: 50 will be a dangerously high default for most other windows (than Ladder),
-      // so be careful if you add other WindowProps members.
-      optional uint32 max_total_open = 1 [default = 50];
-   }
-
-   optional WindowProps ladder = 1;
-}
-
 message GridSettings {
     optional string font_family = 1 [default = "Calibri"];
     optional float font_size = 2 [default = 10];
@@ -62,9 +48,11 @@ message ShellLimits {
    optional uint64 limit_group_memory_mb = 3;
 }
 
-// priorities (ordered): SystemIdle, ApplicationIdle, ContextIdle, Background, Input, Loaded, Render, DataBind, Normal, Send
-// It seems like all priorities higher than ApplicationIdle choke the main thread.
-message LadderTimerParams {
+message LadderKnobs {
+  // priorities (ordered): SystemIdle, ApplicationIdle, ContextIdle, Background, Input, Loaded, Render, DataBind, Normal, Send
+  // It seems like all priorities higher than ApplicationIdle choke the main thread.
   optional string priority = 1 [default = "ApplicationIdle"];
   optional double interval_ms = 2 [default = 100];
+  optional uint32 seconds_to_recenter = 3 [default = 300];
+  optional uint32 max_open_ladders = 4 [default = 75];
 }

src/lib/md/relay/QueryServer.cpp

@@ -15,6 +15,9 @@
 #include "lib/api/apicore.pb.h"
 #include "lib/api/services/MarketDataRelay.server.pb.h"
 #include "lib/api/public/ServiceContext.h"
+#ifndef _WIN32
+#include "server/sparkd/metrics/metrics.h"
+#endif
 
 #include <vector>
 #include <list>
@@ -61,13 +64,29 @@ struct DatedHandle {
 
 struct MDConsumerGroup {
    // probably just have to send the consumer in from outside
-   MDConsumerGroup(const UserSymbol& symbol, const MDConsumer& consumer) :
+   MDConsumerGroup(const UserSymbol& symbol, const MDConsumer& consumer, metrics::MetricsContainer& stats) :
       symbol(symbol),
       platform(platform_for_exchange(symbol.exchange())),
       channel(consumer.get_channel(symbol)),
       handles()
    {
       set_consumer(consumer);
+      
+      // create and publish a stat for this group
+#ifndef _WIN32
+      if (symbol.type() != SecurityType::SECURITYTYPE_SYNTHETIC) {
+         std::vector<metrics::Label> labels;
+         labels.push_back({ "symbol", symbol.symbol() });
+         labels.push_back({ "type", SecurityType_Name(symbol.type()).substr(13) /*SECURITYTYPE_*/ });
+         labels.push_back({ "exch", Venue_Name(symbol.exchange()).substr(6) /*VENUE_*/ });
+         stats.publish(
+            std::make_shared<metrics::LazyCounterMetric>("spark_md_book_updates", [=]() { return std::max(channel.bbo_count(), channel.depth_count());  }, labels));
+         stats.publish(
+            std::make_shared<metrics::LazyCounterMetric>("spark_md_security_count", [=]() { return channel.security_count();  }, labels));
+         stats.publish(
+            std::make_shared<metrics::LazyCounterMetric>("spark_md_trade_count", [=]() { return channel.trade_count();  }, labels));
+      }
+#endif
    }
    MDConsumerGroup(const MDConsumerGroup& other) = delete;
 
@@ -109,6 +128,7 @@ void set_price(PriceSize ps, co2::TradeMessage* ptrade) {
    ptrade->set_exponent(ps.exponent());
    ptrade->set_size(ps.size());
 }
+
 void copy_misc(MiscData misc, co2::MiscMessage* pmisc) {
    pmisc->set_settlement_price(misc.settlement_price);
    pmisc->set_settlement_date(misc.settlement_date.to_short());
@@ -134,14 +154,14 @@ class QueryServerImpl : public IQueryServer {
    boost::container::flat_map<UserSymbol,shared_ptr<MDConsumerGroup>> _groups;
 
 public:
-   QueryServerImpl(const std::set<UserSymbol>& symbols, std::shared_ptr<api::ISyntheticSecurityService>& ss) :
+   QueryServerImpl(const std::set<UserSymbol>& symbols, std::shared_ptr<api::ISyntheticSecurityService>& ss, metrics::MetricsContainer& stats) :
       _consumer(symbols, ss),
       _symbols(symbols),
       _ss(ss),
       _groups()
    {
       for (const auto& sym : symbols) {
-         _groups.emplace(std::make_pair(sym, std::make_unique<MDConsumerGroup>(sym, _consumer)));
+         _groups.emplace(std::make_pair(sym, std::make_unique<MDConsumerGroup>(sym, _consumer, stats)));
       }
       _groups.shrink_to_fit();
 
@@ -336,24 +356,36 @@ public:
 struct MDRelayServer : bts::co2::MarketDataRelayBase {
    const QueryServerImpl& _db;
    utils::Dispatcher& _threadpool;
-    MDRelayServer(const QueryServerImpl& db, utils::Dispatcher& threadpool, bts::api::ServiceContext ctx) : 
+#ifndef _WIN32
+   std::shared_ptr<metrics::HistogramMetric<5>> _time,_size;
+#endif
+
+    MDRelayServer(const QueryServerImpl& db, utils::Dispatcher& threadpool, bts::api::ServiceContext ctx, metrics::MetricsContainer& stats) : 
        bts::co2::MarketDataRelayBase(ctx),
        _db(db), _threadpool(threadpool)
-    { }
+#ifndef _WIN32
+       , _time(std::make_shared<metrics::HistogramMetric<5>>("spark_qserver_response_time_ms", std::array<double,5>({ 10.0, 50.0, 100.0, 500.0, 1000.0 }))),
+       _size(std::make_shared<metrics::HistogramMetric<5>>("spark_qserver_size_bytes", std::array<double,5>({ 1024 /*kb*/, 1024*1024 /*mb*/, (1024*1024)*4, (1024*1024)*16, (1024*1024)*32 })))
+    {
+       stats.publish(_time);
+       stats.publish(_size);
+    }
+#else
+   {}
+#endif
 
     virtual void requestmd(const MDRequest & request, const std::function<void(const MDReply&)>& reply) override {
+        TimeStamp start = TimeStamp::now();
        _threadpool.post([=]() {
           google::protobuf::Arena arena;
           MDReply* preply = google::protobuf::Arena::CreateMessage<MDReply>(&arena);
-          TimeStamp start = TimeStamp::now();
           _db.query(request, preply);
-          auto elapsed = (TimeStamp::now() - start);
-          LOG(md, Diag,
-             "Request for: {} took {}us and returned {} bytes before compression",
-             request.host(),
-             elapsed.microseconds(),
-             preply->ByteSize());
           reply(*preply);
+          auto elapsed = (TimeStamp::now() - start);
+#ifndef _WIN32
+          _time->observe(elapsed.milliseconds());
+          _size->observe(preply->ByteSize());
+#endif
        });
     }
     
@@ -382,11 +414,15 @@ struct MDRelayServer : bts::co2::MarketDataRelayBase {
 };
 
 struct QueryServer {
-    QueryServer(std::set<UserSymbol> symbols, utils::Dispatcher& threadpool, const bts::api::ServiceContext& ctx, std::shared_ptr<api::ISyntheticSecurityService>& ss):
-    db(symbols, ss),
-    relay(db, threadpool, ctx),
-    security_timer(),
-    health_timer()
+    QueryServer(std::set<UserSymbol> symbols, 
+                utils::Dispatcher& threadpool, 
+                const bts::api::ServiceContext& ctx, 
+                std::shared_ptr<api::ISyntheticSecurityService>& ss,
+                metrics::MetricsContainer& stats):
+      db(symbols, ss, stats),
+      relay(db, threadpool, ctx, stats),
+      security_timer(),
+      health_timer()
     { }
     QueryServerImpl db;
     MDRelayServer relay;
@@ -394,7 +430,7 @@ struct QueryServer {
     std::unique_ptr<utils::DispatcherTimer> health_timer;
 };
    
-QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared_ptr<api::ISyntheticSecurityService>& ss) {
+QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared_ptr<api::ISyntheticSecurityService>& ss, metrics::MetricsContainer& stats) {
    auto& threadpool = utils::Dispatcher::get_named("MDQuery");
 
    std::set<UserSymbol> all_symbols;
@@ -403,7 +439,8 @@ QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared
    SecurityType type;
    bts_assert(api::SecurityType_Parse(config::get<PlatformConfig>().type(), &type));
    get_system_md_symbols(venue, type, all_symbols);
-   QueryServerPtr container = std::make_shared<QueryServer>(all_symbols, threadpool, ctx, ss);
+
+   QueryServerPtr container = std::make_shared<QueryServer>(all_symbols, threadpool, ctx, ss, stats);
    auto& db = container->db;
 
     // Scan for new securities every half second
@@ -426,8 +463,8 @@ QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared
    return container;
 }
 
-std::unique_ptr<IQueryServer> create_query_server(const std::set<UserSymbol>& symbols, std::shared_ptr<api::ISyntheticSecurityService>& ss) {
-   return std::make_unique<QueryServerImpl>(symbols, ss);
+std::unique_ptr<IQueryServer> create_query_server(const std::set<UserSymbol>& symbols, std::shared_ptr<api::ISyntheticSecurityService>& ss, metrics::MetricsContainer& stats) {
+   return std::make_unique<QueryServerImpl>(symbols, ss, stats);
 }
 
 }}//namespace

src/lib/md/relay/QueryServer.h

@@ -11,6 +11,7 @@
 
 namespace bts {
     namespace api { class ServiceContext; }
+    namespace metrics { class MetricsContainer; }
 namespace md {
 
 struct IQueryServer {
@@ -18,11 +19,11 @@ struct IQueryServer {
    virtual void scan_securities() = 0;
    virtual void query(const co2::MDRequest& request, co2::MDReply* reply) const = 0;
 };
-std::unique_ptr<IQueryServer> create_query_server(const std::set<UserSymbol>&, std::shared_ptr<api::ISyntheticSecurityService>&);
+std::unique_ptr<IQueryServer> create_query_server(const std::set<UserSymbol>&, std::shared_ptr<api::ISyntheticSecurityService>&, metrics::MetricsContainer&);
 
 struct QueryServer;
 typedef std::shared_ptr<QueryServer> QueryServerPtr;
-QueryServerPtr run_query_server(const bts::api::ServiceContext&, std::shared_ptr<api::ISyntheticSecurityService>&);
+QueryServerPtr run_query_server(const bts::api::ServiceContext&, std::shared_ptr<api::ISyntheticSecurityService>&, metrics::MetricsContainer&);
 
 }}//namespace
 

src/lib/md/test/md2_tests.cpp

@@ -10,6 +10,7 @@
 #include "lib/utils/defer.h"
 #include "lib/utils/warn_ctrl.h"
 #include "lib/utils/bts_chrono.h"
+#include "server/sparkd/metrics/metrics.h"
 
 #include "lib/config/protos/MarketData.pb.h"
 
@@ -30,6 +31,15 @@
 using namespace std;
 
 namespace bts {
+
+#ifdef _WIN32
+namespace metrics {
+struct MetricCollection{};
+MetricsContainer::MetricsContainer() {}
+MetricsContainer::~MetricsContainer() {}
+}
+#endif
+
 int md_tests = 0;
 namespace md {
 // import this from RelayClient.cpp
@@ -1275,7 +1285,8 @@ struct Co2RelayTests {
       set<UserSymbol> symbols;
       symbols.insert(symbol);
       shared_ptr<api::ISyntheticSecurityService> empty;
-      auto server = create_query_server(symbols, empty);
+      metrics::MetricsContainer stats;
+      auto server = create_query_server(symbols, empty, stats);
       REQUIRE(server);
       server->scan_securities();
 

src/lib/quickbus/network/QBConnection.cpp

@@ -193,10 +193,18 @@ void QBConnection::handle_msg(msgs::QBMsg *msgs)
             me->intro=op.intro();
             me->my_id = NodeID(me->intro.node_id());
 
-            if (!me->router.add_neighbour(me->my_id,shared_from_this())) {
-               close_connection("Node is already connected");
+            if (me->router.is_neighbour(me->my_id)) {
+               LOG(QBus,Warn, "Refusing connection from {} to {} reason: {} is already connected" ,
+                   me->my_desc , me->remote_desc , me->my_id.value());
+               if (me->socket.is_open()) 
+                  me->socket.close();
                return;
             }
+            else {
+               LOG(QBus,Warn, "Accepted connection from {} to {}. {} is now connected." ,
+                   me->my_desc , me->remote_desc , me->my_id.value());
+               me->router.add_neighbour(me->my_id,shared_from_this());
+            }
          }
          break;
       }

src/lib/quickbus/routing/Router.cpp

@@ -184,6 +184,11 @@ void Router::remove_neighbour(NodeID const&removed_neighbour_id)
    me->neighbours.erase(removed_neighbour_id);
 }
 
+bool Router::is_neighbour(NodeID const&neighbour_id) const
+{
+   return me->neighbours.find(neighbour_id) != me->neighbours.end();
+}
+
 void Router::advertise(NodeID const& sender,TableID const&tid,Advertisement const&ad) 
 {
    LOG(QBus,Diag, "{} received advertisement from nid: {}  tid: {} ad topic: {}",

src/lib/quickbus/routing/Router.h

@@ -15,7 +15,8 @@ public:
    NodeID id() const ;
    bool add_neighbour (NodeID const& neighbour_id, INodePtr const& neighbour);
    void remove_neighbour (NodeID const& neighbour_id);
-
+   bool is_neighbour(NodeID const&neighbour_id) const;
+   
    // from INode 
    void advertise(NodeID const& sender,TableID const&id,Advertisement const&adv) override;
    void unadvertise(NodeID const& sender,TableID const&tid) override;

src/server/sparkd/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/Md.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;dspark/mirage.cpp;dspark/mirage/routing/Router.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;main.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/MDP3Utils.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;mercury/actors/Actor.cpp;mercury/broker/AsyncEventBroker.cpp;mercury/broker/CoreEventBroker.cpp;mercury/broker/IEventBroker.cpp;mercury/events/TimerEvent.cpp;mercury/mercury_utils/Executor.cpp;mercury/mercury_utils/IPollableActor.cpp;mercury/mercury_utils/mercury_common.cpp;mercury/synopsis/SynopsisBuilder.cpp;mercury/topics/ITopic.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/TickOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/Line truncated
+set(sources cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/Md.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage.cpp;dspark/mirage/routing/Router.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;main.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/MDP3Utils.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;mercury/actors/Actor.cpp;mercury/broker/AsyncEventBroker.cpp;mercury/broker/CoreEventBroker.cpp;mercury/broker/IEventBroker.cpp;mercury/events/TimerEvent.cpp;mercury/mercury_utils/Executor.cpp;mercury/mercury_utils/IPollableActor.cpp;mercury/mercury_utils/mercury_common.cpp;mercury/synopsis/SynopsisBuilder.cpp;mercury/topics/ITopic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/TickOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreaderLine truncated

src/server/sparkd/cmds/Run.cpp

@@ -16,6 +16,8 @@
 #include "server/sparkd/persistence/PersistenceServiceServer.h"
 #include "server/sparkd/support/ErrorReportService.h"
 #include "server/sparkd/http/WebServer.h"
+#include "server/sparkd/metrics/metrics.h"
+#include "server/sparkd/metrics/system_metrics.h"
 #include "server/sparkd/user/AdminServiceServer.h"
 #include "server/sparkd/user/UserServiceServer.h"
 
@@ -37,7 +39,9 @@ int Run::execute(std::vector<std::string> const &) {
    qbus::Quickbus quickbus(qb_cfg);
    api::ServiceContext sc(&quickbus);
 
-   http::SparkWebServer webserver(sc, ports.config_port());
+   metrics::MetricsContainer stats;
+   metrics::publish_system_metrics(stats);
+   http::SparkWebServer webserver(sc, stats, ports.config_port());
 
    persistence::PersistenceServiceServer settings_saver(sc, "settings");
    user::UserServiceServer login_server(sc);
@@ -45,7 +49,7 @@ int Run::execute(std::vector<std::string> const &) {
    std::shared_ptr<api::ISyntheticSecurityService> synthetic_service =
        std::make_shared<api::synthetic::SyntheticSecurityService>(sc);
    md::QueryServerPtr md_query_server(
-       md::run_query_server(sc, synthetic_service));
+       md::run_query_server(sc, synthetic_service, stats));
    orderman::OrderServiceServer order_service_server(sc, synthetic_service);
    support::ErrorReportServer error_report_service(sc);
    admin::AdminServiceServer admin_service(sc);

src/server/sparkd/cmds/Web.cpp

@@ -0,0 +1,56 @@
+#include "Web.h"
+
+#include "lib/config/config.h"
+#include "lib/config/protos/quickbus_cfg.pb.h"
+#include "lib/config/protos/Ports.pb.h"
+#include "lib/quickbus/quickbus.h"
+#include "lib/api/public/ServiceContext.h"
+
+#include "server/sparkd/metrics/metrics.h"
+#include "server/sparkd/metrics/system_metrics.h"
+#include "server/sparkd/http/WebServer.h"
+#include "lib/md/relay/QueryServer.h"
+#include "lib/md/SyntheticSecurityService.h"
+#include "server/sparkd/user/AdminServiceServer.h"
+#include "server/sparkd/user/UserServiceServer.h"
+#include "server/sparkd/md/run_md.h"
+
+#include <vector>
+#include <string>
+
+namespace bts {
+namespace cmd {
+
+int Web::execute(const std::vector<std::string>& args) {
+   config::QuickbusNodeCfg qb_cfg;
+   auto& ports = config::get<config::Ports>();
+   qb_cfg.set_my_port(ports.qbus_port());
+   qbus::Quickbus quickbus(qb_cfg);
+   api::ServiceContext sc(&quickbus);
+
+   metrics::MetricsContainer stats;
+   metrics::publish_system_metrics(stats);
+   http::SparkWebServer webserver(sc, stats, ports.config_port());
+
+   user::UserServiceServer login_server(sc);
+   if (std::find(args.begin(), args.end(), "-md") != args.end()) {
+      md::run_market_datapublishers();
+   }
+
+   std::shared_ptr<api::ISyntheticSecurityService> synthetic_service =
+       std::make_shared<api::synthetic::SyntheticSecurityService>(sc);
+   md::QueryServerPtr md_query_server(
+       md::run_query_server(sc, synthetic_service, stats));
+
+   admin::AdminServiceServer admin_service(sc);
+
+
+   
+   std::this_thread::sleep_until(
+       std::chrono::time_point<std::chrono::system_clock>::max());
+   return 0;
+}
+
+std::string Web::help() { return "\t run only the services required for the web admin.  This cannot be run concurrently with 'sparkd run'\n\tAdd -md to start market data too."; }
+
+}}//namespace

src/server/sparkd/cmds/Web.h

@@ -0,0 +1,17 @@
+
+#pragma once
+
+#include "ICmd.h"
+
+namespace bts {
+namespace cmd {
+
+class Web : public ICmd 
+{
+public:
+   int execute(std::vector<std::string> const &args);
+   std::string help ();
+};
+
+}}
+

src/server/sparkd/http/WebServer.cpp

@@ -5,6 +5,7 @@
 #include "lib/api/apigateway.pb.h"
 #include "lib/logging/logging.h"
 #include "lib/os/os.h"
+#include "lib/utils/TimeStamp.h"
 
 #include <websocketpp/config/asio_no_tls.hpp>
 #include <websocketpp/server.hpp>
@@ -49,24 +50,33 @@ namespace http {
       std::string _www;
       std::string _conf;
       const api::ServiceContext& _ctx;
+      metrics::MetricsContainer& _metrics;
+      std::shared_ptr<metrics::HistogramMetric<4>> _mtimer;
 
-      ServerState(const api::ServiceContext& ctx, int port) :
+      ServerState(const api::ServiceContext& ctx, metrics::MetricsContainer& metrics, int port) :
          _m(),
          _dispatcher(utils::Dispatcher::get_named("Webserver")), 
          _session(), 
          _server(),
          _port(port),
          _www(),
-         _ctx(ctx)
+         _ctx(ctx),
+         _metrics(metrics),
+         _mtimer(std::make_shared<metrics::HistogramMetric<4>>("spark_metrics_query_duration_ms", 
+                                                               std::array<double,4>({ 0.1, 0.5, 1.0, 5.0 })))
       {
          _www = os::sparkdata_path("www");
          _conf = os::sparkdata_path("conf");
 
+         _metrics.publish(_mtimer);
+
          _server.init_asio(_dispatcher.io_service());
          _server.set_open_handler(bind(&ServerState::on_open, this, ::_1));
          _server.set_close_handler(bind(&ServerState::on_close, this, ::_1));
          _server.set_message_handler(bind(&ServerState::on_message, this, ::_1, ::_2));
          _server.set_http_handler(bind(&ServerState::on_http, this, ::_1));
+         // Turn off logging
+         _server.clear_access_channels(websocketpp::log::alevel::all);
          run();
       }
 
@@ -137,6 +147,15 @@ namespace http {
          con->set_status(websocketpp::http::status_code::ok);
       }
 
+      void return_metrics(connection_ptr& con) {
+         utils::TimeStamp start = utils::TimeStamp::now();
+         std::stringstream ss;
+         _metrics.print(ss);
+         con->set_body(ss.str());
+         con->set_status(websocketpp::http::status_code::ok);
+         _mtimer->observe((utils::TimeStamp::now() - start).microseconds() / 1000.0);
+      }
+
       void directory_index(connection_ptr& con, const std::string& resource, const boost::filesystem::path& path) {
          // directory listing
          std::string body = "<!doctype html>\n";
@@ -198,11 +217,22 @@ namespace http {
          con->set_body(read_file(path.string()));
          con->set_status(websocketpp::http::status_code::ok);
       }
+
       void return_bts_installer(connection_ptr& con) {
          std::string installer_bits = read_file(_www+"/bts_installer.exe");
          const std::string installer_config = _www+"install_config.json";
          if (boost::filesystem::is_regular_file(installer_config)) {
-            installer_bits += read_file(installer_config);
+            auto conf = read_file(installer_config);
+            std::string needle = "$HOST";
+            std::size_t pos;
+            do {
+               pos = conf.find(needle,0);
+               if (pos != std::string::npos) {
+                  auto host = con->get_uri()->get_host();
+                  conf.replace(pos, needle.size(), host);
+               }
+            } while(pos != std::string::npos);
+            installer_bits += conf;
          }
          con->set_body(installer_bits);
          con->set_status(websocketpp::http::status_code::ok);
@@ -219,10 +249,13 @@ namespace http {
       void on_http(connection_hdl hdl) {
          auto con = _server.get_con_from_hdl(hdl);
          auto resource = strip_query_str(con->get_resource());
-         LOG(Api,Info, "REQUEST: {}", resource);
+         LOG(Api,Diag, "HTTP GET: {}", resource);
          if (boost::iequals(resource, "/config/.all")) {
             return_dotall(con);
          }
+         else if (boost::iequals(resource, "/metrics")) {
+            return_metrics(con);
+         }
          else if (boost::iequals(resource, "/bts_installer.exe") && boost::filesystem::is_regular_file(_www+"/bts_installer.exe")) {
             return_bts_installer(con);
          }
@@ -256,8 +289,8 @@ namespace http {
 
 
 
-SparkWebServer::SparkWebServer(const api::ServiceContext& ctx, int port) :
-   _me(std::make_shared<ServerState>(ctx, port))
+SparkWebServer::SparkWebServer(const api::ServiceContext& ctx, metrics::MetricsContainer& metrics, int port) :
+   _me(std::make_shared<ServerState>(ctx, metrics, port))
 { 
 }
 

src/server/sparkd/http/WebServer.h

@@ -1,5 +1,6 @@
 #pragma once
 #include "lib/api/public/ServiceContext.h"
+#include "server/sparkd/metrics/metrics.h"
 #include <memory>
 
 namespace bts {
@@ -10,7 +11,7 @@ namespace http {
    class SparkWebServer {
       std::shared_ptr<ServerState> _me;
       public:
-      SparkWebServer(const api::ServiceContext& ctx, int port);
+      SparkWebServer(const api::ServiceContext& ctx, metrics::MetricsContainer& metrics, int port);
    };
 
 }}

src/server/sparkd/jetfix/gateway/CoreGateway.cpp

@@ -449,7 +449,7 @@ void CoreGateway::sync_time(EventData const &ev) {
                me->session_info.reset();
                me->start_of_pause = datetime::now();
                LOG(Jetfix, Error,
-                   "Failed to connect to exchange gateway {} too many"
+                   "Failed to connect to exchange gateway {} too many "
                    "times. Giving up permanently. You must bounce "
                    "the server to recover.",
                    me->cfg.name());

src/server/sparkd/jetfix/store/FIXFileStore.cpp

@@ -127,17 +127,6 @@ std::string store_path (std::string base_path,
    return result.string ();
 }
 
-void create_current_simlink (std::string const &path)
-{
-   auto sym_link_path = boost::filesystem::path (path);
-   sym_link_path /= "..";
-   sym_link_path /= "current";
-   if (boost::filesystem::exists (sym_link_path)) 
-      boost::filesystem::remove (sym_link_path);
-   
-   boost::filesystem::create_symlink (boost::filesystem::path (path),sym_link_path);
-}
-
 std::map<uint32_t,InMsgBuffer>
 load_replay (config::JetFIX::Connection const &cfg,std::string const&path, uint32_t max_sn)
 {
@@ -227,8 +216,6 @@ FIXFileStore::FIXFileStore (config::JetFIX::Connection const&cfg,std::string bas
          }
       }
       me->fix_store.open (fix_log_p,std::ofstream::app);
-
-      create_current_simlink (path);
       me->writer_thread = std::thread (std::bind (&FIXFileStore::write,this));
    }
 }

src/server/sparkd/main.cpp

@@ -15,6 +15,7 @@
 #include "server/sparkd/cmds/AutoCert.h"
 #include "server/sparkd/cmds/QuickTest.h"
 #include "server/sparkd/cmds/Span.h"
+#include "server/sparkd/cmds/Web.h"
 
 int main(int argc, char **argv) {
    using namespace bts;
@@ -33,6 +34,7 @@ int main(int argc, char **argv) {
    REGISTER(QuickTest);
    REGISTER(AutoCert);
    REGISTER(Span);
+   REGISTER(Web);
    END_CMD;
 
    utils::finit_spark_lib();

src/server/sparkd/metrics/metrics.cpp

@@ -0,0 +1,77 @@
+#include "metrics.h"
+#include <sstream>
+#include <tbb/concurrent_vector.h>
+
+using std::vector;
+using std::shared_ptr;
+
+namespace bts {
+namespace metrics {
+
+Metric::Metric(const std::string& cname, const std::vector<Label>& clabels):
+   name(cname),
+   labels(clabels) {
+}
+
+std::string Metric::str() const {
+   std::stringstream ss;
+   print(ss);
+   return ss.str();
+}
+
+std::ostream& operator<<(std::ostream& os, const std::vector<Label>& labels) {
+   if (!labels.empty()) {
+      os << "{";
+      for (auto iter = labels.begin(); iter != labels.end(); ++iter) {
+         if (iter != labels.begin()) os << ",";
+         os << iter->name << "=\"" << iter->value << "\"";
+      }
+      os << "}";
+   }
+   return os;
+}
+
+const std::string double_to_string(double d) {
+   std::stringstream ss;
+   ss << d;
+   return ss.str();
+}
+
+const std::vector<Label> vector_concat(const std::vector<Label>& orig, const std::vector<Label>& extra) {
+   if (orig.empty()) return extra; 
+   std::vector<Label> result;
+   result.reserve(orig.size() + extra.size());
+   result.insert(result.end(), orig.begin(), orig.end());
+   result.insert(result.end(), extra.begin(), extra.end());
+   return result;
+}
+
+
+void LazyCounterMetric::print(std::ostream& os) const {
+   os << name 
+       << labels
+       << " " 
+       << _get()
+       << std::endl;
+}
+
+//
+// Histogram
+// -- collection of CounterMetrics?
+
+struct MetricCollection {
+   tbb::concurrent_vector<shared_ptr<Metric>> metrics;
+};
+
+MetricsContainer::MetricsContainer() : _me(std::make_unique<MetricCollection>()) {}
+MetricsContainer::~MetricsContainer() {}
+void MetricsContainer::print(std::ostream& out) const {
+   for (const auto& m : _me->metrics) {
+      m->print(out);
+   }
+}
+void MetricsContainer::publish(std::shared_ptr<Metric> m) {
+   _me->metrics.push_back(m);
+}
+
+}}//namespace

src/server/sparkd/metrics/metrics.h

@@ -0,0 +1,149 @@
+#pragma once
+
+#include <iostream>
+#include <string>
+#include <memory>
+#include <vector>
+#include <functional>
+#include <atomic>
+#include <sstream>
+
+namespace bts {
+namespace metrics {
+class MetricsContainer;
+
+struct Label {
+   std::string name;
+   std::string value;
+};
+std::ostream& operator<<(std::ostream& os, const std::vector<Label>& l);
+
+struct Metric {
+   const std::string name;
+   const std::vector<Label> labels;
+   Metric(const std::string&, const std::vector<Label>&);
+   virtual void print(std::ostream&) const = 0;
+   std::string str() const;
+   virtual ~Metric() {}
+};
+
+template <class T>
+struct CounterMetric : public Metric {
+   T _val;
+   CounterMetric(const std::string& name) : Metric(name, {}), _val() {}
+   CounterMetric(const std::string& name, const std::vector<Label>& labels) : Metric(name, labels), _val() {}
+   ~CounterMetric() {};
+   void print(std::ostream& out) const override {
+      out << name << labels << " " << _val << std::endl;
+   }
+   void increment() { _val++; }
+   void increment(T val) { _val += val; }
+   void decrement() { _val--; }
+   void decrement(T val) { _val -= val; }
+   void set(T val) { _val = val; }
+};
+
+template <class T>
+class AtomicCounterMetric : public Metric {
+   std::atomic<T> _val;
+   public:
+   AtomicCounterMetric(const std::string& name) : Metric(name, {}), _val() {}
+   AtomicCounterMetric(const std::string& name, const std::vector<Label>& labels) : Metric(name, labels), _val() {}
+   AtomicCounterMetric(const AtomicCounterMetric& o) : Metric(o.name, o.labels), _val(o._val.load()) { }
+   ~AtomicCounterMetric() {};
+   void print(std::ostream& out) const override {
+      out << name << labels << " " << _val << std::endl;
+   }
+   void increment() { _val++; }
+   void increment(T val) { _val += val; }
+   void decrement() { _val--; }
+   void decrement(T val) { _val -= val; }
+   void set(T val) { _val = val; }
+};
+
+class LazyCounterMetric : public Metric {
+   const std::function<long()> _get;
+   public:
+   LazyCounterMetric(
+      const std::string& name, 
+      const std::function<long()>& fn) : Metric(name, {}), _get(fn) {}
+   LazyCounterMetric(
+      const std::string& name, 
+      const std::function<long()>& fn,
+      const std::vector<Label>& labels) : Metric(name, labels), _get(fn) {}
+   void print(std::ostream&) const override;
+};
+
+const std::string double_to_string(double d);
+const std::vector<Label> vector_concat(const std::vector<Label>& orig, const std::vector<Label>& extra);
+
+template <size_t N, size_t... Is>
+std::array<CounterMetric<long>,N> mk_counters(const std::string& name,
+                                              const std::vector<Label>& labels,
+                                              const std::array<double,N>& in,
+                                              std::index_sequence<Is...>) {
+   return {{ CounterMetric<long>(name+"_bucket", vector_concat(labels, {{ "le", double_to_string(in[Is]) }}))... }};
+}
+template <size_t N, size_t... Is> 
+std::array<CounterMetric<long>,N> mk_counters(const std::string& name,
+                                              const std::vector<Label>& labels,
+                                              const std::array<double,N>& in) {
+   return mk_counters(name, labels, in, std::make_index_sequence<N>());
+}
+
+
+// Calculate the 90th percentile of response times
+// histogram_quantile(0.9, sum(rate(spark_metrics_query_duration_us_bucket[5m])) by (le))
+template <size_t N>
+class HistogramMetric : public Metric {
+   std::array<CounterMetric<long>,N> _buckets;
+   std::array<double,N> _bucket_values;
+   CounterMetric<double> _sum;
+   CounterMetric<long> _count;
+   public:
+   HistogramMetric(
+      const std::string& name,
+      const std::array<double,N>& buckets) 
+         : Metric(name,{}), _buckets(mk_counters(name, {}, buckets)), _bucket_values(buckets), _sum(name+"_sum"), _count(name+"_count") {}
+   HistogramMetric(
+      const std::string& name,
+      const std::array<double,N>& buckets,
+      const std::vector<Label>& labels) 
+         : Metric(name,labels), _buckets(mk_counters(name, labels, buckets)), _bucket_values(buckets), _sum(name+"_sum"), _count(name+"_count") {}
+   void print(std::ostream& os) const override {
+      for (size_t x = 0; x < N; ++x) {
+         _buckets[x].print(os);
+      }
+      os << name << "_bucket{le=\"+Inf\"";
+      for (auto iter = labels.begin(); iter != labels.end(); ++iter) {
+         os << "," << iter->name << "=\"" << iter->value << "\"";
+      }
+      os << "} " << _count._val << std::endl;
+      _sum.print(os);
+      _count.print(os);
+   }
+   void observe(double v) {
+      for (size_t x = 0; x < N; ++x) {
+         if (v <= _bucket_values[x]) {
+            _buckets[x].increment();
+         }
+      }
+      _sum.increment(v);
+      _count.increment();
+   }
+};
+
+struct MetricCollection;
+class MetricsContainer {
+   std::unique_ptr<MetricCollection> _me;
+   public:
+   MetricsContainer();
+   ~MetricsContainer();
+
+   void print(std::ostream&) const;
+   void publish(std::shared_ptr<Metric>);
+};
+
+
+}}//namespace
+

src/server/sparkd/metrics/system_metrics.cpp

@@ -0,0 +1,86 @@
+#include "system_metrics.h"
+
+#include "lib/config/config.h"
+#include "lib/config/protos/system.pb.h"
+
+#include <memory>
+#include <functional>
+#include <unistd.h>
+#include <fstream>
+
+namespace bts {
+namespace metrics {
+
+struct ProcessStats {
+   int pid;
+   std::string comm, state, ppid, pgrp, session, tty_nr;
+   std::string tpgid, flags, minflt, cminflt, majflt, cmajflt;
+   // utime - user time
+   // stime - kernel time
+   // cutime - cumulative user time
+   // cstime - cumulative system time
+   long utime, stime, cutime, cstime, priority, nice;
+   std::string O, itrealvalue, starttime;
+   // virtual size
+   uint64_t vsize;
+   // resident set
+   int64_t  rss;
+};
+
+void read_stats(ProcessStats& stats) {
+   std::ifstream stat_stream("/proc/self/stat", std::ios_base::in);
+   stat_stream 
+            >> stats.pid >> stats.comm >> stats.state >> stats.ppid >> stats.pgrp >> stats.session >> stats.tty_nr
+            >> stats.tpgid >> stats.flags >> stats.minflt >> stats.cminflt >> stats.majflt >> stats.cmajflt
+            >> stats.utime >> stats.stime >> stats.cutime >> stats.cstime >> stats.priority >> stats.nice
+            >> stats.O >> stats.itrealvalue >> stats.starttime >> stats.vsize >> stats.rss;
+   stat_stream.close();
+}
+
+void process_mem_usage(double& vm, double& resident) {
+   using std::ios_base;
+   using std::ifstream;
+   vm = resident = 0.0;
+
+   ProcessStats stats;
+   read_stats(stats);
+
+   long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024;
+   vm = stats.vsize / 1024.0;
+   resident = stats.rss * page_size_kb;
+}
+
+
+void publish_system_metrics(MetricsContainer& stats) {
+   auto version = std::make_shared<CounterMetric<int>>("spark_version", std::vector<Label> ({{"version", config::get<config::System>().version()}}));
+   version->set(1);
+   stats.publish(version);
+
+   // publish system cpu use
+
+   // publish sparkd cpu use and thread use (/proc/self/task/*)
+
+
+   //
+   // sparkd memory use
+   // Note:  It would be nice to have 2 metrics work on the same 'read' call
+   stats.publish(
+      std::make_shared<LazyCounterMetric>("spark_memory_res_kb", []() {
+         double vm,res;
+         process_mem_usage(vm,res);
+         return res;
+      }));
+
+   stats.publish(
+      std::make_shared<LazyCounterMetric>("spark_memory_vm_kb", []() {
+         double vm,res;
+         process_mem_usage(vm,res);
+         return vm;
+      }));
+
+
+}
+
+
+
+}}

src/server/sparkd/metrics/system_metrics.h

@@ -0,0 +1,9 @@
+#pragma once
+#include "metrics.h"
+
+namespace bts {
+namespace metrics {
+
+void publish_system_metrics(MetricsContainer&);
+
+}}

src/server/sparkd/orderman/orderman_utils/fix_order_codec.h

@@ -14,7 +14,7 @@ namespace api {class Security;}
 namespace orderman {
 
 constexpr int clordid_sz() {return 20;}
-constexpr int price_sz() {return 15;}
+constexpr int price_sz() {return 17;}
 constexpr int quantity_sz() {return 6;}
 
 inline void encode_clorid(OrderKey ok, SecurityKey sk,std::string *out) {

src/server/sparkd/orderman/orders/NativeOrder.cpp

@@ -282,7 +282,7 @@ void NativeOrder::on_activate(Price p, Quantity q,
       me->state.set_filled_quantity(0);
       me->state.set_last_price(0);
       me->state.set_avg_price(0);
-      
+
       me->session_services = &me->services->get_session_services(
           get_native_state(me->state).routing_record().session());
       me->price_encoding_precision =
@@ -303,8 +303,7 @@ void NativeOrder::on_activate(Price p, Quantity q,
           AccountID(me->definition.account_id()));
       if (!lqr.ok) {
          LOG(OrderMan, Warn, "Unauthorized order attempted. User: {} on "
-                             "Account: {}. Msg: {} Order: {} "
-                             "Dropping order {}",
+             "Account: {}. Msg: {} Order: {}. Dropping order",
              me->definition.user(), me->definition.account(), lqr.error_msg,
              key().value());
 
@@ -357,8 +356,8 @@ void NativeOrder::on_activate(Price p, Quantity q,
       case CustomOrderDefinition::kStopLimit:
          me->create_price_bm = encoding.overwrite(
              me->create_price_bm,
-             securities::encode_exchange_price(*me->sec, p).value(), 7,
-             price_sz());
+             securities::encode_exchange_price(*me->sec, p).value(),
+             me->price_encoding_precision, price_sz());
          break;
       case CustomOrderDefinition::kStop:
       case CustomOrderDefinition::kMarket:
@@ -484,11 +483,12 @@ void NativeOrder::on_update(Price p, Quantity q, TickToTradeCorID cid) {
 
       // are we under the messaging threshold?
       if (!me->session_services->governor().can_send()) {
-         LOG(OrderMan, Warn, "Cannot update order {} because of messaging limits. ",
+         LOG(OrderMan, Warn,
+             "Cannot update order {} because of messaging limits. ",
              key().value());
          auto msg = "Messaging threshold breached";
          if (parent_key() == null_order_key())
-            transition(me->state.status(),msg);
+            transition(me->state.status(), msg);
          else {
             me->state.set_last_msg(msg);
             on_cancel(cid);
@@ -500,8 +500,8 @@ void NativeOrder::on_update(Price p, Quantity q, TickToTradeCorID cid) {
       auto &encoding = me->update_encoding;
       me->update_price_bm = encoding.overwrite(
           me->update_price_bm,
-          securities::encode_exchange_price(*me->sec, p).value(), 7,
-          price_sz());
+          securities::encode_exchange_price(*me->sec, p).value(),
+          me->price_encoding_precision, price_sz());
       me->update_quantity_bm =
           encoding.overwrite(me->update_quantity_bm, q.value(), quantity_sz());
 
@@ -829,8 +829,10 @@ void NativeOrder::init_fix() {
       cancel_encoding.add<fix::OrdType>(NATIVE_ORDER_TYPE_LIMIT);
 
       // price
-      me->create_price_bm = create_encoding.add<fix::Price>(0, 7, price_sz());
-      me->update_price_bm = update_encoding.add<fix::Price>(0, 7, price_sz());
+      me->create_price_bm = create_encoding.add<fix::Price>(
+          0, me->price_encoding_precision, price_sz());
+      me->update_price_bm = update_encoding.add<fix::Price>(
+          0, me->price_encoding_precision, price_sz());
       break;
 
    case CustomOrderDefinition::kStop: {
@@ -862,8 +864,10 @@ void NativeOrder::init_fix() {
       update_encoding.add<fix::StopPx>(encoded_stop_px, -1, -1);
 
       // price
-      me->create_price_bm = create_encoding.add<fix::Price>(0, 7, price_sz());
-      me->update_price_bm = update_encoding.add<fix::Price>(0, 7, price_sz());
+      me->create_price_bm = create_encoding.add<fix::Price>(
+          0, me->price_encoding_precision, price_sz());
+      me->update_price_bm = update_encoding.add<fix::Price>(
+          0, me->price_encoding_precision, price_sz());
       break;
    }
    default:

src/server/sparkd/orderman/orders/spreader_orders/SpreaderOrder.h

@@ -133,10 +133,11 @@ private:
    void _transform_to_END(std::string* m = nullptr);
    // swtches to the state passed in the arg
    void _switch_state (ISpreaderStateWorker* new_state);
-   // this means on_init_new has been called and we have the
-   // the structure built
    // misc ////////////////////
    ////////////////////////////
+   // this means on_init_new or on_init_recovered
+   // has been called and we have the bookkeeping
+   // data built
    bool _inited () const {
       return _bookkeeping_data_ptr != nullptr;
    }

src/server/sparkd/orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp

@@ -40,13 +40,16 @@ void SpreaderStartStateWorker::on_init_recovered(const bts::api::orderman::Order
                                                  const SpreaderPreActivationOrderServices& spreader_pao_services)
 {
    bts_assert(!_spreader->_inited());
-   // we are not handling this yet, so lets send it directly to END
+   // we are not handling recovery, so we go directly to the end state
+   // and the status of the order to cancelled
+   bts::api::orderman::OrderRecord order_record_copy = order_record;   
+   order_record_copy.mutable_state()->set_status(bts::api::orderman::OrderStatus::ORDER_STATUS_CANCELED);
    _spreader->_bookkeeping_data_ptr = std::make_unique<SpreaderBookkeepingData>(*_spreader,
-                                                                                order_record,
+                                                                                order_record_copy,
                                                                                 spreader_pao_services,
                                                                                 std::vector<LegPropertyData>());
    LOG(OrderMan, Warn, "SpreaderStartStateWorker.on_init_recovered key: {} , going to END", _spreader->key().value());
-   std::string msg = "Recovered spreader orders and not supported";
+   std::string msg = "Recovered spreader orders (from last session) are cancelled";
    _spreader->_transform_to_END (&msg);
 }
 void SpreaderStartStateWorker::on_activate(Price price, Quantity quantity, SpreaderOrderServices& order_services){

src/server/sparkd/orderman/perf_analysis/load.sh

@@ -4,7 +4,12 @@ sqlite3 trace_data/trace.sqlite <<EOF
 
 .separator ","
 
-create table trace (seq integer, ts integer, hts text, msg text, cpu integer, tid integer,arg1 integer);
-create index trace_seq_idx on trace(seq);
-.import trace_data/parsed_trace.txt trace
+create table raw_trace (seq integer, ts integer, hts text, msg text, cpu integer, tid integer,arg1 integer);
+.import trace_data/parsed_trace.txt raw_trace
+create table trace as
+select
+   (seq % 65535) seq16, seq, ts, hts, msg, cpu, tid, arg1
+from
+   raw_trace;
+create index trace_seq_idx on trace(seq16);
 EOF

src/server/sparkd/persistence/PersistenceServiceServer.cpp

@@ -88,7 +88,7 @@ void PersistenceServiceServer::retrieve(const RetrieveRequest& request,
 
    auto it = me->data.find(request.ns());
    if (it == me->data.end()) {
-      LOG(persist,Error, "Client tried to retrieve specific rows from unknown namespace {}/{}",
+      LOG(persist,Warn, "Client tried to retrieve specific rows from unknown namespace {}/{}",
           me->db,request.ns());
       response.set_result ( RetrieveResponse::NO_SUCH_NS);
       reply(response);
@@ -108,7 +108,7 @@ void PersistenceServiceServer::retrieve(const RetrieveRequest& request,
       }
       auto it = known_rows.find(k);
       if (it == known_rows.end()) {
-         LOG(persist,Error, "Client tried to retrieve unknown row with key {} in {}/{}", k,
+         LOG(persist,Warn, "Client tried to retrieve unknown row with key {} in {}/{}", k,
              me->db,request.ns());
          response.set_result ( RetrieveResponse::DUPLICATE_KEY);
          reply(response);
@@ -132,7 +132,7 @@ void PersistenceServiceServer::retrieve_all(const RetrieveAllRequest& request,
    RetrieveResponse response;
    
    if (request.ns().empty()) {
-      LOG(persist,Error, "Client used empty namespace while trying to retrieve rows in database {}",
+      LOG(persist,Warn, "Client used empty namespace while trying to retrieve rows in database {}",
           me->db);
       response.set_result ( RetrieveResponse::EMPTY_NS);
       reply(response);
@@ -171,7 +171,7 @@ void PersistenceServiceServer::remove(const RemoveRequest& request,
    
    auto it = me->data.find(request.ns());
    if (it == me->data.end()) {
-      LOG(persist,Error, "Client used unknown namespace {} while trying to remove rows in database {}",
+      LOG(persist,Warn, "Client used unknown namespace {} while trying to remove rows in database {}",
           request.ns(),
           me->db);
       response.set_result ( RemoveResponse::NO_SUCH_NS);
@@ -184,7 +184,7 @@ void PersistenceServiceServer::remove(const RemoveRequest& request,
    for (auto &k : request.key()) {
 
       if (keys.count(k)) {
-         LOG(persist,Error, "Client used duplicate key {} while trying to remove rows in {}/{}",
+         LOG(persist,Warn, "Client used duplicate key {} while trying to remove rows in {}/{}",
              k,me->db,request.ns());
          response.set_result ( RemoveResponse::DUPLICATE_KEY);
          reply(response);
@@ -194,7 +194,7 @@ void PersistenceServiceServer::remove(const RemoveRequest& request,
       keys.insert(k);
 
       if (k.empty()) {
-         LOG(persist,Error, "Client used empty key while trying to remove rows in {}/{}",
+         LOG(persist,Warn, "Client used empty key while trying to remove rows in {}/{}",
              me->db,request.ns());
          response.set_result ( RemoveResponse::EMPTY_KEY);
          reply(response);
@@ -202,7 +202,7 @@ void PersistenceServiceServer::remove(const RemoveRequest& request,
       }
 
       if (ns_rec.data.find(k) == ns_rec.data.end()) {
-         LOG(persist,Error, "Client tried to remove unknown row with key {} in {}/{}",
+         LOG(persist,Info, "Client tried to remove unknown row with key {} in {}/{}",
              k, me->db, request.ns());
          response.set_result ( RemoveResponse::NO_SUCH_KEY);
          reply(response);

src/server/sparkd/user/tests/admin_service_tests.cpp

@@ -268,4 +268,71 @@ TEST_CASE ("Bootstrapping", "[admin_service]")
       REQUIRE(account.connections().exchange_connection_names_size()==2);
    }
 }
+TEST_CASE("Sorting Limit","[admin_service]"){
+   bts::api::admin::AdminData admin_data;
+   bts::api::admin::AdminServiceCreateAccountResult account_result;
+   auto account = AdminServiceUtils::create_account(admin_data,
+                                                    0, 
+                                                    std::string("A"), 
+                                                    std::string("description"), 
+                                                    true,
+                                                    account_result);
+   auto l1 = account->mutable_security_clip_limits()->add_limit();
+   l1->mutable_security()->set_exchange(api::Venue::VENUE_GLBX);
+   l1->mutable_security()->set_security_type(api::SecurityType::SECURITYTYPE_FUTURE);
+   l1->mutable_security()->set_security_symbol("*");
+   l1->mutable_security()->set_security_id(0);
+   l1->set_clip_size(100);
+   auto l2 = account->mutable_security_clip_limits()->add_limit();
+   l2->mutable_security()->set_exchange(api::Venue::VENUE_GLBX);
+   l2->mutable_security()->set_security_type(api::SecurityType::SECURITYTYPE_FUTURE);
+   l2->mutable_security()->set_security_symbol("SI");
+   l2->mutable_security()->set_security_id(0);
+   l2->set_clip_size(101);
+   auto l3 = account->mutable_security_clip_limits()->add_limit();
+   l3->mutable_security()->set_exchange(api::Venue::VENUE_GLBX);
+   l3->mutable_security()->set_security_type(api::SecurityType::SECURITYTYPE_FUTURE);
+   l3->mutable_security()->set_security_symbol("CL");
+   l3->mutable_security()->set_security_id(0);
+   l3->set_clip_size(102);
+   auto l4 = account->mutable_security_clip_limits()->add_limit();
+   l4->mutable_security()->set_exchange(api::Venue::VENUE_GLBX);
+   l4->mutable_security()->set_security_type(api::SecurityType::SECURITYTYPE_FUTURE);
+   l4->mutable_security()->set_security_symbol("ZC");
+   l4->mutable_security()->set_security_id(10);
+   l4->set_clip_size(103);
+   auto l5 = account->mutable_security_clip_limits()->add_limit();
+   l5->mutable_security()->set_exchange(api::Venue::VENUE_GLBX);
+   l5->mutable_security()->set_security_type(api::SecurityType::SECURITYTYPE_FUTURE);
+   l5->mutable_security()->set_security_symbol("ZN");
+   l5->mutable_security()->set_security_id(5);
+   l5->set_clip_size(104);
+   auto l6 = account->mutable_security_clip_limits()->add_limit();
+   l6->mutable_security()->set_exchange(api::Venue::VENUE_GLBX);
+   l6->mutable_security()->set_security_type(api::SecurityType::SECURITYTYPE_SPREAD);
+   l6->mutable_security()->set_security_symbol("SI");
+   l6->mutable_security()->set_security_id(0);
+   l6->set_clip_size(105);
+   // tests
+   auto sorted_admin_data = AdminServiceUtils::sort_limits(admin_data);
+   const auto& r_account = sorted_admin_data.accounts(0);
+   const auto& r_l1 = r_account.security_clip_limits().limit(0);   
+   REQUIRE(r_l1.security().security_id()==5);
+   REQUIRE(r_l1.clip_size()==104);
+   const auto& r_l2 = r_account.security_clip_limits().limit(1);
+   REQUIRE(r_l2.security().security_id()==10);
+   REQUIRE(r_l2.clip_size()==103);
+   const auto& r_l3 = r_account.security_clip_limits().limit(2);
+   REQUIRE(r_l3.security().security_symbol()=="CL");
+   REQUIRE(r_l3.clip_size()==102);
+   const auto& r_l4 = r_account.security_clip_limits().limit(3);
+   REQUIRE(r_l4.security().security_symbol()=="SI");
+   REQUIRE(r_l4.clip_size()==101);
+   const auto& r_l5 = r_account.security_clip_limits().limit(4);
+   REQUIRE(r_l5.security().security_symbol()=="SI");
+   REQUIRE(r_l5.clip_size()==105);
+   const auto& r_l6 = r_account.security_clip_limits().limit(5);
+   REQUIRE(r_l6.security().security_symbol()=="*");
+   REQUIRE(r_l6.clip_size()==100);
+}
 }}

src/web/ui/src/Accounts/View.fs

@@ -207,7 +207,11 @@ let clipLimitsView model dispatch =
   let addRow _ = Array.append model.scratch.securityClipLimits.limit [| emptyLimit |] |> (ClipLimits >> dispatch)
   div []
     [
-      p [ClassName "is-size-7 has-text-grey-mid"][ str "Clip limits can be set for one security, for all securities in a product (by selecting \"All\" for securities), or for all products within a security type (by selecting \"All\" for both symbol and security)"]
+      div [ ClassName "columns" ][     
+            div [ ClassName "column is-size-7 has-text-grey-mid" ]
+              [str "Clip limits can be set for one security, for all securities in a product (by selecting \"All\" for securities), or for all products within a security type (by selecting \"All\" for both symbol and security)"]
+            div [ ClassName "column" ][]
+      ]
       TableControl.createTableWithStyle (Some headerRow) rws "table-bordered" [Width "100%"] 
       span [ClassName "is-pulled-right"] [ Button.button "Add" "contrast-button" "plus" Button.Enabled addRow ]
       span [ClassName "is-clearfix"][]
@@ -311,7 +315,7 @@ let productLimits model dispatch =
     product = {
                 exchange = bts.api.Venue.GLBX
                 securityType = bts.api.SecurityType.FUTURE
-                securitySymbol = model.AllValueDisplay
+                securitySymbol = model.AllValue
               }
     longPositionLimit = 0u
     shortPositionLimit = 0u

src/web/ui/src/Login/Login.fs

@@ -43,7 +43,10 @@ let update msg model =
         model, Cmd.Empty
     | Submit -> 
         printfn "Login with %s %s" model.user model.password
-        model, loginMsg (model.user.ToLower()) model.password
+        let isAdmin = ((model.user.ToLower().Contains "bts") || (model.user.ToLower().Contains "admin"))
+        match isAdmin with
+        | true -> model, loginMsg (model.user.ToLower()) model.password
+        | false -> { model with error = "Only administrators are allowed to login" }, Cmd.Empty
     | LoginFailure msg ->
         { model with error = msg }, Cmd.Empty
     | LoginSuccess _ -> failwith "Handled externally. Should not get here."

testing/cycle_info.txt

@@ -4,10 +4,10 @@ testers=(
 #'pedro.pinto'
 # 'eric.harding'
  'jake.norbut'
-# 'michael.harding'
+ 'michael.harding'
  'karl.ellefsen'
  'pete.fisher'
-# 'taha.afzal'
+ 'taha.afzal'
  'john.colvin'
  'kevin.darby'
  'brian.muhr'

testing/docs/spreader_common_setup_1.txt

@@ -2,7 +2,7 @@ This is not a test, but common setup instructions that will be used for a few te
 
 0. This needs sim market data **not** to be merged with replay (or cert), so we can create our own markets.
 
-Add or update this line in your spark.conf (probably in ~/www/config)
+Add or update this line in your spark.conf (probably in ~/sparkdata/config)
 ```
 ExchangeSim {value {merge_with_md:false}}
 ```

testing/manual/admin/clip_limits.txt

@@ -1,24 +1,24 @@
 This test verifies that clip limits are followed, and rule precedence works correctly
 
-1. Pick a future, lets say ESU8.
+1. Pick a future, lets say SIZ1.
 
 2. In the Spark Admin, go to accounts and select the default account for your user (let say ACC A1).
-Go to Position Limits and enter large values (for both security and product level), so that
-position limits are bypassed.
+Go to Position Limits and enter large values for the Product Limits and remove all rows for the Security Limits
 
 4. Now go to Clip Limits and delete all entries.
 
-5. Enter an order and it should fail with a message that should tell exactly why.
+5. Enter an order and it should fail with a message that should tell exactly why. Make sure the orderbook and the
+ladder show the message.
 
-6. Now enter a rule for ESU8 (in the first version we can not specify securities, so go as far as the symbol (ES)).
+6. Now enter a Clip Limit for SIZ1.
 Enter a Clip Size of 10. Make sure orders > 10 are rejected and < 10 are allowed.
 
-7. Now enter a non ES future order, lets say SI. Enter a 1 lot and it should be rejected.
+7. Now enter an order for another future, lets say SIN1. Enter a 1 lot and it should be rejected.
 
-8. Enter another rule, GLBX, FUTURES, * and enter a clip limit of 100.
+8. Enter another rule, GLBX, FUTURES, All, All and enter a clip limit of 100.
 
-9. Now try the SI order again. Verify it does the right thing for the clip limit you entered in 8.
+9. Now try the SIN1 order again. Verify it does the right thing for the clip limit you entered in 8.
 
-10. Verify that ESU8 is still controlled by a clip size of 10.
+10. Verify that SIZ1 is still controlled by a clip size of 10.
 
 11. Enter a future spread order. It should be rejected since we don't have any clip limits for futures.

testing/manual/admin/security_position_limits.txt

@@ -1,5 +1,5 @@
 This test verifies that security position limits are enforced correctly. (Position limits are
-enforced at both the security and account level)
+enforced (per account) at the product level and optionally at the security level)
 
 0. Clear up your orders and fills so we don't have any lingering positions
 (run rm -r orders and rm -r fills from /sparkdata/data/databases). Restart your sparkd
@@ -14,7 +14,7 @@ clip limits don't interfere with this test.
 4. Enter an ESU8 order. It should be rejected. Make sure the rejection message makes sense
 
 5. Now enter a product level rule for GLBX, Future, ES, 1,000,000, 1,000,000 (to bypass it),
-and a security level rule GLBX, Future, ES, 50, 100
+and a security level rule GLBX, Future, ES, ESU8, 50, 100.
 
 6. Enter a 10 lot. This brings our 'position' to 10 (where 'positions' = fills + working orders).
 Enter more orders to get it to 50. Now try to get past 50 and you should be stopped with a clear message.
@@ -24,12 +24,13 @@ don't cancel these yet.
 the way to 150 (net 100 short), but not beyond that. Verify that.
 
 8. Cancel all your buy orders. Cancel some of your sell orders to get the net short position to 50.
-Start selling again, and make sure you are stopped at 100.
+Start selling again, and make sure you are stopped at 100. Cancel all ESU8 orders.
 
 9. Try another future, lets say SIU8. Since we don't have a position rule for it, it should fail.
 
 10. Enter a new rule for the product (and use All for the symbol) and enter 10 for the long and short limits.
 
-11. Now enter a 1 lot SIU8 again. Keep buying more and make sure you are stopped at the long limit of 10.
+11. Now enter a 1 lot SIU8 again. Keep buying more and make sure you are stopped at the long limit of 10. Keep
+these working
 
-12. Start selling and make sure you are stopped at the short limit.
+12. Start selling and make sure you are stopped at the short limit (remember to account the buys from 11)

testing/manual/admin/spread_position_limits.txt

@@ -1,29 +1,36 @@
 This test verifies the following 2 things
-i. spread legs fills (and working orders) are applied toward enforcing
+i. future and spread product limits can co-exist and work correctly
+ii. spread legs fills (and working orders) are applied toward enforcing
 the position limits for the legs
-ii. leg limits are checked when entering spread orders
+iii. leg limits are checked when entering spread orders
 
 i:
 
-1. Pick a 2 legged spread (lets say SIU8-SIZ8). Set the clip limits high so that they don't interfere.
-Set the product position limits high as well so they don't interfere either.
+1. Pick a 2 legged spread (lets say SIU8-SIZ8). Set the clip limits high for both spreads and futures
+so that they don't interfere. Remove all Security Position limits.
 
-2. For the security limits (which is what we will be testing), we want Max Long and Max Short of 20 and 30
-for the SI futures and 100 and 110 for the SI spreads respectively.
+2. For the Product Limits, set Max Long and Max Short of 100 and 110
+for the SI futures and 20 and 30 for the SI spreads respectively.
 
+3. Verify that the limits set in 2 work for futures(remember working orders count toward limit enforcement so you dont need
+fills). Cancel all the orders.
 
-3. Enter a 15 lot buy order for the spread. This 15 should count toward the leg limits. The leg long limit
-is 20. So enter a 5 lot buy for SIU8 which should be allowed, but a subsequent 1 lot will violate the 20 long
-and should be rejected.
+4. Verify that the limits set in 2. work for spreads. Cancel all the orders.
 
-4. Now enter a 15 lot sell for SIZ8 which should be allowed but a subsequent one lot should not be allowed, as it
-will violate the short limit of 30.
+5. Make these product limits 1,000,000 to make them irrelevant
 
 ii:
 
-5. Cancel all orders
+6. For security limits enter long 10 and short 20 for SIU8 and SIZ8 futures respectively and 50 and 60 for
+the SIU8-SIZ8 spread.
 
-6. Enter a 15 lot buy for SIU8. Any spread buy order of > 5 should be rejected, since this will violate the 20 long limit.
+7. Enter a 5 lot buy for SIU8-SIZ8. This 5 should count toward SIU8 long limit (which is 10). Enter
+a 5 lot SIU8 buy, which should be allowed but a subsequent 1 lot buy should be rejected. Similiary, the
+spreader order should count toward the SIZ8 short limit of 20. Enter a 15 lot sell on SIZ8, and a subsequent
+1 lot should be rejected.
 
-8. Enter a 20 sell for SIZ8. Any spread buy order of > 10 should be rejected since it will
-violate the 30 short limit.
+8. Cancel all orders
+
+iii:
+
+Enter a 11 lot spread order. It should be rejected because of the 10 lot limit on SIU8.

testing/manual/algos/join.txt

@@ -0,0 +1,35 @@
+
+- Open up a ladder for a liquid future
+- Make sure you have limits to trade this future
+- Select Join order from the menu
+Label 1) Set join thresh to absolute 10
+Label 2) Set initial tick offset to 2
+
+
+The goal of the join order is to watch the opposite-inside market and join when the market *flips and 
+has at least threshold size
+
+*flip is when the best offer becomes the best bid, or the market trades away from you (the line moves up in the gui)
+
+Label 3) Put an order in to way down in the bid side of the book
+- Because of the parameter in label 2, the order should instead join two levels below the best bid
+- Watch the opposite inside market, if it flips make sure the order moves itself to join the new best bid (former offer)
+
+
+Goto Label 2 and set tick offset to 0
+   {
+      Orders should then go in where you put them 
+   }
+Goto Label l and test the ratio threshold
+   {
+      - Use a threshold of at least 1.0      
+   }
+Goto Label 3 and try a sell order   
+   {
+   }
+
+Now do this test again with "Enable Hedge" checked and hedge ticks set to 1, hedge type set to limit
+
+This should modify the behavior such that the algorithm will send a limit order on the opposite 
+side of the market, one tick away from fill price in a quantity equivalent to the amount of the fill   
+

testing/manual/algos/lean.txt

@@ -0,0 +1,28 @@
+
+- Open up a ladder for a liquid future
+- Make sure you have limits to trade this future
+- Select Lean order from the menu
+Label 1) Set cross thresh to absolute 10
+
+The goal of the lean order is to watch the opposite-inside market size and *cross when that size goes below the threshold 
+(this also implies a cross if that size goes to zero and the market trades away from you)
+(if the market trades toward you, we just start watching the new ask)
+
+*cross means take an order working on the bid and 'cross' by buying into the asks.  This is also called 'paying up'
+
+- Open up order book and fills
+Label 2) Put an order in to Join the best bid
+- Watch the opposite inside market, if it dips below the threshold (or trades away from you) make sure the order moves itself to cross
+- Verify that the order filled at the offer price
+- If the opposite inside market does not dip below threshhold, you should expect to be eventually filled at the bid price
+- Verify that these fills come in at the proper price, crosses at the offer, working fills at the bid
+
+
+Goto Label 2 and join the ask instead of the bid
+   {
+   }
+Goto Label l and test the ratio threshold
+   {
+      - Use a threshold of at least 1.0
+      - Test with a larger order (say 100) to trigger crosses more quickly and across different price levels
+   }

testing/manual/algos/tick.txt

@@ -0,0 +1,32 @@
+
+- Open up a ladder for a liquid future
+- Make sure you have limits to trade this future
+- Select Tick order from the menu
+Label 1) Set cross thresh to absolute 10
+Label 2) Set initial tick offset to 1
+
+
+The goal of the tick order is to watch the opposite-inside market size and *cross when that size goes below the threshold 
+(this also implies a cross if that size goes to zero and the market trades away from you)
+(if the market trades toward you, the algo will 'detach' or become a regular limit order by default)
+
+*cross means take an order working on the bid and 'cross' by buying into the asks.  This is also called 'paying up'
+
+- Open up order book and fills
+Label 3) Put an order in to way down in the bid side of the book
+- Because of the parameter in label 2, the order should instead join at the best bid
+- Watch the opposite inside market, if it dips below the threshold or the market trades away make sure the order moves itself to cross
+- Verify that the order filled at the offer price
+- If the opposite inside market does not dip below threshhold, you should expect to be eventually filled at the bid price
+- If the opposite inside market trades toward you, you should see the algo detach and the limit be left working
+- Verify that these fills come in at the proper price, crosses at the offer, working fills at the bid
+
+
+Goto Label 2 and join the ask instead of the bid
+   {
+   }
+Goto Label l and test the ratio threshold
+   {
+      - Use a threshold of at least 1.0
+      - Test with a larger order (say 100) to trigger crosses more quickly and across different price levels
+   }

testing/manual/algos/top.txt

@@ -0,0 +1,16 @@
+
+- Open up a ladder for a liquid future
+- Make sure you have limits to trade this future
+- Select Top order from the menu
+Label 1) Set top thresh to absolute 50, start with a bid at the best, 10 lot
+
+The goal of the top order is to not trade by yourself. 
+IF the order is on the market (current bid price) AND the quantity on that bid price is below top 
+thresh then cancel the order else do nothing
+
+
+- Verify that orders immediately cancel if you put them in and inside market with less then the threshold
+- Verify that orders cancel if they end up working at an inside market with less than threshold size
+
+Goto Label l and test a sell
+   

testing/manual/forms2/PAndL.txt

@@ -12,9 +12,12 @@ clear your fills database:
 
 and you may also want to clear your orders for a clean slate:
 	rm ~/sparkdata/data/databases/orders/*
-	
-Throughout the test we'll refert to accounts BTS1 and BTS2. These specific accouts aren't necessary, but rather that we have
-fills in multiple distinct accounts.	
+
+If your Ladder is not centered on at 2500 after entering your first order you may need to clear other databases
+  rm ~/sparkdata/data/databases/*
+
+Throughout the test we'll refer to accounts BTS1 and BTS2. These specific accounts aren't necessary, but rather that we have
+fills in multiple distinct accounts.
 
 Bring up the pnl form (empty).
 Bring up a ladder form. It should be completely empty since we're controlling the md.  For this exercise we'll be using ES.
@@ -58,14 +61,14 @@ The simulator might not work for orders left up overnight
 	If your ladder shows no working orders, place a buy order with price 2500 and qty 100
 Sell a 1 lot as the current bid
 	The SOD should still show -1
-	The Buy and Sell Qty should show 1 (we crossed ourself)
+	The Buy and Sell Qty should show 1 (we crossed ourselves)
 	The net position will still say -1
 	Now switch to BTS2
 Sell a 5 lot at the current bid
 	The SOD should still show -1
-	BTS1 Buy qty should show 5
+	BTS1 Buy qty should show 6 (1 (crossed) + 5 (BTS2))
 	BTS1 Sell Qty should show 1
-	Net Pos should show 3 (-1 (SOD) + 5 (Buys) -1 (Sells))
+	Net Pos should show 4 (-1 (SOD) + 6 (Buys) -1 (Sells))
 	BTS1 PnL should show 12.5
 
 Test sorting and filtering

testing/post_to_jira.sh

@@ -22,10 +22,13 @@ function post_issue()
 source cycle_info.txt
 
 # invoke with ./post_to_jira.sh <allocation_file> <ppinto_password>
+# if authentication fails try to log out / log back in to jira using 
+# their web page
 for assignment in $( cat $1); do
     echo $assignment
     IFS=: read -a fields <<<"$assignment"
     tester=${fields[0]}
     test=${fields[1]}
     post_issue $2
+    sleep 5
 done

tools/linux_control/build_in_docker.sh

@@ -1,3 +1,7 @@
 #!/bin/bash
-cid=`docker ps | grep bluetradingsystems/linux_docker | awk '{print $1}'`
-docker exec $cid /bin/bash -c "cd build/clang_debug/ && ninja"
+if [ -f /.dockerenv ]; then
+    ninja
+else
+    cid=`docker ps | grep bluetradingsystems/linux_docker | awk '{print $1}'`
+    docker exec $cid /bin/bash -c "cd build/clang_debug/ && ninja"
+fi

tools/setup_builds.sh

@@ -6,7 +6,7 @@ cd $DIR/../build/
 rm -rf clang_debug 
 mkdir clang_debug
 cd clang_debug
-cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER=`realpath ../../tools/linux_control/rtags.sh` ../..
+cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DCMAKE_CXX_COMPILER=`realpath ../../tools/linux_control/rtags.sh` ../..
 cd ..
 
 rm -rf ninja_debug
@@ -18,7 +18,7 @@ cd ..
 rm -rf ninja_release
 mkdir ninja_release
 cd ninja_release
-cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DCMAKE_CXX_COMPILER=g++ ../..
+cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=g++ ../..
 cd ..