Merged in SPARK-1376 (pull request #42)

SPARK-1176 Named Window

Approved-by: Michael Harding <michael.harding@bluetradesys.com>
Approved-by: Jake Norbut <jake.norbut@bluetradesys.com>

src/client/spark/Images/Icons.cs

@@ -47,7 +47,7 @@ namespace mbts.ui
       //public static ImageSource Ticket { get { return load_image("Ticket.png"); } } // replaced with AddOrders
       public static ImageSource Date { get { return load_image("Date.png"); } }
 
-      public static ImageSource Cube3d { get { return (DrawingImage)_resources["Cube"];  /*return load_image("Cube-Molecule.png");*/ } }
+      public static ImageSource Cube3d { get { return (DrawingImage)_resources["Cube"]; } }
       public static ImageSource Algo { get { return Cube3d; } }
       public static ImageSource Quote { get { return Math; } }
       public static ImageSource Math { get { return (DrawingImage)_resources["Surface"]; } }
@@ -62,9 +62,7 @@ namespace mbts.ui
 #pragma warning restore CS0618
 
       // Vector icons
-      public static ImageSource Favorites { get { return Favorites_Gold; } }
-      public static ImageSource Favorites_Gold { get { return (DrawingImage)_resources["FavoritesGold"]; } }
-      public static ImageSource Favorites_White { get { return (DrawingImage)_resources["FavoritesWhite"]; } }
+      public static ImageSource Bookmarks { get { return (DrawingImage)_resources["Bookmarks"]; } }
       public static ImageSource Window { get { return (DrawingImage)_resources["Window"]; } }
       public static ImageSource ActiveWindow { get { return (DrawingImage)_resources["ActiveWindow"]; } }
       public static ImageSource Trash { get { return (DrawingImage)_resources["Trash"]; } }

src/client/spark/Images/Vector/Icons.xaml

@@ -3,33 +3,17 @@
              xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
              xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" 
              xmlns:mbts="clr-namespace:mbts.ui">
-  <DrawingImage x:Key="FavoritesGold">
-    <DrawingImage.Drawing>
-      <DrawingGroup>
-        <DrawingGroup.Children>
-          <GeometryDrawing Geometry="M1540.22,2082.07L1546.95,2102.78 1568.73,2102.78 1551.11,2115.58 1557.84,2136.29 1540.22,2123.49 1522.6,2136.29 1529.33,2115.58 1511.71,2102.78 1533.49,2102.78 1540.22,2082.07z">
-            <GeometryDrawing.Pen>
-              <Pen Thickness="1" Brush="Black" />
-            </GeometryDrawing.Pen>
-            <GeometryDrawing.Brush>
-              <SolidColorBrush Color="#ffd932" />
-            </GeometryDrawing.Brush>
-          </GeometryDrawing>
-        </DrawingGroup.Children>
-      </DrawingGroup>
-    </DrawingImage.Drawing>
-  </DrawingImage>
 
-  <DrawingImage x:Key="FavoritesWhite">
+  <DrawingImage x:Key="Bookmarks">
     <DrawingImage.Drawing>
       <DrawingGroup>
         <DrawingGroup.Children>
           <GeometryDrawing Geometry="M1540.22,2082.07L1546.95,2102.78 1568.73,2102.78 1551.11,2115.58 1557.84,2136.29 1540.22,2123.49 1522.6,2136.29 1529.33,2115.58 1511.71,2102.78 1533.49,2102.78 1540.22,2082.07z">
             <GeometryDrawing.Pen>
-              <Pen Thickness="1" Brush="Black" />
+              <Pen Thickness="3" Brush="Black" />
             </GeometryDrawing.Pen>
             <GeometryDrawing.Brush>
-              <SolidColorBrush Color="#ffd932" />
+              <SolidColorBrush Color="Transparent" />
             </GeometryDrawing.Brush>
           </GeometryDrawing>
         </DrawingGroup.Children>

src/client/spark/Modules/Bookmarks/BookmarkShellModule.cs

@@ -0,0 +1,81 @@
+using bts.spark;
+using mbts.spark;
+using mbts.ui;
+using mbts.ui.settings;
+using mbts.ui.shell;
+using System;
+using System.Collections.Generic;
+using System.ComponentModel.Composition;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Windows.Media;
+
+namespace bts.ui {
+   [Export(typeof(IShellModule))]
+   public class BookmarkShellModule : ILazyMenuModule {
+      readonly WindowManager _wm;
+      readonly IBookmarks _bookmarks;
+      readonly ISavedWindowFactory _window_factory;
+      readonly WorkspacesModel _workspaces;
+
+      [ImportingConstructor]
+      public BookmarkShellModule(
+         WorkspacesModel workspaces, 
+         WindowManager wm, 
+         IBookmarks bm,
+         ISavedWindowFactory swf) {
+         _workspaces = workspaces;
+         _wm = wm;
+         _bookmarks = bm;
+         _window_factory = swf;
+      }
+
+      public ImageSource icon => Icons.Bookmarks;
+
+      public string name => "My Windows";
+
+      public int priority => 35;
+
+      public IEnumerable<SparkCommand> actions {
+         get {
+            // The windows that are open
+            var open =
+               _wm.list_documents()
+               .Select(td => td.ViewModel)
+               .Where(vm => vm.HasCustomTitle)
+               .Select(vm =>
+                  SparkCommand.create(vm.Title, vm.Icon, SparkTags.ActivateWindow, () => vm.Model))
+               .Cast<SparkCommand>();
+
+            var open_set = new HashSet<string>(open.Select(c => c.title));
+
+            // The windows that are not open
+            var closed =
+               _bookmarks.bookmarks
+               .Where(state => !open_set.Contains(state.title))
+               .Select(state =>
+                  SparkCommand.create(state.title, Icons.Bookmarks, SparkTags.ShowWindowAsync,
+                     () => {
+                        _bookmarks.remove(state);
+                        return _window_factory.restore(state);
+                     }))
+               .Cast<SparkCommand>();
+
+            // todo: The windows that are closed
+            foreach (var w in open.Concat(closed).OrderBy(w => w.title)) {
+               yield return w;
+            }
+         }
+      }
+
+      public Task<IShellWindow> create(Stream load_stream) {
+         // This module shouldn't have any windows
+         throw new NotImplementedException();
+      }
+
+      public void Dispose() { }
+
+   }
+}

src/client/spark/Modules/Bookmarks/BookmarksModel.cs

@@ -0,0 +1,48 @@
+using mbts.ui.settings;
+using System;
+using System.Collections.Generic;
+using System.ComponentModel.Composition;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace mbts.ui.shell {
+
+   public interface IBookmarks {
+      void save_bookmark(IShellWindow window);
+      void save_bookmark(WindowState window);
+      void remove(WindowState window);
+      void clear();
+      IEnumerable<WindowState> bookmarks { get; }
+   }
+
+   [Export(typeof(IBookmarks))]
+   public class BookmarksModel : IBookmarks {
+      readonly ISettingsCollector _settings_collector;
+      readonly Dictionary<string, WindowState> _bookmarks = new Dictionary<string, WindowState>();
+
+      [ImportingConstructor]
+      public BookmarksModel(ISettingsCollector settings) {
+         _settings_collector = settings;
+      }
+      public void save_bookmark(IShellWindow window) {
+         var state = _settings_collector.get_window_state(window);
+         if (state != null) {
+            _bookmarks[state.title] = state;
+         }
+      }
+      public void save_bookmark(WindowState window) {
+         _bookmarks[window.title] = window;
+      }
+
+      public void clear() {
+         _bookmarks.Clear();
+      }
+
+      public void remove(WindowState window) {
+         _bookmarks.Remove(window.title);
+      }
+
+      public IEnumerable<WindowState> bookmarks => _bookmarks.Values;
+   }
+}

src/client/spark/Modules/Settings/SettingsShellModule.cs

@@ -34,7 +34,7 @@ namespace bts.ui {
 
       public int priority => 40;
 
-      public IEnumerable<SparkCommand> actions => new []{
+      public IEnumerable<SparkCommand> actions => new SparkCommand[]{
          SparkCommands.ShowWorkspaces
          //SparkCommands.ShowSettings
       };

src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs

@@ -173,7 +173,8 @@ namespace mbts.spark {
                }
                _is_setting_price_mults = false;
             };
-            _can_add_leg = backing(nameof(can_add_leg), leg_tuples.ObserveSelf().Select(_ => leg_count < 2), leg_count < 2);
+            //_can_add_leg = backing(nameof(can_add_leg), leg_tuples.ObserveSelf().Select(_ => leg_count < 2), leg_count < 2);
+            _can_add_leg = backing(nameof(can_add_leg), leg_tuples.ObserveSelf().Select(_ => true), true);
          } else {
             _can_add_leg = backing(nameof(can_add_leg), false);
          }

src/client/spark/Modules/Trading/Fills/FillsVM.cs

@@ -130,7 +130,7 @@ namespace mbts.spark {
       protected override IEnumerable<SortingCriteria> get_default_sorting() { yield return new SortingCriteria(nameof(Row.fill_time), false); }
 
       static IObservable<IIncrementalUpdate<FillTuple, FillTupleKey>> _get_observable(IBus bus, OrdersModel model, StartupData state) {
-         if (state.orders != null && state.orders.Length == 0) {
+         if (!state.orders.is_null_or_empty()) {
             var orders_hash = new HashSet<uint>(state.orders);
             return model.on_fill_tuples.filter(f => orders_hash.Contains(f.fill.OrderKey));
          } else return model.on_fill_tuples;

src/client/spark/Modules/Trading/LadderWindowFactory.cs

@@ -30,8 +30,7 @@ namespace bts.shell {
          _wm = wm;
          _settings_hub = settings_hub;
          _orders_model = orders_model;
-         var window_props = Configuration.get<bts.config.IShellWindowProps>();
-         _max_ladders_open = (int)(window_props?.ladder?.max_total_open ?? 75);
+         _max_ladders_open = (int)(Configuration.get<bts.config.LadderKnobs>().max_open_ladders);
          _dispose = Disposable.Create(
             bus.subscribe(SparkTags.SecuritiesTag).Subscribe(s => _securities = s),
             bus.subscribe(SparkTags.ShowLadderTag).OnDispatcher().Subscribe(s => show_windows(new[] { s })),

src/client/spark/Orders/OrderActivitySummarizer.cs

@@ -114,7 +114,7 @@ namespace bts.spark {
             (var security, var spec) = get_security(neworder.Def.SecurityKey);
             //
             // description
-            var mod_desc = modified_description(oldorder, neworder, spec.Value);
+            var mod_desc = modified_description(oldorder, neworder, spec);
             if (!string.IsNullOrEmpty(mod_desc)) {
                log_status(ActivityLogCategory.modify, security, mod_desc);
             }

src/client/spark/Tests/ViewTests/WindowManagementTests.cs

@@ -75,7 +75,7 @@ namespace mbts.ui.tests
             Logger.log(bts.log.Level.Info, bts.log.Subsystem.Ui, "Begin: test_windows_disposed");
             // Make sure that an IShellWindow and ISlowWindow added to the Windows collection is eventually disposed
             var thread_factory = new ShellThreadFactory();
-            IWindowManager wm = new WindowManager(thread_factory, new Lazy<ISettingsCollector>(() => new TestSettingsCollector()), new TestSavedWindowFactory(), new TestUndo(), new SimpleBus());
+            IWindowManager wm = new WindowManager(thread_factory, new Lazy<ISettingsCollector>(() => new TestSettingsCollector()), new TestSavedWindowFactory(), null, new SimpleBus());
             Assert.IsNotNull(Application.Current);
             Assert.IsNotNull(Application.Current.Windows);
             Assert.AreEqual(Application.Current.Windows.Count, 0);
@@ -100,7 +100,7 @@ namespace mbts.ui.tests
             Logger.log(bts.log.Level.Info, bts.log.Subsystem.Ui, "Begin: test_slow_windows_disposed");
             // Make sure that an IShellWindow and ISlowWindow added to the Windows collection is eventually disposed
             var thread_factory = new ShellThreadFactory();
-            IWindowManager wm = new WindowManager(thread_factory, new Lazy<ISettingsCollector>(() => new TestSettingsCollector()), new TestSavedWindowFactory(), new TestUndo(), new SimpleBus());
+            IWindowManager wm = new WindowManager(thread_factory, new Lazy<ISettingsCollector>(() => new TestSettingsCollector()), new TestSavedWindowFactory(), null, new SimpleBus());
             Assert.AreEqual(Application.Current.Windows.Count, 0);
 
             // an ISlowWindow gets its own thread

src/client/spark/core/Bus/SparkTags.cs

@@ -5,6 +5,7 @@ using bts.spark;
 using mbts.ui;
 using System;
 using System.Collections.Generic;
+using System.Threading.Tasks;
 using static mbts.spark.SparkBusData;
 
 namespace mbts.spark
@@ -34,6 +35,8 @@ namespace mbts.spark
       public static ITag<(PartialCommand<SecurityMD>, SecurityCriteria)> FilterPickOne { get; } = new SparkTag<(PartialCommand<SecurityMD>, SecurityCriteria)>("FilterPick1");
 
       public static ITag<IShellWindow> ShowWindow { get; } = new SparkTag<IShellWindow>("Show Window");
+      public static ITag<Task<IShellWindow>> ShowWindowAsync { get; } = new SparkTag<Task<IShellWindow>>("ShowWindowAsync");
+      public static ITag<IShellWindow> ActivateWindow { get; } = new SparkTag<IShellWindow>("Activate Window");
       public static ITag<Nothing> ShowWorkspaces { get; } = new SparkTag<Nothing>(nameof(ShowWorkspaces));
       public static ITag<Nothing> ShowSettings { get; } = new SparkTag<Nothing>(nameof(ShowSettings));
       public static ITag<Uri> ShowUri { get; } = new SparkTag<Uri>("ShowUri");

src/client/spark/core/GlobalExtensions/GenericCollectionExtensions.cs

@@ -48,10 +48,10 @@ namespace System.Collections.Generic {
          return rv;
       }
 
-
-      public static IEnumerable<T> ToIEnumerable<T>(this T item) {
-         yield return item;
-      }
+      // use "new [] { item }" - avoid putting this on the intellisense for every item everywhere
+      //public static IEnumerable<T> ToIEnumerable<T>(this T item) {
+      //   yield return item;
+      //}
 
       public static IEnumerable<T> TakeWhile<T, TAccumulate>(this IEnumerable<T> source, TAccumulate seed, Func<TAccumulate, T, TAccumulate> accumulate, Func<TAccumulate, T, bool> predicate) {
          TAccumulate a = seed;

src/client/spark/core/Settings/ISettingsCollector.cs

@@ -69,16 +69,18 @@ namespace mbts.ui.settings
             if (window == null || window.Owner == null) return null;
             MemoryStream out_stream = new MemoryStream();
             window.GetState(out_stream);
+
+            var pos = _interop.GetPosition(window);
             var state = new WindowState() {
                owner = window.Owner.key(),
                app_data = out_stream.ToArray(),
-               position = _interop.GetPosition(window),
-               title = window.Title
+               position = pos,
+               title = string.IsNullOrEmpty(pos.custom_title) ? window.Title : pos.custom_title
             };
             return state;
          }
          catch (Exception e) {
-            Logger.log(bts.log.Level.Error, bts.log.Subsystem.Ui, "Error saving window state for type {0}.\n {1}".fmt(window.GetType(), e.Message));
+            Logger.log(Level.Error, Subsystem.Ui, "Error saving window state for type {0}.\n {1}".fmt(window.GetType(), e.Message));
          }
          return null;
       }

src/client/spark/core/Settings/Workspaces/WorkspaceProtos.cs

@@ -39,6 +39,9 @@ namespace mbts.ui.shell {
       [ProtoMember(102), DefaultValue(true)]
       public bool always_on_top = true;
       //[ProtoMember(103)] //used
+
+      [ProtoMember(104)]
+      public string custom_title;
    }
 
    [ProtoContract]
@@ -58,7 +61,7 @@ namespace mbts.ui.shell {
       public WindowPosition position;
 
       [ProtoMember(9)]
-      public string title;
+      public string title; // for debugging
    }
 
    [ProtoContract]

src/client/spark/core/ShellInterface/IShellModule.cs

@@ -23,6 +23,9 @@ namespace mbts.ui
       Task<IShellWindow> create(Stream load_stream);
    }
 
+   // ILazyModule may also impement IShellMenu to get opening/closing notification
+   public interface ILazyMenuModule : IShellModule { }
+
    public interface IPersistShellModule : IShellModule {
       // Called at startup before the UI is shown
       // This is an opportunity for you to kick off background services

src/client/spark/core/ShellInterface/IWindowsInterop.cs

@@ -17,5 +17,5 @@ namespace mbts.ui
       WindowPosition GetPosition(IShellWindow window);
       bool IsVisible(IShellWindow window);
       void PinWindow(IShellWindow window);
-   }
+  }
 }

src/client/spark/core/Utility/AnonymousEqualityComparer.cs

@@ -3,6 +3,8 @@ using System.Collections.Generic;
 
 namespace mbts.utils {
    public static class EqualityComparer {
+      const int _hash_seed = 63607, _hash_multiplier = 6451;
+
       public static IEqualityComparer<T> Create<T>(Func<T, T, bool> equals) =>
          new AnonymousEqualityComparer<T>(equals);
 
@@ -29,7 +31,7 @@ namespace mbts.utils {
          non_null_equals(x, y);
 
       public static int aggregate_hash(int item1_hash, int item2_hash, int item3_hash) {
-         int rv = 63607;
+         int rv = _hash_seed;
          unchecked {
             _accumulate_hash(ref rv, ref item1_hash);
             _accumulate_hash(ref rv, ref item2_hash);
@@ -38,9 +40,20 @@ namespace mbts.utils {
          return rv;
       }
 
+      // Note: be wary of the performance cost this method may have, as frequently creating large arrays can be expensive.
+      // If you wish to avoid this cost, consider making an overload like the method above, with the appropriate number of item-hash arguments
+      public static int aggregate_hash(params int[] item_hashes) {
+         int hash = _hash_seed;
+         unchecked {
+            foreach (int item_hash in item_hashes)
+               hash = hash * _hash_multiplier + item_hash;
+         }
+         return hash;
+      }
+
       // WARNING: be sure to call this within in an unchecked scope
       static void _accumulate_hash(ref int aggregate_hash, ref int item_hash) =>
-         aggregate_hash = aggregate_hash * 6451 + item_hash;
+         aggregate_hash = aggregate_hash * _hash_multiplier + item_hash;
 
       private class AnonymousEqualityComparer<T> : IEqualityComparer<T> {
          readonly Func<T, T, bool> _equals;

src/client/spark/view/Controls/DataGridEx/Filters/Views/TimeFilter.cs

@@ -54,8 +54,9 @@ namespace mbts.ui {
       public static TimeTextTuple FromTicks(long ticks) =>
          FromDateTime(new DateTime(ticks));
 
-      public static TimeTextTuple FromDateTime(DateTime date_time) =>
-         new TimeTextTuple(DateTimeFormatInfo.CurrentInfo.AbbreviatedMonthNames[date_time.Month - 1], date_time.Day.ToString(), date_time.Year.ToString(), date_time.ToString("t"));
+      public static TimeTextTuple FromDateTime(DateTime date_time) => FromDateTime(date_time, "t");
+      public static TimeTextTuple FromDateTime(DateTime date_time, string time_format) =>
+         new TimeTextTuple(DateTimeFormatInfo.CurrentInfo.AbbreviatedMonthNames[date_time.Month - 1], date_time.Day.ToString(), date_time.Year.ToString(), date_time.ToString(time_format));
 
       public TimeTextTuple(string month_text, string day_text, string year_text, string time_text) {
          _month_text = month_text;
@@ -107,14 +108,23 @@ namespace mbts.ui {
 
 
       static DateTime? _parse(string month_text, string day_text, string year_text, string time_text) {
-         DateTime? t = _parse(time_text);
+         TimeSpan? t = _parse_time(time_text);
          if (t.HasValue) {
             DateTime? d = _parse($"{month_text}/{day_text}/{year_text}");
-            if (d.HasValue) return d.Value.Date.AddDays(t.Value.TimeOfDay.TotalDays);
+            if (d.HasValue) return d.Value.Date.Add(t.Value);
          }
          return null;
       }
 
+      static TimeSpan? _parse_time(string text) {
+         DateTime? dt = _parse(text);
+         if (dt.HasValue) {
+            return dt.Value.TimeOfDay;
+         } else if (TimeSpan.TryParse(text, out TimeSpan result) && result < TimeSpan.FromDays(1)) {
+            return result;
+         } else return null;
+      }
+
       static DateTime? _parse(string text) {
          if (!string.IsNullOrEmpty(text) && DateTime.TryParse(text, out DateTime result))
             return result;

src/client/spark/view/Controls/PriceEntryControl.xaml

@@ -28,7 +28,14 @@
         BorderBrush="{TemplateBinding BorderBrush}"
         BorderThickness="{TemplateBinding BorderThickness}"
         Text="{Binding Value, RelativeSource={RelativeSource TemplatedParent}, Converter={StaticResource TickDisplayTwoWayConverter}, Mode=TwoWay}"
-      />
+      >
+        <TextBox.CommandBindings>
+          <CommandBinding Command="{x:Static u:TextBoxBehavior.SelectAllCommand}" Executed="{x:Static u:TextBoxBehavior.SelectAllCommandExecutedHandler}"/>
+        </TextBox.CommandBindings>
+        <TextBox.InputBindings>
+          <MouseBinding MouseAction="LeftDoubleClick" Command="{x:Static u:TextBoxBehavior.SelectAllCommand}"/>
+        </TextBox.InputBindings>
+      </TextBox>
     </c:TickUpDown>
   </ControlTemplate>
   <Style TargetType="usp:PriceEntryControl">

src/client/spark/view/Controls/PriceEntryControl.xaml.cs

@@ -47,24 +47,30 @@ namespace mbts.ui.spark {
       #endregion
 
       TextBox _text;
-      IDisposable _text_input_hook;
 
       public PriceEntryControl() {
          TickUpCommand = new DelegateCommand(_tick_up);
          TickDownCommand = new DelegateCommand(_tick_down);
       }
 
+      IDisposable _keyboard_focus_sub;
+      bool _has_keyboard_focus;
+
       public override void OnApplyTemplate() {
+         Disposable.dispose(ref _keyboard_focus_sub);
          _text = this.assert_template_child<TextBox>("_text");
-         Disposable.dispose(ref _text_input_hook);
-         _text_input_hook = _text.SubscribePreviewTextInput((s, e) => e.Handled = !RegexFilters.PriceInput.IsMatch(_text.PeekText(e)));
+         _keyboard_focus_sub = _text.SubscribeKeyboardFocus(val => _has_keyboard_focus = val);
          _update_text_binding();
       }
 
       void _tick_up() {
+         if (_has_keyboard_focus) {
+            _text.GetBindingExpression(TextBox.TextProperty)?.UpdateSource();
+         }
          double v = Value;
-         if (double.IsNaN(v)) SetCurrentValue(ValueProperty, TickDefaultValue);
-         else {
+         if (double.IsNaN(v)) {
+            SetCurrentValue(ValueProperty, TickDefaultValue);
+         } else {
             var spec = ContractSpec;
             if (spec.HasValue) SetCurrentValue(ValueProperty, spec.Value.tickup(v));
             else SetCurrentValue(ValueProperty, TickDefaultValue);
@@ -72,9 +78,13 @@ namespace mbts.ui.spark {
       }
 
       void _tick_down() {
+         if (_has_keyboard_focus) {
+            _text.GetBindingExpression(TextBox.TextProperty)?.UpdateSource();
+         }
          double v = Value;
-         if (double.IsNaN(v)) SetCurrentValue(ValueProperty, TickDefaultValue);
-         else {
+         if (double.IsNaN(v)) {
+            SetCurrentValue(ValueProperty, TickDefaultValue);
+         } else {
             var spec = ContractSpec;
             if (spec.HasValue) SetCurrentValue(ValueProperty, spec.Value.tickdown(v));
             else SetCurrentValue(ValueProperty, TickDefaultValue);

src/client/spark/view/Controls/SparkMenu/SparkMenu.cs

@@ -10,9 +10,11 @@ namespace mbts.ui.controls {
       static SparkMenu() =>
          DefaultStyleKeyProperty.OverrideMetadata(typeof(SparkMenu), new FrameworkPropertyMetadata(typeof(SparkMenu)));
 
-      public SparkMenu() =>
+      public SparkMenu() {
          SetCurrentValue(FocusableProperty, false);
+      }
 
+      
       public SparkMenu(IEnumerable<SparkCommand> commands, IBus bus) : this() =>
          ItemsSource = commands.Select(c => new SparkCommandVM(c, bus)).ToList();
 

src/client/spark/view/Controls/TickUpDown/TickUpDown.xaml.cs

@@ -23,38 +23,42 @@ namespace mbts.ui.controls {
       }
 
       protected override void OnPreviewKeyDown(KeyEventArgs e) {
-         switch (Keyboard.Modifiers) {
-            case ModifierKeys.None: {
-                  switch (e.Key) {
-                     case Key.Up:
-                        TickUpCommand.TryExecuteIfCan(TickUpParameter);
-                        e.Handled = true;
-                        break;
-                     case Key.Down:
-                        TickDownCommand.TryExecuteIfCan(TickDownParameter);
-                        e.Handled = true;
-                        break;
+         if (EnableArrowKeyTick) {
+            switch (Keyboard.Modifiers) {
+               case ModifierKeys.None: {
+                     switch (e.Key) {
+                        case Key.Up:
+                           TickUpCommand.TryExecuteIfCan(TickUpParameter);
+                           e.Handled = true;
+                           break;
+                        case Key.Down:
+                           TickDownCommand.TryExecuteIfCan(TickDownParameter);
+                           e.Handled = true;
+                           break;
+                     }
                   }
-               }
-               break;
-            case ModifierKeys.Alt: {
-                  // see: https://stackoverflow.com/questions/3099472/previewkeydown-is-not-seeing-alt-modifiers
-                  switch (e.SystemKey) {
-                     case Key.Up:
-                        TickUpCommand.TryExecuteIfCan(TickUpAltParameter);
-                        e.Handled = true;
-                        break;
-                     case Key.Down:
-                        TickDownCommand.TryExecuteIfCan(TickDownAltParameter);
-                        e.Handled = true;
-                        break;
+                  break;
+               case ModifierKeys.Alt: {
+                     // see: https://stackoverflow.com/questions/3099472/previewkeydown-is-not-seeing-alt-modifiers
+                     switch (e.SystemKey) {
+                        case Key.Up:
+                           TickUpCommand.TryExecuteIfCan(TickUpAltParameter);
+                           e.Handled = true;
+                           break;
+                        case Key.Down:
+                           TickDownCommand.TryExecuteIfCan(TickDownAltParameter);
+                           e.Handled = true;
+                           break;
+                     }
                   }
-               }
-               break;
+                  break;
+            }
          }
          base.OnPreviewKeyDown(e);
       }
 
+      public bool EnableArrowKeyTick { get; set; } = true;
+
       #region TickCommand
       public ICommand TickCommand {
          get { return (ICommand)GetValue(TickCommandProperty); }

src/client/spark/view/Controls/Time/DateTimePicker.xaml

@@ -31,8 +31,14 @@
               <ColumnDefinition Width="Auto"/>
               <ColumnDefinition/>
             </Grid.ColumnDefinitions>
-            <c:TickUpDown x:Name="PART_time_tick" Grid.Column="0" MinWidth="85" HorizontalAlignment="Stretch">
-              <TextBox x:Name="PART_time_text" Height="20" u:TextBoxBehavior.EnableArrowKeyTraverse="True" u:Behavior.DisablePaste="True"/>
+            <c:TickUpDown x:Name="PART_time_tick" Grid.Column="0" MinWidth="85" HorizontalAlignment="Stretch" EnableArrowKeyTick="False">
+              <TextBox x:Name="PART_time_text" Height="20" u:TextBoxBehavior.EnableArrowKeyTraverse="True" u:Behavior.DisablePaste="True">
+                <TextBox.InputBindings>
+                  <KeyBinding Key="Up" Command="{x:Static c:DateTimePicker.TickTimeComponentCommand}" CommandParameter="{u:Int 1}"/>
+                  <KeyBinding Key="Down" Command="{x:Static c:DateTimePicker.TickTimeComponentCommand}" CommandParameter="{u:Int -1}"/>
+                  <u:CustomInputBinding Gesture="{x:Static u:InputGestures.LeftTripleClick}" Command="{x:Static u:TextBoxBehavior.SelectAllCommand}"/>
+                </TextBox.InputBindings>
+              </TextBox>
             </c:TickUpDown>
             <c:TickUpDown x:Name="PART_month_tick" Grid.Column="2" Width="50">
               <TextBox x:Name="PART_month_text" Height="20" u:TextBoxBehavior.EnableArrowKeyTraverse="True" u:Behavior.DisablePaste="True"/>

src/client/spark/view/Controls/Time/DateTimePicker.xaml.cs

@@ -3,6 +3,7 @@ using mbts.ui.shell;
 using System;
 using System.Collections.Generic;
 using System.ComponentModel;
+using System.Linq;
 using System.Windows;
 using System.Windows.Controls;
 using System.Windows.Data;
@@ -14,7 +15,9 @@ namespace mbts.ui.controls {
    /// Interaction logic for DateTimePicker.xaml
    /// </summary>
    public class DateTimePicker : Control {
-      TimeTupleVM _vm;
+      public static readonly RoutedCommand TickTimeComponentCommand = new RoutedCommand();
+
+      DTPController _controller;
       bool _suspend_notify_vm;
 
       static DateTimePicker() =>
@@ -24,8 +27,8 @@ namespace mbts.ui.controls {
          new FrameworkPropertyMetadata(default(TimeTextTuple), FrameworkPropertyMetadataOptions.BindsTwoWayByDefault, OnTimeTupleChanged));
       static void OnTimeTupleChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
          var dtp = (DateTimePicker)d;
-         if (!dtp._suspend_notify_vm && dtp._vm != null)
-            dtp._vm.result = (TimeTextTuple)e.NewValue;
+         if (!dtp._suspend_notify_vm && dtp._controller != null)
+            dtp._controller.result = (TimeTextTuple)e.NewValue;
       }
       public TimeTextTuple TimeTuple { get => (TimeTextTuple)GetValue(TimeTupleProperty); set => SetValue(TimeTupleProperty, value); }
 
@@ -41,59 +44,74 @@ namespace mbts.ui.controls {
          var time = this.assert_template_child<TickUpDown>("PART_time_tick");
          var time_text = this.assert_template_child<TextBox>("PART_time_text");
 
-         if (_vm != null) _vm.PropertyChanged -= _vm_PropertyChanged;
-         _vm = new TimeTupleVM();
-         _vm.result = TimeTuple;
-         _vm.PropertyChanged += _vm_PropertyChanged;
+         if (_controller != null) _controller.PropertyChanged -= _vm_PropertyChanged;
+         _controller = new DTPController(time_text);
+         _controller.result = TimeTuple;
+         _controller.PropertyChanged += _vm_PropertyChanged;
+         time.CommandBindings.Add(new CommandBinding(TickTimeComponentCommand, (s, e) => {
+            _controller.tick_time_component((int)e.Parameter);
+         }));
+         time.CommandBindings.Add(new CommandBinding(TextBoxBehavior.SelectAllCommand, TextBoxBehavior.SelectAllCommandExecutedHandler));
 
          const int delay = 500;
-         time.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_time") { Source = _vm });
+         time.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_time") { Source = _controller });
          time.TickUpParameter = 1;
          time.TickDownParameter = -1;
          time.TickUpAltParameter = 60;
          time.TickDownAltParameter = -60;
-         time_text.SetBinding(TextBox.TextProperty, new Binding("time_text") { Source = _vm, Delay = delay, ValidatesOnExceptions = true });
-         _hooks.Add(time_text.SubscribePreviewTextInput((s, e) =>
-            e.Handled = !MatchAny(time_text.PeekText(e), RegexFilters.Time.StandardTime, RegexFilters.Time.MilitaryTime)
-         ));
+         time_text.SetBinding(TextBox.TextProperty, new Binding("time_text") { Source = _controller, Delay = delay, ValidatesOnExceptions = true });
+         _hooks.Add(time_text.SubscribePreviewTextInput((s, e) => {
+            string preview = time_text.PeekText(e);
+            if (RegexFilters.Time.StandardTimeOptionalSeconds.IsMatch(preview)) {
+               _controller.use_military_time = false;
+            } else if (RegexFilters.Time.MilitaryTimeOptionalSeconds.IsMatch(preview)) {
+               _controller.use_military_time = true;
+            } else {
+               e.Handled = true;
+            }
+         }));
 
-         month.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_month") { Source = _vm });
+         month.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_month") { Source = _controller });
          month.TickUpParameter = 1;
          month.TickDownParameter = -1;
-         month_text.SetBinding(TextBox.TextProperty, new Binding("month_text") { Source = _vm, Delay = delay });
+         month_text.SetBinding(TextBox.TextProperty, new Binding("month_text") { Source = _controller, Delay = delay });
          _hooks.Add(month_text.SubscribePreviewTextInput((s, e) =>
             e.Handled = !RegexFilters.Time.MonthAbbrev.IsMatch(month_text.PeekText(e))
          ));
 
-         day.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_day") { Source = _vm });
+         day.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_day") { Source = _controller });
          day.TickUpParameter = 1;
          day.TickDownParameter = -1;
-         day_text.SetBinding(TextBox.TextProperty, new Binding("day_text") { Source = _vm, Delay = delay });
+         day_text.SetBinding(TextBox.TextProperty, new Binding("day_text") { Source = _controller, Delay = delay });
          _hooks.Add(day_text.SubscribePreviewTextInput((s, e) =>
             e.Handled = !RegexFilters.Time.DaysInMonth.IsMatch(day_text.PeekText(e))
          ));
 
-         year.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_year") { Source = _vm });
+         year.SetBinding(TickUpDown.TickCommandProperty, new Binding("tick_year") { Source = _controller });
          year.TickUpParameter = 1;
          year.TickDownParameter = -1;
-         year_text.SetBinding(TextBox.TextProperty, new Binding("year_text") { Source = _vm, Delay = delay });
+         year_text.SetBinding(TextBox.TextProperty, new Binding("year_text") { Source = _controller, Delay = delay });
          _hooks.Add(year_text.SubscribePreviewTextInput((s, e) =>
             e.Handled = !RegexFilters.Time.Year.IsMatch(year_text.PeekText(e))
          ));
       }
 
       void _vm_PropertyChanged(object sender, PropertyChangedEventArgs e) {
-         if (e.PropertyName == nameof(TimeTupleVM.result)) {
+         if (e.PropertyName == nameof(DTPController.result)) {
             _suspend_notify_vm = true;
-            TimeTuple = _vm.result;
+            TimeTuple = _controller.result;
             _suspend_notify_vm = false;
          }
       }
 
-      private class TimeTupleVM : ViewModelBase {
+      // This is not exactly a view-model, bc it accesses the time TextBox directly so it can get/set the CaretIndex,
+      // which is used to determine which time-componenet - hh:mm:ss AM/PM - should be incremented/decremented via the up/down arrow-keys.
+      class DTPController : ViewModelBase {
+         readonly TextBox _time_text_box;
          TimeTextTuple _result;
 
-         public TimeTupleVM() {
+         public DTPController(TextBox time_text_box) {
+            _time_text_box = time_text_box;
             tick_time = new DelegateCommand<int>(_tick_time);
             tick_month = new DelegateCommand<int>(_tick_month);
             tick_day = new DelegateCommand<int>(_tick_day);
@@ -135,12 +153,52 @@ namespace mbts.ui.controls {
                RaisePropertyChanged(nameof(month_text), nameof(day_text), nameof(year_text), nameof(time_text));
             }
          }
+
+         public bool use_military_time { get; set; }
          public ICommand tick_time { get; }
          public ICommand tick_month { get; }
          public ICommand tick_day { get; }
          public ICommand tick_year { get; }
+         string _time_format => use_military_time ? StringFormats.Time.MilitaryWithSeconds : StringFormats.Time.StandardWithSeconds;
 
+         public void tick_time_component(int increment) {
+            _ensure_time_text_up_to_date();
+            var r = _result.result;
+            if (r.HasValue) {
+               int index = _time_text_box.CaretIndex;
+               using (detect_property_changes()) {
+                  DateTime dt;
+                  if (index <= 0) {
+                     dt = r.Value.AddHours(increment);
+                  } else {
+                     int segment = _time_text_box.Text.Take(index).Count(c => c == ':');
+                     switch (segment) {
+                        case 0: dt = r.Value.AddHours(increment); break;
+                        case 1: dt = r.Value.AddMinutes(increment); break;
+                        case 2:
+                           if (_time_text_box.Text.Take(index).Contains(' ')) {
+                              // toggle AM/PM
+                              dt = r.Value.AddHours(r.Value.Hour < 12 ? 12 : -12);
+                           } else {
+                              dt = r.Value.AddSeconds(increment);
+                           }
+                           break;
+                        default:
+                           dt = r.Value.AddSeconds(increment);
+                           break;
+                     }
+                  }
+                  _result = TimeTextTuple.FromDateTime(dt, _time_format);
+               }
+               _time_text_box.CaretIndex = index;
+            } else {
+               using (detect_property_changes())
+                  _force_valid_inputs();
+            }
+         }
+ 
          void _tick_time(int increment) {
+            _ensure_time_text_up_to_date();
             using (detect_property_changes()) {
                var r = _result.result;
                if (r.HasValue) {
@@ -187,6 +245,11 @@ namespace mbts.ui.controls {
                }
             }
          }
+ 
+         void _ensure_time_text_up_to_date() {
+            if (_time_text_box.IsKeyboardFocused)
+               _time_text_box.GetBindingExpression(TextBox.TextProperty)?.UpdateSource();
+         }
 
          void _force_valid_inputs() {
             var dt = DateTime.Today.Date;

src/client/spark/view/DragDrop/DragDropHelper.cs

@@ -61,16 +61,16 @@ namespace mbts.ui
          _source_window = window as FloatingWindow;
          if (_source != null) {
             _mouse_start = e.GetPosition(window);
-            if (!is_button_descendant((DependencyObject)e.OriginalSource)) {
+            if (!is_clickable_descendant((DependencyObject)e.OriginalSource)) {
                Mouse.Capture(_source);
             }
          }
       }
 
-      static bool is_button_descendant(DependencyObject target) {
+      static bool is_clickable_descendant(DependencyObject target) {
          while (target != null && !(target is TabDocument)) {
             target = VisualTreeHelper.GetParent(target);
-            if (target is Button) return true;
+            if (target is Button || target is TextBox) return true;
          }
          return false;
       }

src/client/spark/view/Extensions/AsyncExtensions.cs

@@ -24,38 +24,7 @@ namespace mbts.ui
          }
       }
 
-      public static async Task FromEvent(this object target, string event_name) {
-         var evt = target.GetType().GetEvent(event_name);
-         var delegate_type = evt.EventHandlerType;
-         var tcs = new TaskCompletionSource<object>();
-         Action result = () => { tcs.TrySetResult(null); };
-         var d = Delegate.CreateDelegate(evt.EventHandlerType, result, "Invoke");
-         try {
-            evt.AddEventHandler(target, d);
-            await tcs.Task;
-         }
-         finally {
-            evt.RemoveEventHandler(target, d);
-         }
-      }
-
-      public static async Task<T> FromEvent<T>(this object target, string event_name) {
-         var evt = target.GetType().GetEvent(event_name);
-         var delegate_type = evt.EventHandlerType;
-         var tcs = new TaskCompletionSource<T>();
-         Action<T> result = (arg) => { tcs.TrySetResult(arg); };
-         var d = Delegate.CreateDelegate(evt.EventHandlerType, result, "Invoke");
-         try {
-            evt.AddEventHandler(target, d);
-            await tcs.Task;
-            return tcs.Task.Result;
-         }
-         finally {
-            evt.RemoveEventHandler(target, d);
-         }
-      }
-
-      public static async Task<Tuple<T1,T2>> FromEvent<T1,T2>(this object target, string event_name) {
+      public static async Task<Tuple<T1,T2>> FromEvent<T1,T2>(this DependencyObject target, string event_name) {
          var evt = target.GetType().GetEvent(event_name);
          var delegate_type = evt.EventHandlerType;
          var tcs = new TaskCompletionSource<Tuple<T1,T2>>();

src/client/spark/view/Extensions/InputGestures.cs

@@ -0,0 +1,16 @@
+using System.Windows.Input;
+
+namespace mbts.ui {
+   class CustomInputBinding : InputBinding { }
+
+   static class InputGestures {
+      public static readonly InputGesture LeftTripleClick = new LeftTripleClickGesture();
+
+      class LeftTripleClickGesture : InputGesture {
+         public override bool Matches(object targetElement, InputEventArgs inputEventArgs) =>
+            inputEventArgs is MouseButtonEventArgs mouse_args &&
+            mouse_args.ChangedButton == MouseButton.Left &&
+            mouse_args.ClickCount == 3;
+      }
+   }
+}

src/client/spark/view/Extensions/TextElement/TextBoxBehavior.cs

@@ -7,7 +7,14 @@ using System.Windows.Input;
 
 namespace mbts.ui {
    static class TextBoxBehavior {
-      public static readonly RoutedCommand ReleaseCaptureCommand = new RoutedCommand();
+      #region RoutedCommands/Handlers
+      public static readonly RoutedCommand
+         ReleaseCaptureCommand = new RoutedCommand(),
+         SelectAllCommand = new RoutedCommand();
+
+      public static readonly ExecutedRoutedEventHandler SelectAllCommandExecutedHandler =
+         new ExecutedRoutedEventHandler((s, e) => (e.OriginalSource as TextBox ?? e.Source as TextBox ?? s as TextBox)?.SelectAll());
+      #endregion
 
       #region SelectAllOnFocus
 
@@ -168,7 +175,7 @@ namespace mbts.ui {
       public static void SetInputFilter(DependencyObject d, TextBoxInputFilter value) => d.SetValue(InputFilterProperty, value);
       #endregion
 
-      #region
+      #region IncrementDecrementOnMouseClick
       public static readonly DependencyProperty IncrementDecrementOnMouseClickProperty = DependencyProperty.RegisterAttached(
          "IncrementDecrementOnMouseClick", typeof(bool), typeof(TextBoxBehavior), new PropertyMetadata(false, IncrementDecrementOnMouseClickPropertyChanged)
          );

src/client/spark/view/Extensions/TextElement/TextElementHelper.cs

@@ -2,11 +2,28 @@
 using System.Windows;
 using System.Windows.Controls;
 using System.Windows.Data;
+using System.Windows.Input;
+
+namespace mbts.ui {
+   static class TextElementHelper {
+      public static IDisposable SubscribeKeyboardFocus(this TextBox text_box, Action<bool> callback) {
+         text_box.GotKeyboardFocus += on_got_keyboard_focus;
+         text_box.LostKeyboardFocus += on_lost_keyboard_focus;
+         return Disposable.Create(() => {
+            text_box.GotKeyboardFocus -= on_got_keyboard_focus;
+            text_box.LostKeyboardFocus -= on_lost_keyboard_focus;
+         });
+         void on_got_keyboard_focus(object sender, KeyboardFocusChangedEventArgs e) {
+            if (e.NewFocus == text_box)
+               callback(true);
+
+         }
+         void on_lost_keyboard_focus(object sender, KeyboardFocusChangedEventArgs e) {
+            if (e.OldFocus == text_box)
+               callback(false);
+         }
+      }
 
-namespace mbts.ui
-{
-   public class TextElementHelper
-   {
       public static void GetTargetProperty(DependencyObject d, out FrameworkElement text_element, out DependencyProperty text_property) {
          text_element = (FrameworkElement)d;
          if (text_element is TextBox) text_property = TextBox.TextProperty;

src/client/spark/view/Shell/FloatingWindow.xaml.cs

@@ -14,18 +14,19 @@ namespace mbts.ui.shell
    public partial class FloatingWindow : Window
    {
       bool _closed = false;
-      private IUndoActions _undo;
+      readonly IBookmarks _bookmark;
       private IWindowManager _wm;
       private ISavedWindowFactory _window_factory;
       private ISettingsCollector _window_settings;
       private Stack<Action> _recently_closed_windows = new Stack<Action>();
 
       // if is_loaded is true, will NOT set width/height
-      internal FloatingWindow(TabDocument document, IWindowManager wm, IUndoActions undo, ISettingsCollector settings, ISavedWindowFactory factory, bool is_loaded=false) {
+      internal FloatingWindow(TabDocument document, 
+         IWindowManager wm, IBookmarks bookmarks, ISettingsCollector settings, ISavedWindowFactory factory, bool is_loaded=false) {
          _wm = wm;
-         _undo = undo;
          _window_settings = settings;
          _window_factory = factory;
+         _bookmark = bookmarks;
 
          InitializeComponent();
          if (string.IsNullOrEmpty(document.ViewModel.host_id)) {
@@ -67,6 +68,7 @@ namespace mbts.ui.shell
 
 
       private ItemCollection Items { get { return _tabs.Items; } }
+      public IEnumerable<TabDocument> Documents => _tabs.Items.OfType<TabDocument>();
       public bool IsEmpty { get { return _tabs.Items.Count == 0; } }
       public bool SingleChild { get { return _tabs.Items.Count == 1; } }
       public string host_id { get; private set; }
@@ -88,10 +90,10 @@ namespace mbts.ui.shell
          if (sender != this) {
             var undoaction = create_undo_action(document);
             if (undoaction != null) {
-               IDisposable undoitem = _undo.push("Restore Tab '{0}'".fmt(document.Title), undoaction);
-               _recently_closed_windows.Push(() => { undoaction(); undoitem.Dispose(); });
+               _recently_closed_windows.Push(() => { undoaction(); });
             }
          }
+         _bookmark.save_bookmark(document.Model);
       }
 
       public void remove_document(TabDocument document) {
@@ -144,22 +146,9 @@ namespace mbts.ui.shell
 
       protected override void OnClosed(EventArgs e) {
          _closed = true;
-         TabDocument[] items = _tabs.Items.OfType<TabDocument>().ToArray();
-         if (items.Length > 0) {
-            // If we're closing and still have Tabs open we save the whole "window" worth of state in the undo stack
-            // If the application is shutting down this might be unnecessary work but it's not too costly
-            var undo_actions = items.Select(t => create_undo_action(t.ViewModel)).Where(item => item != null).ToArray();
-            if (undo_actions.Length > 0) {
-               string name = "Restore Window: '{0}'".fmt(string.Join("', '", items.Select(t => t.ViewModel.Title)));
-               _undo.push(name, () => {
-                  foreach (var u in undo_actions) { u(); }
-               });
-            }
-         }
-
          base.OnClosed(e);
 
-         foreach (TabDocument document in items) {
+         foreach (TabDocument document in _tabs.Items.OfType<TabDocument>().ToList()) {
             if (document != null && document.ViewModel.CloseCommand.CanExecute(this)) {
                document.ViewModel.CloseCommand.Execute(this);
             }

src/client/spark/view/Shell/TabDocument.xaml

@@ -3,7 +3,9 @@
          xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
          xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" 
          xmlns:d="http://schemas.microsoft.com/expression/blend/2008" 
-         mc:Ignorable="d" Style="{StaticResource {x:Type TabItem}}" d:DesignWidth="125" 
+         xmlns:ui="clr-namespace:mbts.ui"
+         xmlns:convert="clr-namespace:mbts.ui.converters"
+         mc:Ignorable="d" d:DesignWidth="125" 
          Header="{Binding}" Content="{Binding AppContent}"
          Background="#e6e6e6">
 
@@ -18,6 +20,18 @@
       <Storyboard x:Key="_slide_back">
         <DoubleAnimation Storyboard.TargetName="translate_transform" Storyboard.TargetProperty="X" To="100" Duration="0:0:0.25" FillBehavior="Stop" />
       </Storyboard>
+      <DataTemplate x:Key="_edit_template">
+        <TextBox Text="{Binding TitleOverride}" 
+                 ui:TextBoxBehavior.CommitOnEnter="True" 
+                 ui:TextBoxBehavior.FocusOnLoaded="True"
+                 ui:TextBoxBehavior.SelectAllOnFocus="True"
+                 ui:TextBoxBehavior.ReleaseOnEscape="True"
+                 Width="150" HorizontalAlignment="Left" VerticalAlignment="Stretch" Grid.Column="0" Margin="0,0,4,0" />
+      </DataTemplate>
+      <DataTemplate x:Key="_template">
+        <TextBlock Text="{Binding Title}" TextTrimming="CharacterEllipsis" HorizontalAlignment="Left" VerticalAlignment="Stretch" Grid.Column="0" Margin="0,0,4,0"></TextBlock>
+      </DataTemplate>
+      <convert:BooleanToObjectConverter x:Key="MenuHeaderConverter" ValueForTrue="Rename Window" ValueForFalse="Add to My Windows" />
     </ResourceDictionary>
   </TabItem.Resources>
   <TabItem.LayoutTransform>
@@ -28,25 +42,32 @@
   <TabItem.RenderTransform>
     <TranslateTransform x:Name="translate_transform" X="0" />
   </TabItem.RenderTransform>
-  <TabItem.HeaderTemplate>
-    <DataTemplate>
-      <TextBlock Text="{Binding Title}" TextTrimming="CharacterEllipsis" HorizontalAlignment="Left" VerticalAlignment="Stretch" Grid.Column="0" Margin="0,0,4,0"> </TextBlock>
-    </DataTemplate>
-  </TabItem.HeaderTemplate>
-
   <TabItem.Template>
     <ControlTemplate TargetType="{x:Type TabItem}">
       <Border x:Name="bd" BorderBrush="Black" BorderThickness="0,0,0.5,0" Background="#d6d6d6" Width="{DynamicResource TabWidth}" Height="{DynamicResource TabHeight}" MinWidth="40" Margin="0" SnapsToDevicePixels="True" DockPanel.Dock="Left">
         <Border.ToolTip>
           <TextBlock Text="{Binding Title}" />
         </Border.ToolTip>
+        <Border.ContextMenu>
+          <ContextMenu>
+            <MenuItem Header="{Binding HasCustomTitle, Converter={StaticResource MenuHeaderConverter}}" Command="{Binding rename_command}">
+              <MenuItem.Icon>
+                <Image Source="{StaticResource Bookmarks}" />
+              </MenuItem.Icon>
+            </MenuItem>
+            <MenuItem Header="Remove from My Windows" Command="{Binding unrename_command}" IsEnabled="{Binding HasCustomTitle}" />
+          </ContextMenu>
+        </Border.ContextMenu>
         <Grid>
           <Grid.ColumnDefinitions>
             <ColumnDefinition Width="Auto" />
             <ColumnDefinition Width="*" />
             <ColumnDefinition Width="Auto" />
           </Grid.ColumnDefinitions>
-          <Image SnapsToDevicePixels="True" MaxHeight="20" VerticalAlignment="Center" HorizontalAlignment="Left" Stretch="Uniform" Margin="5,0,5,0" Source="{Binding Icon, Mode=OneTime}" RenderOptions.BitmapScalingMode="HighQuality" />
+          <Image SnapsToDevicePixels="True" MaxHeight="20" VerticalAlignment="Center" HorizontalAlignment="Left" Stretch="Uniform" Margin="5,0,5,0" 
+                 Source="{Binding Icon, Mode=OneTime}" RenderOptions.BitmapScalingMode="HighQuality" />
+          <!--<Image MaxHeight="10" HorizontalAlignment="Left" VerticalAlignment="Bottom" Stretch="Uniform" Margin="5"
+                 Source="{StaticResource Bookmarks}" Visibility="{Binding HasCustomTitle, Converter={StaticResource BoolToVisibility}}" />-->
           <ContentPresenter Grid.Column="1" x:Name="content" ContentSource="Header" TextBlock.Foreground="#666"
                                  RecognizesAccessKey="True" SnapsToDevicePixels="{TemplateBinding SnapsToDevicePixels}" 
                                  HorizontalAlignment="Left"
@@ -62,12 +83,18 @@
           <Setter Property="Panel.ZIndex" Value="1" />
           <Setter Property="Background" TargetName="bd" Value="#F9F9F9" />
           <Setter Property="TextBlock.Foreground" TargetName="content" Value="black" />
-          <!--<Setter Property="BorderThickness" TargetName="bd" Value="0,4,0.5,0" />-->
-          <!--<Setter Property="Margin" Value="-2,-2,-2,-1" />
-               <Setter Property="Margin" TargetName="content" Value="0,0,0,1" />
-               <Setter Property="Margin" TargetName="bd" Value="0,0,0,0" />-->
         </Trigger>
       </ControlTemplate.Triggers>
     </ControlTemplate>
   </TabItem.Template>
+  <TabItem.Style>
+    <Style TargetType="TabItem" BasedOn="{StaticResource {x:Type TabItem}}">
+      <Setter Property="HeaderTemplate" Value="{StaticResource _template}" />
+      <Style.Triggers>
+        <DataTrigger Binding="{Binding EditMode}" Value="True">
+          <Setter Property="HeaderTemplate" Value="{StaticResource _edit_template}" />
+        </DataTrigger>
+      </Style.Triggers>
+    </Style>
+  </TabItem.Style>
 </TabItem>

src/client/spark/view/Shell/ViewModels/ShellModuleVM.cs

@@ -18,8 +18,15 @@ namespace mbts.ui.shell {
       public ImageSource IconSource { get { return _module.icon; } }
       public UIElement MenuContent => new SparkMenu(_module.actions, _bus);
 
-      public void opening() { }
+      public void opening() {
+         (_module as IShellMenu)?.opening();
+         if (_module is ILazyMenuModule) {
+            RaisePropertyChanged(nameof(MenuContent));
+         }
+      }
 
-      public void closing() { }
+      public void closing() {
+         (_module as IShellMenu)?.closing();
+      }
    }
 }

src/client/spark/view/Shell/ViewModels/ShellWindowVM.cs

@@ -24,10 +24,11 @@ namespace mbts.ui.shell
       Dispatcher _dispatcher;
       HwndContentHost _hwnd_host;
 
+
       public ShellWindowVM(IShellWindow model, IWindowManager wm, IShellThreadFactory threads, string host = "") {
          Logger.assert(model != null);
          _model = model;
-         _model.TitleChanged += () => RaisePropertyChanged("Title");
+         _model.TitleChanged += () => { if (!HasCustomTitle) RaisePropertyChanged("Title"); };
          _model.IconChanged += () => { _icon = null; RaisePropertyChanged("Icon"); };
          _wm = wm;
          host_id = host;
@@ -40,13 +41,55 @@ namespace mbts.ui.shell
          if (HasTabAffinity) {
             this.host_id = ((ITabAffineWindow)model).tab_group;
          }
+
+         _edit_mode = backing(nameof(EditMode), false);
       }
 
       public string host_id { get; set; }
       public Dispatcher Dispatcher { get { return _dispatcher; } }
-      public string Title { get { return _model.Title; } }
+
+      public string Title { get { return HasCustomTitle? TitleOverride : _model.Title; } }
+
+      string _title_override;
+      public string TitleOverride {
+         get => _title_override;
+         set {
+            if (_title_override != value) {
+               if (value == _model.Title) {
+                  _title_override = null;
+               }
+               else {
+                  _title_override = value;
+               }
+               RaisePropertyChanged(nameof(TitleOverride), nameof(Title), nameof(HasCustomTitle));
+            }
+            _edit_mode.set(false);
+         }
+      }
+      public bool HasCustomTitle => !string.IsNullOrWhiteSpace(TitleOverride);
+
+      Property<bool> _edit_mode;
+      public bool EditMode => _edit_mode.Value;
+
+
+      public ICommand rename_command => new DelegateCommand(begin_rename);
+      public ICommand unrename_command => new DelegateCommand(remove_custom);
+
+      private void begin_rename() {
+         if (string.IsNullOrEmpty(TitleOverride)) {
+            _title_override = Title;
+         }
+         _edit_mode.set(true);
+      }
+
+      private void remove_custom() {
+         TitleOverride = string.Empty;
+         //begin_close(this);
+      }
+
       public string OwnerName { get { return _model.Owner == null ? "" : _model.Owner.name; } }
       public ICommand CloseCommand { get { return new DelegateCommand(begin_close); } }
+
       public double InitialWidth { get; private set; }
       public double InitialHeight { get; private set; }
 
@@ -155,18 +198,14 @@ namespace mbts.ui.shell
          }
       }
 
-      public byte[] GetSaveState() {
-         MemoryStream stream = new MemoryStream();
-         _model.GetState(stream);
-         return stream.ToArray();
-      }
-
       public IShellWindow Model { get { return _model; } }
 
       public bool IsSlow { get { return _model is ISlowWindow; } }
       public bool IsShared { get { return _model is ISharedWindow; } }
       public bool HasTabAffinity { get { return _model is ITabAffineWindow; } }
 
+
+
       public void Dispose() {
          if (IsShared) {
             // Shared threads continue running

src/client/spark/view/Shell/WindowManager.cs

@@ -20,25 +20,31 @@ namespace mbts.ui.shell {
 
    [Export(typeof(IWindowManager))]
    [Export(typeof(IWindowsInterop))]
+   [Export]
    public class WindowManager : IWindowManager, IWindowsInterop, IDisposable {
       Dictionary<IShellWindow, TabDocument> _documents = new Dictionary<IShellWindow, TabDocument>();
       Dictionary<string, FloatingWindow> _windows = new Dictionary<string, FloatingWindow>();
-      IShellThreadFactory _threads;
+      readonly IShellThreadFactory _threads;
       Lazy<ISettingsCollector> _window_settings;
-      ISavedWindowFactory _window_factory;
-      IUndoActions _undo;
+      readonly ISavedWindowFactory _window_factory;
+      readonly Lazy<IBookmarks> _bookmarks;
       IDisposable _dispose;
       object _lock = new object();
 
       [ImportingConstructor]
       public WindowManager(
          IShellThreadFactory threads, Lazy<ISettingsCollector> settings, 
-         ISavedWindowFactory factory, IUndoActions undo, IBus bus) {
+         ISavedWindowFactory factory, Lazy<IBookmarks> bookmarks, IBus bus) {
          _threads = threads;
          _window_settings = settings;
          _window_factory = factory;
-         _undo = undo;
-         _dispose = bus.subscribe(SparkTags.ShowWindow).OnDispatcher().Subscribe(w => add(w).ignore());
+         _bookmarks = bookmarks;
+         _dispose =
+            Disposable.Create(
+               bus.subscribe(SparkTags.ShowWindow).OnDispatcher().Subscribe(w => add(w).ignore()),
+               bus.subscribe(SparkTags.ShowWindowAsync).OnDispatcher().Subscribe(w => add(w).ignore()),
+               bus.subscribe(SparkTags.ActivateWindow).OnDispatcher().Subscribe(MakeVisible)
+            );
       }
 
       #region Window creation
@@ -48,6 +54,7 @@ namespace mbts.ui.shell {
          var state = model.start_position();
          if (state != null) {
             viewmodel.host_id = state.host_id ?? "";
+            viewmodel.TitleOverride = state.custom_title;
          }
          var window = CreateWindow(content, state);
 
@@ -66,10 +73,10 @@ namespace mbts.ui.shell {
             }
          } else {
             if (position != null) {
-               window = new FloatingWindow(document, this, _undo, _window_settings.Value, _window_factory, true);
+               window = new FloatingWindow(document, this, _bookmarks.Value, _window_settings.Value, _window_factory, true);
                window.set_position(position);
             } else
-               window = new FloatingWindow(document, this, _undo, _window_settings.Value, _window_factory);
+               window = new FloatingWindow(document, this, _bookmarks.Value, _window_settings.Value, _window_factory);
             _windows.Add(window.host_id, window);
             window.Closed += (o, e) => _windows.Remove(window.host_id);
          }
@@ -81,12 +88,19 @@ namespace mbts.ui.shell {
       public IReadOnlyList<IShellWindow> list_windows() =>
          _documents.Keys.ToList();
 
+      public IReadOnlyList<TabDocument> list_documents() => _documents.Values.ToList();
+
       public async Task add(IShellWindow window) {
          if (!Application.Current.Dispatcher.CheckAccess())
             await Application.Current.Dispatcher.InvokeAsync(() => add(window));
          Logger.info(bts.log.Subsystem.Ui, $"Opening window {window.diagnostic_name()}.");
-         lock (_lock)
+         lock (_lock) {
             _documents[window] = _create_document(window);
+         }
+      }
+
+      public async Task add(Task<IShellWindow> window) {
+         await add(await window);
       }
 
       public void remove(IShellWindow window) {
@@ -139,14 +153,13 @@ namespace mbts.ui.shell {
 
       #region IWindowsInterop
       public WindowPosition GetPosition(IShellWindow shell_window) {
-         return GetPosition(shell_window, null);
-      }
-      public WindowPosition GetPosition(IShellWindow shell_window, FloatingWindow owning_window) {
-         FloatingWindow window = owning_window == null ? window_for(shell_window) : owning_window;
+         FloatingWindow window = window_for(shell_window);
+         var doc = window.Documents.FirstOrDefault(td => td.ViewModel.Model == shell_window);
          if (window != null) {
             WindowPosition pos = window.get_position();
             pos.tab_index = window.document_index(shell_window);
             pos.host_id = window.host_id;
+            pos.custom_title = doc?.ViewModel?.TitleOverride;
             return pos;
          }
          Logger.log(bts.log.Level.Warn, bts.log.Subsystem.Ui, "IShellWindow '{0}' does not have a valid position. It may not be displayed".fmt(shell_window.Title));

src/client/spark/view/Shell/Workspaces/WorkspacesModel.cs

@@ -19,6 +19,7 @@ using mbts.persist;
 
 namespace mbts.ui.shell
 {
+
    [Export]
    public class WorkspacesModel : ViewModelBaseWithValidation
    {
@@ -35,10 +36,11 @@ namespace mbts.ui.shell
       private bool _closing = false;
       private ILauncher _launcher;
       private IUndoActions _undo;
+      readonly IBookmarks _bookmarks;
 
       [ImportingConstructor]
       public WorkspacesModel(ISettingsCollector settings, ISavedWindowFactory window_restore, 
-         IWindowManager windows, ObservableCollection<IShellModule> modules, IPersist persist,
+         IWindowManager windows, ObservableCollection<IShellModule> modules, IPersist persist, IBookmarks bm,
          IUserPersist user_persist, ILauncher launcher, IUndoActions undo, [Import("workspace")]string initial_workspace) {
          _settings_collector = settings;
          _window_restore = window_restore;
@@ -48,6 +50,7 @@ namespace mbts.ui.shell
          _modules = modules;
          _launcher = launcher;
          _undo = undo;
+         _bookmarks = bm;
          _launcher.closing += _launcher_closing;
          Workspaces = new ObservableCollection<WorkspaceViewModel>();
          init(initial_workspace).ignore();
@@ -150,6 +153,7 @@ namespace mbts.ui.shell
          workspace.windows = _settings_collector.get_window_state().ToList();
          workspace.modules = _settings_collector.get_module_state().ToList();
          workspace.launcher_state = _settings_collector.get_launcher_state();
+         workspace.named_windows = _bookmarks.bookmarks.ToList();
          return workspace;
       }
 
@@ -166,6 +170,7 @@ namespace mbts.ui.shell
             await Task.Yield();
          }
          workspace.launcher_state = _settings_collector.get_launcher_state();
+         workspace.named_windows = _bookmarks.bookmarks.ToList();
          return workspace;
       }
 
@@ -226,10 +231,15 @@ namespace mbts.ui.shell
             }
          }
 
-         // Notify workspace changed
-         if (WorkspaceLoaded != null) {
-            WorkspaceLoaded(name);
+         _bookmarks.clear();
+         if (def.named_windows != null) {
+            foreach (var window in def.named_windows) {
+               _bookmarks.save_bookmark(window);
+            }
          }
+
+         // Notify workspace changed
+         WorkspaceLoaded?.Invoke(name);
       }
 
       public Task delete_workspace(string name, WorkspaceDefinition def) {
@@ -244,18 +254,20 @@ namespace mbts.ui.shell
          }
          return (def.is_shared ? _public : _private).clear(WORKSPACE_NAMESPACE, name);
       }
+
    }
 
    // The serialized data for the Workspace
    [ProtoContract(Name = "WorkspaceDefinition")]
    public class WorkspaceDefinition
    {
-      // Converted the protobuf type ShellSettings to c#
-      // Avoid re-using id: 1,3,4,5,6,7
-
       [ProtoMember(2, IsRequired = false)]
       public List<WindowState> windows { get; set; }
 
+      // named windows which are not currently open
+      [ProtoMember(3)]
+      public List<WindowState> named_windows { get; set; }
+
       [ProtoMember(9, IsRequired = false)]
       public List<ModuleState> modules { get; set; }
 

src/client/spark/view/Spark/ContractPicker/ContractPickerModel.cs

@@ -197,10 +197,6 @@ namespace mbts.spark
          selected_securities = secs.Select(s => s.Id).ToList();
          ok_button_command.TryRaiseCanExecuteChanged();
          _compose_criteria_clause_name();
-
-         // _security_in_criteria is our fun to be returned to our callers
-         // test on click -- verify our func works
-         //var results = all_securities.Select(all => all.Value.security).Where(x => _security_in_criteria(x)).ToList();
       }
 
       private void on_securities_changed(IReadOnlyDictionary<uint, SecurityMD> changed_securities) {
@@ -212,10 +208,6 @@ namespace mbts.spark
             all_securities = changed_securities;
          }
 
-         securities.sync_with(all_securities
-            .Select(entry => entry.Value.security)
-         );
-
          _update_exchanges();
          _update_security_types();
          _update_symbols();
@@ -327,7 +319,6 @@ namespace mbts.spark
             .Select(s => s.Value.security.Symbol)
             .Distinct()
          );
-            
       }
 
       private void _update_securities() {

src/client/spark/view/Spark/Ladder/LadderGridHost.cs

@@ -533,6 +533,7 @@ namespace mbts.spark {
          _visualHost.AddVisualChild(_mid_mkt_break_vis);
          _visualHost.AddVisualChild(_ask_mkt_break_vis);
          _visualHost.AddVisualChild(_bid_mkt_break_vis);
+         _render_grid_background();
          _init_tactile_adorner();
          // this effectively sets the background of the "Last" column
          if (_scrollViewer != null) {

src/client/spark/view/Spark/Ladder/LadderSettings.cs

@@ -1,5 +1,6 @@
 using bts.api.orderman;
 using mbts.ui;
+using mbts.utils;
 using ProtoBuf;
 using System;
 using System.Linq;
@@ -124,8 +125,6 @@ namespace mbts.spark {
                last_column = last_column,
                calculator_buttons = new[] { 1, 5, 10, 25, 50, 100 },
                scroll_bar_visible = false,
-               ticks_to_recenter = 10,
-               seconds_to_recenter = 300,
                default_order_type = CustomOrderDefinition.CustomOneofCase.Limit,
                default_time_in_force = TimeInForce.Day,
                animate_recenter = true,
@@ -163,10 +162,8 @@ namespace mbts.spark {
       public int[] calculator_buttons { get; set; }
       [ProtoMember(10)]
       public bool scroll_bar_visible { get; set; }
-      [ProtoMember(11)]
-      public int ticks_to_recenter { get; set; }
-      [ProtoMember(12)]
-      public int seconds_to_recenter { get; set; }
+      // USED [ProtoMember(11)]
+      // USED [ProtoMember(12)]
       [ProtoMember(13)]
       public CustomOrderDefinition.CustomOneofCase default_order_type { get; set; }
       [ProtoMember(14)]
@@ -197,11 +194,8 @@ namespace mbts.spark {
          other.price_column.safe_equals(price_column) &&
          other.calculator_buttons.safe_equals(calculator_buttons, (a, b) => a.SequenceEqual(b)) &&
          other.scroll_bar_visible == scroll_bar_visible &&
-         other.ticks_to_recenter == ticks_to_recenter &&
-         other.seconds_to_recenter == seconds_to_recenter &&
          other.default_order_type == default_order_type &&
          other.default_time_in_force == default_time_in_force &&
-         other.seconds_to_recenter == seconds_to_recenter &&
          other.size == size &&
          other.animate_recenter == animate_recenter &&
          other.badge_style == badge_style &&
@@ -213,26 +207,24 @@ namespace mbts.spark {
       public override bool Equals(object obj) => Equals(obj as LadderSettings);
 
       public override int GetHashCode() =>
-         row_height.GetHashCode() +
-         grid_line_color.GetHashCode() +
-         orders_column.GetHashCode() +
-         bid_column.GetHashCode() +
-         ask_column.GetHashCode() +
-         price_column.GetHashCode() +
-         last_column.GetHashCode() +
-         calculator_buttons.GetHashCode() +
-         scroll_bar_visible.GetHashCode() +
-         ticks_to_recenter.GetHashCode() +
-         seconds_to_recenter.GetHashCode() +
-         default_order_type.GetHashCode() +
-         default_time_in_force.GetHashCode() +
-         seconds_to_recenter.GetHashCode() +
-         size.GetHashCode() +
-         animate_recenter.GetHashCode() +
-         badge_style.GetHashCode() +
-         bid_break_line.GetHashCode() +
-         ask_break_line.GetHashCode() +
-         mid_break_line.GetHashCode() +
-         alternate_order_size.GetHashCode();
+         EqualityComparer.aggregate_hash(
+            row_height.GetHashCode(),
+            grid_line_color.GetHashCode(),
+            orders_column.GetHashCode(),
+            bid_column.GetHashCode(),
+            ask_column.GetHashCode(),
+            price_column.GetHashCode(),
+            last_column.GetHashCode(),
+            calculator_buttons.GetHashCode(),
+            scroll_bar_visible.GetHashCode(),
+            default_order_type.GetHashCode(),
+            default_time_in_force.GetHashCode(),
+            size.GetHashCode(),
+            animate_recenter.GetHashCode(),
+            badge_style.GetHashCode(),
+            bid_break_line.GetHashCode(),
+            ask_break_line.GetHashCode(),
+            mid_break_line.GetHashCode(),
+            alternate_order_size.GetHashCode());
    }
 }

src/client/spark/view/Spark/Ladder/LadderVM.cs

@@ -16,8 +16,6 @@ using System.Reactive.Linq;
 using System.Threading.Tasks;
 using System.Windows.Input;
 using System.Windows.Media;
-using mbts.utils;
-using mbts.spark.utilities;
 
 namespace mbts.spark {
    class LadderVM : DisposableViewModelBase, IKeyListener {
@@ -25,7 +23,7 @@ namespace mbts.spark {
       readonly LadderModel _model;
       readonly Property<double> _high, _low, _change, _open, _settlement;
       readonly Property<ulong> _volume;
-      readonly Property<bool> _revert;
+      readonly Property<bool> _auto_clear;
       readonly Property<Brush> _bid_background, _bid_foreground, _ask_background, _ask_foreground;
       IWindowSettingsManager<LadderSettings> _settings_manager;
       ulong _misc_time = 0;
@@ -44,7 +42,7 @@ namespace mbts.spark {
          _open = backing(nameof(open), double.NaN);
          _settlement = backing(nameof(settlement), double.NaN);
          _volume = backing(nameof(volume), 0ul);
-         _revert = backing(nameof(revert), false);
+         _auto_clear = backing(nameof(auto_clear), false);
          _status = backing(nameof(status_message), "");
          _status_brush = backing(nameof(status_background), (Brush)BTSColors.UI.spark_menu_background);
          // brush props
@@ -176,7 +174,7 @@ namespace mbts.spark {
       public double open => _open.Value;
       public double settlement => _settlement.Value;
       public ulong volume => _volume.Value;
-      public bool revert { get => _revert.Value; set => _revert.set(value); }
+      public bool auto_clear { get => _auto_clear.Value; set => _auto_clear.set(value); }
       public CalculatorLogic size_calculator { get; }
       public OrderPropertiesEditorVM order_properties_vm { get; }
       public ObservableCollection<LadderOrderVM> orders { get; }
@@ -232,16 +230,8 @@ namespace mbts.spark {
                
             uint size = args.size_from_alternate ? s.alternate_order_size : (uint)Math.Abs(size_calculator.value.Value);
             await _model.order_gateway.send_order(order, args.price, size);
-            if (revert) {
-               size_calculator.value = s.size;
-               if (order_properties_vm.selected_time_in_force != s.default_time_in_force && order_properties_vm.time_in_forces.Contains(s.default_time_in_force))
-                  order_properties_vm.selected_time_in_force = s.default_time_in_force;
-               foreach (var ot in order_properties_vm.order_types) {
-                  ot.load_from(_model.order_gateway.factory.custom_order(ot.order_type));
-                  if (ot.order_type == s.default_order_type)
-                     order_properties_vm.selected_order_type = ot;
-               }
-            }
+            if (auto_clear)
+               size_calculator.value = 0;
          }
       }
 

src/client/spark/view/Spark/Ladder/LadderView.xaml

@@ -394,7 +394,7 @@
         </ItemsControl>
         <Button Margin="2" HorizontalAlignment="Stretch" Command="{Binding size_calculator.clear}">Clear</Button>
         <!-- reset to defaults after order -->
-        <CheckBox Margin="2" IsChecked="{Binding revert}">Auto Clear</CheckBox>
+        <CheckBox Margin="2" IsChecked="{Binding auto_clear}">Auto Clear</CheckBox>
       </StackPanel>
       <Grid Grid.Row="1">
         <Grid.RowDefinitions>

src/client/spark/view/Spark/Ladder/RecenterCountdown.cs

@@ -17,16 +17,17 @@ namespace mbts.spark {
       };
       SecurityImmutable? _security;
       MDConsumerHandle _handle;
-      LadderSettings _settings;
       DateTime _last_recenter = DateTime.MaxValue;
       double _last_mid = 0.0;
       IDisposable _dispose;
+      readonly TimeSpan _seconds_to_center;
 
       public RecenterCountdown() {
          DataContextChanged += RecenterCountdown_DataContextChanged;
          Unloaded += (o,e) => on_unload(false);
          _timer.Tick += _timer_Tick;
          _timer.Start();
+         _seconds_to_center = TimeSpan.FromSeconds(config.Configuration.get<bts.config.LadderKnobs>().seconds_to_recenter);
       }
 
       static (double,double) tick_range(double price, int tick_count, ContractSpecificationImmutable spec) {
@@ -34,11 +35,10 @@ namespace mbts.spark {
       }
 
       private void _timer_Tick(object sender, EventArgs e) {
-         if (_settings != null && _security != null) {
+         if (_security != null) {
             var mid = _handle.mid();
-            (double low, double high) = tick_range(_last_mid, _settings.ticks_to_recenter, _security.Value.ContractSpec);
             if (//mid < low || mid > high ||
-               (_last_recenter + TimeSpan.FromSeconds(_settings.seconds_to_recenter) < DateTime.UtcNow)) {
+               (_last_recenter + _seconds_to_center < DateTime.UtcNow)) {
                trigger_recenter(mid);
             }
          }
@@ -59,7 +59,6 @@ namespace mbts.spark {
             vm.OnDisposed += on_unload;
             _dispose =
                Disposable.Create(
-                  vm.settings.Subscribe(s => _settings = s),
                   Disposable.Create(() => vm.OnDisposed -= on_unload));
          }
       }

src/client/spark/view/Spark/Shell/OrderBook/OrderNode.cs

@@ -37,13 +37,13 @@ namespace bts.spark
             if (this.IsRoot) {
                return Enumerable.Empty<OrderNode>();
             }
-            return this.Parent.ToIEnumerable().Concat(this.Parent.Ancestors);
+            return new[] { this.Parent }.Concat(this.Parent.Ancestors);
          }
       }
 
       public IEnumerable<OrderNode> MeAndDescedants {
          get {
-            return this.ToIEnumerable().Concat(this.children.SelectMany(c => c.MeAndDescedants));
+            return new[] { this }.Concat(this.children.SelectMany(c => c.MeAndDescedants));
          }
       }
 

src/client/spark/view/Spark/Utilities/WorkLoop.cs

@@ -10,7 +10,7 @@ namespace mbts.spark {
       static readonly TimeSpan default_interval;
 
       static WorkLoop() {
-         var timer_params = config.Configuration.get<bts.config.LadderTimerParams>();
+         var timer_params = config.Configuration.get<bts.config.LadderKnobs>();
          default_priority = Enum.TryParse(timer_params.priority, true, out DispatcherPriority p) ? p : DispatcherPriority.ApplicationIdle;
          default_interval = TimeSpan.FromMilliseconds(timer_params.interval_ms);
       }

src/client/spark/view/Utility/TextUtils.cs

@@ -130,12 +130,22 @@ namespace mbts.ui.utility.text {
          public static class Time {
             public static readonly Regex
                StandardTime = new Regex(@"^((0?[0-9])|(1?[0-2]?))(:?|:([0-5]?|[0-5][0-9]?)) ?(?i)((a|p)m?)?$"),
+               StandardTimeOptionalSeconds = new Regex(@"^((\d?|0\d|1[0-2])|((\d|0\d|1[0-2])(:?|(:([0-5]?|[0-5]\d(( ?|( ([aApP]?|[aApP][mM])))(:?|(: ?)|(: [aApP]?[mM]?)|(:[0-5]?\d? ?[aApP]?[mM]?))|:([0-5]?|([0-5]\d( ?|( ([aApP]?|([aApP][mM]))))))))))))$"),
                MilitaryTime = new Regex(@"^(([0-1]?[0-9])|(2?[0-3]?))(:?|:([0-5]?|[0-5][0-9]?))$"),
+               MilitaryTimeOptionalSeconds = new Regex(@"^((\d?|[0-1]\d|2[0-3])|((\d|[0-1]\d|2[0-3])(:?|(:([0-5]?|[0-5]\d(:?|:([0-5]?|[0-5]\d)))))))$"),
                DaysInMonth = new Regex(@"^([0-9]?|[0-2][0-9]|3[0-1])?$"),
                MonthAbbrev = new Regex(@"^(?i)(a(ug?|pr?)?|d(e|ec)?|f(e?b)?|j(an?|u[nl]?)?|m(a[ry]?)?|(o(ct?)?)|s(ep?)?|n(ov?)?)?$"),
                Year = new Regex(@"^\d{0,4}$"),
                TimeSpan = new Regex(@"^(\d+(:?|:([0-5]?|[0-5]\d(:?|:([0-5]?|[0-5]\d)))))?$");
          }
       }
+
+      public static class StringFormats {
+         public static class Time {
+            public const string
+               MilitaryWithSeconds = "HH:mm:ss",
+               StandardWithSeconds = "hh:mm:ss tt";
+         }
+      }
    }
 }

src/client/spark_api/security/SecurityFormatter.cs

@@ -1,18 +1,18 @@
-using System;
-using System.Collections.Concurrent;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using static bts.api.Security;
 
 namespace bts.api {
    struct FutureLeg {
       public readonly Venue exchange;
-      public readonly string symbol;
+      public readonly string symbol, exchange_desc;
       public readonly FutureContractImmutable future;
       public readonly int quantity;
 
-      public FutureLeg(Venue exchange, string symbol, FutureContractImmutable future, int quantity) {
+      public FutureLeg(Venue exchange, string symbol, string exchange_desc, FutureContractImmutable future, int quantity) {
          this.exchange = exchange;
          this.symbol = symbol;
+         this.exchange_desc = exchange_desc;
          this.future = future;
          this.quantity = quantity;
       }
@@ -32,12 +32,14 @@ namespace bts.api {
          return string.Join("", _format_future_legs(future_legs));
       }
 
-      public static string format_future(string symbol, FutureContractImmutable future) =>
-         $"{symbol}{DateFormatter.Format(future.ExpirationYear, future.ExpirationMonth, future.ExpirationDay, DateFormat.MonthCodeLastDigitYear)}";
+      public static string format_future(string symbol, string exch_desc, FutureContractImmutable future) =>
+         string.IsNullOrEmpty(exch_desc) ?
+            $"{symbol}{DateFormatter.Format(future.ExpirationYear, future.ExpirationMonth, future.ExpirationDay, DateFormat.MonthCodeLastDigitYear)}" :
+            exch_desc;
 
       static string _format(SecurityImmutable security) {
          switch (security.ContractCase) {
-            case ContractOneofCase.Future: return format_future(security.Symbol, security.Future);
+            case ContractOneofCase.Future: return format_future(security.Symbol, security.glbx_desc(), security.Future);
             case ContractOneofCase.Spread:
                if (security.HasGlbx) {
                   // In case we ever bring it back, pretty printed spread names can be found here:
@@ -61,7 +63,7 @@ namespace bts.api {
       }
 
       static string _get_first_leg_name(FutureLeg fl) {
-         string sym_exp_name = format_future(fl.symbol, fl.future);
+         string sym_exp_name = format_future(fl.symbol, fl.exchange_desc, fl.future);
          switch (fl.quantity) {
             case 1: return sym_exp_name;
             case -1: return $"-{sym_exp_name}";
@@ -70,7 +72,7 @@ namespace bts.api {
       }
 
       static string _get_subsequent_leg_name(FutureLeg fl) {
-         string sym_exp_name = format_future(fl.symbol, fl.future);
+         string sym_exp_name = format_future(fl.symbol, fl.exchange_desc, fl.future);
          switch (fl.quantity) {
             case 1: return $"+{sym_exp_name}";
             case -1: return $"-{sym_exp_name}";

src/client/spark_api/security/SecurityHelper.cs

@@ -19,7 +19,7 @@ namespace bts.api {
          foreach (var l in legs) {
             switch (l.Security.ContractCase) {
                case ContractOneofCase.Future:
-                  yield return new FutureLeg(l.Security.Exchange, l.Security.Symbol, l.Security.Future, qty * l.SignedQuantity());
+                  yield return new FutureLeg(l.Security.Exchange, l.Security.Symbol, l.Security.glbx_desc(), l.Security.Future, qty * l.SignedQuantity());
                   break;
                case ContractOneofCase.Spread:
                   foreach (var fl in _get_flat_future_legs(l.Security.Spread.Legs, qty * l.SignedQuantity()))
@@ -56,6 +56,9 @@ namespace bts.api {
 
       public static string DisplayExchange(this SecurityImmutable security) =>
          security.ContractCase == ContractOneofCase.Synthetic ? $"[{string.Join(",", get_flat_future_legs(security).Select(flt => flt.exchange.ToString()).Distinct().OrderBy(e => e))}]" : security.Exchange.ToString();
+
+      public static string glbx_desc(this SecurityImmutable security) =>
+         security.HasGlbx ? security.Glbx.SecurityDesc ?? string.Empty : string.Empty;
       #endregion
 
       public static class Comparer {

src/lib/api/service_helpers/AdminServiceUtils.cpp

@@ -820,19 +820,49 @@ void AdminServiceUtils::delete_user (bts::api::admin::AdminData& admin_data,
 ////////////////////////////////////////////
 bool AdminServiceUtils::_sort_security_scopes (const bts::api::admin::SecurityScope& a,
                                                const bts::api::admin::SecurityScope& b){
-   if (has_id(a.security_id()))
-      return true;
+   // for both the sec and product scope sorting, the more
+   // qualified arg is sent to the top (vs wildcard). the
+   // order of qualification is sec_id, symbol, type, exchange.
+   // if both a and b are equally qualified, it is a
+   // a dont-care case (in terms of sec/prod res) and 
+   // ascending orderring is used
+   if (has_id(a.security_id())){
+      if (has_id(b.security_id()))
+         return a.security_id() < b.security_id();
+      else
+         return true;
+   }
    if (has_id(b.security_id()))
      return false;
-   if (a.security_symbol() != "*")
-      return true; 
-   return false;
+   // symbol
+   if (is_symbol_set(a.security_symbol())){
+      if (is_symbol_set(b.security_symbol()))
+         return a.security_symbol() < b.security_symbol();
+      else
+         return true;
+   }
+   if (is_symbol_set(b.security_symbol()))
+      return false;
+   // sec type
+   if (a.security_type() != b.security_type())
+      return a.security_type() < b.security_type();
+   return a.exchange() < b.exchange();
 }
 bool AdminServiceUtils::_sort_product_scopes (const bts::api::admin::ProductScope& a,
-                                              const bts::api::admin::ProductScope&){
-   if (a.security_symbol() != "*")
-      return true;
-   return false;
+                                              const bts::api::admin::ProductScope& b){
+   // symbol
+   if (is_symbol_set(a.security_symbol())){
+      if (is_symbol_set(b.security_symbol()))
+         return a.security_symbol() < b.security_symbol();
+      else
+         return true;
+   }
+   if (is_symbol_set(b.security_symbol()))
+      return false;
+   // sec type
+   if (a.security_type() != b.security_type())
+      return a.security_type() < b.security_type();
+   return a.exchange() < b.exchange();
 }
 void AdminServiceUtils::sort_limits (bts::api::admin::BTSAccount& account){
    std::sort (account.mutable_security_position_limits()->begin(),
@@ -901,7 +931,7 @@ bool AdminServiceUtils::match_security_scope (const bts::api::admin::SecuritySco
 bool AdminServiceUtils::match_product_scope (const bts::api::admin::ProductScope& scope,
                                              const bts::api::Product& product){
    return product.exchange() == scope.exchange() &&
-          product.type() && scope.security_type() &&
+          product.type() == scope.security_type() &&
           _match_limit_rule_symbol(product.symbol(), scope.security_symbol());
 }
 bool AdminServiceUtils::match_product(const api::Security& security,
@@ -1025,4 +1055,7 @@ void AdminServiceUtils::bootstrap_data (bts::api::admin::AdminData& data){
    acct_a1_clip_size->set_clip_size(100);
 }
 bool AdminServiceUtils::has_id (uint32_t id){return id>0;}
+bool AdminServiceUtils::is_symbol_set (const std::string& symbol){
+   return symbol !="*";
+}
 }}

src/lib/api/service_helpers/AdminServiceUtils.h

@@ -240,6 +240,7 @@ public:
    // misc//////// ////////////////////////////
    ////////////////////////////////////////////
    static bool has_id (uint32_t id);
+   static bool is_symbol_set (const std::string& symbol);
    static void bootstrap_data (bts::api::admin::AdminData& data);
 };
 }}

src/lib/api/services/OrderService.proto

@@ -233,7 +233,7 @@ enum CTICode {
   CTI_CODE_CTI_CODE1 = 49;
   CTI_CODE_CTI_CODE2 = 50;
   CTI_CODE_CTI_CODE3 = 51;
-  CTI_CODE_CTI4 = 52;
+  CTI_CODE_CTI_CODE4 = 52;
 }
 
 enum NativeOrderStatus 

src/lib/config/protos/MarketData.proto

@@ -76,6 +76,10 @@ message CMEOptions
       repeated uint32 affinity = 10;
       optional uint32 udp_socket_buffer_size = 11 [default = 33554432]; //32mb
    }
+   enum BookType {
+       BOOKTYPE_CONSOLIDATED = 0;
+       BOOKTYPE_DIRECT = 1;
+   }
    message CMEChannel {
       repeated string id = 2;
       optional string feed_engine = 3 [default = "default"];
@@ -98,6 +102,7 @@ message CMEOptions
       optional bool join_snapshot_on_gap = 32 [default = false];
 
       optional bool load_security_db = 33 [default = true];
+      optional BookType book_type = 34 [default = BOOKTYPE_CONSOLIDATED];
 
    }
    optional string onixs_license = 1 [default = "{default}"];

src/lib/config/protos/UI.proto

@@ -28,20 +28,6 @@ message IShellModuleProps
    repeated ModuleProps module = 1;
 }
 
-message IShellWindowProps {
-   // TODO: Maybe IWindowManager should be responsible for total-windows-open limits?
-   // To achieve that we would need to use a sort of type-windowprops map.
-   // However, that makes the problem more complicated, and at this point we are just aiming to limit n_ladders open.
-   // So for now, just let the LadderWindowFactory manage this.
-   message WindowProps {
-      // NOTE: 50 will be a dangerously high default for most other windows (than Ladder),
-      // so be careful if you add other WindowProps members.
-      optional uint32 max_total_open = 1 [default = 50];
-   }
-
-   optional WindowProps ladder = 1;
-}
-
 message GridSettings {
     optional string font_family = 1 [default = "Calibri"];
     optional float font_size = 2 [default = 10];
@@ -62,9 +48,11 @@ message ShellLimits {
    optional uint64 limit_group_memory_mb = 3;
 }
 
-// priorities (ordered): SystemIdle, ApplicationIdle, ContextIdle, Background, Input, Loaded, Render, DataBind, Normal, Send
-// It seems like all priorities higher than ApplicationIdle choke the main thread.
-message LadderTimerParams {
+message LadderKnobs {
+  // priorities (ordered): SystemIdle, ApplicationIdle, ContextIdle, Background, Input, Loaded, Render, DataBind, Normal, Send
+  // It seems like all priorities higher than ApplicationIdle choke the main thread.
   optional string priority = 1 [default = "ApplicationIdle"];
   optional double interval_ms = 2 [default = 100];
+  optional uint32 seconds_to_recenter = 3 [default = 300];
+  optional uint32 max_open_ladders = 4 [default = 75];
 }

src/lib/md/SecurityDB.cpp

@@ -89,7 +89,7 @@ size_t load_securities(
 
 void store_if_new(const std::vector<Security*>& securities) {
    auto dbptr = open_db();
-   auto insert = utils::prepare_query(dbptr, "insert into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)");
+   auto insert = utils::prepare_query(dbptr, "insert or ignore into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)");
    sqlite3_exec_nofail(dbptr, "BEGIN TRANSACTION");
    std::string buffer;
    for (const api::Security* psecurity : securities) {
@@ -100,10 +100,7 @@ void store_if_new(const std::vector<Security*>& securities) {
          SQLITE_CHK(sqlite3_bind_int(insert.get(), 4, securitytype_of(*psecurity)), dbptr);
          SQLITE_CHK(sqlite3_bind_blob(insert.get(), 5, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr);
          SQLITE_CHK(sqlite3_bind_int(insert.get(), 6, expiration(*psecurity).to_short()), dbptr);
-         auto result = sqlite3_step(insert.get());
-         if (result != SQLITE_DONE) {
-            LOG(md, Warn, "Failed to insert security into security db. key={}", psecurity->id());
-         }
+         sqlite3_step(insert.get());
          sqlite3_reset(insert.get());
       }
    }

src/lib/md/SecurityIDRecorder.cpp

@@ -133,6 +133,8 @@ void SecurityIDRecorder::work_db() {
    // Once a security has a key it is public so record it in the security database
    std::shared_ptr<SecurityIDEntry> temp;
    vector<std::shared_ptr<SecurityIDEntry>> new_entries;
+   // SPARK-1446 Need to try to add existing securities to the db in case we deleted them by hand
+   vector<std::shared_ptr<SecurityIDEntry>> existing_securities;
    vector<std::shared_ptr<SecurityIDEntry>> defer;
    vector<string> new_descriptors;
    size_t count = 0;
@@ -150,6 +152,7 @@ void SecurityIDRecorder::work_db() {
          if (key != invalid_security_key) {
             temp->security.set_id(key);
             temp->callback(temp->security);
+            existing_securities.push_back(move(temp));
          }
          else {
             // always push in parallel
@@ -168,7 +171,7 @@ void SecurityIDRecorder::work_db() {
    }
    
 
-   if (new_entries.size() > 0) {
+   if (!new_entries.empty()) {
       sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
 
       for (size_t x = 0; x < new_entries.size(); ++x) {
@@ -179,15 +182,22 @@ void SecurityIDRecorder::work_db() {
       }
 
       sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
+   }
 
-      // record the securities
+   // record the securities
+   if (!new_entries.empty() || !existing_securities.empty()) {
       std::vector<Security*> save_these;
-      save_these.reserve(new_entries.size());
+      save_these.reserve(new_entries.size() + existing_securities.size());
       for (const auto& e : new_entries) {
          save_these.push_back(&e->security);
       }
+      for (const auto& e : existing_securities) {
+         save_these.push_back(&e->security);
+      }
       db::store_if_new(save_these);
+   }
 
+   if (!new_entries.empty()) {
       for (size_t x = 0; x < new_entries.size(); ++x) {
          Security& security = new_entries[x]->security;
          new_entries[x]->callback(security);
@@ -199,9 +209,18 @@ void SecurityIDRecorder::bulk_insert(const std::vector<const api::Security*>& da
    if (data.size() == 0) return;
    sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
    std::string descriptor;
-   for (const api::Security* sec : data) {
-      descriptor = security_descriptor(*sec);
-      insert(prefix + descriptor, securitytype_of(*sec));
+   {
+      auto insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)");
+      for (const api::Security* sec : data) {
+         descriptor = prefix + security_descriptor(*sec);
+         auto type = securitytype_of(*sec);
+
+         sqlite3_bind_int(insert_stmt.get(), 1, sec->id());
+         sqlite3_bind_text(insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(), SQLITE_STATIC);
+         sqlite3_bind_int(insert_stmt.get(), 3, (int)type);
+         sqlite3_step(insert_stmt.get());
+         sqlite3_reset(insert_stmt.get());
+      }
    }
    sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
 }

src/lib/md/relay/QueryServer.cpp

@@ -15,6 +15,9 @@
 #include "lib/api/apicore.pb.h"
 #include "lib/api/services/MarketDataRelay.server.pb.h"
 #include "lib/api/public/ServiceContext.h"
+#ifndef _WIN32
+#include "server/sparkd/metrics/metrics.h"
+#endif
 
 #include <vector>
 #include <list>
@@ -61,13 +64,29 @@ struct DatedHandle {
 
 struct MDConsumerGroup {
    // probably just have to send the consumer in from outside
-   MDConsumerGroup(const UserSymbol& symbol, const MDConsumer& consumer) :
+   MDConsumerGroup(const UserSymbol& symbol, const MDConsumer& consumer, metrics::MetricsContainer& stats) :
       symbol(symbol),
       platform(platform_for_exchange(symbol.exchange())),
       channel(consumer.get_channel(symbol)),
       handles()
    {
       set_consumer(consumer);
+      
+      // create and publish a stat for this group
+#ifndef _WIN32
+      if (symbol.type() != SecurityType::SECURITYTYPE_SYNTHETIC) {
+         std::vector<metrics::Label> labels;
+         labels.push_back({ "symbol", symbol.symbol() });
+         labels.push_back({ "type", SecurityType_Name(symbol.type()).substr(13) /*SECURITYTYPE_*/ });
+         labels.push_back({ "exch", Venue_Name(symbol.exchange()).substr(6) /*VENUE_*/ });
+         stats.publish(
+            std::make_shared<metrics::LazyCounterMetric>("spark_md_book_updates", [=]() { return std::max(channel.bbo_count(), channel.depth_count());  }, labels));
+         stats.publish(
+            std::make_shared<metrics::LazyCounterMetric>("spark_md_security_count", [=]() { return channel.security_count();  }, labels));
+         stats.publish(
+            std::make_shared<metrics::LazyCounterMetric>("spark_md_trade_count", [=]() { return channel.trade_count();  }, labels));
+      }
+#endif
    }
    MDConsumerGroup(const MDConsumerGroup& other) = delete;
 
@@ -109,6 +128,7 @@ void set_price(PriceSize ps, co2::TradeMessage* ptrade) {
    ptrade->set_exponent(ps.exponent());
    ptrade->set_size(ps.size());
 }
+
 void copy_misc(MiscData misc, co2::MiscMessage* pmisc) {
    pmisc->set_settlement_price(misc.settlement_price);
    pmisc->set_settlement_date(misc.settlement_date.to_short());
@@ -134,14 +154,14 @@ class QueryServerImpl : public IQueryServer {
    boost::container::flat_map<UserSymbol,shared_ptr<MDConsumerGroup>> _groups;
 
 public:
-   QueryServerImpl(const std::set<UserSymbol>& symbols, std::shared_ptr<api::ISyntheticSecurityService>& ss) :
+   QueryServerImpl(const std::set<UserSymbol>& symbols, std::shared_ptr<api::ISyntheticSecurityService>& ss, metrics::MetricsContainer& stats) :
       _consumer(symbols, ss),
       _symbols(symbols),
       _ss(ss),
       _groups()
    {
       for (const auto& sym : symbols) {
-         _groups.emplace(std::make_pair(sym, std::make_unique<MDConsumerGroup>(sym, _consumer)));
+         _groups.emplace(std::make_pair(sym, std::make_unique<MDConsumerGroup>(sym, _consumer, stats)));
       }
       _groups.shrink_to_fit();
 
@@ -336,24 +356,36 @@ public:
 struct MDRelayServer : bts::co2::MarketDataRelayBase {
    const QueryServerImpl& _db;
    utils::Dispatcher& _threadpool;
-    MDRelayServer(const QueryServerImpl& db, utils::Dispatcher& threadpool, bts::api::ServiceContext ctx) : 
+#ifndef _WIN32
+   std::shared_ptr<metrics::HistogramMetric<5>> _time,_size;
+#endif
+
+    MDRelayServer(const QueryServerImpl& db, utils::Dispatcher& threadpool, bts::api::ServiceContext ctx, metrics::MetricsContainer& stats) : 
        bts::co2::MarketDataRelayBase(ctx),
        _db(db), _threadpool(threadpool)
-    { }
+#ifndef _WIN32
+       , _time(std::make_shared<metrics::HistogramMetric<5>>("spark_qserver_response_time_ms", std::array<double,5>({ 10.0, 50.0, 100.0, 500.0, 1000.0 }))),
+       _size(std::make_shared<metrics::HistogramMetric<5>>("spark_qserver_size_bytes", std::array<double,5>({ 1024 /*kb*/, 1024*1024 /*mb*/, (1024*1024)*4, (1024*1024)*16, (1024*1024)*32 })))
+    {
+       stats.publish(_time);
+       stats.publish(_size);
+    }
+#else
+   {}
+#endif
 
     virtual void requestmd(const MDRequest & request, const std::function<void(const MDReply&)>& reply) override {
+        TimeStamp start = TimeStamp::now();
        _threadpool.post([=]() {
           google::protobuf::Arena arena;
           MDReply* preply = google::protobuf::Arena::CreateMessage<MDReply>(&arena);
-          TimeStamp start = TimeStamp::now();
           _db.query(request, preply);
-          auto elapsed = (TimeStamp::now() - start);
-          LOG(md, Diag,
-             "Request for: {} took {}us and returned {} bytes before compression",
-             request.host(),
-             elapsed.microseconds(),
-             preply->ByteSize());
           reply(*preply);
+          auto elapsed = (TimeStamp::now() - start);
+#ifndef _WIN32
+          _time->observe(elapsed.milliseconds());
+          _size->observe(preply->ByteSize());
+#endif
        });
     }
     
@@ -382,11 +414,15 @@ struct MDRelayServer : bts::co2::MarketDataRelayBase {
 };
 
 struct QueryServer {
-    QueryServer(std::set<UserSymbol> symbols, utils::Dispatcher& threadpool, const bts::api::ServiceContext& ctx, std::shared_ptr<api::ISyntheticSecurityService>& ss):
-    db(symbols, ss),
-    relay(db, threadpool, ctx),
-    security_timer(),
-    health_timer()
+    QueryServer(std::set<UserSymbol> symbols, 
+                utils::Dispatcher& threadpool, 
+                const bts::api::ServiceContext& ctx, 
+                std::shared_ptr<api::ISyntheticSecurityService>& ss,
+                metrics::MetricsContainer& stats):
+      db(symbols, ss, stats),
+      relay(db, threadpool, ctx, stats),
+      security_timer(),
+      health_timer()
     { }
     QueryServerImpl db;
     MDRelayServer relay;
@@ -394,7 +430,7 @@ struct QueryServer {
     std::unique_ptr<utils::DispatcherTimer> health_timer;
 };
    
-QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared_ptr<api::ISyntheticSecurityService>& ss) {
+QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared_ptr<api::ISyntheticSecurityService>& ss, metrics::MetricsContainer& stats) {
    auto& threadpool = utils::Dispatcher::get_named("MDQuery");
 
    std::set<UserSymbol> all_symbols;
@@ -403,7 +439,8 @@ QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared
    SecurityType type;
    bts_assert(api::SecurityType_Parse(config::get<PlatformConfig>().type(), &type));
    get_system_md_symbols(venue, type, all_symbols);
-   QueryServerPtr container = std::make_shared<QueryServer>(all_symbols, threadpool, ctx, ss);
+
+   QueryServerPtr container = std::make_shared<QueryServer>(all_symbols, threadpool, ctx, ss, stats);
    auto& db = container->db;
 
     // Scan for new securities every half second
@@ -426,8 +463,8 @@ QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared
    return container;
 }
 
-std::unique_ptr<IQueryServer> create_query_server(const std::set<UserSymbol>& symbols, std::shared_ptr<api::ISyntheticSecurityService>& ss) {
-   return std::make_unique<QueryServerImpl>(symbols, ss);
+std::unique_ptr<IQueryServer> create_query_server(const std::set<UserSymbol>& symbols, std::shared_ptr<api::ISyntheticSecurityService>& ss, metrics::MetricsContainer& stats) {
+   return std::make_unique<QueryServerImpl>(symbols, ss, stats);
 }
 
 }}//namespace

src/lib/md/relay/QueryServer.h

@@ -11,6 +11,7 @@
 
 namespace bts {
     namespace api { class ServiceContext; }
+    namespace metrics { class MetricsContainer; }
 namespace md {
 
 struct IQueryServer {
@@ -18,11 +19,11 @@ struct IQueryServer {
    virtual void scan_securities() = 0;
    virtual void query(const co2::MDRequest& request, co2::MDReply* reply) const = 0;
 };
-std::unique_ptr<IQueryServer> create_query_server(const std::set<UserSymbol>&, std::shared_ptr<api::ISyntheticSecurityService>&);
+std::unique_ptr<IQueryServer> create_query_server(const std::set<UserSymbol>&, std::shared_ptr<api::ISyntheticSecurityService>&, metrics::MetricsContainer&);
 
 struct QueryServer;
 typedef std::shared_ptr<QueryServer> QueryServerPtr;
-QueryServerPtr run_query_server(const bts::api::ServiceContext&, std::shared_ptr<api::ISyntheticSecurityService>&);
+QueryServerPtr run_query_server(const bts::api::ServiceContext&, std::shared_ptr<api::ISyntheticSecurityService>&, metrics::MetricsContainer&);
 
 }}//namespace
 

src/lib/md/relay/RelayRecorder.cpp

@@ -121,9 +121,6 @@ void record_security_keys(md::SecurityIDRecorder& idrec, const co2::MDReply& rep
          }
       }
    }
-   // Sort by id only actually works if the recording was done on a fresh machine
-   std::sort(data.begin(), data.end(), [](const Security* a, const Security* b) { return a->id() < b->id(); });
-   // We could insert garbage until data[0]->id()
    idrec.bulk_insert(data, "REPLAY: ");
 }
 #endif

src/lib/md/test/md2_tests.cpp

@@ -10,6 +10,7 @@
 #include "lib/utils/defer.h"
 #include "lib/utils/warn_ctrl.h"
 #include "lib/utils/bts_chrono.h"
+#include "server/sparkd/metrics/metrics.h"
 
 #include "lib/config/protos/MarketData.pb.h"
 
@@ -30,6 +31,15 @@
 using namespace std;
 
 namespace bts {
+
+#ifdef _WIN32
+namespace metrics {
+struct MetricCollection{};
+MetricsContainer::MetricsContainer() {}
+MetricsContainer::~MetricsContainer() {}
+}
+#endif
+
 int md_tests = 0;
 namespace md {
 // import this from RelayClient.cpp
@@ -1275,7 +1285,8 @@ struct Co2RelayTests {
       set<UserSymbol> symbols;
       symbols.insert(symbol);
       shared_ptr<api::ISyntheticSecurityService> empty;
-      auto server = create_query_server(symbols, empty);
+      metrics::MetricsContainer stats;
+      auto server = create_query_server(symbols, empty, stats);
       REQUIRE(server);
       server->scan_securities();
 

src/lib/quickbus/network/QBConnection.cpp

@@ -193,10 +193,18 @@ void QBConnection::handle_msg(msgs::QBMsg *msgs)
             me->intro=op.intro();
             me->my_id = NodeID(me->intro.node_id());
 
-            if (!me->router.add_neighbour(me->my_id,shared_from_this())) {
-               close_connection("Node is already connected");
+            if (me->router.is_neighbour(me->my_id)) {
+               LOG(QBus,Warn, "Refusing connection from {} to {} reason: {} is already connected" ,
+                   me->my_desc , me->remote_desc , me->my_id.value());
+               if (me->socket.is_open()) 
+                  me->socket.close();
                return;
             }
+            else {
+               LOG(QBus,Warn, "Accepted connection from {} to {}. {} is now connected." ,
+                   me->my_desc , me->remote_desc , me->my_id.value());
+               me->router.add_neighbour(me->my_id,shared_from_this());
+            }
          }
          break;
       }

src/lib/quickbus/routing/Router.cpp

@@ -184,6 +184,11 @@ void Router::remove_neighbour(NodeID const&removed_neighbour_id)
    me->neighbours.erase(removed_neighbour_id);
 }
 
+bool Router::is_neighbour(NodeID const&neighbour_id) const
+{
+   return me->neighbours.find(neighbour_id) != me->neighbours.end();
+}
+
 void Router::advertise(NodeID const& sender,TableID const&tid,Advertisement const&ad) 
 {
    LOG(QBus,Diag, "{} received advertisement from nid: {}  tid: {} ad topic: {}",

src/lib/quickbus/routing/Router.h

@@ -15,7 +15,8 @@ public:
    NodeID id() const ;
    bool add_neighbour (NodeID const& neighbour_id, INodePtr const& neighbour);
    void remove_neighbour (NodeID const& neighbour_id);
-
+   bool is_neighbour(NodeID const&neighbour_id) const;
+   
    // from INode 
    void advertise(NodeID const& sender,TableID const&id,Advertisement const&adv) override;
    void unadvertise(NodeID const& sender,TableID const&tid) override;

src/server/sparkd/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/Md.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;dspark/mirage.cpp;dspark/mirage/routing/Router.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;main.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/MDP3Utils.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;mercury/actors/Actor.cpp;mercury/broker/AsyncEventBroker.cpp;mercury/broker/CoreEventBroker.cpp;mercury/broker/IEventBroker.cpp;mercury/events/TimerEvent.cpp;mercury/mercury_utils/Executor.cpp;mercury/mercury_utils/IPollableActor.cpp;mercury/mercury_utils/mercury_common.cpp;mercury/synopsis/SynopsisBuilder.cpp;mercury/topics/ITopic.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/TickOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/Line truncated
+set(sources cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/Md.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage.cpp;dspark/mirage/routing/Router.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;main.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/MDP3Utils.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;mercury/actors/Actor.cpp;mercury/broker/AsyncEventBroker.cpp;mercury/broker/CoreEventBroker.cpp;mercury/broker/IEventBroker.cpp;mercury/events/TimerEvent.cpp;mercury/mercury_utils/Executor.cpp;mercury/mercury_utils/IPollableActor.cpp;mercury/mercury_utils/mercury_common.cpp;mercury/synopsis/SynopsisBuilder.cpp;mercury/topics/ITopic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/TickOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreaderLine truncated

src/server/sparkd/cmds/Run.cpp

@@ -8,6 +8,7 @@
 #include "lib/logging/logging.h"
 #include "lib/os/os.h"
 #include "lib/quickbus/quickbus.h"
+#include "lib/utils/Dispatcher.h"
 
 #include "lib/md/SyntheticSecurityService.h"
 #include "server/sparkd/exchange_sim/ExchangeSim.h"
@@ -16,6 +17,8 @@
 #include "server/sparkd/persistence/PersistenceServiceServer.h"
 #include "server/sparkd/support/ErrorReportService.h"
 #include "server/sparkd/http/WebServer.h"
+#include "server/sparkd/metrics/metrics.h"
+#include "server/sparkd/metrics/system_metrics.h"
 #include "server/sparkd/user/AdminServiceServer.h"
 #include "server/sparkd/user/UserServiceServer.h"
 
@@ -37,7 +40,9 @@ int Run::execute(std::vector<std::string> const &) {
    qbus::Quickbus quickbus(qb_cfg);
    api::ServiceContext sc(&quickbus);
 
-   http::SparkWebServer webserver(sc, ports.config_port());
+   metrics::MetricsContainer stats;
+   metrics::publish_system_metrics(stats);
+   http::SparkWebServer webserver(sc, stats, ports.config_port());
 
    persistence::PersistenceServiceServer settings_saver(sc, "settings");
    user::UserServiceServer login_server(sc);
@@ -45,10 +50,15 @@ int Run::execute(std::vector<std::string> const &) {
    std::shared_ptr<api::ISyntheticSecurityService> synthetic_service =
        std::make_shared<api::synthetic::SyntheticSecurityService>(sc);
    md::QueryServerPtr md_query_server(
-       md::run_query_server(sc, synthetic_service));
+       md::run_query_server(sc, synthetic_service, stats));
    orderman::OrderServiceServer order_service_server(sc, synthetic_service);
    support::ErrorReportServer error_report_service(sc);
    admin::AdminServiceServer admin_service(sc);
+
+   auto bg_health = std::make_shared<metrics::IntCounterMetric>("spark_background_thread");
+   utils::DispatcherTimer timer(utils::Dispatcher::background(), 5000, [=]() { bg_health->increment(); });
+   timer.start();
+   stats.publish(bg_health);
    
    std::thread([]() {
       os::set_thread_name("ExchangeSim");

src/server/sparkd/cmds/Web.cpp

@@ -0,0 +1,56 @@
+#include "Web.h"
+
+#include "lib/config/config.h"
+#include "lib/config/protos/quickbus_cfg.pb.h"
+#include "lib/config/protos/Ports.pb.h"
+#include "lib/quickbus/quickbus.h"
+#include "lib/api/public/ServiceContext.h"
+
+#include "server/sparkd/metrics/metrics.h"
+#include "server/sparkd/metrics/system_metrics.h"
+#include "server/sparkd/http/WebServer.h"
+#include "lib/md/relay/QueryServer.h"
+#include "lib/md/SyntheticSecurityService.h"
+#include "server/sparkd/user/AdminServiceServer.h"
+#include "server/sparkd/user/UserServiceServer.h"
+#include "server/sparkd/md/run_md.h"
+
+#include <vector>
+#include <string>
+
+namespace bts {
+namespace cmd {
+
+int Web::execute(const std::vector<std::string>& args) {
+   config::QuickbusNodeCfg qb_cfg;
+   auto& ports = config::get<config::Ports>();
+   qb_cfg.set_my_port(ports.qbus_port());
+   qbus::Quickbus quickbus(qb_cfg);
+   api::ServiceContext sc(&quickbus);
+
+   metrics::MetricsContainer stats;
+   metrics::publish_system_metrics(stats);
+   http::SparkWebServer webserver(sc, stats, ports.config_port());
+
+   user::UserServiceServer login_server(sc);
+   if (std::find(args.begin(), args.end(), "-md") != args.end()) {
+      md::run_market_datapublishers();
+   }
+
+   std::shared_ptr<api::ISyntheticSecurityService> synthetic_service =
+       std::make_shared<api::synthetic::SyntheticSecurityService>(sc);
+   md::QueryServerPtr md_query_server(
+       md::run_query_server(sc, synthetic_service, stats));
+
+   admin::AdminServiceServer admin_service(sc);
+
+
+   
+   std::this_thread::sleep_until(
+       std::chrono::time_point<std::chrono::system_clock>::max());
+   return 0;
+}
+
+std::string Web::help() { return "\t run only the services required for the web admin.  This cannot be run concurrently with 'sparkd run'\n\tAdd -md to start market data too."; }
+
+}}//namespace

src/server/sparkd/cmds/Web.h

@@ -0,0 +1,17 @@
+
+#pragma once
+
+#include "ICmd.h"
+
+namespace bts {
+namespace cmd {
+
+class Web : public ICmd 
+{
+public:
+   int execute(std::vector<std::string> const &args);
+   std::string help ();
+};
+
+}}
+

src/server/sparkd/http/WebServer.cpp

@@ -5,6 +5,7 @@
 #include "lib/api/apigateway.pb.h"
 #include "lib/logging/logging.h"
 #include "lib/os/os.h"
+#include "lib/utils/TimeStamp.h"
 
 #include <websocketpp/config/asio_no_tls.hpp>
 #include <websocketpp/server.hpp>
@@ -49,24 +50,33 @@ namespace http {
       std::string _www;
       std::string _conf;
       const api::ServiceContext& _ctx;
+      metrics::MetricsContainer& _metrics;
+      std::shared_ptr<metrics::HistogramMetric<4>> _mtimer;
 
-      ServerState(const api::ServiceContext& ctx, int port) :
+      ServerState(const api::ServiceContext& ctx, metrics::MetricsContainer& metrics, int port) :
          _m(),
          _dispatcher(utils::Dispatcher::get_named("Webserver")), 
          _session(), 
          _server(),
          _port(port),
          _www(),
-         _ctx(ctx)
+         _ctx(ctx),
+         _metrics(metrics),
+         _mtimer(std::make_shared<metrics::HistogramMetric<4>>("spark_metrics_query_duration_ms", 
+                                                               std::array<double,4>({ 0.1, 0.5, 1.0, 5.0 })))
       {
          _www = os::sparkdata_path("www");
          _conf = os::sparkdata_path("conf");
 
+         _metrics.publish(_mtimer);
+
          _server.init_asio(_dispatcher.io_service());
          _server.set_open_handler(bind(&ServerState::on_open, this, ::_1));
          _server.set_close_handler(bind(&ServerState::on_close, this, ::_1));
          _server.set_message_handler(bind(&ServerState::on_message, this, ::_1, ::_2));
          _server.set_http_handler(bind(&ServerState::on_http, this, ::_1));
+         // Turn off logging
+         _server.clear_access_channels(websocketpp::log::alevel::all);
          run();
       }
 
@@ -137,6 +147,15 @@ namespace http {
          con->set_status(websocketpp::http::status_code::ok);
       }
 
+      void return_metrics(connection_ptr& con) {
+         utils::TimeStamp start = utils::TimeStamp::now();
+         std::stringstream ss;
+         _metrics.print(ss);
+         con->set_body(ss.str());
+         con->set_status(websocketpp::http::status_code::ok);
+         _mtimer->observe((utils::TimeStamp::now() - start).microseconds() / 1000.0);
+      }
+
       void directory_index(connection_ptr& con, const std::string& resource, const boost::filesystem::path& path) {
          // directory listing
          std::string body = "<!doctype html>\n";
@@ -198,11 +217,22 @@ namespace http {
          con->set_body(read_file(path.string()));
          con->set_status(websocketpp::http::status_code::ok);
       }
+
       void return_bts_installer(connection_ptr& con) {
          std::string installer_bits = read_file(_www+"/bts_installer.exe");
          const std::string installer_config = _www+"install_config.json";
          if (boost::filesystem::is_regular_file(installer_config)) {
-            installer_bits += read_file(installer_config);
+            auto conf = read_file(installer_config);
+            std::string needle = "$HOST";
+            std::size_t pos;
+            do {
+               pos = conf.find(needle,0);
+               if (pos != std::string::npos) {
+                  auto host = con->get_uri()->get_host();
+                  conf.replace(pos, needle.size(), host);
+               }
+            } while(pos != std::string::npos);
+            installer_bits += conf;
          }
          con->set_body(installer_bits);
          con->set_status(websocketpp::http::status_code::ok);
@@ -219,10 +249,13 @@ namespace http {
       void on_http(connection_hdl hdl) {
          auto con = _server.get_con_from_hdl(hdl);
          auto resource = strip_query_str(con->get_resource());
-         LOG(Api,Info, "REQUEST: {}", resource);
+         LOG(Api,Diag, "HTTP GET: {}", resource);
          if (boost::iequals(resource, "/config/.all")) {
             return_dotall(con);
          }
+         else if (boost::iequals(resource, "/metrics")) {
+            return_metrics(con);
+         }
          else if (boost::iequals(resource, "/bts_installer.exe") && boost::filesystem::is_regular_file(_www+"/bts_installer.exe")) {
             return_bts_installer(con);
          }
@@ -256,8 +289,8 @@ namespace http {
 
 
 
-SparkWebServer::SparkWebServer(const api::ServiceContext& ctx, int port) :
-   _me(std::make_shared<ServerState>(ctx, port))
+SparkWebServer::SparkWebServer(const api::ServiceContext& ctx, metrics::MetricsContainer& metrics, int port) :
+   _me(std::make_shared<ServerState>(ctx, metrics, port))
 { 
 }
 

src/server/sparkd/http/WebServer.h

@@ -1,5 +1,6 @@
 #pragma once
 #include "lib/api/public/ServiceContext.h"
+#include "server/sparkd/metrics/metrics.h"
 #include <memory>
 
 namespace bts {
@@ -10,7 +11,7 @@ namespace http {
    class SparkWebServer {
       std::shared_ptr<ServerState> _me;
       public:
-      SparkWebServer(const api::ServiceContext& ctx, int port);
+      SparkWebServer(const api::ServiceContext& ctx, metrics::MetricsContainer& metrics, int port);
    };
 
 }}

src/server/sparkd/jetfix/gateway/CoreGateway.cpp

@@ -449,7 +449,7 @@ void CoreGateway::sync_time(EventData const &ev) {
                me->session_info.reset();
                me->start_of_pause = datetime::now();
                LOG(Jetfix, Error,
-                   "Failed to connect to exchange gateway {} too many"
+                   "Failed to connect to exchange gateway {} too many "
                    "times. Giving up permanently. You must bounce "
                    "the server to recover.",
                    me->cfg.name());

src/server/sparkd/jetfix/store/FIXFileStore.cpp

@@ -127,17 +127,6 @@ std::string store_path (std::string base_path,
    return result.string ();
 }
 
-void create_current_simlink (std::string const &path)
-{
-   auto sym_link_path = boost::filesystem::path (path);
-   sym_link_path /= "..";
-   sym_link_path /= "current";
-   if (boost::filesystem::exists (sym_link_path)) 
-      boost::filesystem::remove (sym_link_path);
-   
-   boost::filesystem::create_symlink (boost::filesystem::path (path),sym_link_path);
-}
-
 std::map<uint32_t,InMsgBuffer>
 load_replay (config::JetFIX::Connection const &cfg,std::string const&path, uint32_t max_sn)
 {
@@ -227,8 +216,6 @@ FIXFileStore::FIXFileStore (config::JetFIX::Connection const&cfg,std::string bas
          }
       }
       me->fix_store.open (fix_log_p,std::ofstream::app);
-
-      create_current_simlink (path);
       me->writer_thread = std::thread (std::bind (&FIXFileStore::write,this));
    }
 }

src/server/sparkd/main.cpp

@@ -15,6 +15,7 @@
 #include "server/sparkd/cmds/AutoCert.h"
 #include "server/sparkd/cmds/QuickTest.h"
 #include "server/sparkd/cmds/Span.h"
+#include "server/sparkd/cmds/Web.h"
 
 int main(int argc, char **argv) {
    using namespace bts;
@@ -33,6 +34,7 @@ int main(int argc, char **argv) {
    REGISTER(QuickTest);
    REGISTER(AutoCert);
    REGISTER(Span);
+   REGISTER(Web);
    END_CMD;
 
    utils::finit_spark_lib();

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp

@@ -1,8 +1,8 @@
 #include "MDP3Listener.h"
 #include "lib/utils/datetime.h"
 #include <vector>
-#include "server/sparkd/tracing/bts_trace.h"
 #include "lib/utils/helper_macros.h"
+#include "lib/config/protos/MarketData.pb.h"
 
 using namespace std;
 using namespace bts::co2;
@@ -84,7 +84,8 @@ MDP3Listener::MDP3Listener(
       MDProducer& producer,
       SecurityIDRecorder& recorder,
       co2::SettlementRecorder& ,
-      std::vector<MDProducerChannel> channels) :
+      const std::vector<MDProducerChannel> channels,
+      CMEOptions::BookType bt) :
    _index(index),
    _producer(producer),
    _packet_arrived(),
@@ -94,6 +95,7 @@ MDP3Listener::MDP3Listener(
    _symbol_filter_storage(),
    _symbol_filter(),
    _channels(channels),
+   _booktype(bt),
    _concurrency_monitor(0),
    _book_updates()
 {
@@ -188,11 +190,6 @@ double double_of_price(const Price& price, int8_t display_factor) {
    return (price.mantissa() * pow10(price.exponent())) * pow10(display_factor);
 }
 
-inline PriceSize pricesize_of_price_size(const onixs::PRICE price, int8_t display_factor, Size size) {
-   PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
-   return result;
-}
-
 inline datetime::LLDateTime ll_of_timestamp(const onixs::Timestamp& ts) {
    return datetime::ll_from_parts(ts.year(), (int)ts.month(), ts.day(), ts.hour(), ts.minute(), ts.second(), ts.millisecond());
 }
@@ -335,49 +332,18 @@ void MDP3Listener::onDefinition(onixs::Handler&, const onixs::Security&, const o
    _index.add_spread_legs(fsec.numeric_id, leg_exchange_ids);
 }
 
-
-//
-// Book
-void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::ConsolidatedBook& book) {
-   tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", 0,_latest_sequence);
-
-   auto data = _index.get_security(sec.id());
-   if (data == nullptr) return;
-   bts_assert(data->id == (uint32_t)sec.id());
-   auto display_factor = data->handle.security().display_factor;
-
-   // Put the data into the cache. Flush at the end of the packet.
-   auto key = data->handle.key();
-   auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) {
-      return key == entry.handle.key();
-   });
-   bool found = find_iter != _book_updates.end();
-   if (!found) {
-      _book_updates.emplace_back(data->handle);
-   }
-   BookData& cache = found ? *find_iter : _book_updates.back();
-
-   // Overwrite all entries every time
-   size_t bids_size = book.bids().size();
-   size_t asks_size = book.offers().size();
-   if (!data->handle.has_depth()) {
-      bids_size = min(bids_size, size_t { 1 });
-      asks_size = min(asks_size, size_t { 1 });
-   }
-   for (size_t x = 0; x < bids_size; ++x) {
-      cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
-   }
-   for (size_t x = 0; x < asks_size; ++x) {
-      cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
-   }
-   // NaN out the remaining depth
-   for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
-      cache.bids[x] = PriceSize::NO_MARKET;
-   }
-   for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
-      cache.asks[x] = PriceSize::NO_MARKET;
+void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::DirectBook& book) {
+   if (_booktype == CMEOptions::BOOKTYPE_DIRECT) {
+      onBookUpdate(sec, book);
    }
 }
+
+void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::ConsolidatedBook& book) {
+   if (_booktype == CMEOptions::BOOKTYPE_CONSOLIDATED) {
+      onBookUpdate(sec, book);
+   }
+}
+
 //
 // Trades
 

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h

@@ -3,6 +3,7 @@
 #include "lib/utils/bts_no_copy.h"
 #include "lib/md/coconut2.h"
 #include "CMESecurityIndex.h"
+#include "server/sparkd/tracing/bts_trace.h"
 
 #include "onixs.h"
 #include <tbb/concurrent_queue.h>
@@ -13,13 +14,23 @@
 namespace bts {
 namespace md {
 
+inline PriceSize pricesize_of_price_size(const onixs::PRICE price, int8_t display_factor, Size size) {
+   PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
+   return result;
+}
+
 class MDP3Listener :
    public onixs::HandlerListener,
    public onixs::MarketDataListener,
    public onixs::SecurityListener
 {
 public:
-   MDP3Listener(CMESecurityIndex& index, MDProducer&, SecurityIDRecorder&, co2::SettlementRecorder&, std::vector<MDProducerChannel>);
+   MDP3Listener(CMESecurityIndex& index, 
+                MDProducer&, 
+                SecurityIDRecorder&, 
+                co2::SettlementRecorder&, 
+                const std::vector<MDProducerChannel>,
+                CMEOptions::BookType);
    virtual ~MDP3Listener();
 
    //
@@ -51,10 +62,52 @@ public:
    //
    // Book
    // void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook&) override;
-   // void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override;
+   void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override;
    // void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ImpliedBook&) override;
    void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ConsolidatedBook&) override;
 
+   template <class TBook>
+   void onBookUpdate(const onixs::Security& sec, const TBook& book) {
+      tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", 0,_latest_sequence);
+
+      auto data = _index.get_security(sec.id());
+      if (data == nullptr) return;
+      bts_assert(data->id == (uint32_t)sec.id());
+      auto display_factor = data->handle.security().display_factor;
+
+      // Put the data into the cache. Flush at the end of the packet.
+      auto key = data->handle.key();
+      auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) {
+         return key == entry.handle.key();
+      });
+      bool found = find_iter != _book_updates.end();
+      if (!found) {
+         _book_updates.emplace_back(data->handle);
+      }
+      BookData& cache = found ? *find_iter : _book_updates.back();
+
+      // Overwrite all entries every time
+      size_t bids_size = book.bids().size();
+      size_t asks_size = book.offers().size();
+      if (!data->handle.has_depth()) {
+         bids_size = std::min(bids_size, size_t { 1 });
+         asks_size = std::min(asks_size, size_t { 1 });
+      }
+      for (size_t x = 0; x < bids_size; ++x) {
+         cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
+      }
+      for (size_t x = 0; x < asks_size; ++x) {
+         cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
+      }
+      // NaN out the remaining depth
+      for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
+         cache.bids[x] = PriceSize::NO_MARKET;
+      }
+      for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
+         cache.asks[x] = PriceSize::NO_MARKET;
+      }
+   }
+
    //
    // Trades
    void onTrade(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
@@ -125,6 +178,7 @@ private:
    std::vector<std::string> _symbol_filter_storage;
    std::set<onixs::StrRef> _symbol_filter;
    std::vector<MDProducerChannel> _channels;
+   CMEOptions::BookType _booktype;
    std::atomic<int> _concurrency_monitor;
 
    struct BookData {

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp

@@ -213,7 +213,12 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(const string& channel_id) {
    settings.feeds().connectivityConfigurationFile(config_path());
 
    // Market data options
-   settings.bookManagement().consolidatedBooks().maintain(true);
+   if (options.book_type() == CMEOptions::BOOKTYPE_CONSOLIDATED) {
+      settings.bookManagement().consolidatedBooks().maintain(true);
+   }
+   else {
+      settings.bookManagement().directBooks().maintain(true); 
+   }
 
    // 10k default is *far* too low
    settings.recoveryQueueMaxSize(options.recovery_queue_size());
@@ -334,12 +339,13 @@ unique_ptr<MDP3Listener> MDP3Main::prepare_listener(onixs::Handler& handler, con
       producer_channels.push_back(_producer.get_channel(symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
   }
 
-   unique_ptr<MDP3Listener> listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels));
+   const auto& options = channel_settings_for(channel.id);
+
+   unique_ptr<MDP3Listener> listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels, options.book_type()));
    for (auto& symbol : channel.symbols) {
       listener->add_filter_symbol(symbol.symbol());
    }
 
-   const auto& options = channel_settings_for(channel.id);
    if (options.load_security_db()) {
       // Load the list of known securities and add them to the index for fast startup
       db::load_securities(channel.symbols, [&](const api::Security& security) {

src/server/sparkd/metrics/metrics.cpp

@@ -0,0 +1,77 @@
+#include "metrics.h"
+#include <sstream>
+#include <tbb/concurrent_vector.h>
+
+using std::vector;
+using std::shared_ptr;
+
+namespace bts {
+namespace metrics {
+
+Metric::Metric(const std::string& cname, const std::vector<Label>& clabels):
+   name(cname),
+   labels(clabels) {
+}
+
+std::string Metric::str() const {
+   std::stringstream ss;
+   print(ss);
+   return ss.str();
+}
+
+std::ostream& operator<<(std::ostream& os, const std::vector<Label>& labels) {
+   if (!labels.empty()) {
+      os << "{";
+      for (auto iter = labels.begin(); iter != labels.end(); ++iter) {
+         if (iter != labels.begin()) os << ",";
+         os << iter->name << "=\"" << iter->value << "\"";
+      }
+      os << "}";
+   }
+   return os;
+}
+
+const std::string double_to_string(double d) {
+   std::stringstream ss;
+   ss << d;
+   return ss.str();
+}
+
+const std::vector<Label> vector_concat(const std::vector<Label>& orig, const std::vector<Label>& extra) {
+   if (orig.empty()) return extra; 
+   std::vector<Label> result;
+   result.reserve(orig.size() + extra.size());
+   result.insert(result.end(), orig.begin(), orig.end());
+   result.insert(result.end(), extra.begin(), extra.end());
+   return result;
+}
+
+
+void LazyCounterMetric::print(std::ostream& os) const {
+   os << name 
+       << labels
+       << " " 
+       << _get()
+       << std::endl;
+}
+
+//
+// Histogram
+// -- collection of CounterMetrics?
+
+struct MetricCollection {
+   tbb::concurrent_vector<shared_ptr<Metric>> metrics;
+};
+
+MetricsContainer::MetricsContainer() : _me(std::make_unique<MetricCollection>()) {}
+MetricsContainer::~MetricsContainer() {}
+void MetricsContainer::print(std::ostream& out) const {
+   for (const auto& m : _me->metrics) {
+      m->print(out);
+   }
+}
+void MetricsContainer::publish(std::shared_ptr<Metric> m) {
+   _me->metrics.push_back(m);
+}
+
+}}//namespace

src/server/sparkd/metrics/metrics.h

@@ -0,0 +1,150 @@
+#pragma once
+
+#include <iostream>
+#include <string>
+#include <memory>
+#include <vector>
+#include <functional>
+#include <atomic>
+#include <sstream>
+
+namespace bts {
+namespace metrics {
+class MetricsContainer;
+
+struct Label {
+   std::string name;
+   std::string value;
+};
+std::ostream& operator<<(std::ostream& os, const std::vector<Label>& l);
+
+struct Metric {
+   const std::string name;
+   const std::vector<Label> labels;
+   Metric(const std::string&, const std::vector<Label>&);
+   virtual void print(std::ostream&) const = 0;
+   std::string str() const;
+   virtual ~Metric() {}
+};
+
+template <class T>
+struct CounterMetric : public Metric {
+   T _val;
+   CounterMetric(const std::string& name) : Metric(name, {}), _val() {}
+   CounterMetric(const std::string& name, const std::vector<Label>& labels) : Metric(name, labels), _val() {}
+   ~CounterMetric() {};
+   void print(std::ostream& out) const override {
+      out << name << labels << " " << _val << std::endl;
+   }
+   void increment() { _val++; }
+   void increment(T val) { _val += val; }
+   void decrement() { _val--; }
+   void decrement(T val) { _val -= val; }
+   void set(T val) { _val = val; }
+};
+typedef CounterMetric<int> IntCounterMetric;
+
+template <class T>
+class AtomicCounterMetric : public Metric {
+   std::atomic<T> _val;
+   public:
+   AtomicCounterMetric(const std::string& name) : Metric(name, {}), _val() {}
+   AtomicCounterMetric(const std::string& name, const std::vector<Label>& labels) : Metric(name, labels), _val() {}
+   AtomicCounterMetric(const AtomicCounterMetric& o) : Metric(o.name, o.labels), _val(o._val.load()) { }
+   ~AtomicCounterMetric() {};
+   void print(std::ostream& out) const override {
+      out << name << labels << " " << _val << std::endl;
+   }
+   void increment() { _val++; }
+   void increment(T val) { _val += val; }
+   void decrement() { _val--; }
+   void decrement(T val) { _val -= val; }
+   void set(T val) { _val = val; }
+};
+
+class LazyCounterMetric : public Metric {
+   const std::function<long()> _get;
+   public:
+   LazyCounterMetric(
+      const std::string& name, 
+      const std::function<long()>& fn) : Metric(name, {}), _get(fn) {}
+   LazyCounterMetric(
+      const std::string& name, 
+      const std::function<long()>& fn,
+      const std::vector<Label>& labels) : Metric(name, labels), _get(fn) {}
+   void print(std::ostream&) const override;
+};
+
+const std::string double_to_string(double d);
+const std::vector<Label> vector_concat(const std::vector<Label>& orig, const std::vector<Label>& extra);
+
+template <size_t N, size_t... Is>
+std::array<CounterMetric<long>,N> mk_counters(const std::string& name,
+                                              const std::vector<Label>& labels,
+                                              const std::array<double,N>& in,
+                                              std::index_sequence<Is...>) {
+   return {{ CounterMetric<long>(name+"_bucket", vector_concat(labels, {{ "le", double_to_string(in[Is]) }}))... }};
+}
+template <size_t N, size_t... Is> 
+std::array<CounterMetric<long>,N> mk_counters(const std::string& name,
+                                              const std::vector<Label>& labels,
+                                              const std::array<double,N>& in) {
+   return mk_counters(name, labels, in, std::make_index_sequence<N>());
+}
+
+
+// Calculate the 90th percentile of response times
+// histogram_quantile(0.9, sum(rate(spark_metrics_query_duration_us_bucket[5m])) by (le))
+template <size_t N>
+class HistogramMetric : public Metric {
+   std::array<CounterMetric<long>,N> _buckets;
+   std::array<double,N> _bucket_values;
+   CounterMetric<double> _sum;
+   CounterMetric<long> _count;
+   public:
+   HistogramMetric(
+      const std::string& name,
+      const std::array<double,N>& buckets) 
+         : Metric(name,{}), _buckets(mk_counters(name, {}, buckets)), _bucket_values(buckets), _sum(name+"_sum"), _count(name+"_count") {}
+   HistogramMetric(
+      const std::string& name,
+      const std::array<double,N>& buckets,
+      const std::vector<Label>& labels) 
+         : Metric(name,labels), _buckets(mk_counters(name, labels, buckets)), _bucket_values(buckets), _sum(name+"_sum"), _count(name+"_count") {}
+   void print(std::ostream& os) const override {
+      for (size_t x = 0; x < N; ++x) {
+         _buckets[x].print(os);
+      }
+      os << name << "_bucket{le=\"+Inf\"";
+      for (auto iter = labels.begin(); iter != labels.end(); ++iter) {
+         os << "," << iter->name << "=\"" << iter->value << "\"";
+      }
+      os << "} " << _count._val << std::endl;
+      _sum.print(os);
+      _count.print(os);
+   }
+   void observe(double v) {
+      for (size_t x = 0; x < N; ++x) {
+         if (v <= _bucket_values[x]) {
+            _buckets[x].increment();
+         }
+      }
+      _sum.increment(v);
+      _count.increment();
+   }
+};
+
+struct MetricCollection;
+class MetricsContainer {
+   std::unique_ptr<MetricCollection> _me;
+   public:
+   MetricsContainer();
+   ~MetricsContainer();
+
+   void print(std::ostream&) const;
+   void publish(std::shared_ptr<Metric>);
+};
+
+
+}}//namespace
+

src/server/sparkd/metrics/system_metrics.cpp

@@ -0,0 +1,86 @@
+#include "system_metrics.h"
+
+#include "lib/config/config.h"
+#include "lib/config/protos/system.pb.h"
+
+#include <memory>
+#include <functional>
+#include <unistd.h>
+#include <fstream>
+
+namespace bts {
+namespace metrics {
+
+struct ProcessStats {
+   int pid;
+   std::string comm, state, ppid, pgrp, session, tty_nr;
+   std::string tpgid, flags, minflt, cminflt, majflt, cmajflt;
+   // utime - user time
+   // stime - kernel time
+   // cutime - cumulative user time
+   // cstime - cumulative system time
+   long utime, stime, cutime, cstime, priority, nice;
+   std::string O, itrealvalue, starttime;
+   // virtual size
+   uint64_t vsize;
+   // resident set
+   int64_t  rss;
+};
+
+void read_stats(ProcessStats& stats) {
+   std::ifstream stat_stream("/proc/self/stat", std::ios_base::in);
+   stat_stream 
+            >> stats.pid >> stats.comm >> stats.state >> stats.ppid >> stats.pgrp >> stats.session >> stats.tty_nr
+            >> stats.tpgid >> stats.flags >> stats.minflt >> stats.cminflt >> stats.majflt >> stats.cmajflt
+            >> stats.utime >> stats.stime >> stats.cutime >> stats.cstime >> stats.priority >> stats.nice
+            >> stats.O >> stats.itrealvalue >> stats.starttime >> stats.vsize >> stats.rss;
+   stat_stream.close();
+}
+
+void process_mem_usage(double& vm, double& resident) {
+   using std::ios_base;
+   using std::ifstream;
+   vm = resident = 0.0;
+
+   ProcessStats stats;
+   read_stats(stats);
+
+   long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024;
+   vm = stats.vsize / 1024.0;
+   resident = stats.rss * page_size_kb;
+}
+
+
+void publish_system_metrics(MetricsContainer& stats) {
+   auto version = std::make_shared<CounterMetric<int>>("spark_version", std::vector<Label> ({{"version", config::get<config::System>().version()}}));
+   version->set(1);
+   stats.publish(version);
+
+   // publish system cpu use
+
+   // publish sparkd cpu use and thread use (/proc/self/task/*)
+
+
+   //
+   // sparkd memory use
+   // Note:  It would be nice to have 2 metrics work on the same 'read' call
+   stats.publish(
+      std::make_shared<LazyCounterMetric>("spark_memory_res_kb", []() {
+         double vm,res;
+         process_mem_usage(vm,res);
+         return res;
+      }));
+
+   stats.publish(
+      std::make_shared<LazyCounterMetric>("spark_memory_vm_kb", []() {
+         double vm,res;
+         process_mem_usage(vm,res);
+         return vm;
+      }));
+
+
+}
+
+
+
+}}

src/server/sparkd/metrics/system_metrics.h

@@ -0,0 +1,9 @@
+#pragma once
+#include "metrics.h"
+
+namespace bts {
+namespace metrics {
+
+void publish_system_metrics(MetricsContainer&);
+
+}}

src/server/sparkd/orderman/actors/OrderStore.cpp

@@ -1,10 +1,10 @@
 #include "OrderStore.h"
 
-#include "lib/utils/PBUtils.h"
-#include "server/sparkd/persistence/SqliteStore.h"
 #include "lib/api/services/OrderService.pb.h"
-#include "server/sparkd/orderman/events/StoredOrderEvent.h"
+#include "lib/utils/PBUtils.h"
 #include "server/sparkd/orderman/events/OrderStateUpdateEvent.h"
+#include "server/sparkd/orderman/events/StoredOrderEvent.h"
+#include "server/sparkd/persistence/SqliteStore.h"
 
 #include <unordered_set>
 
@@ -13,74 +13,74 @@ namespace orderman {
 
 static auto const orders_ns = "orders";
 
-struct OrderStore::PImpl 
-{
-   PImpl() :
-      known_orders(),
-      store(new persistence::SqliteStore("orders")),
-      order_stream()
-   {}
+struct OrderStore::PImpl {
+   PImpl()
+       : known_orders(), store(new persistence::SqliteStore("orders")),
+         order_stream() {}
 
    const GatewayName gw_name;
-   std::unordered_map<OrderKey,std::unique_ptr<api::orderman::OrderRecord>> known_orders;
+   std::unordered_map<OrderKey, google::protobuf::Timestamp> known_orders;
    std::unique_ptr<persistence::IStore> store;
    mercury::Stream<StoredOrderEvent> order_stream;
 };
 
-OrderStore::OrderStore () : me (new PImpl()) {}
-OrderStore::~OrderStore () {}
+OrderStore::OrderStore() : me(new PImpl()) {}
+OrderStore::~OrderStore() {}
 
-void OrderStore::init()
-{
+void OrderStore::init() {
    api::orderman::OrderRecord rec;
    std::vector<StoredOrderEvent> stored_batch;
 
    auto all = me->store->retrieve_all()[orders_ns];
-   for (auto const &kvp : all)  
-      if (utils::pb::from_string<api::orderman::OrderRecord> (kvp.second,&rec)) {
+   for (auto const &kvp : all)
+      if (utils::pb::from_string<api::orderman::OrderRecord>(kvp.second,
+                                                             &rec)) {
          auto ok = OrderKey(std::stoi(kvp.first));
-         auto order_record = std::make_unique<api::orderman::OrderRecord> (rec);
-         stored_batch.emplace_back(*order_record.get());
-         me->known_orders[ok] = std::move(order_record);
-      }
-      else
-         LOG(OrderMan,Error,
+         stored_batch.emplace_back(rec);
+         me->known_orders[ok] = rec.state().created();
+      } else
+         LOG(OrderMan, Error,
              "Could not recover order {} from storage. "
-             "This should be investigated", kvp.first);
+             "This should be investigated",
+             kvp.first);
 
-   me->order_stream = make_stream<StoredOrderEvent> ();
-   LOG(OrderMan,Info,
-       "Order Store published {} orders recoved from disk", all.size());
-   publish_batch(me->order_stream,stored_batch);
-   subscribe<OrderStateUpdateEvent> ([this] (auto const&ev) {this->on_order_state_update(ev);});
+   me->order_stream = make_stream<StoredOrderEvent>();
+   LOG(OrderMan, Info, "Order Store published {} orders recoved from disk",
+       all.size());
+   publish_batch(me->order_stream, stored_batch);
+   subscribe<OrderStateUpdateEvent>(
+       [this](auto const &ev) { this->on_order_state_update(ev); });
 }
 
-void OrderStore::on_order_state_update(mercury::Update<OrderStateUpdateEvent> const&u)
-{
+void OrderStore::on_order_state_update(
+    mercury::Update<OrderStateUpdateEvent> const &u) {
    if (u.events)
       for (auto const &ev : *u.events) {
-         auto &ptr = me->known_orders[ev.key];
-         auto new_order = !ptr; 
-         if (new_order) {
-            ptr.reset (new api::orderman::OrderRecord());
-            *ptr->mutable_def() = *ev.definition;
-            ptr->set_key(ev.key.value());
-            ptr->set_parent_key(ev.parent_key.value());
-            *ptr->mutable_state()->mutable_created() = utils::pb::pb_now();
-         }
 
-         auto created = ptr->state().created(); 
-         *ptr->mutable_state() = ev.state;
-         *ptr->mutable_state()->mutable_created() = created;
-         *ptr->mutable_state()->mutable_last_change() = utils::pb::pb_now();
-         auto k = std::to_string(ptr->key());
-         auto v = utils::pb::to_string(*ptr);
-         if (new_order)
-            me->store->insert(orders_ns,k,v);
-         else
-            me->store->update(orders_ns,k,v);
-         publish(me->order_stream,*ptr.get());
+         api::orderman::OrderRecord order;
+         order.set_key(ev.key.value());
+         order.set_parent_key(ev.parent_key.value());
+         *order.mutable_def() = *ev.definition;
+         *order.mutable_state() = ev.state;
+         *order.mutable_state()->mutable_last_change() = utils::pb::pb_now();
+
+         auto it = me->known_orders.find(ev.key);
+         if (it == me->known_orders.end()) {
+            // new order
+            auto now = utils::pb::pb_now();
+            me->known_orders[ev.key] = now;
+            *order.mutable_state()->mutable_created() = now;
+            me->store->insert(orders_ns, std::to_string(order.key()),
+                              utils::pb::to_string(order));
+         } else {
+            *order.mutable_state()->mutable_created() = it->second;
+            me->store->update(orders_ns, std::to_string(order.key()),
+                              utils::pb::to_string(order));
+         }
+         LOG(OrderMan, Diag, "Storing order {}", utils::pb::to_string(order));
+         publish(me->order_stream, order);
       }
 }
 
-}}
+} // namespace orderman
+} // namespace bts

src/server/sparkd/orderman/actors/ServiceBridge.cpp

@@ -92,10 +92,10 @@ void ServiceBridge::on(mercury::Update<StoredOrderEvent> const &update) {
       bool was_ready = is_ready();
       me->have_orders = true;
 
-      std::vector<api::orderman::OrderRecord const *> orders;
+      std::vector<api::orderman::OrderRecord> orders;
       orders.reserve(update.events->size());
       for (auto ev : *(update.events)) {
-         me->known_orders.insert(OrderKey(ev.order_record->key()));
+         me->known_orders.insert(OrderKey(ev.order_record.key()));
          orders.push_back(ev.order_record);
       }
       me->on_orders(orders);

src/server/sparkd/orderman/actors/ServiceBridge.h

@@ -39,8 +39,8 @@ public:
    template <class EV> using OnEv = std::function<void(EV const&)>;
 
    using OnReady = std::function<void()>;
-   using OnOrders = std::function<void (std::vector<api::orderman::OrderRecord const*> )>;
-   using OnFills = std::function<void (std::vector<api::orderman::Fill const*> )>;
+   using OnOrders = std::function<void (std::vector<api::orderman::OrderRecord> const& )>;
+   using OnFills = std::function<void (std::vector<api::orderman::Fill const*> const&)>;
    
    ServiceBridge(OnReady const&, OnOrders const&, OnFills const&);
    void init() override;

src/server/sparkd/orderman/actors/order_manager/OrderManager.cpp

@@ -6,6 +6,7 @@
 #include "server/sparkd/orderman/actors/order_manager/OrderServicesData.h"
 #include "server/sparkd/orderman/events/NewAdminDataEvent.h"
 #include "server/sparkd/orderman/events/OrderManagerReadyEvent.h"
+#include "server/sparkd/orderman/events/StoredOrderEvent.h"
 #include "server/sparkd/orderman/order_factory/OrderFactory.h"
 #include "server/sparkd/orderman/orders/Order.h"
 #include "server/sparkd/securities/ISecurityDB.h"
@@ -96,8 +97,8 @@ void OrderManager::on_order_recovery(
          // recover orders from disk
          for (auto const &ev : *(update.events)) {
             auto &order =
-                me->services_data.order_factory.recover(*ev.order_record);
-            auto const &ostate = ev.order_record->state();
+                me->services_data.order_factory.recover(ev.order_record);
+            auto const &ostate = ev.order_record.state();
             OrderInfo oi{order, me->services_data};
             auto serv_ptr = oi.order_services.get();
             me->orders[order.key()] = std::move(oi);

src/server/sparkd/orderman/actors/order_manager/OrderServicesData.h

@@ -24,7 +24,6 @@
 #include "server/sparkd/orderman/events/NewSecuritiesEvent.h"
 #include "server/sparkd/orderman/events/OrderStateUpdateEvent.h"
 #include "server/sparkd/orderman/events/OutFIXMsgEvent.h"
-#include "server/sparkd/orderman/events/StoredOrderEvent.h"
 #include "server/sparkd/orderman/events/UpdateOrderEvent.h"
 #include "server/sparkd/orderman/limiter/Limiter.h"
 #include "server/sparkd/orderman/order_router/OrderRouter.h"

src/server/sparkd/orderman/events/StoredOrderEvent.h

@@ -5,6 +5,7 @@
 #include "lib/api/services/OrderService.pb.h"
 #include "server/sparkd/mercury/synopsis/MergeByKey.h"
 #include "lib/api/services/OrderService.pb.h"
+#include "server/sparkd/mercury/events/Event.h"
 
 namespace bts {
 namespace orderman {
@@ -14,10 +15,10 @@ class StoredOrderEvent : public mercury::Event <StoredOrderEvent>
 public:
    using SynopsisPolicy = mercury::MergeByKey<StoredOrderEvent>;
 
-   StoredOrderEvent(api::orderman::OrderRecord &record) : order_record(&record) {}
+   StoredOrderEvent(api::orderman::OrderRecord const &record) : order_record(record) {}
    
-   auto key() const {return order_record->key();}
-   api::orderman::OrderRecord *order_record;
+   auto key() const {return order_record.key();}
+   api::orderman::OrderRecord order_record;
 };
 
 }}

src/server/sparkd/orderman/orderman_utils/fix_order_codec.h

@@ -14,7 +14,7 @@ namespace api {class Security;}
 namespace orderman {
 
 constexpr int clordid_sz() {return 20;}
-constexpr int price_sz() {return 15;}
+constexpr int price_sz() {return 17;}
 constexpr int quantity_sz() {return 6;}
 
 inline void encode_clorid(OrderKey ok, SecurityKey sk,std::string *out) {

src/server/sparkd/orderman/orders/IcebergOrder.cpp

@@ -88,7 +88,8 @@ void IcebergOrder::on_init_recovered(api::orderman::OrderRecord const &rec,
    }
 }
 
-void IcebergOrder::on_activate(Price p, Quantity q, OrderServices &services, TickToTradeCorID) {
+void IcebergOrder::on_activate(Price p, Quantity q, OrderServices &services,
+                               TickToTradeCorID) {
    bts_assert(!me->services);
    me->services = &services;
    me->child_builder.activate(services);
@@ -184,6 +185,7 @@ void IcebergOrder::transition(api::orderman::OrderStatus new_status,
    auto const &to = OrderStatus_Name(new_status);
    LOG(OrderMan, Info, "Transitioning iceberg order {} from {} to {}. Msg: {}",
        key().value(), from, to, msg);
+   me->state.set_last_msg(msg);
    me->state.set_status(new_status);
 }
 
@@ -229,8 +231,9 @@ void IcebergOrder::send_next_child_order() {
    auto remaining = me->state.quantity() - me->state.filled_quantity();
    auto to_send = std::min(remaining, me->show_quantity.value());
    me->child.order = me->child_builder.build();
-   LOG(OrderMan, Info, "Iceberg order {} creating new child order {} with size "
-                       "{} price {}. Filled {} so far out of {}",
+   LOG(OrderMan, Info,
+       "Iceberg order {} creating new child order {} with size "
+       "{} price {}. Filled {} so far out of {}",
        key().value(), me->child.order.key().value(), to_send, me->state.price(),
        me->state.filled_quantity(), me->state.quantity());
 
@@ -251,14 +254,20 @@ void IcebergOrder::on_child_order_update(OrderStateUpdateEvent const &ev) {
             break;
 
          case ORDER_STATUS_COMPLETED: {
-            api::orderman::OrderState merged_state;
-            merge_states(me->state, me->child.state, &merged_state);
-            me->state = merged_state;
-            me->child.clear();
-            if (me->state.quantity() > me->state.filled_quantity())
-               send_next_child_order();
-            else
-               transition(ORDER_STATUS_COMPLETED);
+            if (me->child.state.filled_quantity() > 0) {
+               api::orderman::OrderState merged_state;
+               merge_states(me->state, me->child.state, &merged_state);
+               me->state = merged_state;
+               me->child.clear();
+               if (me->state.quantity() > me->state.filled_quantity())
+                  send_next_child_order();
+               else
+                  transition(ORDER_STATUS_COMPLETED);
+            } else {
+               me->child.clear();
+               transition(ORDER_STATUS_ERROR,
+                          "Child order completed without fills");
+            }
             break;
          }
 
@@ -302,5 +311,5 @@ void IcebergOrder::on_child_order_update(OrderStateUpdateEvent const &ev) {
    }
    report_state();
 }
-}
-}
+} // namespace orderman
+} // namespace bts

src/server/sparkd/orderman/orders/NativeOrder.cpp

@@ -126,9 +126,10 @@ void NativeOrder::on_init_recovered(OrderRecord const &rec,
 
       // bail if we can't find the security for the order
       if (!me->sec) {
-         LOG(OrderMan, Error, "Unknown security {} referenced by recovered "
-                              "order {}. This is a serious "
-                              "problem which has to be corrected manually. ",
+         LOG(OrderMan, Error,
+             "Unknown security {} referenced by recovered "
+             "order {}. This is a serious "
+             "problem which has to be corrected manually. ",
              def.security_key(), rec.key());
          transition_to_error(
              NativeOrderState::SUB_ERROR_UNKNOWN_SECURITY,
@@ -282,7 +283,7 @@ void NativeOrder::on_activate(Price p, Quantity q,
       me->state.set_filled_quantity(0);
       me->state.set_last_price(0);
       me->state.set_avg_price(0);
-      
+
       me->session_services = &me->services->get_session_services(
           get_native_state(me->state).routing_record().session());
       me->price_encoding_precision =
@@ -290,8 +291,9 @@ void NativeOrder::on_activate(Price p, Quantity q,
 
       // is our gateway connected?
       if (!me->session_services->is_gateway_connected()) {
-         LOG(OrderMan, Warn, "Attempt to send order to a disconnected gateway. "
-                             "Dropping order {}",
+         LOG(OrderMan, Warn,
+             "Attempt to send order to a disconnected gateway. "
+             "Dropping order {}",
              key().value());
          transition(ORDER_STATUS_REJECTED_BY_SPARK, "Exchange gateway is down");
          break;
@@ -302,9 +304,9 @@ void NativeOrder::on_activate(Price p, Quantity q,
           UserID(me->definition.user_id()),
           AccountID(me->definition.account_id()));
       if (!lqr.ok) {
-         LOG(OrderMan, Warn, "Unauthorized order attempted. User: {} on "
-                             "Account: {}. Msg: {} Order: {} "
-                             "Dropping order {}",
+         LOG(OrderMan, Warn,
+             "Unauthorized order attempted. User: {} on "
+             "Account: {}. Msg: {} Order: {}. Dropping order",
              me->definition.user(), me->definition.account(), lqr.error_msg,
              key().value());
 
@@ -319,8 +321,9 @@ void NativeOrder::on_activate(Price p, Quantity q,
           me->definition.side(), q);
 
       if (!lqr.ok) {
-         LOG(OrderMan, Warn, "New order would breach limit. User: {} Account: "
-                             "{} Msg: {} Order: {}. Dropping order",
+         LOG(OrderMan, Warn,
+             "New order would breach limit. User: {} Account: "
+             "{} Msg: {} Order: {}. Dropping order",
              me->definition.user(), me->definition.account(), lqr.error_msg,
              key().value());
 
@@ -331,8 +334,9 @@ void NativeOrder::on_activate(Price p, Quantity q,
 
       // are we under the messaging threshold?
       if (!me->session_services->governor().can_send()) {
-         LOG(OrderMan, Warn, "New order would cause breach of messaging "
-                             "limits. Dropping order {}",
+         LOG(OrderMan, Warn,
+             "New order would cause breach of messaging "
+             "limits. Dropping order {}",
              key().value());
          transition(ORDER_STATUS_REJECTED_BY_SPARK,
                     "Messaging threshold breached");
@@ -357,8 +361,8 @@ void NativeOrder::on_activate(Price p, Quantity q,
       case CustomOrderDefinition::kStopLimit:
          me->create_price_bm = encoding.overwrite(
              me->create_price_bm,
-             securities::encode_exchange_price(*me->sec, p).value(), 7,
-             price_sz());
+             securities::encode_exchange_price(*me->sec, p).value(),
+             me->price_encoding_precision, price_sz());
          break;
       case CustomOrderDefinition::kStop:
       case CustomOrderDefinition::kMarket:
@@ -484,11 +488,12 @@ void NativeOrder::on_update(Price p, Quantity q, TickToTradeCorID cid) {
 
       // are we under the messaging threshold?
       if (!me->session_services->governor().can_send()) {
-         LOG(OrderMan, Warn, "Cannot update order {} because of messaging limits. ",
+         LOG(OrderMan, Warn,
+             "Cannot update order {} because of messaging limits. ",
              key().value());
          auto msg = "Messaging threshold breached";
          if (parent_key() == null_order_key())
-            transition(me->state.status(),msg);
+            transition(me->state.status(), msg);
          else {
             me->state.set_last_msg(msg);
             on_cancel(cid);
@@ -500,8 +505,8 @@ void NativeOrder::on_update(Price p, Quantity q, TickToTradeCorID cid) {
       auto &encoding = me->update_encoding;
       me->update_price_bm = encoding.overwrite(
           me->update_price_bm,
-          securities::encode_exchange_price(*me->sec, p).value(), 7,
-          price_sz());
+          securities::encode_exchange_price(*me->sec, p).value(),
+          me->price_encoding_precision, price_sz());
       me->update_quantity_bm =
           encoding.overwrite(me->update_quantity_bm, q.value(), quantity_sz());
 
@@ -747,8 +752,9 @@ void NativeOrder::report_state() {
              me->definition.side(),
              Quantity(previous_position.value() - current_position.value()));
    } else
-      LOG(OrderMan, Error, "Order {} ignored for purposes of limit enforcing "
-                           "because it references unknown security {} ",
+      LOG(OrderMan, Error,
+          "Order {} ignored for purposes of limit enforcing "
+          "because it references unknown security {} ",
           key().value(), me->definition.security_key());
 
    me->services->report_state(me->state);
@@ -829,8 +835,10 @@ void NativeOrder::init_fix() {
       cancel_encoding.add<fix::OrdType>(NATIVE_ORDER_TYPE_LIMIT);
 
       // price
-      me->create_price_bm = create_encoding.add<fix::Price>(0, 7, price_sz());
-      me->update_price_bm = update_encoding.add<fix::Price>(0, 7, price_sz());
+      me->create_price_bm = create_encoding.add<fix::Price>(
+          0, me->price_encoding_precision, price_sz());
+      me->update_price_bm = update_encoding.add<fix::Price>(
+          0, me->price_encoding_precision, price_sz());
       break;
 
    case CustomOrderDefinition::kStop: {
@@ -843,8 +851,10 @@ void NativeOrder::init_fix() {
       auto stop_px = def.custom_def().stop().stop_price();
       auto encoded_stop_px =
           securities::encode_exchange_price(*me->sec, Price(stop_px)).value();
-      create_encoding.add<fix::StopPx>(encoded_stop_px, -1, -1);
-      update_encoding.add<fix::StopPx>(encoded_stop_px, -1, -1);
+      create_encoding.add<fix::StopPx>(encoded_stop_px,
+                                       me->price_encoding_precision, -1);
+      update_encoding.add<fix::StopPx>(encoded_stop_px,
+                                       me->price_encoding_precision, -1);
       break;
    }
 
@@ -858,12 +868,16 @@ void NativeOrder::init_fix() {
       auto stop_px = def.custom_def().stop_limit().stop_price();
       auto encoded_stop_px =
           securities::encode_exchange_price(*me->sec, Price(stop_px)).value();
-      create_encoding.add<fix::StopPx>(encoded_stop_px, -1, -1);
-      update_encoding.add<fix::StopPx>(encoded_stop_px, -1, -1);
+      create_encoding.add<fix::StopPx>(encoded_stop_px,
+                                       me->price_encoding_precision, -1);
+      update_encoding.add<fix::StopPx>(encoded_stop_px,
+                                       me->price_encoding_precision, -1);
 
       // price
-      me->create_price_bm = create_encoding.add<fix::Price>(0, 7, price_sz());
-      me->update_price_bm = update_encoding.add<fix::Price>(0, 7, price_sz());
+      me->create_price_bm = create_encoding.add<fix::Price>(
+          0, me->price_encoding_precision, price_sz());
+      me->update_price_bm = update_encoding.add<fix::Price>(
+          0, me->price_encoding_precision, price_sz());
       break;
    }
    default:
@@ -1017,5 +1031,5 @@ NativeOrder::get_native_state(api::orderman::OrderState &st) {
    bts_assert(false);
    return *st.mutable_stop_limit()->mutable_native_state();
 }
-}
-}
+} // namespace orderman
+} // namespace bts

src/server/sparkd/orderman/orders/spreader_orders/SpreaderOrder.cpp

@@ -53,7 +53,7 @@ void SpreaderOrder::on_activate(Price price, Quantity quantity,
 }
 void SpreaderOrder::on_update(Price price, Quantity quantity,TickToTradeCorID){
    if (!_active) return;
-   LOG(OrderMan, Info, "SpreaderOrder.on_update key: {} price: {} quantity: {}", key().value(), price.value(), quantity.value());   
+   LOG(OrderMan, Info, "SpreaderOrder.on_update key: {} price: {} quantity: {}", key().value(), price.value(), quantity.value());
    _state_worker->on_update(price, quantity);
    _report_state();
 }
@@ -242,45 +242,50 @@ bool SpreaderOrder::_quoters_can_accept_modification (){
    }
    return true;
 }
-void SpreaderOrder::_send_hedge(size_t filled_leg_index,
-                                Price filled_price,
-                                Quantity filled_quantity, 
-                                size_t hedge_leg_index){
-   LOG(OrderMan, Info, "SpreaderOrder::_send_hedge key: {}, filled_leg_index: {} filled_price: {} filled_quantity: {} hedge_leg_index: {}",
-                        key().value(), filled_leg_index, filled_price.value(), filled_quantity.value(), hedge_leg_index);
-   Price hedge_price;
-   Quantity hedge_quantity;
-   _compute_hedge_price_quantity (filled_leg_index, filled_price, 
-                                  filled_quantity, hedge_leg_index,
-                                  hedge_price, hedge_quantity);
-   SingleFillHedger& hedger = _bookkeeping_data().add_new_hedger (*this, 
+void SpreaderOrder::_send_hedge(size_t hedge_leg_index,
+                                Price hedge_price,
+                                Quantity hedge_qty){
+   LOG(OrderMan, Info, "SpreaderOrder::_send_hedge key: {}, hedge_leg_index: {} hedge_price: {} hedge_qty: {}",
+                        key().value(), hedge_leg_index, hedge_price, hedge_qty);
+   SingleFillHedger& hedger = _bookkeeping_data().add_new_hedger (*this,
                                                                   hedge_leg_index,
                                                                   hedge_price,
-                                                                  hedge_quantity);
+                                                                  hedge_qty);
    hedger.init ();
 }
 void SpreaderOrder::_compute_hedge_price_quantity(size_t filled_leg_index,
-                                                  Price fill_price, Quantity fill_quantity,
+                                                  Price fill_price,
+                                                  double fill_quantity,
                                                   size_t hedge_leg_index,
-                                                  Price& hedge_price, Quantity& hedge_quantity){
+                                                  Price& hedge_price,
+                                                  double& hedge_quantity){
    Price computed_hedge_price;
    SpreaderLegData& filled_leg_data = _bookkeeping_data().get_leg_data(filled_leg_index);
    SpreaderLegData& hedged_leg_data = _bookkeeping_data().get_leg_data(hedge_leg_index);
    if (_bookkeeping_data().n_legs() == 2)
       SpreaderUtils::compute_hedge_price_size_2_legs (_bookkeeping_data().price(),
-                                                      filled_leg_data.leg_property_data(), fill_price, fill_quantity,
-                                                      hedged_leg_data.leg_property_data(), computed_hedge_price, hedge_quantity);
+                                                      filled_leg_data.leg_property_data(),
+                                                      fill_price,
+                                                      fill_quantity,
+                                                      hedged_leg_data.leg_property_data(),
+                                                      computed_hedge_price,
+                                                      hedge_quantity);
    else{
-      const QuoterLegDriverData& driver_data = filled_leg_data.quoter().driver_data(hedge_leg_index);
-      SpreaderUtils::compute_hedge_price_size_n_legs (filled_leg_data.leg_property_data(), fill_price, fill_quantity,
-                                                      hedged_leg_data.leg_property_data(), driver_data,
-                                                      computed_hedge_price, hedge_quantity); 
+      const QuoteLegDriver& driver_data = filled_leg_data.quoter().driver_data(fill_price, hedge_leg_index);
+      SpreaderUtils::compute_hedge_price_size_n_legs (filled_leg_data.leg_property_data(),
+                                                      fill_price,
+                                                      fill_quantity,
+                                                      hedged_leg_data.leg_property_data(),
+                                                      driver_data,
+                                                      computed_hedge_price,
+                                                      hedge_quantity); 
    }
    hedge_price = SpreaderUtils::apply_pre_hedge_parameters (hedged_leg_data.leg_property_data().leg_params().pre_hedge_params(), 
                                                             hedged_leg_data.book(),
                                                             hedged_leg_data.leg_property_data().trading_side(),
                                                             hedged_leg_data.leg_property_data().tick_size(),
-                                                            computed_hedge_price, hedge_quantity);
+                                                            Price(computed_hedge_price),
+                                                            Quantity(hedge_quantity));
 }
 void SpreaderOrder::_report_state (){ 
    if (_activated()){ 

src/server/sparkd/orderman/orders/spreader_orders/SpreaderOrder.h

@@ -133,10 +133,11 @@ private:
    void _transform_to_END(std::string* m = nullptr);
    // swtches to the state passed in the arg
    void _switch_state (ISpreaderStateWorker* new_state);
-   // this means on_init_new has been called and we have the
-   // the structure built
    // misc ////////////////////
    ////////////////////////////
+   // this means on_init_new or on_init_recovered
+   // has been called and we have the bookkeeping
+   // data built
    bool _inited () const {
       return _bookkeeping_data_ptr != nullptr;
    }
@@ -165,17 +166,15 @@ private:
    // on leg 'leg_to_hedge_index' in response to a fill
    // on leg 'filled_leg_index' with price 'filled_price'
    // quantity 'filled_quantity'
-   void _send_hedge(size_t filled_leg_index, Price filled_price,
-                    Quantity filled_quantity,
-                    size_t leg_to_hedge_index);
-   // purely a computating function that computes the
-   // hedge price and quantity (last 2 args) for leg
-   // 'hedged_leg_index' for a fill on leg 'filled_leg_index'
-   // with price 'filled_price' and quantity 'filled_quantity'
+   void _send_hedge(size_t hedge_leg_index,
+                    Price hedge_price,
+                    Quantity hedge_qty);
    void _compute_hedge_price_quantity(size_t filled_leg_index,
-                                      Price filled_price, Quantity filled_quantity,
+                                      Price filled_price,
+                                      double filled_quantity,
                                       size_t hedged_leg_index,
-                                      Price& hedge_price, Quantity& hedge_quantity);
+                                      Price& hedge_price,
+                                      double& hedge_quantity);
    // sends the report out
    void _report_state ();      
    SpreaderBookkeepingData& _bookkeeping_data(){

src/server/sparkd/orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp

@@ -20,6 +20,7 @@ _spreader(spreader),
 _this_leg_data (this_leg_property),
 _n_spreader_legs (n_spreader_legs),
 _child_order_state (child_order_state),
+_driver_cache(n_spreader_legs,n_spreader_legs>2?10:0,this_leg_property.tick_size()),
 _price_quantity_state(Price(0), Quantity(0), child_order_state),
 _is_permenantly_cancelled (false),
 _worker(nullptr),
@@ -33,8 +34,7 @@ _end_state_worker(this){
    if (_active_quoting){
       if (!_two_legged){
          for (size_t q_idx = 0; q_idx < _n_spreader_legs; q_idx++){
-            _drivers_commited.push_back (QuoterLegDriverData());
-            _drivers_temp_usage.push_back (QuoterLegDriverData());          
+            _drivers_temp_usage.push_back (QuoteLegDriver());
          }
       }
    }
@@ -49,7 +49,7 @@ _this_leg_data (other._this_leg_data),
 _n_spreader_legs (other._n_spreader_legs),
 _child_order_state (other._child_order_state),
 _drivers_temp_usage(other._drivers_temp_usage),
-_drivers_commited (other._drivers_commited),   
+_driver_cache(other._driver_cache), 
 _price_quantity_state(other._price_quantity_state.target_spread_price(), 
                       other._price_quantity_state.target_leg_quantity(),
                       other._price_quantity_state.order_state()), 
@@ -131,13 +131,8 @@ bool SingleLegQuoter::is_terminal_state () const {
 bool SingleLegQuoter::accept_modifications() const {
     return _worker->state() == QuoterState::WAITING_FOR_SIZE || _worker->state() == QuoterState::WORKING;
 }
-Price SingleLegQuoter::driver_price (size_t leg_idx) const{
-   bts_assert (leg_idx < _n_spreader_legs);
-   return _drivers_commited[leg_idx].driver_price;
-}
-void SingleLegQuoter::_incorporate_other_leg_fill (size_t leg_index, Quantity filled_quantity){
-   Quantity this_leg_reduction = SpreaderUtils::get_leg_quantity_from_another_leg_quantity (_spreader.get_leg_data(leg_index), filled_quantity, _this_leg_data);
-   _price_quantity_state.add_hedged_reduction (this_leg_reduction);
+void SingleLegQuoter::_incorporate_other_leg_fill (size_t, Quantity hedge_reduction){
+   _price_quantity_state.add_hedged_reduction (hedge_reduction);
 }
 void SingleLegQuoter::_compute_price_quantity(Price& price, Quantity& quantity){
    Quantity remaining_quantity = _price_quantity_state.remaining_to_fill();
@@ -152,11 +147,9 @@ void SingleLegQuoter::_compute_price_quantity(Price& price, Quantity& quantity){
        SpreaderUtils::compute_quoting_price_size_n_legs (_price_quantity_state.target_spread_price(), remaining_quantity,
                                                          _this_leg_data.leg_index(), _spreader, _drivers_temp_usage, price, quantity);
 }
-void SingleLegQuoter::_commit_drivers(){
-    if (!_two_legged)
-    // this could be done with a pointer swap too
-       for (size_t idx = 0; idx < _n_spreader_legs; idx++)
-         _drivers_commited[idx] = _drivers_temp_usage[idx];
+void SingleLegQuoter::_commit_drivers(Price quote_price){
+   if (!_two_legged)
+      _driver_cache.add(_drivers_temp_usage,quote_price);
 }
 /*********************************************************************************/
 /*******************************state transformations*****************************/

src/server/sparkd/orderman/orders/spreader_orders/spreader/SingleLegQuoter.h

@@ -14,9 +14,10 @@
 #include "server/sparkd/orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.h"
 #include "server/sparkd/orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.h"
 #include "server/sparkd/orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.h"
-#include "server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoterLegDriverData.h"
+#include "server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.h"
 #include "server/sparkd/orderman/actors/order_manager/ChildOrder.h"
 #include "server/sparkd/orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.h"
+#include "server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.h"
 #include <sstream>
 
 
@@ -24,7 +25,6 @@ namespace bts {
 namespace orderman {
 
 class ISpreaderManager;
-//class SpreaderChildOrderState;
 class SingleLegQuoter
 {
 friend QuoterStartStateWorker;
@@ -45,8 +45,8 @@ private:
    // legs requires auxiliary storage for drivers, which
    // _drivers_temp_usage provides. once we are ready
    // to send the orders, the drivers are committed to
-   // _drivers_commited
-   std::vector<QuoterLegDriverData> _drivers_temp_usage, _drivers_commited;
+   std::vector<QuoteLegDriver> _drivers_temp_usage;
+   QuoteDriverSetHistory _driver_cache;
    QuoteLegPriceQuantityState _price_quantity_state;
    bool _is_permenantly_cancelled;
    // state workers
@@ -108,13 +108,10 @@ public:
    // the quoter returns true if it is in a state where orders
    // can be modified
    bool accept_modifications() const;
-   // for a given leg, returns the driver price that was used
-   // to price the current order
-   Price driver_price (size_t leg_idx) const;
-   const QuoterLegDriverData& driver_data (size_t leg_idx) const
-   {
-      bts_assert (leg_idx < _drivers_commited.size());
-      return _drivers_commited[leg_idx];
+   const QuoteLegDriver& driver_data (Price quote_price, size_t leg_idx) const{
+      const QuoteLegDriver* leg_driver = _driver_cache.find_driver(quote_price,leg_idx);
+      bts_assert(leg_driver!=nullptr);
+      return *leg_driver;
    }
    // tells if it is expecting to get a cancel (this helps catch
    // unsolicited cancels). returns true is its expecting a cancel
@@ -152,12 +149,12 @@ private:
    }
    void _cancel_order () {_child_order_state.cancel_order();}
    // this updates the hedge reduction in _price_quantity_state
-   void _incorporate_other_leg_fill (size_t leg_index, Quantity filled_quantity);
+   void _incorporate_other_leg_fill (size_t leg_index, Quantity hedge_reduction);
    // this is the function that computes the quoting price and the   
    // quantity for this leg.
    void _compute_price_quantity(Price& price, Quantity& quantity);
    // copies the computed driver to committed dri
-   void _commit_drivers ();
+   void _commit_drivers (Price quote_price);
    void _transform_to_START ();
    void _transform_to_WAITING_FOR_SIZE ();
    void _transform_to_WAITING_FOR_FIRST_ACK ();

src/server/sparkd/orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp

@@ -23,6 +23,7 @@ _reported_state(){
    _quoter_child_order_states.reserve(_n_legs);
    _quoters.reserve (_n_legs);
    _hedgers.reserve (_n_legs);
+   _unhedged_deltas.reserve(_n_legs);
    _leg_data.reserve (_n_legs);
    for (const auto& lpd_iter : legs){
       _leg_prop_data.push_back(lpd_iter);
@@ -34,9 +35,15 @@ _reported_state(){
       _quoter_child_order_states.emplace_back (empty_order,leg_property_data.leg_security_key(),_is_live,true);
       _quoters.emplace_back(spreader_manager, leg_property_data, _n_legs, _quoter_child_order_states.back());
       _hedgers.emplace_back ();
-      _leg_data.emplace_back(leg_property_data, _quoters.back(), _quoter_order_builders.back(), 
-                             _hedger_order_builders.back(), _hedgers.back (), _quoter_child_order_states.back(),
-                             _books.back());
+      _unhedged_deltas.emplace_back();
+      _leg_data.emplace_back(leg_property_data,
+                             _quoters.back(),
+                             _quoter_order_builders.back(), 
+                             _hedger_order_builders.back(),
+                             _hedgers.back (),
+                             _quoter_child_order_states.back(),
+                             _books.back(),
+                             _unhedged_deltas.back());
       bts_assert (_sec_key_to_index.find(leg_property_data.security_key()) == _sec_key_to_index.end());
       _sec_key_to_index.insert(std::pair<uint32_t, size_t>(leg_property_data.security_key(), leg_property_data.leg_index()));
    }

src/server/sparkd/orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.h

@@ -8,6 +8,7 @@
 #include "server/sparkd/orderman/actors/order_manager/OrderBuilder.h"
 #include "server/sparkd/orderman/orders/PreActivationOrderServices.h"
 #include "server/sparkd/orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.h"
+#include "server/sparkd/orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.h"
 
 namespace bts{
 namespace orderman { 
@@ -32,7 +33,8 @@ private:
    std::vector<SpreaderChildOrderState> _quoter_child_order_states;
    std::vector<SingleLegQuoter> _quoters;
    std::vector<std::vector<SingleFillHedger>> _hedgers;
-   std::vector<SpreaderLegData> _leg_data;
+   std::vector<UnhedgedDelta> _unhedged_deltas;
+   std::vector<SpreaderLegData> _leg_data;   
    boost::container::flat_map<uint32_t, size_t> _sec_key_to_index;
    // state thats sent out in report (its kept here amd modified
    // vs creating a new one everytime)

src/server/sparkd/orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp

@@ -9,7 +9,8 @@ SpreaderLegData::SpreaderLegData (const LegPropertyData& leg_property_data,
                                   SpreaderOrderBuilder& hedger_order_builder,
                                   std::vector<SingleFillHedger>& hedger_data_vector,
                                   SpreaderChildOrderState& quoter_child_order_state,
-                                  Book& book)
+                                  Book& book,
+                                  UnhedgedDelta& unhedged_delta)
 :
 _leg_property_data (leg_property_data),
 _quoter(quoter),
@@ -17,7 +18,8 @@ _quoter_order_builder (quoter_order_builder),
 _hedger_order_builder (hedger_order_builder),
 _quoter_child_order_state  (quoter_child_order_state),
 _hedger_data_vector (hedger_data_vector),     
-_book(book)
+_book(book),
+_unhedged_delta(unhedged_delta)
 {}
 
 SpreaderLegData::SpreaderLegData(SpreaderLegData&& other)
@@ -28,7 +30,8 @@ _quoter_order_builder(other._quoter_order_builder),
 _hedger_order_builder(other._hedger_order_builder),
 _quoter_child_order_state (other._quoter_child_order_state),
 _hedger_data_vector (other._hedger_data_vector),
-_book(other._book){
+_book(other._book),
+_unhedged_delta(other._unhedged_delta){
    bts_assert (false);
 }
 

src/server/sparkd/orderman/orders/spreader_orders/spreader/SpreaderLegData.h

@@ -5,6 +5,7 @@
 #include "SpreaderChildOrderState.h"
 #include "server/sparkd/orderman/actors/order_manager/OrderBuilder.h"
 #include "server/sparkd/orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.h"
+#include "server/sparkd/orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.h"
 
 namespace bts{
 namespace orderman{
@@ -28,6 +29,7 @@ private:
    SpreaderChildOrderState& _quoter_child_order_state;
    std::vector<SingleFillHedger>& _hedger_data_vector;   
    Book& _book;
+   UnhedgedDelta& _unhedged_delta;
 public:  
    SpreaderLegData (const LegPropertyData& leg_property_data,
                     SingleLegQuoter& quoter,                    
@@ -35,7 +37,8 @@ public:
                     SpreaderOrderBuilder& hedger_order_builder,
                     std::vector<SingleFillHedger>& hedger_data_vector,
                     SpreaderChildOrderState& quoter_child_order_state,
-                    Book& book);
+                    Book& book,
+                    UnhedgedDelta& unhedged_delta);
    SpreaderLegData() = delete;
    SpreaderLegData(const SpreaderLegData&) = delete;
    // this cons is not deleted since for each leg,
@@ -51,12 +54,13 @@ public:
    /////////////////////////////////
    const LegPropertyData& leg_property_data() const {return _leg_property_data;}
    SingleLegQuoter& quoter() {return _quoter;}
-   const bts::orderman::Book& book () const {return _book;}   
+   const bts::orderman::Book& book () const {return _book;}
    SpreaderChildOrderState& order_update_message () {return _quoter_child_order_state;}
    bts::orderman::OrderKey key() {return _quoter_child_order_state.order_key();}
    bts::orderman::OrderKey order_key() {return _quoter_child_order_state.order_key();}
    SpreaderOrderBuilder& quoter_order_builder() {return _quoter_order_builder;}
    SpreaderOrderBuilder& hedger_order_builder() {return _hedger_order_builder;}
+   UnhedgedDelta& unhedged_delta() {return _unhedged_delta;}
    // returns the number of active hedgers
    size_t n_hedgers() const { return _hedger_data_vector.size(); }
    // returns the hedger at a given index

src/server/sparkd/orderman/orders/spreader_orders/spreader/SpreaderUtils.h

@@ -6,7 +6,7 @@
 #include "server/sparkd/orderman/orderman_utils/Book.h"
 #include "server/sparkd/orderman/orders/spreader_orders/spreader/LegPropertyData.h"
 #include "server/sparkd/orderman/orders/spreader_orders/spreader/ISpreaderManager.h"
-#include "server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoterLegDriverData.h"
+#include "server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.h"
 #include "lib/api/services/OrderService.pb.h"
 #include "server/sparkd/orderman/actors/order_manager/OrderBuilder.h"
 
@@ -423,17 +423,28 @@ public:
    // so if we get an N lot fill in l2, we send a hedge on l1
    // sized s1 * N / s2
    ////////////////////////general functions
+   static double get_leg_quantity_from_another_leg_quantity (const LegPropertyData& source_leg,
+                                                             double source_leg_quantity,
+                                                             const LegPropertyData& target_leg){
+      return std::abs(target_leg.size_multiplier () * source_leg_quantity / source_leg.size_multiplier ());
+   }
    static Quantity get_leg_quantity_from_another_leg_quantity (const LegPropertyData& source_leg,
                                                                Quantity source_leg_quantity,
                                                                const LegPropertyData& target_leg){
-      double rv = std::abs(target_leg.size_multiplier () * source_leg_quantity.value () / source_leg.size_multiplier ());
+      double rv = get_leg_quantity_from_another_leg_quantity(source_leg,
+                                                             static_cast<double>(source_leg_quantity.value()),
+                                                             target_leg);            
       return Quantity(_round_leg_qty_computed_from_another_leg_qty(rv));
-   }
+   }   
    // given the quantity of the spreader order, determines the quantity
    // of the leg
+   static double get_leg_quantity_from_spreader_quantity(const LegPropertyData& leg,
+                                                         double spreader_order_quantity){
+      return std::abs (leg.size_multiplier () * spreader_order_quantity);
+   }
    static Quantity get_leg_quantity_from_spreader_quantity(const LegPropertyData& leg,
                                                            Quantity spreader_order_quantity){
-      double rv = std::abs (leg.size_multiplier () * spreader_order_quantity.value());
+      double rv = get_leg_quantity_from_spreader_quantity(leg,static_cast<double>(spreader_order_quantity.value()));
       return Quantity (_round_leg_qty_computed_from_spreader_qty (rv));
    }
    // finds the average weighted price and total quantity
@@ -450,53 +461,71 @@ public:
          out_avg_price = Price((base_price.value() * base_qty.value() + inc_price.value() * inc_qty.value())/out_total_qty.value());
    }
    ////////////////////////2 legged spreads
+   // given a quoting target qty, finds the "driver" price
+   // and qty, thats needed to hedge the quote. It also
+   // finds the ammended quoting target (since the hedge 
+   // book might not have enough to support the entire
+   // quoting target qty)
+   static void compute_driver(Quantity quote_target,
+                              const LegPropertyData& quote_leg_data,
+                              const LegPropertyData& hedge_leg_data,
+                              const bts::orderman::Book& hedge_leg_book,
+                              Price& out_hedge_price,
+                              Quantity& out_hedge_quantity,
+                              Quantity& out_quote_target){
+      // effective_mult is the number of contracts needed to
+      // hedge one quoting contract
+      double effective_mult = std::abs(hedge_leg_data.leg_params().quote_params().lean_quantity_ratio() * hedge_leg_data.size_multiplier() / quote_leg_data.size_multiplier());
+      uint32_t hedge_requirement = _round_hedge_target (effective_mult * quote_target.value());       // rounding hedging up
+      SpreaderUtils::traverse_depth_for_quantity(hedge_leg_book,
+                                                 SpreaderUtils::invert_side (hedge_leg_data.trading_side()),
+                                                 Quantity(hedge_requirement),
+                                                 out_hedge_price,
+                                                 out_hedge_quantity);
+      // when we floor the quoting target (ie _round_quoting_qty)
+      // we are essentially demanding less in terms of hedging, ex :
+      // consider the hedge book depth with qty's 5, 10, 5, 20, 10 (5 is the inside)
+      // and hedge ratio 20 and target qty = 5, so effectively target = 100,
+      // we get 50 at the last level, out_qty = _round_quoting_qty (50/20) = 2,
+      // which means we need 2 * 20 and should re-evaluate
+      uint32_t out_quote_target_uint = _round_quoting_qty (out_hedge_quantity.value() / effective_mult);    // rounding quoting down
+      out_quote_target_uint = std::min(out_quote_target_uint,quote_target.value());
+      out_quote_target = Quantity (out_quote_target_uint);
+   }
    // calculates the computed price and quantity for a quoting
    // leg given the data for the other leg
    static void compute_price_quantity_2_legs(Price target_spread_price,
-                                             Quantity this_leg_target_quantity,
-                                             const LegPropertyData& this_leg_data,
-                                             const LegPropertyData& other_leg_data,
-                                             const bts::orderman::Book& other_leg_book,
-                                             Price& price, Quantity& quantity){
-      // other_leg_price and other_leg_quantity are the price and quantity calculated
-      // from traversing the other book, for the requires size   
-      Price other_leg_price; Quantity other_leg_quantity;
-      // this is the size ratio of the other leg vs this leg
-      double effective_mult = std::abs(other_leg_data.leg_params().quote_params().lean_quantity_ratio() * other_leg_data.size_multiplier() / this_leg_data.size_multiplier());
-      // to be conservative round the required size up
-      uint32_t other_leg_target_quantity = _round_hedge_target (effective_mult * this_leg_target_quantity.value());
-      SpreaderUtils::traverse_depth_for_quantity(other_leg_book, SpreaderUtils::invert_side (other_leg_data.trading_side()), Quantity(other_leg_target_quantity), other_leg_price, other_leg_quantity);
-      // when we floor we essentially go back up the hedging ladder
-      // so we should re-evaluate 
-      // ex, consider the hedge book depth with qty's 5, 10, 5, 20, 10 (5 is the inside)
-      // and hedge ratio 20 and target qty = 5, so effectively target = 100,
-      // we get 50 at the last level, out_qty = floor (50/20) = 2, so the
-      // price should not be the last level but the second last level
-      uint32_t out_quantity = _round_quoting_qty (other_leg_quantity.value() / effective_mult);
-      quantity = Quantity (out_quantity);
-      double out_price = (target_spread_price.value() - other_leg_price.value() * other_leg_data.price_multiplier())/this_leg_data.price_multiplier();
-      out_price = bts::securities::raw_round_to_quote_price (out_price, this_leg_data.tick_size(), this_leg_data.trading_side());
-      price = Price(out_price);
+                                             Quantity quote_target,
+                                             const LegPropertyData& quote_leg_data,
+                                             const LegPropertyData& hedge_leg_data,
+                                             const bts::orderman::Book& hedge_leg_book,
+                                             Price& quote_price,
+                                             Quantity& quote_qty){
+      Price hedge_price;
+      Quantity hedge_qty;
+      compute_driver(quote_target,quote_leg_data,hedge_leg_data,hedge_leg_book,
+                     hedge_price,hedge_qty,quote_qty);
+      double out_price = (target_spread_price.value() - hedge_price.value() * hedge_leg_data.price_multiplier())/quote_leg_data.price_multiplier();
+      out_price = bts::securities::raw_round_to_quote_price (out_price, quote_leg_data.tick_size(), quote_leg_data.trading_side());
+      quote_price = Price(out_price);      
    }
    static void compute_hedge_price_size_2_legs (Price spread_price,
                                                 const LegPropertyData& quote_filled_leg,
                                                 Price quote_fill_price,
-                                                Quantity quote_fill_quantity,
+                                                double quote_fill_quantity,
                                                 const LegPropertyData& hedge_leg,
                                                 Price& out_hedge_price,
-                                                Quantity& out_hedge_quantity){
-      // for ratio spreads hedge sizes are simply rounded off (the fraction can be a setting)
+                                                double& out_hedge_quantity){
       out_hedge_quantity = get_leg_quantity_from_another_leg_quantity (quote_filled_leg, quote_fill_quantity, hedge_leg);
       double out_hedge_price_d = (spread_price.value() - quote_fill_price.value() * quote_filled_leg.price_multiplier())/hedge_leg.price_multiplier();
-      out_hedge_price_d = bts::securities::raw_round_to_quote_price (out_hedge_price_d, hedge_leg.tick_size(), hedge_leg.trading_side());
-      out_hedge_price = Price(out_hedge_price_d);
+      out_hedge_price = Price(bts::securities::raw_round_to_quote_price (out_hedge_price_d, hedge_leg.tick_size(), hedge_leg.trading_side()));
    }
    ////////////////////////N legged case
    static void compute_quoting_price_size_n_legs (Price target_spread_price,
                                                   Quantity quoting_leg_target_quantity,
                                                   size_t quoting_leg_index,
                                                   const ISpreaderManager& spreader,
-                                                  std::vector<QuoterLegDriverData>& driver_vector,
+                                                  std::vector<QuoteLegDriver>& driver_vector,
                                                   Price& out_price,
                                                   Quantity& out_quantity){
       compute_drivers (quoting_leg_target_quantity, quoting_leg_index, spreader, driver_vector);
@@ -514,19 +543,23 @@ public:
    static void compute_drivers(Quantity quoting_leg_target_quantity,
                                size_t quoting_leg_index,
                                const ISpreaderManager& spreader,
-                               std::vector<QuoterLegDriverData>& out_driver_vector){
-      const LegPropertyData& quoting_leg_data = spreader.get_leg_data (quoting_leg_index);
+                               std::vector<QuoteLegDriver>& out_driver_vector){
+      const LegPropertyData& quote_leg_data = spreader.get_leg_data (quoting_leg_index);
       size_t n_legs = out_driver_vector.size();
       for (size_t idx = 0; idx < n_legs; idx++)
          if (idx != quoting_leg_index){
-            const bts::orderman::Book& driver_leg_book = spreader.get_book(idx);
-            const LegPropertyData& driver_leg_data = spreader.get_leg_data(idx);            
-            double driver_effective_mult = std::abs(driver_leg_data.leg_params().quote_params().lean_quantity_ratio() * driver_leg_data.size_multiplier() / quoting_leg_data.size_multiplier());
-            uint32_t driver_leg_target_quantity = _round_hedge_target (driver_effective_mult * quoting_leg_target_quantity.value());
-            Price driver_price;  Quantity driver_quantity;
-            SpreaderUtils::traverse_depth_for_quantity(driver_leg_book, SpreaderUtils::invert_side (driver_leg_data.trading_side()), Quantity(driver_leg_target_quantity), driver_price, driver_quantity);
-            Quantity quoting_leg_calc_quantity = Quantity (_round_quoting_qty (driver_quantity.value() / driver_effective_mult));
-            out_driver_vector[idx].set (driver_price, driver_quantity, quoting_leg_calc_quantity);
+            const bts::orderman::Book& hedge_leg_book = spreader.get_book(idx);
+            const LegPropertyData& hedge_leg_data = spreader.get_leg_data(idx);
+            Price hedge_price;
+            Quantity hedge_qty, quote_qty;
+            compute_driver(quoting_leg_target_quantity,
+                           quote_leg_data,
+                           hedge_leg_data,
+                           hedge_leg_book,
+                           hedge_price,
+                           hedge_qty,
+                           quote_qty);
+            out_driver_vector[idx].set (hedge_price, hedge_qty, quote_qty);
          }
    }
    // computes the quoting price from the leg drivers
@@ -534,13 +567,13 @@ public:
    static Price compute_quoting_price_from_drivers(Price target_spread_price,
                                                    size_t quoting_leg_index,
                                                    const ISpreaderManager& spreader,
-                                                   const std::vector<QuoterLegDriverData>& driver_vector){
+                                                   const std::vector<QuoteLegDriver>& driver_vector){
       const LegPropertyData& quoting_leg_data = spreader.get_leg_data (quoting_leg_index);
       size_t n_legs = driver_vector.size();
       double sum = 0;
       for (size_t idx = 0; idx < n_legs; idx++)
          if (idx != quoting_leg_index)
-            sum += driver_vector[idx].driver_price.value() * spreader.get_leg_data (idx).price_multiplier();
+            sum += driver_vector[idx].driver_price().value() * spreader.get_leg_data (idx).price_multiplier();
       double rv = (target_spread_price.value() - sum)/quoting_leg_data.price_multiplier();
       rv = bts::securities::raw_round_to_quote_price (rv, quoting_leg_data.tick_size(), quoting_leg_data.trading_side());
       return Price(rv);
@@ -549,26 +582,40 @@ public:
    // what each hedging leg can support. this function
    // simply finds that minimum    
    static Quantity compute_quoting_quantity_from_drivers (size_t quoting_leg_index,
-                                                          const std::vector<QuoterLegDriverData>& driver_vector){
+                                                          const std::vector<QuoteLegDriver>& driver_vector){
       uint32_t rv = std::numeric_limits<uint32_t>::max();
       for (size_t idx = 0; idx < driver_vector.size(); idx++)
          if (idx != quoting_leg_index)
-            if (driver_vector[idx].quoter_leg_quantity.value() < rv)
-               rv = driver_vector[idx].quoter_leg_quantity.value();
+            if (driver_vector[idx].quoter_leg_quantity().value() < rv)
+               rv = driver_vector[idx].quoter_leg_quantity().value();
       return Quantity(rv);
    }
    static void compute_hedge_price_size_n_legs (const LegPropertyData& quote_filled_leg,
                                                 Price,
-                                                Quantity quote_fill_quantity,
+                                                double quote_fill_quantity,
                                                 const LegPropertyData& hedge_leg,
-                                                const QuoterLegDriverData& leg_driver_data,
-                                                Price& out_hedge_price, Quantity& out_hedge_quantity){
-      out_hedge_price = leg_driver_data.driver_price;
-      out_hedge_quantity = get_leg_quantity_from_another_leg_quantity (quote_filled_leg, quote_fill_quantity, hedge_leg);
+                                                const QuoteLegDriver& leg_driver_data,
+                                                Price& out_hedge_price,
+                                                double& out_hedge_quantity){
+      out_hedge_price = Price(leg_driver_data.driver_price().value());
+      out_hedge_quantity = get_leg_quantity_from_another_leg_quantity (quote_filled_leg, 
+                                                                       quote_fill_quantity,
+                                                                       hedge_leg);
    }
    ////////////////////////////////////////////////////
    // Rounding functions //////////////////////////////
    ////////////////////////////////////////////////////
+   // rounds a double to a whole number if its too close
+   // to it (this is ued in the case of rounding errors
+   // when 0.9999999 or 1.0000001 is really 1)
+   static double round_if_near_whole(double qty,
+                                     double vicinity=0.000000001){
+      double round_qty = std::round(qty);
+      if (std::abs(round_qty - qty) <= vicinity)
+         return round_qty;
+      else
+         return qty;
+   }
    static uint32_t _round_quantity(double qty, bool is_up){
       double round_err = 0.000000001;
       double round_qty = std::round(qty);

src/server/sparkd/orderman/orders/spreader_orders/spreader/hedger/IHedgerStateWorker.h

@@ -10,8 +10,7 @@ namespace orderman {
 class SpreaderChildOrderState;
 class Book;
 class SingleFillHedger;
-class IHedgerStateWorker
-{   
+class IHedgerStateWorker{
 public:  
    // called once when transition to this state happens   
    virtual void on_transition () = 0;

src/server/sparkd/orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp

@@ -0,0 +1,29 @@
+#include "UnhedgedDelta.h"
+#include <cmath>
+#include "lib/utils/bts_assert.h"
+#include "server/sparkd/orderman/orders/spreader_orders/spreader/SpreaderUtils.h"
+
+namespace bts{
+namespace orderman{
+
+UnhedgedDelta::UnhedgedDelta()
+:
+_qty(0),
+_avg_price(0){
+}
+void UnhedgedDelta::add_unhedged(double price,double qty){
+   bts_assert(qty >= 0);
+   auto total_qty=_qty+qty;
+   _avg_price = (_qty*_avg_price + qty*price)/total_qty;
+   _qty=total_qty;
+}
+uint32_t UnhedgedDelta::subtract_hedge_if_available(){
+   uint32_t rv = 0;
+   double qty = SpreaderUtils::round_if_near_whole(_qty);
+   if (qty >= 1){
+      rv = std::floor(qty);
+      _qty = qty-rv;
+   }
+   return rv;
+}
+}}

src/server/sparkd/orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <cstdint>
+
+namespace bts{
+namespace orderman{
+// in the case when the hedge quantity is not a full contract
+// (ex if we get a 1 lot fill on a 1 X 3 spreader, where 3 is 
+// a quoting leg), this class is used to accumulate the unhedged
+// delta
+class UnhedgedDelta{
+private:
+   double _qty;
+   double _avg_price;
+public:
+   UnhedgedDelta();
+   // adds to the unhedged qty and updates the
+   // running average price
+   void add_unhedged (double price, double qty);
+   // if >= 1 unhedged delta is available, it
+   // subtracts the integer part and returns it.
+   // else returns 0
+   uint32_t subtract_hedge_if_available();
+};
+}}

src/server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp

@@ -0,0 +1,29 @@
+#include "QuoteDriverSet.h"
+
+namespace bts {
+namespace orderman {
+
+QuoteDriverSet::QuoteDriverSet(size_t n_legs)
+:
+_quote_seq_no(0),
+_quote_price(0.0),
+_drivers(){
+   _drivers.reserve(n_legs);
+   for (size_t i=0; i<n_legs; i++)
+      _drivers.emplace_back();
+}
+void QuoteDriverSet::update(uint16_t quote_seq_no,
+                             Price quote_price,
+                             const std::vector<QuoteLegDriver>& drivers){
+   _quote_seq_no=quote_seq_no;
+   _quote_price=quote_price;
+   bts_assert(_drivers.size()==drivers.size());
+   int i=0;
+   for(const auto& d:drivers)
+      _drivers[i++]=d;
+}
+const QuoteLegDriver& QuoteDriverSet::leg_driver(size_t leg_index) const{
+   bts_assert(leg_index<_drivers.size());
+   return _drivers[leg_index];   
+}
+}}

src/server/sparkd/orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "QuoteLegDriver.h"
+
+namespace bts {
+namespace orderman {
+
+// this is the list of all drivers for a 
+// single quoting leg
+class QuoteDriverSet{
+private:
+   uint16_t _quote_seq_no;
+   Price _quote_price;
+   std::vector<QuoteLegDriver> _drivers;
+public:   
+   QuoteDriverSet (size_t n_legs);
+   uint16_t quote_seq_no() const{
+      return _quote_seq_no; 
+   }
+   Price quote_price() const{
+      return _quote_price;
+   }
+   void update(uint16_t quote_seq_no,
+               Price quote_price,
+               const std::vector<QuoteLegDriver>& drivers);
+   const QuoteLegDriver& leg_driver (size_t leg_index) const;
+};
+}}