Updated release notes.

guard against UI setting cash and leverage factors for accounts I don't know about when span is off

Approved-by: Pedro Pinto <pedro.pinto+atlassian@bluetradesys.com>

.gitignore

@@ -32,4 +32,4 @@ trace_data/
 /src/lib/orderman/perf_analysis/last_pretty_log.csv
 /src/lib/orderman/perf_analysis/log.csv
 /doc/user/public/*
-.ionide 
+.ionide/*

doc/user/content/release_notes/_index.md

@@ -4,11 +4,55 @@ weight = 3
 sort_by = "weight"
 insert_anchor_links = "right"
 +++
+## 2.7.1
+### User Visible Changes
+  - Fixed some issues with the Cover algo, where in some circumstances
+    it would erroneously cancel launched hedges on completion.
+	
+## 2.7.0
+### User Visible Changes
+  - Allowed algos to process "in-flight" fills while shutting down
+  - Changed the cover algo to continue to hedge while shutting down
+### API Changes
+  - Changed the API so that algos do not fully terminate until all
+    their children are terminated. Algos that are in the process of
+    shutting down in response to a complete or cancel call will still
+    receive child updates, which they can use to calculate and report
+    an accurate fill count (during this shutdown stage algos cannot
+    create, cancel or update children). A new IAlgoServices method,
+    is_shutting_down(), can be used to determine if an algo is
+    shutting down.
+
+## 2.6.2
+### User Visible Changes
+  - Fixed an issue in the ladder where in some cases the mini order book could not be brought up from a badge
+  - Fixed a rare race condition that could lead to erroneous limit enforcement
+  - Fixed an issue with the margin limits where an error could occur if an account name with more than 12 chars was used  
+
+## 2.6.1
+### User Visible Changes
+  - Fixed an issue in the ladder where an error could occur when trying to change ladder settings for the first time
+  
+## 2.6.0
+### User Visible Changes
+  - In the ladder made last trade more distinctive when overlayed on the volumn column
+  - In the ladder, corrected display of volume tool tip 
+  - In the ladder, changed the consolidation of last trade to consider only the price ignoring aggressor changes
+  - Added an option for the Top Algo to delete itself if within N ticks from top of book
+  - Address a rare issue that could lead to an error report on UI exit
+  - Added margin risk limits
+  - Added the "Offset" algo. Offset prices a child algo at a parameterizable distance from the top of the book
+  - Avoided switching the current account and order type on open ladders when limits are changed 
+### Internal Changes
+  - Ensure that admin logins are always allowed to login, regardless of keycode status.
+  
 ## 2.5.0
-### User Visible Settings
+### User Visible Changes
+  - Added loss limit risk check
   - Made algos, synthetics and user names private so that multiple users can share a single system privately.
   - Allowed the mini order book defaults for column widths to be changed from the mini order book
   - Allowed the mini order book to shrink to the minimum size that will be needed to display the selected columns
+  - Improved accuracy of the directional arrow when displaying last trades in the volumn column
 ## 2.4.0
 ### User Visible Settings
   - In the ladder, changed the last trade column to accumulate successive trades at the same price
@@ -291,7 +335,7 @@ insert_anchor_links = "right"
   - The zoom slider can be enabled from Ladder Form -> Settings button -> Left Pane tab -> check Zoom Slider checkbox
   - The zoom functionality can also be performed via hotkey and mouse wheel combinations
     - Zoom In = CTRL + Mouse wheel up
-    - Zoom Out = CTRL + Mouse wheel down
+    - Zoom Out = CTRL + Mouse wheel down␍
     - Reset zoom = CTRL + Mouse wheel click or double click on the zoom slider
 ### Exchange simulator
   - Improved the exchange simulator to track real market data more closely
@@ -396,7 +440,8 @@ insert_anchor_links = "right"
   - This is configurable from a ladder's settings as well
 ### Customized settings in all windows
  - All windows now contain a settings page where grid settings (font color, size, etc) are applied
- - These settings can be saved on a per-window basis or as a Default for newly-launched windows␍
+ - These settings can be saved on a per-window basis or as a Default for newly-launched windows
+
 
 ## 1.3.1 Through 1.3.4
 

src/client/spark/Modules/Trading/Ladder/LadderSettings.cs

@@ -339,6 +339,7 @@ namespace mbts.spark {
                zoom_slider_visible = false,
                zoom_factor = _default_zoom_factor,
                net_change = NetChangeTerms.None,
+               mini_ob = new LadderMiniOrderBookSettings(bts.shell.orderbook.OrderBookStartupData.default_mini_grid.general),
             };
             rv.row_height = LadderRender.measure_row_height(rv);
             return rv;

src/client/spark/Modules/Trading/Ladder/LadderWindowFactory.cs

@@ -113,7 +113,7 @@ namespace bts.shell {
                });
             });
             // vm
-            vm = LadderVM.create(_bus, model, last_value ?? _settings_hub.current.ladder, settings_source);
+            vm = LadderVM.create(_bus, model, LadderSettings.sanitize(last_value ?? _settings_hub.current.ladder), settings_source);
             vm.settings_invalidated += on_settings_invalidated;
             vm.order_properties_vm.open_algo_popup_on_select = _ladder_conf.open_algo_popup_on_select;
             vm.order_properties_vm.use_algo_param_grid = _ladder_conf.use_algo_param_grid;

src/client/spark/Modules/Trading/Ladder/VM/LadderSettingsVM.cs

@@ -363,7 +363,9 @@ namespace mbts.spark {
          } else {
             clone = SparkUtils.clone_proto(_current);
 #pragma warning disable CS0612 // Type or member is obsolete
-            clone.mini_ob = new LadderMiniOrderBookSettings(mini_ob.create().with(sorting_criterion: _current.mini_ob.value.sorting_criterion));
+            clone.mini_ob = new LadderMiniOrderBookSettings(_current.mini_ob.value != null ? 
+               mini_ob.create().with(sorting_criterion: _current.mini_ob.value.sorting_criterion) :
+               mini_ob.create());
 #pragma warning restore CS0612 // Type or member is obsolete
          }
          clone.badge_style = badge_style;

src/client/spark/Modules/Trading/Ladder/VM/LadderVMContainer.cs

@@ -51,7 +51,7 @@ namespace mbts.spark {
                LadderVM.create_readonly(_bus,
                   new LadderModel(security_md.Value, _model, _model.mbo_service,
                   new Func<OrderSecurity, bool>(osec => _selected_order_tree.Contains(osec.order.Key))),
-                  Observable.Return(LadderSettings.Default)) : null);
+                  Observable.Return(LadderSettings.sanitize(LadderSettings.Default))) : null);
          RaisePropertyChanged(nameof(value));
       }
    }

src/client/spark/Modules/Trading/Ladder/View/LadderGridHost.cs

@@ -67,7 +67,8 @@ namespace mbts.spark {
       NativePriceSize _last_trade;
       Func<NativePriceSize, string> _fmt_size = MDDisplayHelper.get_formatter_nps(MarketSizeDisplay.get_default());
       int _old_tooltip_row = int.MaxValue;
-      double _old_last_trade_price = 0.0;
+      NativePriceSize? _old_last_trade;
+      string _direction_arrow = string.Empty;
       bool _tooltips_are_hooked_up = false;
       ToolTip tooltip_view = new ToolTip();
       IDisposable _parameterized_order_popup;
@@ -693,12 +694,14 @@ namespace mbts.spark {
                   // At any rate, we need this part hiding the tooltip so that we don't leave random tooltips around
                   // and we have to reset to old row so that when we go back to what could be our preivous row, the tooltip gets drawn again
                   //System.Diagnostics.Debug.WriteLine("   Tooltip - Volume is 0");
+                  _old_tooltip_row = int.MaxValue;
                   _hide_tool_tip();
                   //_old_tooltip_row = Int32.MaxValue;
                }
             }
          } else {
             //System.Diagnostics.Debug.WriteLine("   Tooltip - not over Volume column");
+            _old_tooltip_row = int.MaxValue;
             _hide_tool_tip();
          }
       }
@@ -861,21 +864,43 @@ namespace mbts.spark {
       void _refresh_volume_at_price() {
          if (!_settings.volume_column.general.show)
             return;
+
+         // if the last trade has a new size then we need to explicitly
+         // mark the volume container as dirty because there is a chance
+         // we have arrived here off a size change and not a volume change
+         // or a random change, like applying new settings
+         bool last_trade_size_changed = _old_last_trade.HasValue ? _last_trade.size != _old_last_trade.Value.size : true;
+         if (last_trade_size_changed && _settings.volume_column.show_ltq) {
+            _volume_container.mark_dirty();
+         }
          var vapls = _volume_container.read_new_value();
          if (vapls != null) {
             Log.assert_d(_price_list != null);
-            var ltp = _last_trade.price;
-            var last_index = LadderRender.index_of(_price_list, ltp, _price_comparer);
-            string direction_arrow = string.Empty;
-            if (MathUtils.epsilon_equals(ltp, _old_last_trade_price)) {
-               // do nothing if in here b/c direction_arrow already string.empty
-            } else if (ltp > _old_last_trade_price) {
-               direction_arrow = "▲";
-            } else {
-               direction_arrow = "▼";
-            }
-            LadderRender.render_volumes(_volume_vis_tup.visual, _volume_vis_tup.glyph_context, _price_list, vapls, _settings.row_height, security.ContractSpec, _settings.volume_column, last_index, _last_trade, direction_arrow);
-            _old_last_trade_price = ltp;
+            var last_index = LadderRender.index_of(_price_list, _last_trade.price, _price_comparer);
+            LadderRender.render_volumes(_volume_vis_tup.visual, _volume_vis_tup.glyph_context, _price_list, vapls, _settings.row_height, security.ContractSpec, _settings.volume_column, last_index, _get_last_arrow_string());
+            _old_last_trade = _last_trade;
+         }
+
+         string _get_last_arrow_string() {
+            // if _old_last_trade doesn't have a value, this is the first time the ladder has been launched
+            // and so the direction should be drawn without an arrow since there is no basis for comparison
+            if (!_old_last_trade.HasValue) {
+               _direction_arrow = string.Empty;
+               // if the Lasts are different then we need to reformat the direction arrow in some fashion
+            } else if (_old_last_trade != _last_trade) {
+               var ltp = _last_trade.price;
+               if (MathUtils.epsilon_equals(ltp, _old_last_trade.Value.price)) {
+                  // update on the same price, so size must be different
+                  _direction_arrow = string.Empty;
+               } else if (ltp > _old_last_trade.Value.price) {
+                  _direction_arrow = "▲";
+               } else {
+                  _direction_arrow = "▼";
+               }
+            } 
+            // regardless of if we've arrived here via a new Last (and new direction arrow)
+            // or we're reusing the old one, this format bit is the same
+            return $"{_direction_arrow} {_last_trade.size} ";
          }
       }
 

src/client/spark/Modules/Trading/Ladder/View/LadderRender.cs

@@ -196,8 +196,7 @@ namespace mbts.spark {
         ContractSpecificationImmutable spec,
         VolumeColumnSettings settings,
         int last_index,
-        NativePriceSize last_trade, 
-        string direction_arrow
+        string last_size_and_direction
         ) {
          if (!data.Any()) {
             _clear_visual(dv);
@@ -216,16 +215,16 @@ namespace mbts.spark {
          Brush total_brush = null;
          if (settings.total.bars.result.HasValue) {
             total_brush = new SolidColorBrush(settings.total.bars.result.Value);
-         } else if (components.Count == 0) {
+         } else if (components.Count == 0 && !settings.show_ltq) {
             _clear_visual(dv);
             return;
          }
          // inflate the max so that the graph never fills the available space
          double max = data.Max(v => v.Volume) * 1.1;
-         Brush text_brush = settings.general.foreground.rgb_brush();
          _render_cells(dv, row_height, data.Select(d => (d.Price, d)), settings.general, prices, spec, render_cell);
          /// local methods
          void render_cell(DrawingContext dc, int r, VolumeAtPriceLevelImmutable vap) {
+            Brush text_brush = settings.general.foreground.rgb_brush();
             double y = row_height * r;
             var last_trade_here = r == last_index;
 
@@ -259,7 +258,8 @@ namespace mbts.spark {
             }
             if (settings.show_ltq && last_trade_here) {
                numbers_text.Clear();
-               numbers_text.Add($"{direction_arrow} {last_trade.size} ");
+               numbers_text.Add(last_size_and_direction);
+               text_brush = BTSCoreColors.white;
             }
             if (numbers_text.Count > 0) {
                string str = string.Join("/", numbers_text);

src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs

@@ -994,11 +994,13 @@ namespace mbts.spark {
             var join = config.Configuration.get<bts.config.JoinAttachUIOptions>();
             var lean = config.Configuration.get<bts.config.LeanAttachUIOptions>();
             var top = config.Configuration.get<bts.config.TopAttachUIOptions>();
+            var offset = config.Configuration.get<bts.config.OffsetAttachUIOptions>();
             _algo_type_conf = new Dictionary<CustomOrderDefinition.CustomOneofCase, (bool, bool)> {
                { CustomOrderDefinition.CustomOneofCase.Cover, (cover.show_price, cover.show_quantity) },
                { CustomOrderDefinition.CustomOneofCase.Join, (join.show_price, join.show_quantity) },
                { CustomOrderDefinition.CustomOneofCase.Lean, (lean.show_price, lean.show_quantity) },
-               { CustomOrderDefinition.CustomOneofCase.Top, (top.show_price, top.show_quantity) }
+               { CustomOrderDefinition.CustomOneofCase.Top, (top.show_price, top.show_quantity) },
+               { CustomOrderDefinition.CustomOneofCase.Offset, (offset.show_price, offset.show_quantity) }
             };
          }
 

src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookFilterStates.cs

@@ -144,7 +144,7 @@ namespace bts.shell.orderbook {
          pb.add_discrete_values(fs.symbols, o => o.security?.Symbol);
          // numeric
          pb.add_numeric(fs.qty, o => o.order.quantity());
-         pb.add_numeric(fs.price, o => o.order.price());
+         pb.add_price(fs.price, o => o.order.price(), o => o.security);
          pb.add_numeric(fs.qty_filled, o => o.order.filled());
          pb.add_numeric(fs.avg_price, o => o.order.avg_price());
          pb.add_numeric(fs.last_price, o => o.order.last_price());

src/client/spark/Orders/OrderDataHelper.cs

@@ -602,7 +602,8 @@ namespace mbts.spark.utilities {
             CustomOrderDefinition.CustomOneofCase.Lean,
             CustomOrderDefinition.CustomOneofCase.Join,
             CustomOrderDefinition.CustomOneofCase.Cover,
-            CustomOrderDefinition.CustomOneofCase.Top
+            CustomOrderDefinition.CustomOneofCase.Top,
+            CustomOrderDefinition.CustomOneofCase.Offset
          },
          // The following spreader types are not yet ready for production use:
          // CustomOrderDefinition.CustomOneofCase.Dynamic,
@@ -619,7 +620,8 @@ namespace mbts.spark.utilities {
             CustomOrderDefinition.CustomOneofCase.Lean,
             CustomOrderDefinition.CustomOneofCase.Join,
             CustomOrderDefinition.CustomOneofCase.Cover,
-            CustomOrderDefinition.CustomOneofCase.Top
+            CustomOrderDefinition.CustomOneofCase.Top,
+            CustomOrderDefinition.CustomOneofCase.Offset
          },
          outright_order_types = new[] {
             CustomOrderDefinition.CustomOneofCase.Market,
@@ -715,7 +717,7 @@ namespace mbts.spark.utilities {
                !b.HasChild);
 
          public static bool top_equals(TopOrderDefinitionImmutable a, TopOrderDefinitionImmutable b) =>
-            a.WatchLevelTickOffset == b.WatchLevelTickOffset &&
+            a.TicksOff == b.TicksOff &&
             (a.HasTopThreshold ?
                b.HasTopThreshold && quantity_measure_equals(a.TopThreshold, b.TopThreshold) :
                !b.HasTopThreshold) &&
@@ -906,7 +908,7 @@ namespace mbts.spark.utilities {
                      add_component(_get_hash(_get_tick_child(tick)));
                      break;
                   case CustomOrderDefinition.CustomOneofCase.Top:
-                     add_component(def.Top.WatchLevelTickOffset.GetHashCode());
+                     add_component(def.Top.TicksOff.GetHashCode());
                      add_component(def.Top.HasTopThreshold ? def.Top.TopThreshold.GetHashCode() : 0);
                      add_component(_get_top_child(def.Top).GetHashCode());
                      break;

src/client/spark/Orders/OrderPropertiesEditorVM.cs

@@ -105,13 +105,18 @@ namespace mbts.spark {
          var inline_account_src = order_properties.AggregateLatest(
             model.factory.subscribe_accounts(),
             (last, change, cache) => convert_inline_account(change.New.account, cache),
-            (last, props, change) => convert_inline_account(props.account, change.New),
+            (last, props, change) => convert_inline_account(props.account, change.New, false),
             () => new InlineUserProperty<AccountTuple>(),
             new InlineOrderProperties(),
             DictionaryHelper.empty<uint, string>());
+         // Note that with the .Where on the account_tuples, we will ignore any
+         // updates for which all accounts have been removed.
+         // This was a safer change than changing SelectorProperty's IEnumberable.OnNext
+         // so that it would not reset the selected item on server reboots; i.e. this
+         // change only affects this IObservable.
          _accounts = backing(
             nameof(selected_account),
-            model.factory.subscribe_account_tuples(),
+            model.factory.subscribe_account_tuples().Where(items => items.Any()),
             inline_account_src,
             a => new AccountTuple?(a),
             () => accounts.Cast<AccountTuple?>().FirstOrDefault(),
@@ -263,9 +268,18 @@ namespace mbts.spark {
                   break;
             }
          }
-         InlineUserProperty<AccountTuple> convert_inline_account(InlineUserProperty<uint> account_id, IReadOnlyDictionary<uint, string> account_map) =>
+         //
+         //  Regarding the 3rd parameter, force_override_from_inline_property:
+         //    When using an InlineUserProperty, in the context of assigning a default vs selected item,
+         //    there is somewhat of a conflict between the IObserver<InlineUserProperty<TItem>> and the 
+         //    IObserver<IEnumerable<TItem>> OnNext methods.  This 3rd paramter provides yet another override
+         //    to say if the force_override should come from the inline property or if the force_override should
+         //    be disbled, by way of the immediate return in ViewModelBase._try_auto_select when force_override
+         //    is false.  Our intent is then accomplished because we are no longer attempting to set a default/
+         //    selected item when the account_tuples potentially changes
+         InlineUserProperty<AccountTuple> convert_inline_account(InlineUserProperty<uint> account_id, IReadOnlyDictionary<uint, string> account_map, bool force_override_from_inline_property = true) =>
             account_id.has_value && account_map.TryGetValue(account_id.value, out string name) ?
-               new InlineUserProperty<AccountTuple>(new AccountTuple(account_id.value, name), account_id.force_override) :
+               new InlineUserProperty<AccountTuple>(new AccountTuple(account_id.value, name), force_override_from_inline_property ? account_id.force_override : false) :
                new InlineUserProperty<AccountTuple>();
       }
 

src/client/spark/Orders/OrderTypeVMs/TopOrderVM.cs

@@ -5,13 +5,16 @@ using TopDefi = bts.api.orderman.TopOrderDefinitionImmutable;
 namespace mbts.spark {
    static partial class OrderTypeVMFactory {
       class TopOrderVM : OrderTypeVMBase<TopDef, TopDefi> {
+         readonly Property<uint?> _ticks_off;
          readonly Property<QuantityMeasureImmutable?> _top_threshold;
 
          public TopOrderVM(AlgoKey key, string display_name, AlgoEditingPermissions editing_permissions)
             : base(key, display_name, editing_permissions) {
+            _ticks_off = add_nullable_property<uint>(nameof(ticks_off), TopDef.TicksOffFieldNumber);
             _top_threshold = add_property<QuantityMeasure, QuantityMeasureImmutable>(nameof(top_threshold), TopDef.TopThresholdFieldNumber, new QuantityMeasureImmutable(new QuantityMeasure { Ratio = 1.0, Absolute = 1 }));
          }
 
+         public uint? ticks_off { get => _ticks_off.Value; set => _ticks_off.set(value); }
          public QuantityMeasureImmutable? top_threshold { get => _top_threshold.Value; set => _top_threshold.set(value); }
       }
    }

src/client/spark/core/Utility/ViewModelBase.cs

@@ -820,6 +820,11 @@ namespace mbts.ui.shell {
 
          void _try_auto_select() {
             if (_is_user_selected_item) {
+               // if we have a user-selected item and the _desired_item for which we might want to set as the selected item
+               // isn't force_override, then no change
+               if (!_desired_item.force_override) {
+                  return;
+               }
                if (desired_item_valid() && _desired_item.force_override && _try_match_item(_desired_item.value, out TValue match)) {
                   _set_selected_item(match, SelectionReason.SettingsMandatedSelection);
                   return;

src/client/spark/view/Controls/DIYGrid/Columns/ColumnPredicateBuilder.cs

@@ -37,6 +37,51 @@ namespace mbts.spark.filters {
             _predicates.Add(impl);
             bool impl(TRow o) => max.CompareTo(selector(o)) >= 0;
          }
+      }
+
+      // this is a copy of add_numeric except we do a tick_equals comparsion
+      // if the ticks are not equals, then it falls back to a regular double comparison
+      //
+      // if the SecurityImmutable? doesn't have a value
+      // then we fallback to double comparison
+      public void add_price(DoubleMinMaxState value, Func<TRow, double> selector, Func<TRow, SecurityImmutable?> si_selector) {
+         if (value == null)
+            return;
+         if (value.minimum.active) {
+            var min = value.minimum.value.Value;
+            if (value.maximum.active) {
+               var max = value.maximum.value.Value;
+               _predicates.Add(impl);
+               bool impl(TRow o) {
+                  var v = selector(o);
+                  var si = si_selector(o);
+                  if (si.HasValue) {
+                     return si.Value.ContractSpec.compare_tick_fallback_to_double(min, v) <= 0 &&
+                        si.Value.ContractSpec.compare_tick_fallback_to_double(max, v) >= 0;
+                  }
+                  return min.CompareTo(v) <= 0 && max.CompareTo(v) >= 0;
+               }
+            } else {
+               _predicates.Add(impl);
+               bool impl(TRow o) {
+                  var si = si_selector(o);
+                  if (si.HasValue) {
+                     return si.Value.ContractSpec.compare_tick_fallback_to_double(min, selector(o)) <= 0;
+                  }
+                  return min.CompareTo(selector(o)) <= 0;
+               }
+            }
+         } else if (value.maximum.active) {
+            var max = value.maximum.value.Value;
+            _predicates.Add(impl);
+            bool impl(TRow o) {
+               var si = si_selector(o);
+               if (si.HasValue) {
+                  return si.Value.ContractSpec.compare_tick_fallback_to_double(max, selector(o)) >= 0;
+               }
+               return max.CompareTo(selector(o)) >= 0;
+            }
+         }
       }
 
       public void add_regex(string str, Func<TRow, string> selector) {

src/client/spark/view/Controls/Order/AlgoContentTemplateSelector.xaml

@@ -489,6 +489,8 @@
             </VisualStateGroup>
           </VisualStateManager.VisualStateGroups>
           <Grid.RowDefinitions>
+            <RowDefinition Height="Auto"/>
+            <RowDefinition Height="{Binding Source={StaticResource VerticalPush1}}"/>
             <RowDefinition Height="Auto"/>
             <RowDefinition Height="{Binding Source={StaticResource VerticalPush3}}"/>
             <RowDefinition Height="*"/>
@@ -503,9 +505,11 @@
             <c:HelpButton Margin="{StaticResource Margin.LP1}" ToolTip="If the order is at the top of book, the quantity remaining below which the order will be canceled."/>
           </StackPanel>
           <uag:QuantityMeasureControl x:Name="PART_TopThreshold_QuantityMeasureControl" Grid.Column="2" QuantityMeasure="{Binding top_threshold}"/>
+          <TextBlock Grid.Row="2">Ticks Off</TextBlock>
+          <c:TickUpDownUInt Grid.Row="2" Grid.Column="2" Value="{Binding ticks_off}"/>
           <uag:AlgoPickerNodeView
             x:Name="PART_Child_AlgoPickerNodeView"
-            Grid.Row="2"
+            Grid.Row="4"
             Grid.ColumnSpan="3"
             ItemsSource="{Binding children_vm.child.algos_view}"
             SelectedItem="{Binding children_vm.child.selected_algo}"

src/client/spark/view/Spark/Core/TickUtils.cs

@@ -128,6 +128,9 @@ namespace mbts.spark {
       public static IEqualityComparer<double> tick_equality_comparer(this ContractSpecificationImmutable contract) =>
          EqualityComparer.Create((double a, double b) => tick_equals(contract, a, b), x => MathUtils.RoundFractionalUnits(x, contract.LowTick));
 
+      public static int compare_tick_fallback_to_double(this ContractSpecificationImmutable contract, double x, double y) =>
+         tick_equals(contract, x, y) ? 0 : x.CompareTo(y);
+
       public static double round_to_tick(this ContractSpecificationImmutable spec, double value) {
          double intpart = (long)value;
          double floatpart = value % 1;

src/client/spark_api/Co2/MDConsumerHandle.cs

@@ -253,14 +253,13 @@ namespace mbts.co2 {
          }
       }
 
-      [StructLayout(LayoutKind.Sequential, Size = 712)]
+      [StructLayout(LayoutKind.Sequential, Size = 704)]
       internal struct ColdEntry {
          public int exchange_status;
          // Last trade
          public AtomicPriceSize last_trade;
          public ulong total_volume;
          public utils.LLDateTime last_trade_time;
-         private AggressorSide last_side; 
 
          //// Settlement
          public UInt64 misc_timestamp;

src/client/spark_excel/rtd/formatting/JsonFormatter.cs

@@ -13,6 +13,8 @@ namespace bts.excel.rtd
    class JsonFormatter : IExcelFormatter
    {
       private object _data;
+      static Google.Protobuf.JsonFormatter _protoformatter =
+         new Google.Protobuf.JsonFormatter(Google.Protobuf.JsonFormatter.Settings.Default.WithFormatDefaultValues(true));
       public JsonFormatter(object data)
       {
          _data = data;
@@ -81,7 +83,7 @@ namespace bts.excel.rtd
       {
          string json = null;
          if(data is IMessage imsg)
-            json = Google.Protobuf.JsonFormatter.Default.Format(imsg);
+            json = _protoformatter.Format(imsg);
          else
             json = JsonConvert.SerializeObject(data);
 

src/client/spark_sdk/README.md

@@ -73,6 +73,10 @@ Commands:
    order <exchange> <security_type> <symbol> <security_desc> <signed_quantity> [<price>]
 Arguments:
    --pump=<pump_interval_in_ms=100 default>
+   
+Example:
+   To print simulated market data to the screen:
+   spark_sdk.exe md SIM FUTURE ES SIM:ESU9 --user=bts
 ```
 
 * Check out sample code

src/lib/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServices.cpp;orderman/actors/order_manLine truncated
+set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderMLine truncated

src/lib/CMakeDeps_win.cmake

@@ -1 +1 @@
-set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthSessionManager.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/crypto/CipherType.cpp;utils/crypto/CryptoInit.cpp;utils/crypto/Digest.cpp;utils/crypto/KeyCodeHelper.cpp;utils/crypto/PasswordCallback.cpp;utils/crypto/PrivateKey.cpp;utils/crypto/PublicKey.cpp;utils/crypto/iostreambio.cpp;utils/datetime.cpp;utils/download.cpp;utLine truncated
+set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthSessionManager.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/crypto/CipherType.cpp;utils/crypto/CryptoInit.cpp;utils/crypto/Digest.cpp;utils/crypto/KeyCodeHelper.cpp;utils/crypto/PasswordCallback.cpp;utils/crypto/PrivateKey.cpp;utils/cryptoLine truncated

src/lib/api/apicore.proto

@@ -50,6 +50,7 @@ message VoidMessage {}
 message BytesMessage {
 	bytes buffer = 1;
 }
-message BoolResultMessage {
-	bool success = 1;
+message ResultMessage {
+   bool success = 1;
+   string error_message = 2;
 }

src/lib/api/public/Update.h

@@ -27,7 +27,12 @@ namespace api {
          _removed_items(std::move(removed))
       {}
 
-      Update(const Update& other) = delete;
+      Update(const Update& other) : _all_items(other._all_items),
+                                    _added_items(other._added_items),
+                                    _changed_items(other._changed_items),
+                                    _removed_items(other._removed_items)
+      {}
+      
       Update(Update&& other) :
          _all_items(std::move(other._all_items)),
          _added_items(std::move(other._added_items)),
@@ -104,4 +109,4 @@ namespace api {
    }
 
 
-}}//namespace
+}}//namespace

src/lib/api/services/AdminService.proto

@@ -149,6 +149,7 @@ message BTSAccount{
    ExchangeAccountData exchange_account_data = 110;
    // extra parameters for risk enforcement
    RiskLimitEnforcement risk_limit_enforcement=120;
+   AccountLossLimit account_loss_limit = 130;
 }
 
 message RiskLimitEnforcement{   
@@ -190,6 +191,11 @@ message AccountExchangeConnections{
    repeated string exchange_connection_names = 20;
 }
 
+message AccountLossLimit{
+   bool enforce = 10;
+   uint32 total_loss_limit = 20;
+   // if we need per product we can add it here
+}
 /************************************************************************/
 /***************************Exchange Data********************************/
 /************************************************************************/

src/lib/api/services/MarginService.proto

@@ -0,0 +1,67 @@
+syntax="proto3";
+package bts.api.position;
+option csharp_namespace="bts.api.position";
+import "lib/api/apioptions.proto";
+import "lib/api/apicore.proto";
+import "lib/api/services/PositionService.proto";
+
+message MarginResult {
+   enum ResultStatus { NewValid = 0; CachedValid = 1; Error = 2; }
+   
+   string account = 10 [(bts.api.key)=true];    
+   ResultStatus status = 12;
+   string portfolio_id = 15; 
+   string settle_indicator = 20;
+   string settle_qualifier = 30;   
+   string as_of_time = 40;
+   string calculation_id = 50;   
+   
+   double opt_value = 100;
+   double nonopt_value = 110;
+   double maint_margin = 120;
+   double initial_margin = 130;
+}
+
+message ExcessLiquidityReport { 
+   string account = 10 [(bts.api.key)=true];    
+   uint32 account_id = 11;
+   double leverage_factor = 20;
+   double cash_balance = 30;
+   MarginResult margin_result = 40;
+   AccountPNL account_pnl = 50;
+   bool should_enforce_margin = 60;
+   double excess_liquidity = 100;   
+}
+
+
+message MarginRequest {
+   string account = 10;
+}
+
+message CashFactorUpdate {
+   string account = 10;
+   double leverage_factor = 20;
+}
+
+message CashBalanceUpdate {
+   string account = 10;
+   double cash_balance = 20;
+}
+
+message EnforceUpdate {
+   string account = 10;
+   bool should_enforce_margin = 20;
+}
+
+service MarginService {
+   option (bts.api.service_description) = "Provides CME Margin Information per account";
+
+   rpc get_margin(MarginRequest) returns (MarginResult);
+   rpc get_excess_liquidity_report(MarginRequest) returns (ExcessLiquidityReport);
+   rpc set_should_enforce_margin(EnforceUpdate) returns (bts.api.ResultMessage);
+   rpc set_cash_factor(CashFactorUpdate) returns (bts.api.ResultMessage);
+   rpc set_cash_balance(CashBalanceUpdate) returns (bts.api.ResultMessage);
+   
+   rpc margin_data(bts.api.VoidMessage) returns (stream MarginResult);
+   rpc excess_liquidity_report(bts.api.VoidMessage) returns (stream ExcessLiquidityReport);
+}

src/lib/api/services/OrderService.proto

@@ -101,7 +101,6 @@ message CustomOrderDefinition
     OCOOrderDefinition oco = 25;
     PricerOrderDefinition pricer = 26;
     CompositeOrderDefinition composite = 27;
-    SplitterOrderDefinition   splitter = 29;
     
     DynamicOrderDefinition dynamic = 1000;
   }
@@ -161,8 +160,8 @@ message JoinOrderDefinition {
   CustomOrderDefinition child = 6;
 }
 message TopOrderDefinition {
-  uint32 watch_level_tick_offset = 1;
   spark.QuantityMeasure top_threshold = 2;
+  uint32 ticks_off = 3;
   CustomOrderDefinition child = 6;
 }
 message TriggerOrderDefinition {
@@ -205,12 +204,6 @@ message CompositeOrderDefinition {
   uint32 algo_template_key = 1;
   CustomOrderDefinition definition = 2;
 }
-message SplitterOrderDefinition {
-  uint32 max_order_qty   = 1;
-  uint32 exposed_qty     = 2;
-  // uint32 target_qty      = 3;
-  CustomOrderDefinition child = 10;
-}
 message DynamicOrderDefinition {
   string dynamic_order_type = 1;
   map<string, DynamicOrderParam> order_params = 2;
@@ -260,10 +253,9 @@ message OrderState {
     TriggerOrderState trigger = 21;
     CoverOrderState cover = 22;
     OffsetOrderState offset = 24;
+    QueueHolderState queue_holder = 30;
     OCOOrderState oco = 25;
     PricerOrderState pricer = 26;
-    QueueHolderState queue_holder = 30;
-    SplitterOrderState splitter = 31;
     
     DynamicOrderState dynamic = 255; // do not use anything bigger than 255, or things will break
   }
@@ -342,7 +334,6 @@ message OCOOrderState {}
 message PricerOrderState {}
 message DynamicOrderState {}
 message QueueHolderState {}
-message SplitterOrderState {}
 
 enum OrderStatus {
 

src/lib/api/services/PositionService.proto

@@ -8,6 +8,7 @@ import "lib/domain_model/Security.proto";
 message PositionUnit{
    int32 record_id = 1 [(bts.api.key)=true];
    string account = 10;
+   uint32 account_id = 15;
    uint32 security_id = 20;
    double sod_long_pos = 30;
    double sod_long_avg_price = 40;
@@ -19,6 +20,22 @@ message PositionUnit{
    double day_short_avg_price = 100;   
 }
 
+message PNLUnit{
+   int32 record_id = 1 [(bts.api.key)=true];
+   PositionUnit pos = 10;
+   double cash_value = 20;   
+   double bid_price = 30;
+   double ask_price = 40;
+   double last_price = 50;
+   double settlement_price = 60;
+}
+
+message AccountPNL{
+   uint32 account_id = 1 [(bts.api.key)=true];
+   string account_name = 10;
+   double day_pnl = 20;
+}
+
 message ManualFill {
   int32 record_id = 1 [(bts.api.key)=true];
   uint32 security_key = 2;
@@ -53,5 +70,6 @@ service PositionService {
   // subscription
   rpc manual_fills(bts.api.VoidMessage) returns (stream ManualFill);
   rpc position_data(bts.api.VoidMessage) returns (stream PositionUnit);
-
+  rpc pnl_data(bts.api.VoidMessage) returns (stream PNLUnit);
+  rpc pnl_account_data(bts.api.VoidMessage) returns (stream AccountPNL);
 }

src/lib/config/defaults/span.conf

@@ -0,0 +1,9 @@
+
+SpanMargin {
+   value {
+      enabled: false
+      login: 'api_bts1'
+      password: 'Summer123'
+      url: 'https://cmecorenr.cmegroup.com/MarginServiceApi/1.4/'
+   }
+}

src/lib/config/defaults/toggles.conf

@@ -8,4 +8,4 @@ OrdersSoftCleanup {
    value {
       enabled: true
    }
-}
+}

src/lib/config/protos/UI.proto

@@ -141,6 +141,11 @@ message TopAttachUIOptions {
    optional bool show_quantity = 2 [default = false];
 }
 
+message OffsetAttachUIOptions {
+   optional bool show_price = 1 [default = false];
+   optional bool show_quantity = 2 [default = false];
+}
+
 message ManualFillsOptions {
    optional bool modal_transactions = 1 [default = true];
 }

src/lib/config/protos/span.proto

@@ -0,0 +1,10 @@
+syntax='proto2';
+package bts.config;
+
+message SpanMargin
+{
+   optional bool enabled = 10;
+   optional string login = 20; 
+   optional string password = 30; 
+   optional string url = 40;    
+}

src/lib/config/protos/toggles.proto

@@ -53,4 +53,4 @@ message EnsureOnScreenWithWorkAreaRect {
    // and the new ensure_onscreen code has only been tested with a single monitor arrangement.
    // Should verify that the code works with every monitor arrangement before enabling by default.
    optional bool enabled = 1 [default = false];
-}
+}

src/lib/md/IntPrice.h

@@ -34,9 +34,9 @@ class IntPrice {
 #endif
 
    public:
-   constexpr IntPrice() : _mantissa(0), _exponent(127) {/*nan*/}
-   constexpr IntPrice(int64_t mantissa, int8_t exponent) : _mantissa(mantissa), _exponent(exponent) {}
-   constexpr IntPrice(const IntPrice& other) : _mantissa(other._mantissa), _exponent(other._exponent) {}
+   IntPrice() : _mantissa(0), _exponent(127) {/*nan*/}
+   IntPrice(int64_t mantissa, int8_t exponent) : _mantissa(mantissa), _exponent(exponent) {}
+   IntPrice(const IntPrice& other) : _mantissa(other._mantissa), _exponent(other._exponent) {}
 
    bool operator==(const IntPrice& other) const {
       if(_exponent == other._exponent) {

src/lib/md/MDProducerHandle.cpp

@@ -102,14 +102,13 @@ void MDProducerHandle::set_status(CoconutSecurityStatus status) {
 }
 
 void MDProducerHandle::add_trade(TimeStamp time, const PriceSize price, LLDateTime tradetime, co2::AggressorSide agside, uint32_t volume) {
-   // Add this trade to the last trade if price/side match
+   // Add this trade to the last trade if price match
    auto current = std::get<PriceSize>(cdata().last_trade.load());
-   if (current.int_price() == price.int_price() && agside == cdata().last_side) {
+   if (current.int_price() == price.int_price()) {
       cdata().last_trade.store(PriceSize { price.int_price(), price.size() + current.size() }, time);
    }
    else {
       cdata().last_trade.store(price,time);
-      cdata().last_side = agside;
    }
    cdata().last_trade_time = tradetime;
    cdata().total_volume = volume;

src/lib/md/PriceSize.h

@@ -77,7 +77,7 @@ public:
 extern PriceSize NO_MARKET;
 static_assert(sizeof(PriceSize) == 16, "PriceSize should be small");
    
-template <uint8_t PRECISION=11>
+template <uint8_t PRECISION>
 PriceSize PriceSize_from_double(double price, Size size) {
    if (size == 0 || os::is_nan(price)) return NO_MARKET;
    return { IntPrice_from_double<PRECISION>(price), size };

src/lib/md/SecurityCache.cpp

@@ -22,6 +22,7 @@ void SecurityCacheImpl::enqueue_update_db(const SecurityPtr& security) {
 }
 
 void SecurityCacheImpl::flush() {
+   #ifndef _WIN32
    if (!update_queue.empty()) {
       std::vector<SecurityPtr> work;
       SecurityPtr temp;
@@ -33,6 +34,7 @@ void SecurityCacheImpl::flush() {
          }
       }
    }
+   #endif
 }
 
 //////////////////

src/lib/md/Shm/MDStructure.h

@@ -26,7 +26,7 @@ namespace md {
 
 using utils::TimeStamp;
 
-const uint32_t shm_version = 101;
+const uint32_t shm_version = 102;
 
 struct ALIGNED_(CACHE_LINE) HotHeader {
    size_t count;
@@ -214,7 +214,6 @@ struct ColdEntry {
       last_trade(),
       total_volume(0),
       last_trade_time(),
-      last_side(co2::AggressorSide::AGGRESSOR_SIDE_NONE),
       misc(),
       deleted(false)
    {
@@ -230,7 +229,6 @@ struct ColdEntry {
    AtomicPriceSize last_trade;
    size_t total_volume;
    datetime::LLDateTime  last_trade_time;
-   co2::AggressorSide last_side;
 
    MiscData misc;
 

src/lib/md/relay/RelayRecorder.cpp

@@ -11,7 +11,6 @@
 #include "lib/md/protos/co2.pb.h"
 #include "lib/md/SecurityIDRecorder.h"
 #include "lib/config/protos/system.pb.h"
-#include "lib/config/protos/MarketData.pb.h"
 #include "lib/utils/bts_chrono.h"
 
 #include <boost/filesystem.hpp>
@@ -174,7 +173,7 @@ void replay_file(const std::string& filename, IRelayWriter& writer, int32_t nagg
    } while (loop);
 }
 
-bool replay_url(const std::string& url, IRelayWriter& writer, int32_t naggle_ms, bool loop) {
+void replay_url(const std::string& url, IRelayWriter& writer, int32_t naggle_ms) {
    // split up url parts
    size_t http_len = BTS_ARRAYSIZE("http://") - 1;
    auto after_host = url.find('/', http_len);
@@ -185,53 +184,21 @@ bool replay_url(const std::string& url, IRelayWriter& writer, int32_t naggle_ms,
    filename = os::sparkdata_path("data/replay/") + filename; 
    utils::download_file_if_newer(host, path, filename);
    if (boost::filesystem::exists(filename)) {
-      replay_file(filename, writer, naggle_ms, loop);
-      return true;
-   }
-   return false;
-}
-
-void replay_url_with_fallback(const std::string& url, IRelayWriter& writer, int32_t naggle_ms, bool loop) {
-   if (!replay_url(url, writer, naggle_ms, loop)) {
-      const auto fallback = config::get<ReplayOptions>().fallback();
-      if (boost::filesystem::exists(fallback)) {
-         replay_file(fallback, writer, naggle_ms, loop);
-      }
+      replay_file(filename, writer, naggle_ms, true);
    }
 }
 
-struct SecurityExtractorDone : public std::bad_exception {};
-struct SecurityExtractor : public IRelayWriter {
-   size_t _count;
-   std::function<void(const api::Security&)> _cb;
-   std::set<uint32_t> _keys;
-   SecurityExtractor(std::function<void(const api::Security&)>& cb, size_t count) 
-      : _count(count), _cb(cb), _keys() { }
-   ~SecurityExtractor() override {};
-   void write(const co2::MDReply& data, bool*) override {
-      for (auto const &ch : data.channels()) {
-         for (auto const &cr : ch.updates()) {
-            if (cr.has_security() && _keys.find(cr.security().id()) == _keys.end()) {
-               _keys.insert(cr.security().id());
-               _cb(cr.security());
-            }
-         }
-      }
-      if (_keys.size() >= _count) {
-         throw SecurityExtractorDone();
-      }
+void replay_url_with_fallback(const std::string& url, IRelayWriter& writer, int32_t naggle_ms) {
+   // loops forever
+   replay_url(url, writer, naggle_ms);
+   // If it returns it means it failed to download or play cached
+   const auto fallback = config::get<ReplayOptions>().fallback();
+   if (boost::filesystem::exists(fallback)) {
+      replay_file(fallback, writer, naggle_ms);
    }
-};
 
-void get_securities_from_replay(std::function<void(const api::Security&)> cb) {
-   SecurityExtractor extractor(cb, 100);
-   auto url = config::get<ReplayOptions>().url();
-   try {
-      replay_url_with_fallback(url, extractor, 0, false);
-   } catch(const SecurityExtractorDone&) { }
 }
 
-
 shared_ptr<IRelayWriter> mkrelayproducer(bool shared);
 
 /*this is the public one*/

src/lib/md/relay/RelayRecorder.h

@@ -29,10 +29,9 @@ void print_recording(const std::string&);
 void unprint_recording(const std::string& src, const std::string& dest);
 void trace_recording(const std::string& file, security_key key);
 void replay_recording(const std::string&, int32_t delay_ms, const std::string fallback="");
-bool replay_url(const std::string& url, IRelayWriter& writer, int32_t naggle_ms, bool loop=true);
-void replay_url_with_fallback(const std::string& url, IRelayWriter& writer, int32_t naggle_ms, bool loop=true);
+void replay_url(const std::string& url, IRelayWriter& writer, int32_t naggle_ms);
+void replay_url_with_fallback(const std::string& url, IRelayWriter& writer, int32_t naggle_ms);
 void replay_file(const std::string& filename, IRelayWriter& writer, int32_t naggle_ms, bool loop=false);
-void get_securities_from_replay(std::function<void(const api::Security&)>);
 
 }}//namespace
 

src/lib/md/test/synthetic_book_merge_tests.cpp

@@ -111,7 +111,7 @@ tuple<shared_ptr<SyntheticStrategy>,vector<Co2TestInstrument>> make_securities(c
       l->mutable_security()->mutable_contract_spec()->set_low_tick(tick_size);
       l->mutable_security()->mutable_contract_spec()->set_high_tick(tick_size);
       l->set_quantity(quantity);
-      l->set_price_multiplier(quantity * (side == api::SIDE_BUY ? 1.0 : -1));
+      l->set_price_multiplier(mult);
       l->set_side(side);
       synthetic->legs.push_back(instr.consumer());
       legs.push_back(instr);
@@ -126,7 +126,8 @@ tuple<shared_ptr<SyntheticStrategy>,vector<Co2TestInstrument>> make_securities(c
       double quantity, tick_size;
       api::Side side;
       std::tie(quantity, tick_size, side) = t;
-      legs.push_back({ quantity, quantity, tick_size, side });
+      double mult = quantity * (side == api::SIDE_BUY ? 1.0 : -1);
+      legs.push_back({ quantity, mult, tick_size, side });
    }
    return make_securities(legs);
 }
@@ -457,6 +458,113 @@ TEST_CASE("Synthetic_net_change", "[md][synthetic]") {
    }
 }
 
+TEST_CASE("HCL_SP_104", "[md][synthetic]") {
+   shared_ptr<SyntheticStrategy> synthetic;
+   vector<Co2TestInstrument> test_instr;
+   tie(synthetic, test_instr) = make_securities({ 
+      // (quantity, multiplier, tick_size, side)
+      {1.0, 1.0,  0.01, api::SIDE_BUY},
+      {1.0, 1.0,  0.01, api::SIDE_BUY},
+      {1.0, -1.0, 0.01, api::SIDE_SELL},
+      {1.0, -1.0, 0.01, api::SIDE_SELL }
+   });
+   auto leg1 = test_instr[0]; // CLQ9-CLU9
+   auto leg2 = test_instr[1]; // HCLQ9-CLQ9
+   auto leg3 = test_instr[2]; // HCLQ9-HCLU9
+   auto leg4 = test_instr[3]; // HCLU9-CLU9
+
+   setup_market(leg1.producer(),
+      {
+         { -0.10, 3299 },
+         { -0.11, 2885 },
+         { -0.12, 2558 },
+         { -0.13, 2447 },
+         { -0.14, 1540 },
+         { -0.15, 2334 },
+         { -0.16, 1396 },
+         { -0.17, 1190 },
+         { -0.18, 1221 },
+         { -0.19, 812  }
+      },
+      { 
+         { -0.09, 1318 },
+         { -0.08, 2174 },
+         { -0.07, 3057 },
+         { -0.06, 2331 },
+         { -0.05, 2543 },
+         { -0.04, 2638 },
+         { -0.03, 2232 },
+         { -0.02, 3221 },
+         { -0.01, 2863 },
+         {  0.00, 3327 }
+      });
+   
+   setup_market(leg2.producer(),
+      {
+         { 4.06,  1 },
+         { 4.05,  2 },
+         { 4.04,  3 },
+         { 4.03, 12 },
+         { 4.02,  6 },
+         { 4.01,  1 }
+      },
+      {
+         { 4.07, 1 },
+         { 4.08, 2 },
+         { 4.10, 8 },
+         { 4.11, 10 },
+         { 4.12, 2 },
+         { 4.14, 1 }
+      });
+   
+   setup_market(leg3.producer(),
+      {
+         { 0.20, 2  },
+         { 0.19, 6  },
+         { 0.18, 5  },
+         { 0.17, 18 },
+         { 0.16, 2  },
+         { 0.15, 3  },
+         { 0.14, 2  }
+      },
+      {
+         { 0.22, 2 },
+         { 0.23, 4 },
+         { 0.24, 9 },
+         { 0.25, 17},
+         { 0.26, 5 },
+         { 0.27, 5 },
+         { 0.28, 4 },
+         { 0.29, 2 },
+         { 0.30, 2 }
+      });
+
+   setup_market(leg4.producer(),
+      {
+         { 3.75, 1 },
+         { 3.74, 3 },
+         { 3.73, 4 },
+         { 3.72, 12 },
+         { 3.71, 1 },
+         { 3.70, 3 }
+      },
+      {
+         { 3.78, 1 },
+         { 3.79, 2 },
+         { 3.80, 6 },
+         { 3.81, 1 },
+         { 3.82, 10 },
+         { 3.83, 1 }
+      });
+
+      MDConsumerHandle synthetic_handle(synthetic);
+      // dump_market(synthetic_handle);
+      expect_market(synthetic_handle,
+         { { -0.04,1 },{ -0.06,1 },{ -0.07,1 },{ -0.09,3 },{ -0.11,3 },{ -0.12,1 },{ -0.13,5 },{ -0.14,3 },{ -0.15,2 },{ -0.16,1 } },
+         {  { 0.03,1 },{ 0.05,1 },{ 0.06,1 },{ 0.08,1 },{ 0.09,4 },{ 0.11,3 },{ 0.12,2 },{ 0.13,7 },{ 0.14,1 },{ 0.16,2 } }
+      );
+}
+
 
 //
 // Benchmarks

src/lib/orderman/actors/FillStore.cpp

@@ -126,6 +126,7 @@ void FillStore::on_execution_report(
                if (it != me->known_orders.end()) {
                   auto &order_def = it->second;
                   fill.set_account(order_def.account());
+                  fill.set_internal_account_id(order_def.account_id());
                } else
                   me->fills_with_unresolved_accounts[er.order_key]
                       .fills.push_back(fill);
@@ -153,6 +154,7 @@ void FillStore::on_order_definition_update(
          auto &unresolved_fills = me->fills_with_unresolved_accounts[ev.key];
          for (auto &f : unresolved_fills.fills) {
             f.set_account(ev.def->account());
+            f.set_internal_account_id(ev.def->account_id());
             auto fv = utils::pb::to_string(f);
             me->store->update(fills_ns, f.fill_key(), fv);
             publish(me->fill_stream, f, *unresolved_fills.security);

src/lib/orderman/actors/SODPosition.cpp

@@ -1,6 +1,7 @@
 #include "lib/orderman/actors/SODPosition.h"
 #include "lib/config/config.h"
 #include "lib/config/protos/orders_fills_cleanup.pb.h"
+#include "lib/api/service_helpers/AdminServiceUtils.h"
 #include "lib/config/protos/toggles.pb.h"
 #include "lib/logging/logging.h"
 #include "lib/utils/PBUtils.h"
@@ -131,6 +132,7 @@ void SODPosition::_merge_pos(Fill &pos, const Fill &fill) {
 void SODPosition::_zero_pos(Fill &pos, const Fill &fill, api::Side side) {
    pos.set_security_key(fill.security_key());
    pos.set_account(fill.account());
+   pos.set_internal_account_id(fill.internal_account_id());
    pos.set_side(side);
    pos.mutable_timestamp()->set_seconds(_sod_time.tv_sec);
    pos.mutable_timestamp()->set_nanos(0);
@@ -140,14 +142,20 @@ void SODPosition::_zero_pos(Fill &pos, const Fill &fill, api::Side side) {
    pos.set_last_shares(0);
 }
 
-bool SODPosition::add_fill_to_sod_pos(const Fill &fill) {
+bool SODPosition::add_fill_to_sod_pos(const Fill &fill) {   
    if (_enabled && fill.timestamp().seconds() < _sod_time.tv_sec) {
-      uint32_t seckey = fill.security_key();
-      const std::string &acct = fill.account();
-      auto acct_it = _sod_map.find(acct);
+      if (!bts::admin::AdminServiceUtils::has_id(fill.internal_account_id())){
+         LOG(persist, Error, "SODPosition::add_fill_to_sod_pos Could not find "
+                             "account_id for fill:{} account:{}",
+                             fill.fill_key(), fill.account());
+         return false;
+      }
+      uint32_t seckey = fill.security_key(),
+               account_id = fill.internal_account_id();
+      auto acct_it = _sod_map.find(account_id);
       if (acct_it == _sod_map.end())
-         _sod_map[acct] = SecPosMap();
-      acct_it = _sod_map.find(fill.account());
+         _sod_map[account_id] = SecPosMap();
+      acct_it = _sod_map.find(account_id);
       bts_assert(acct_it != _sod_map.end());
       SecPosMap &sec_map = acct_it->second;
       auto sec_it = sec_map.find(seckey);
@@ -165,7 +173,8 @@ bool SODPosition::add_fill_to_sod_pos(const Fill &fill) {
       else
          _merge_pos(pos.second, fill);
       return true;
-   } else
+   }
+   else
       return false;
 }
 

src/lib/orderman/actors/SODPosition.h

@@ -16,8 +16,7 @@ SecPosMap;
 
 class SODPosition{
 private:
-   std::map<std::string,SecPosMap> _sod_map;
-   std::map<uint32_t, const api::Security*> _sec_ptr_map;
+   std::map<uint32_t,SecPosMap> _sod_map;
    const bool _enabled;
    const timeval _sod_time;   
 private:

src/lib/orderman/actors/ServiceBridge.cpp

@@ -1,10 +1,14 @@
 #include "ServiceBridge.h"
 
 #include "lib/api/public/ServiceContext.h"
+#include "lib/api/services/OrderService.pb.h"
+#include "lib/api/services/PositionService.pb.h"
 #include "lib/mercury/broker/BrokerProxy.h"
+#include "lib/orderman/events/AccountPnLUpdateEvent.h"
 #include "lib/orderman/events/ActivateOrderEvent.h"
 #include "lib/orderman/events/CancelOrderEvent.h"
 #include "lib/orderman/events/DetachOrderEvent.h"
+#include "lib/orderman/events/ExcessLiquidityUpdateEvent.h"
 #include "lib/orderman/events/NewAdminDataEvent.h"
 #include "lib/orderman/events/OrderManagerReadyEvent.h"
 #include "lib/orderman/events/OrderOpResultEvent.h"
@@ -13,13 +17,11 @@
 #include "lib/orderman/events/UpdateOrderEvent.h"
 #include "lib/orderman/order_factory/OrderFactory.h"
 #include "lib/orderman/order_router/OrderRouter.h"
+#include "lib/orderman/orderman_utils/orderman_common.h"
+#include "lib/orderman/orders/Order.h"
 #include "lib/orderman/securities/MapSecurityDB.h"
 #include "lib/tracing/PerfLog.h"
 #include "lib/utils/PBUtils.h"
-
-#include "lib/api/services/OrderService.pb.h"
-#include "lib/orderman/orderman_utils/orderman_common.h"
-#include "lib/orderman/orders/Order.h"
 #include <future>
 
 namespace bts {
@@ -50,6 +52,8 @@ struct ServiceBridge::PImpl {
    mercury::Stream<CancelOrderEvent> cancel_order_pub;
    mercury::Stream<DetachOrderEvent> detach_order_pub;
    mercury::Stream<NewAdminDataEvent> new_admin_data_pub;
+   mercury::Stream<AccountPnLUpdateEvent> account_pnl_pub;
+   mercury::Stream<ExcessLiquidityUpdateEvent> excess_liquidity_pub;
    std::unordered_set<OrderKey> known_orders;
    std::vector<std::shared_ptr<const api::admin::AdminData>> admin_data_updates;
    mercury::BrokerProxy &service_call_marshaller;
@@ -81,6 +85,8 @@ void ServiceBridge::init() {
    me->cancel_order_pub = make_stream<CancelOrderEvent>();
    me->detach_order_pub = make_stream<DetachOrderEvent>();
    me->new_admin_data_pub = make_stream<NewAdminDataEvent>();
+   me->account_pnl_pub = make_stream<AccountPnLUpdateEvent>();
+   me->excess_liquidity_pub = make_stream<ExcessLiquidityUpdateEvent>();
    me->ready.set_value();
 }
 
@@ -217,6 +223,36 @@ void ServiceBridge::update_admin_data(
    });
 }
 
+void ServiceBridge::update_account_pnl(
+    api::position::Pnl_account_dataUpdate const &pnl) {
+   me->service_call_marshaller.schedule_action([this, pnl]() {
+      for (auto const &account_pnl : pnl.added()) {
+         publish(me->account_pnl_pub, AccountID(account_pnl->account_id()),
+                 account_pnl->day_pnl());
+      }
+      for (auto const &account_pnl : pnl.changed()) {
+         publish(me->account_pnl_pub,
+                 AccountID(account_pnl.current()->account_id()),
+                 account_pnl.current()->day_pnl());
+      }
+   });
+}
+
+void ServiceBridge::update_excess_liquidity_report(
+    api::position::Excess_liquidity_reportUpdate const &elr) {
+   me->service_call_marshaller.schedule_action([this, elr]() {
+      for (auto const &elr : elr.added()) {
+         publish(me->excess_liquidity_pub, AccountID(elr->account_id()),
+                 elr->excess_liquidity());
+      }
+      for (auto const &elr : elr.changed()) {
+         publish(me->excess_liquidity_pub,
+                 AccountID(elr.current()->account_id()),
+                 elr.current()->excess_liquidity());
+      }
+   });
+}
+
 bool ServiceBridge::is_ready() const {
    return me->have_orders && me->order_manager_ready;
 }

src/lib/orderman/actors/ServiceBridge.h

@@ -1,18 +1,26 @@
 #pragma once
 
+#include "lib/api/services/MarginService.client.pb.h"
+#include "lib/api/services/PositionService.client.pb.h"
 #include "lib/config/protos/jetfix.pb.h"
-#include "lib/mercury/actors/Actor.h"
 #include "lib/md/SecurityOracle.h"
+#include "lib/mercury/actors/Actor.h"
+
 #include <memory>
 
 // fwds
 namespace bts {
-namespace mercury {class BrokerProxy;}
+namespace mercury {
+class BrokerProxy;
+}
 namespace api {
 class ServiceContext;
 namespace admin {
 class AdminData;
 }
+namespace position {
+class AccountPNL;
+}
 namespace orderman {
 class OrderRecord;
 class Fill;
@@ -45,16 +53,14 @@ class ServiceBridge : public mercury::Actor {
 
    using OnReady = std::function<void()>;
    using OnOrders =
-      std::function<void(std::vector<api::orderman::OrderRecord> const &)>;
+       std::function<void(std::vector<api::orderman::OrderRecord> const &)>;
    using OnFills =
-      std::function<void(std::vector<api::orderman::Fill> const &)>;
+       std::function<void(std::vector<api::orderman::Fill> const &)>;
+
+   using OnOrderOpResult = std::function<void(OrderOpResultEvent const &)>;
 
-   using OnOrderOpResult =
-      std::function<void(OrderOpResultEvent const&)>;
-   
    ServiceBridge(OnReady const &, OnOrders const &, OnFills const &,
-                 OnOrderOpResult const&,
-                 DynamicOrderRegistry const &,
+                 OnOrderOpResult const &, DynamicOrderRegistry const &,
                  mercury::BrokerProxy &service_call_marshaller,
                  md::SecurityOracle o);
    void init() override;
@@ -71,6 +77,9 @@ class ServiceBridge : public mercury::Actor {
 
    void
    update_admin_data(std::shared_ptr<api::admin::AdminData const> const &admin);
+   void update_account_pnl(api::position::Pnl_account_dataUpdate const &);
+   void update_excess_liquidity_report(
+       api::position::Excess_liquidity_reportUpdate const &);
 
  private:
    struct PImpl;

src/lib/orderman/actors/order_manager/OrderManager.cpp

@@ -4,6 +4,8 @@
 #include "lib/mercury/events/TimerEvent.h"
 #include "lib/orderman/actors/order_manager/OrderServices.h"
 #include "lib/orderman/actors/order_manager/OrderServicesData.h"
+#include "lib/orderman/events/AccountPnLUpdateEvent.h"
+#include "lib/orderman/events/ExcessLiquidityUpdateEvent.h"
 #include "lib/orderman/events/NewAdminDataEvent.h"
 #include "lib/orderman/events/NewSecuritiesEvent.h"
 #include "lib/orderman/events/OrderManagerReadyEvent.h"
@@ -52,6 +54,11 @@ OrderManager::~OrderManager() {}
 void OrderManager::init() {
    subscribe<NewAdminDataEvent>(
        [this](auto const &admin_update) { this->on_admin_data(admin_update); });
+   subscribe<AccountPnLUpdateEvent>(
+       [this](auto const &pnl_update) { this->on_pnl_update(pnl_update); });
+   subscribe<ExcessLiquidityUpdateEvent>([this](auto const &liq_update) {
+      this->on_excess_liquidity_update(liq_update);
+   });
 }
 
 void OrderManager::on_admin_data(const mercury::Update<NewAdminDataEvent> &m) {
@@ -74,6 +81,20 @@ void OrderManager::on_admin_data(const mercury::Update<NewAdminDataEvent> &m) {
    }
 }
 
+void OrderManager::on_pnl_update(
+    const mercury::Update<AccountPnLUpdateEvent> &u) {
+   if (u.events)
+      for (auto ev : *u.events)
+         me->services_data.limiter.set_day_pnl(ev.account_id, ev.day_pnl);
+}
+
+void OrderManager::on_excess_liquidity_update(
+    const mercury::Update<ExcessLiquidityUpdateEvent> &u) {
+   if (u.events)
+      for (auto ev : *u.events)
+         me->services_data.limiter.set_excess_liquidity(ev.account_id, ev.excess_liquidity);
+}
+
 void OrderManager::on_securities(
     const mercury::Update<NewSecuritiesEvent> &update) {
 
@@ -116,9 +137,9 @@ void OrderManager::on_fill_recovery(
          for (auto const &ev : *(update.events)) {
             auto trimmed_account_name =
                 boost::algorithm::trim_copy(ev.fill.account());
-            me->services_data.limiter.add(*ev.security, trimmed_account_name,
-                                          ev.fill.side(),
-                                          Quantity(ev.fill.last_shares()),false);
+            me->services_data.limiter.add(
+                *ev.security, trimmed_account_name, ev.fill.side(),
+                Quantity(ev.fill.last_shares()), false);
          }
 
          me->stored_order_sub = subscribe<StoredOrderEvent>(

src/lib/orderman/actors/order_manager/OrderManager.h

@@ -34,6 +34,8 @@ class OrderServices;
 class CreateFIXBufferReplyEvent;
 class GatewayStatusChangeEvent;
 class NewAdminDataEvent;
+class AccountPnLUpdateEvent;
+class ExcessLiquidityUpdateEvent;
 struct SessionServicesData;
 class DynamicOrderRegistry;
 } // namespace orderman
@@ -83,6 +85,8 @@ class OrderManager : public mercury::Actor, public mercury::IPollableActor {
    void on_gateway_state(const mercury::Update<GatewayStatusChangeEvent> &);
    void on_timer(const mercury::Update<mercury::TimerEvent> &);
    void on_admin_data(const mercury::Update<NewAdminDataEvent> &);
+   void on_pnl_update(const mercury::Update<AccountPnLUpdateEvent> &);
+   void on_excess_liquidity_update(const mercury::Update<ExcessLiquidityUpdateEvent> &);
    void on_poll(trace::PerfLog& log) override;
    // end external events
 

src/lib/orderman/events/AccountPnLUpdateEvent.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include "lib/api/services/AdminService.pb.h"
+
+#include "lib/mercury/common/mercury_common.h"
+#include "lib/mercury/events/Event.h"
+#include "lib/mercury/synopsis/MergeByKey.h"
+
+#include <memory>
+
+namespace bts {
+namespace orderman {
+
+class AccountPnLUpdateEvent : public mercury::Event<AccountPnLUpdateEvent> {
+ public:
+   using SynopsisPolicy = mercury::MergeByKey<AccountPnLUpdateEvent>;
+   AccountID key() const { return account_id; }
+
+   AccountPnLUpdateEvent(AccountID id, double pnl)
+       : account_id(id), day_pnl(pnl) {}
+
+   AccountID account_id;
+   double day_pnl;
+};
+
+} // namespace orderman
+} // namespace bts

src/lib/orderman/events/ExcessLiquidityUpdateEvent.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "lib/mercury/common/mercury_common.h"
+#include "lib/mercury/events/Event.h"
+#include "lib/mercury/synopsis/MergeByKey.h"
+
+#include <memory>
+
+namespace bts {
+namespace orderman {
+
+class ExcessLiquidityUpdateEvent : public mercury::Event<ExcessLiquidityUpdateEvent> {
+ public:
+   using SynopsisPolicy = mercury::MergeByKey<ExcessLiquidityUpdateEvent>;
+   AccountID key() const { return account_id; }
+
+   ExcessLiquidityUpdateEvent(AccountID id, double excess_liquidity)
+       : account_id(id), excess_liquidity(excess_liquidity) {}
+
+   AccountID account_id;
+   double excess_liquidity;
+};
+
+} // namespace orderman
+} // namespace bts

src/lib/orderman/events/OrderStateUpdateEvent.h

@@ -153,9 +153,6 @@ class OrderStateUpdateEvent : public mercury::Event<OrderStateUpdateEvent> {
       case api::orderman::OrderState::kDynamic:
          result.mutable_dynamic();
          break;
-      case api::orderman::OrderState::kSplitter:
-         result.mutable_splitter();
-         break;
       case api::orderman::OrderState::CUSTOM_NOT_SET:
          break;
       }

src/lib/orderman/limiter/LimitStore.cpp

@@ -5,8 +5,10 @@ namespace orderman {
 
 LimitStore::LimitStore(const bts::api::admin::AdminData &admin_data,
                        const std::map<uint32_t, api::Security const *> &sec_map)
-    : _admin_data(bts::admin::AdminServiceUtils::sort_limits(admin_data)),
-      _account_map(bts::admin::AdminServiceUtils::accounts_map(_admin_data)) {
+    : _admin_data(bts::admin::AdminServiceUtils::sort_limits(admin_data)) {
+   for (const auto& acct : _admin_data.accounts())
+      _account_data_map.insert(std::pair<uint32_t,AccountData>(
+                                                acct.id(),AccountData(acct)));
    for (auto it = sec_map.begin(); it != sec_map.end(); it++) {
       const api::Security &security = *(it->second);
       _add_product_entry(security, _admin_data);
@@ -22,7 +24,7 @@ LimitStore::LimitStore(const bts::api::admin::AdminData &admin_data,
    }
    _set_user_data();
 }
-void LimitStore::copy_positions_from(LimitStore &source) {
+void LimitStore::copy_data_from(LimitStore &source) {
    // copy securities
    for (auto it = _sec_map.begin(); it != _sec_map.end(); it++) {
       SecurityLimitTree *source_security_limit_tree =
@@ -30,6 +32,25 @@ void LimitStore::copy_positions_from(LimitStore &source) {
       if (source_security_limit_tree != nullptr)
          it->second.copy_positions_from(*source_security_limit_tree);
    }
+   // copy pnl's and liqs
+   for (auto it=_account_data_map.begin();it!=_account_data_map.end();it++){
+      auto source_it = source._account_data_map.find(it->first);
+      if(source_it !=  source._account_data_map.end()) {
+         it->second.curr_day_pnl = source_it->second.curr_day_pnl;
+         it->second.excess_liquidity = source_it->second.excess_liquidity;
+      }
+   }
+}
+void LimitStore::set_pnl (AccountID account_id, double pnl){
+   auto it = _account_data_map.find(account_id.value());
+   bts_assert(it != _account_data_map.end());
+   it->second.curr_day_pnl = pnl;
+}
+void LimitStore::set_excess_liquidity (AccountID account_id,
+                                       double excess_liquidity){
+   auto it = _account_data_map.find(account_id.value());
+   bts_assert(it != _account_data_map.end());
+   it->second.excess_liquidity = excess_liquidity;
 }
 void LimitStore::_add_product_entry(
     const api::Security &security,

src/lib/orderman/limiter/LimitStore.h

@@ -9,6 +9,10 @@
 #include "Limiter.h"
 #include "ProductLimitTree.h"
 #include "SecurityLimitTree.h"
+#include "lib/os/os.h"
+#include "lib/config/config.h"
+#include "lib/config/protos/span.pb.h"
+
 
 namespace bts {
 namespace orderman {
@@ -33,9 +37,22 @@ private:
          return nullptr;
       }
    };
+   
+   class AccountData{
+   public:
+      const bts::api::admin::BTSAccount& account;
+      double curr_day_pnl;
+      double excess_liquidity;
+      AccountData (const bts::api::admin::BTSAccount& account)
+      :
+      account(account),
+      curr_day_pnl (os::nan()),
+      excess_liquidity (os::nan())
+      {}
+   };
    const bts::api::admin::AdminData _admin_data;
-   // just an easy way to find accounts
-   const std::map<uint32_t, bool> _account_map;
+   // easy account access (uses for loss limit)
+   std::map<uint32_t, AccountData> _account_data_map;
    // map of user id's vs the user pointer and acccounts
    std::map<uint32_t, UserData> _user_data_map;
    // map of securities vs their account limits and positions
@@ -52,14 +69,20 @@ public:
 
    // accessors ///////////////
    ////////////////////////////
-   bool has_account (AccountID account_id) const
-   {return _account_map.find(account_id.value()) != _account_map.end();}
+   bool has_account (AccountID account_id) const{
+      return _account_data_map.find(account_id.value()) !=
+            _account_data_map.end();
+   }
    bool has_security (uint32_t sec_id) const
    {return _sec_map.find(sec_id) != _sec_map.end();}
    const bts::api::admin::AdminData& admin_data () const {return _admin_data;}
-   // position updates ////////
-   ////////////////////////////
-   void copy_positions_from(LimitStore& source);
+   // data updates ////////////
+   // once the LimitStore is created its position and
+   // pnl is updated from outside.
+   ////////////////////////////   
+   // copy_data_from copies the position and
+   // data from another source
+   void copy_data_from(LimitStore& source);
    void add_position(const api::Security& security,
                      AccountID account_id, api::Side side,
                      Quantity quantity, bool recurse){
@@ -88,6 +111,8 @@ public:
          }
       }
    }
+   void set_pnl (AccountID account_id, double pnl);
+   void set_excess_liquidity (AccountID account_id, double excess_liquidity);
    // risk limit checks
    ////////////////////
    // checks to see if the user is allowed to trade
@@ -130,11 +155,21 @@ public:
    Limiter::LimiterQueryResult can_send_new_order(AccountID account_id,
                                                    const api::Security& security,
                                                    api::Side side,
-                                                   Quantity quantity) const{
+                                                   Quantity quantity,
+                                                   bool enforce_clip = true) const{
       Limiter::LimiterQueryResult rv; rv.result = Limiter::OK;
+      const auto acct_it = _account_data_map.find(account_id.value());
+      AccountData const* account_ptr = (acct_it == _account_data_map.end()) ? 
+                                       nullptr : &(acct_it->second);
+      _loss_limit(account_ptr, rv);
+      if(!rv.ok())
+         return rv;
+      _margin_limit(account_ptr,rv);
+      if(!rv.ok())
+         return rv;
       const SecurityLimitTree* sec_tree = _find_security(security.id());
       bts_assert(sec_tree != nullptr);
-      rv = sec_tree->can_send_new_order(account_id, side, quantity);
+      rv = sec_tree->can_send_new_order(account_id, side, quantity, enforce_clip);
       if (rv.ok() && security.has_spread()){
          // if its rejected we dont need to validate further,
          // however if its ok and its a spread we need to check
@@ -142,7 +177,12 @@ public:
          for (const api::ContractLeg& leg : security.spread().legs()){
             api::Side effective_leg_side = side == api::Side::SIDE_BUY ? leg.side() : LimiterUtils::invert_side(leg.side());
             Quantity effective_leg_qty = Quantity (quantity.value() * std::abs(leg.quantity()));
-            rv = can_send_new_order (account_id, leg.security(), effective_leg_side, effective_leg_qty);
+            // in the case of a spread with future legs the clip limits 
+            // for the legs are not checked. if the legs are are spreads
+            // themselves then they are checked (eventually it will recuse
+            // down to a non spread)
+            rv = can_send_new_order (account_id, leg.security(), effective_leg_side, effective_leg_qty,
+                                     leg.security().has_spread());
             if (!rv.ok())
                return rv;
          }
@@ -152,18 +192,29 @@ public:
    Limiter::LimiterQueryResult can_update_order(AccountID account_id,
                                                 const api::Security& security,
                                                 api::Side side,
-                                                Quantity from, Quantity to) const{
+                                                Quantity from, Quantity to,
+                                                bool enforce_clip = true) const{
       Limiter::LimiterQueryResult rv; rv.result = Limiter::OK;
+      const auto acct_it = _account_data_map.find(account_id.value());
+      AccountData const* account_ptr = (acct_it == _account_data_map.end()) ? 
+                                       nullptr : &(acct_it->second);
+      // in case of order modifications, loss limits are only checked
+      // for size increases
+      if (to.value() > from.value())
+         _loss_limit(account_ptr, rv);
+      if(!rv.ok())
+         return rv;
       const SecurityLimitTree* sec_tree = _find_security(security.id());
       bts_assert(sec_tree != nullptr);
-      rv = sec_tree->can_update_order(account_id, side, from, to);
+      rv = sec_tree->can_update_order(account_id, side, from, to, enforce_clip);
       // same logic for the spread legs as above
       if (rv.ok() && security.has_spread()){
          for (const api::ContractLeg& leg : security.spread().legs()){
             api::Side effective_leg_side = side == api::Side::SIDE_BUY ? leg.side() : LimiterUtils::invert_side(leg.side());
             Quantity effective_leg_from_qty = Quantity (from.value() * std::abs(leg.quantity()));
             Quantity effective_leg_to_qty = Quantity (to.value() * std::abs(leg.quantity()));
-            rv = can_update_order (account_id, leg.security(), effective_leg_side, effective_leg_from_qty, effective_leg_to_qty);
+            rv = can_update_order (account_id, leg.security(), effective_leg_side, effective_leg_from_qty, effective_leg_to_qty,
+                                   leg.security().has_spread());
             if (!rv.ok())
                return rv;
          }
@@ -190,5 +241,66 @@ private:
    void _add_product_entry(const api::Security& security, 
                            const bts::api::admin::AdminData& admin_data);
    void _set_user_data();
+   void _loss_limit(AccountData const* account_ptr,
+                    Limiter::LimiterQueryResult& out_res) const{
+      // the usual flow should not alter out_res.
+      // its set to an error state iff :
+      // i.  account is not found
+      // ii. limit check is enabled but pnl is nan
+      // iii. limit check is enable and it actually fails
+      if(account_ptr != nullptr){
+         const auto& account_data = *account_ptr;
+         const auto& account = account_data.account;
+         if (account.has_account_loss_limit() && 
+                              account.account_loss_limit().enforce()){
+            double pnl = account_data.curr_day_pnl;
+            if(os::is_nan(pnl)){
+               out_res.result = Limiter::LimiterErrorCode::LOSS_LIMIT_BREACHED;
+               out_res.error_msg = "Could not find account PNL";
+            }
+            else{
+               double loss_limit = -1.0 * static_cast<double>(
+                           account.account_loss_limit().total_loss_limit());
+               if(pnl<loss_limit){
+                  out_res.result = Limiter::LimiterErrorCode::LOSS_LIMIT_BREACHED;
+                  out_res.error_msg = fmt::format("Loss Limit (Max {} Current {})",
+                                                  loss_limit, pnl);
+               }  
+            }
+         }
+      }
+      else{
+         out_res.result = Limiter::LimiterErrorCode::NO_SUCH_ACCOUNT;
+         out_res.error_msg = "Could not find account";
+      }
+   }
+   void _margin_limit(AccountData const* account_ptr,
+                      Limiter::LimiterQueryResult& out_res) const{
+      // similiar flow as the loss_limit above
+      if(account_ptr != nullptr){            
+         const auto& account_data = *account_ptr;
+         const auto& account = account_data.account;
+         if(config::get<bts::config::SpanMargin>().enabled()) {
+            if (account.has_margin_limit() && account.margin_limit().enforce_margin()){
+               double ex_liq = account_data.excess_liquidity;
+               if(os::is_nan(ex_liq)){
+                  out_res.result = Limiter::LimiterErrorCode::MARGIN_LIMIT_BREACHED;
+                  out_res.error_msg = "Could not find account Excess Liquidity";
+               }
+               else{               
+                  if(ex_liq < 0){
+                     out_res.result = Limiter::LimiterErrorCode::MARGIN_LIMIT_BREACHED;
+                     out_res.error_msg = fmt::format("Margin Limit Excess Liquidity : {}",
+                                                   ex_liq); 
+                  }
+               }
+            }
+         }
+      }
+      else{
+         out_res.result = Limiter::LimiterErrorCode::NO_SUCH_ACCOUNT;
+         out_res.error_msg = "Could not find account";
+      }
+   }
 };
 }}

src/lib/orderman/limiter/Limiter.cpp

@@ -73,6 +73,20 @@ void Limiter::remove(const api::Security &security,
    else
       remove(security, AccountID(acct_ptr->id()), side, quantity, recurse);
 }
+void Limiter::set_day_pnl (AccountID account_id, double pnl){
+   if (!me->limit_store->has_account(account_id))
+      _add_account_and_reset(account_id);
+   me->limit_store->set_pnl(account_id, pnl);
+}
+void Limiter::set_excess_liquidity (AccountID account_id, 
+                                    double excess_liquidity) {
+   LOG(OrderMan, Info, 
+       "Limiter::set_excess_liquidity account:{} excess_liquidity: {}",
+       account_id, excess_liquidity);
+   if (!me->limit_store->has_account(account_id))
+      _add_account_and_reset(account_id);
+   me->limit_store->set_excess_liquidity(account_id, excess_liquidity);
+}
 Limiter::LimiterQueryResult Limiter::can_trade(UserID user_id,
                                                AccountID account_id) const {
    LimiterQueryResult rv = me->limit_store->can_trade(user_id, account_id);
@@ -119,7 +133,7 @@ void Limiter::_add_account_and_reset(AccountID account_id) {
 }
 void Limiter::_reset(const bts::api::admin::AdminData &admin_data) {
    LimitStore *new_store = new LimitStore(admin_data, me->sec_map);
-   new_store->copy_positions_from(*me->limit_store);
+   new_store->copy_data_from(*me->limit_store);
    me->limit_store.reset(new_store);
 }
 

src/lib/orderman/limiter/Limiter.h

@@ -33,7 +33,9 @@ class Limiter {
             api::Side, Quantity, bool recurse);
    void remove(api::Security const &, std::string const & /* account_name */,
                api::Side, Quantity, bool recurse);
-
+   void set_day_pnl (AccountID account_id, double pnl);
+   void set_excess_liquidity(AccountID account_id, double excesss_liquidity);
+   
    enum LimiterErrorCode {
       OK,
       NO_SUCH_SECURITY,
@@ -45,7 +47,9 @@ class Limiter {
       ACCOUNT_DEACTIVATED,
       NO_SUCH_ACCOUNT,
       USER_DEACTIVATED,
-      NO_SUCH_USER
+      NO_SUCH_USER,
+      LOSS_LIMIT_BREACHED,
+      MARGIN_LIMIT_BREACHED
    };
 
    struct LimiterQueryResult {

src/lib/orderman/limiter/SecurityLimitNode.h

@@ -63,9 +63,11 @@ public:
    }
    // risk limit checks
    ////////////////////
-   Limiter::LimiterQueryResult can_send_new_order(api::Side side, Quantity quantity) const{
+   Limiter::LimiterQueryResult can_send_new_order(api::Side side, 
+                                             Quantity quantity,
+                                             bool enforce_clip = true) const{
       Limiter::LimiterQueryResult result;result.result= Limiter::OK;
-      if (quantity.value() > _clip_size){
+      if (enforce_clip && quantity.value() > _clip_size){
          result.result = Limiter::CLIP_SIZE_BREACHED;
          result.error_msg = fmt::format("Security Clip Size (Max {} Qty {})", _clip_size, quantity.value());
          return result;
@@ -78,9 +80,10 @@ public:
          result = _product_node.can_send_new_order(side, quantity);
       return result;
    }
-   Limiter::LimiterQueryResult can_update_order(api::Side side, Quantity from, Quantity to) const{ 
+   Limiter::LimiterQueryResult can_update_order(api::Side side, Quantity from, Quantity to,
+                                                bool enforce_clip = true) const{ 
       Limiter::LimiterQueryResult result;result.result=Limiter::OK;
-      if (to.value() > _clip_size){
+      if (enforce_clip && to.value() > _clip_size){
          result.result = Limiter::CLIP_SIZE_BREACHED;
          result.error_msg = fmt::format("Security Clip Size (Max {} Qty {})", _clip_size, to.value());
          return result;

src/lib/orderman/limiter/SecurityLimitTree.h

@@ -50,17 +50,19 @@ public:
    ////////////////////
    Limiter::LimiterQueryResult can_send_new_order(AccountID account_id,
                                                   api::Side side,
-                                                  Quantity quantity) const{
+                                                  Quantity quantity,
+                                                  bool enforce_clip = true) const{
       const SecurityLimitNode* node_ptr = _find_account(account_id.value());
       bts_assert(node_ptr!=nullptr);
-      return node_ptr->can_send_new_order(side, quantity);
+      return node_ptr->can_send_new_order(side, quantity, enforce_clip);
    }
    Limiter::LimiterQueryResult can_update_order(AccountID account_id,
                                                 api::Side side,
-                                                Quantity from, Quantity to) const{
+                                                Quantity from, Quantity to,
+                                                bool enforce_clip = true) const{
       const SecurityLimitNode* node_ptr = _find_account(account_id.value());
       bts_assert(node_ptr!=nullptr);
-      return node_ptr->can_update_order(side, from, to);      
+      return node_ptr->can_update_order(side, from, to, enforce_clip);      
    }
    // accessors
    ////////////

src/lib/orderman/order_factory/OrderFactory.cpp

@@ -19,9 +19,8 @@
 #include "lib/orderman/orders/algo/algo_orders/TopAlgo.h"
 #include "lib/orderman/orders/algo/algo_orders/IcebergAlgo.h"
 #include "lib/orderman/orders/algo/algo_orders/LeanAlgo.h"
-#include "lib/orderman/orders/algo/algo_orders/SplitterAlgo.h"
+#include "lib/orderman/orders/algo/algo_orders/OffsetAlgo.h"
 #include "lib/orderman/orders/book_algos/OCOOrder.h"
-#include "lib/orderman/orders/book_algos/OffsetOrder.h"
 #include "lib/orderman/orders/book_algos/PricerOrder.h"
 #include "lib/orderman/orders/spreader_orders/SpreaderOrder.h"
 #include "lib/orderman/orders/CompositeOrder.h"
@@ -184,7 +183,7 @@ Order &OrderFactory::allocate(OrderKey k, OrderKey pk,
           new orderman::AlgoWrapperOrder(std::make_unique<CoverAlgo>()));
       break;
    case CustomOrderDefinition::kOffset:
-      order_ptr.reset(new orderman::OffsetOrder());
+      order_ptr.reset(new orderman::AlgoWrapperOrder(std::make_unique<OffsetAlgo>()));
       break;
    case CustomOrderDefinition::kQueueHolder:
       order_ptr.reset(new orderman::QueueHolderOrder());
@@ -209,10 +208,6 @@ Order &OrderFactory::allocate(OrderKey k, OrderKey pk,
    case CustomOrderDefinition::kComposite:
       order_ptr.reset(new orderman::CompositeOrder());
       break;
-   case CustomOrderDefinition::kSplitter:
-      order_ptr.reset(
-         new orderman::AlgoWrapperOrder(std::make_unique<splitter::SplitterAlgo>()));
-      break;
    case CustomOrderDefinition::CUSTOM_NOT_SET:
       bts_assert(false);
       break;

src/lib/orderman/orderman_utils/orderman_utils.cpp

@@ -14,6 +14,7 @@ bool can_consume_attachment(api::orderman::CustomOrderDefinition::CustomCase ord
    case CustomOrderDefinition::kTop:
    case CustomOrderDefinition::kCover:
    case CustomOrderDefinition::kLean:
+   case CustomOrderDefinition::kOffset:
       return true;
    default:
       return false;
@@ -44,73 +45,14 @@ bool should_forward_attachment(
       return
          cdef.lean().has_child() &&
          can_consume_attachment(cdef.lean().child().custom_case());
+   case CustomOrderDefinition::kOffset:
+      return
+         cdef.offset().has_child() &&
+         can_consume_attachment(cdef.offset().child().custom_case());
    default:
       return false;
    }
 }
 
-api::orderman::DynamicOrderParam dynamic_parameter(int32_t v) {
-   api::orderman::DynamicOrderParam res;
-   res.set_as_int32(v);
-   return res;
-}
-api::orderman::DynamicOrderParam dynamic_parameter(uint32_t v) {
-   api::orderman::DynamicOrderParam res;
-   res.set_as_uint32(v);
-   return res;
-}
-api::orderman::DynamicOrderParam dynamic_parameter(double v) {
-   api::orderman::DynamicOrderParam res;
-   res.set_as_double(v);
-   return res;
-}
-api::orderman::DynamicOrderParam dynamic_parameter(std::string v) {
-   api::orderman::DynamicOrderParam res;
-   res.set_as_string(v);
-   return res;
-}
-api::orderman::DynamicOrderParam dynamic_parameter(bool v) {
-   api::orderman::DynamicOrderParam res;
-   res.set_as_bool(v);
-   return res;
-}
-api::orderman::DynamicOrderParam dynamic_parameter(const api::orderman::CustomOrderDefinition& v) {
-   api::orderman::DynamicOrderParam res;
-   res.mutable_as_order_def()->CopyFrom(v);
-   return res;
-}
-api::orderman::DynamicOrderParam dynamic_parameter(const api::orderman::DynamicOrderArray& v) {
-   api::orderman::DynamicOrderParam res;
-   res.mutable_as_array()->CopyFrom(v);
-   return res;
-}
-api::orderman::DynamicOrderParam dynamic_parameter(const api::orderman::DynamicOrderStruct& v) {
-   api::orderman::DynamicOrderParam res;
-   res.mutable_as_struct()->CopyFrom(v);
-   return res;
-}
-api::orderman::DynamicOrderParam dynamic_parameter(const api::orderman::DynamicOrderEnumerated& v) {
-   api::orderman::DynamicOrderParam res;
-   res.mutable_as_enumerated()->CopyFrom(v);
-   return res;
-}
-api::orderman::DynamicOrderParam dynamic_array(const std::vector<api::orderman::DynamicOrderParam>& v) {
-   api::orderman::DynamicOrderArray a;
-   for (const auto& item : v) {
-      a.add_element()->CopyFrom(item);
-   }
-   return dynamic_parameter(a);
-}
-api::orderman::DynamicOrderParam dynamic_map(
-   const std::vector<std::tuple<std::string,api::orderman::DynamicOrderParam>>& v) {
-   DynamicOrderStruct res;
-   auto& map = *res.mutable_field();
-   for (const auto& tup : v) {
-      map[std::get<std::string>(tup)] = std::get<1>(tup);
-   }
-   return dynamic_parameter(res);
-}
-
-
 } // namespace orderman
 } // namespace bts

src/lib/orderman/orderman_utils/orderman_utils.h

@@ -1,6 +1,4 @@
 #include "lib/api/services/OrderService.pb.h"
-#include <optional>
-#include <variant>
 
 namespace bts {
 namespace orderman {
@@ -32,64 +30,5 @@ inline bool is_parent_order(const bts::api::orderman::OrderRecord &ord) {
 bool should_forward_attachment(const api::orderman::CustomOrderDefinition &def);
 bool can_consume_attachment(api::orderman::CustomOrderDefinition::CustomCase) ;
 
-
-api::orderman::DynamicOrderParam dynamic_parameter(int32_t);
-api::orderman::DynamicOrderParam dynamic_parameter(uint32_t);
-api::orderman::DynamicOrderParam dynamic_parameter(double);
-api::orderman::DynamicOrderParam dynamic_parameter(std::string);
-api::orderman::DynamicOrderParam dynamic_parameter(bool);
-api::orderman::DynamicOrderParam dynamic_parameter(const api::orderman::CustomOrderDefinition&);
-api::orderman::DynamicOrderParam dynamic_parameter(const api::orderman::DynamicOrderArray&);
-api::orderman::DynamicOrderParam dynamic_parameter(const api::orderman::DynamicOrderStruct&);
-api::orderman::DynamicOrderParam dynamic_parameter(const api::orderman::DynamicOrderEnumerated&);
-api::orderman::DynamicOrderParam dynamic_array(const std::vector<api::orderman::DynamicOrderParam>&);
-api::orderman::DynamicOrderParam dynamic_map( const std::vector<std::tuple<std::string,api::orderman::DynamicOrderParam>>&);
-
-template <class T>
-std::optional<T> get_if(const api::orderman::DynamicOrderParam&) { return nullptr; }
-
-template <>
-inline std::optional<int32_t> get_if(const api::orderman::DynamicOrderParam& p) { 
-   return p.value_case() == api::orderman::DynamicOrderParam::kAsInt32 ? std::optional<int32_t>{p.as_int32()} : std::nullopt;
-}
-template <>
-inline std::optional<uint32_t> get_if(const api::orderman::DynamicOrderParam& p) { 
-   return p.value_case() == api::orderman::DynamicOrderParam::kAsUint32 ? std::optional<uint32_t>{p.as_uint32()} : std::nullopt;
-}
-template <>
-inline std::optional<double> get_if(const api::orderman::DynamicOrderParam& p) { 
-   return p.value_case() == api::orderman::DynamicOrderParam::kAsDouble ? std::optional<double>{p.as_double()} : std::nullopt;
-}
-template <>
-inline std::optional<bool> get_if(const api::orderman::DynamicOrderParam& p) { 
-   return p.value_case() == api::orderman::DynamicOrderParam::kAsBool ? std::optional<bool>{p.as_bool()} : std::nullopt;
-}
-template <>
-inline std::optional<api::orderman::CustomOrderDefinition> get_if(const api::orderman::DynamicOrderParam& p) { 
-   return p.value_case() == api::orderman::DynamicOrderParam::kAsOrderDef ? std::optional<api::orderman::CustomOrderDefinition>{p.as_order_def()} : std::nullopt;
-}
-template <>
-inline std::optional<api::orderman::DynamicOrderArray> get_if(const api::orderman::DynamicOrderParam& p) { 
-   return p.value_case() == api::orderman::DynamicOrderParam::kAsArray ? std::optional<api::orderman::DynamicOrderArray>{p.as_array()} : std::nullopt;
-}
-template <>
-inline std::optional<api::orderman::DynamicOrderStruct> get_if(const api::orderman::DynamicOrderParam& p) { 
-   return p.value_case() == api::orderman::DynamicOrderParam::kAsStruct ? std::optional<api::orderman::DynamicOrderStruct>{p.as_struct()} : std::nullopt;
-}
-template <>
-inline std::optional<api::orderman::DynamicOrderEnumerated> get_if(const api::orderman::DynamicOrderParam& p) { 
-   return p.value_case() == api::orderman::DynamicOrderParam::kAsEnumerated ? std::optional<api::orderman::DynamicOrderEnumerated>{p.as_enumerated()} : std::nullopt;
-}
-
-
-template <class T>
-std::optional<T> get_if(const google::protobuf::Map<std::string,api::orderman::DynamicOrderParam>& map, const std::string& key) { 
-   auto c = map.find(key);
-   if (c != map.end()) {
-      return get_if<T>(c->second);
-   }
-   return std::nullopt;
-}
-
 } // namespace orderman
 } // namespace bts

src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp

@@ -16,7 +16,13 @@ namespace orderman {
 
 using namespace api::orderman;
 template <class K, class V> using Map = boost::container::flat_map<K, V>;
-enum class AlgoWrapperOrder::Status { INACTIVE, W_ATTACH, WORKING, COMPLETED };
+enum class AlgoWrapperOrder::Status {
+   INACTIVE,
+   W_ATTACH,
+   WORKING,
+   SHUTTING_DOWN,
+   COMPLETED
+};
 
 class ChildTemplate {
  public:
@@ -62,6 +68,7 @@ struct AlgoWrapperOrder::PImpl {
    OrderKey attach_target = null_order_key();
 
    std::optional<api::orderman::Update> pending_update;
+   std::optional<std::pair<OrderStatus, std::string>> pending_terminate;
    OrderStatus external_status() const { return _state.status(); }
 
    void set_external_status(OrderStatus s) {
@@ -222,6 +229,7 @@ OrderOpResult AlgoWrapperOrder::on_activate(Price p, Quantity q,
       return {};
 
    case Status::W_ATTACH:
+   case Status::SHUTTING_DOWN:
    case Status::WORKING:
       bts_assert(false);
    }
@@ -238,6 +246,7 @@ OrderOpResult AlgoWrapperOrder::on_update(Update const &update) {
       me->pending_update = update;
       return {};
    case Status::COMPLETED:
+   case Status::SHUTTING_DOWN:
       return {};
    case Status::WORKING: {
       me->set_price_quantity(Price(update.price()),
@@ -275,6 +284,7 @@ orderman::OrderOpResult AlgoWrapperOrder::on_detach() {
    switch (me->status) {
    case Status::W_ATTACH:
    case Status::WORKING:
+   case Status::SHUTTING_DOWN:
       for (auto &[_, c] : me->children) {
          (void)_;
          c.order.mark_detached_from_parent();
@@ -309,6 +319,7 @@ orderman::OrderOpResult AlgoWrapperOrder::on_cancel() {
          break;
       }
       break;
+   case Status::SHUTTING_DOWN:
    case Status::COMPLETED:
       break;
    case Status::INACTIVE:
@@ -347,13 +358,20 @@ Quantity AlgoWrapperOrder::total_fills() const {
    return Quantity(me->state().filled_quantity());
 }
 
+bool AlgoWrapperOrder::is_shutting_down() const {
+   return me->status == Status::SHUTTING_DOWN;
+}
+
 void AlgoWrapperOrder::subscribe_book(SecurityKey k) {
    if (!is_working())
       return;
    bts_assert(me->book_subs.find(k) == me->book_subs.end());
 
    me->book_subs[k] = me->services->subscribe_to_book(
-       k, [this, k](auto const &ev) { me->algo->on_book(k, ev.book); });
+       k, [this, k](auto const &ev) {
+          ReportStateGuard _(*this);
+          me->algo->on_book(k, ev.book);
+       });
 }
 
 void AlgoWrapperOrder::unsubscribe_book(SecurityKey k) {
@@ -392,7 +410,10 @@ void AlgoWrapperOrder::subscribe_timer() {
    bts_assert(!me->has_timer_sub);
    me->has_timer_sub = true;
    me->services->subscribe_to_time(
-       [this](auto const &) { me->algo->on_timer(); });
+       [this](auto const &) {
+          ReportStateGuard _(*this);
+          me->algo->on_timer();
+       });
 }
 
 void AlgoWrapperOrder::unsubscribe_timer() {
@@ -485,10 +506,9 @@ void AlgoWrapperOrder::detach_all_children() {
 }
 
 void AlgoWrapperOrder::update_fill_state(Quantity q, Price p) {
-   if (!is_working())
+   if (!is_working() && !is_shutting_down())
       return;
 
-   ReportStateGuard _(*this);
    auto previous_qty = me->state().filled_quantity();
    if (previous_qty > q.value()) {
       LOG(OrderMan, Error,
@@ -510,35 +530,37 @@ void AlgoWrapperOrder::update_fill_state(Quantity q, Price p) {
 void AlgoWrapperOrder::set_ladder_visibility(bool is_visible) {
    if (!is_working())
       return;
-
-   ReportStateGuard _(*this);
    me->set_ladder_visibility(is_visible);
 }
 
 void AlgoWrapperOrder::cancel(std::string const &msg) {
    if (!is_working())
       return;
-   ReportStateGuard _(*this);
-   terminate(ORDER_STATUS_CANCELED, msg);
+   if (working_children_count() == 0)
+      terminate(ORDER_STATUS_CANCELED, msg);
+   else
+      shutdown(ORDER_STATUS_CANCELED, msg);
 }
 
 void AlgoWrapperOrder::terminate_error(std::string const &msg) {
-   if (!is_working() && me->status != Status::W_ATTACH)
+   if (!is_working() && !is_shutting_down() && me->status != Status::W_ATTACH)
       return;
-   ReportStateGuard _(*this);
    // If we attached and an error was signaled we do not want to
    // cancel our child because that child might be a long running
    // order with a good queue position that we want to preserve
    if (is_attaching())
       me->children.erase(me->attach_target);
+
    terminate(ORDER_STATUS_ERROR, msg);
 }
 
 void AlgoWrapperOrder::complete() {
    if (!is_working())
       return;
-   ReportStateGuard _(*this);
-   terminate(ORDER_STATUS_COMPLETED);
+   if (working_children_count() == 0)
+      terminate(ORDER_STATUS_COMPLETED);
+   else
+      shutdown(ORDER_STATUS_COMPLETED);
 }
 
 // --------------------------------------------------------------------------------
@@ -573,6 +595,7 @@ void AlgoWrapperOrder::on_attach_complete(
       break;
    case Status::COMPLETED:
       break;
+   case Status::SHUTTING_DOWN:
    case Status::INACTIVE:
    case Status::WORKING:
       bts_assert(false);
@@ -587,6 +610,7 @@ void AlgoWrapperOrder::on_child_order_update(
    switch (me->status) {
    case Status::WORKING:
    case Status::W_ATTACH:
+   case Status::SHUTTING_DOWN:
       if (auto it = me->children.find(child_key); it != me->children.end()) {
          auto &child_record = it->second;
          switch (new_child_state.status()) {
@@ -625,16 +649,38 @@ void AlgoWrapperOrder::on_child_order_update(
             break;
          }
       } else
-         terminate_error(
-             fmt::format("Unknown child order update: {}", child_key.value()));
-      return;
+         LOG(OrderMan, Info,
+             "AlgoOrder {} received update for presumably detached order {}. "
+             "Ignoring it",
+             key().value(), child_key.value());
+      break;
    case Status::COMPLETED:
       return;
    case Status::INACTIVE:
       bts_assert(false);
       return;
    }
-   bts_assert(false);
+
+   if (me->status == Status::SHUTTING_DOWN)
+      try_complete_shutdown();
+}
+
+void AlgoWrapperOrder::try_complete_shutdown() {
+   bts_assert(me->status == Status::SHUTTING_DOWN);
+   bts_assert(me->children.size());
+   bts_assert(me->pending_terminate.has_value());
+
+   if (working_children_count() == 0) {
+      LOG(OrderMan, Info, "AlgoOrder {} completed shutdown. Terminated",
+          key().value());
+      terminate(me->pending_terminate.value().first,
+                me->pending_terminate.value().second);
+   } else {
+      LOG(OrderMan, Info,
+          "AlgoOrder {} still has {} orders working. Shutdown still in "
+          "progress",
+          key().value(), working_children_count());
+   }
 }
 
 void AlgoWrapperOrder::setup_transitions() {
@@ -676,8 +722,21 @@ void AlgoWrapperOrder::setup_transitions() {
    // From WORKING
    me->transition_actions[{Status::WORKING, Status::COMPLETED}] = [this]() {
       cleanup();
+      me->children.clear();
       _transition(Status::COMPLETED);
    };
+   me->transition_actions[{Status::WORKING, Status::SHUTTING_DOWN}] = [this]() {
+      cleanup();
+      _transition(Status::SHUTTING_DOWN);
+   };
+
+   // From SHUTTING_DOWN
+   me->transition_actions[{Status::SHUTTING_DOWN, Status::COMPLETED}] =
+       [this]() {
+          me->children.clear();
+          me->pending_terminate.reset();
+          _transition(Status::COMPLETED);
+       };
 }
 
 void AlgoWrapperOrder::check_invariants() {
@@ -697,6 +756,7 @@ void AlgoWrapperOrder::check_invariants() {
       bts_assert(me->book_subs.empty());
       bts_assert(me->children.empty());
       bts_assert(!me->has_timer_sub);
+      bts_assert(!me->pending_terminate.has_value());
       break;
 
    case Status::WORKING:
@@ -704,10 +764,23 @@ void AlgoWrapperOrder::check_invariants() {
       bts_assert(me->security);
       bts_assert(me->external_status() == ORDER_STATUS_WORKING);
       bts_assert(!me->pending_update.has_value());
+      bts_assert(!me->pending_terminate.has_value());
+      break;
+
+   case Status::SHUTTING_DOWN:
+      bts_assert(working_children_count());
+      bts_assert(me->services);
+      bts_assert(me->security);
+      bts_assert(me->external_status() == ORDER_STATUS_WORKING);
+      bts_assert(me->book_subs.empty());
+      bts_assert(!me->has_timer_sub);
+      bts_assert(!me->pending_update.has_value());
+      bts_assert(me->pending_terminate.has_value());
       break;
 
    case Status::COMPLETED:
       bts_assert(me->children.empty());
+      bts_assert(!me->pending_update.has_value());
       bts_assert(me->external_status() == ORDER_STATUS_CANCELED ||
                  me->external_status() == ORDER_STATUS_COMPLETED ||
                  me->external_status() == ORDER_STATUS_ERROR ||
@@ -715,6 +788,7 @@ void AlgoWrapperOrder::check_invariants() {
       bts_assert(me->book_subs.empty());
       bts_assert(!me->has_timer_sub);
       bts_assert(!me->pending_update.has_value());
+      bts_assert(!me->pending_terminate.has_value());
       break;
    }
 }
@@ -745,15 +819,25 @@ void AlgoWrapperOrder::terminate(api::orderman::OrderStatus st,
    transition(Status::COMPLETED);
 }
 
+void AlgoWrapperOrder::shutdown(api::orderman::OrderStatus st,
+                                std::string const &error) {
+   bts_assert(!me->pending_terminate.has_value());
+   bts_assert(working_children_count());
+   me->pending_terminate = std::make_pair(st, error);
+   LOG(OrderMan, Info,
+       "AlgoOrder {} is shutting down. Waiting for {} child orders to complete "
+       "before terminating",
+       key().value(), working_children_count());
+   transition(Status::SHUTTING_DOWN);
+}
+
 void AlgoWrapperOrder::cleanup() {
    for (auto const &s : me->book_subs) {
       me->services->unsubscribe_from_book(s.second);
       me->book_subs.clear();
    }
-   for (auto &c : me->children) {
+   for (auto &c : me->children)
       c.second.order.cancel();
-      me->children.clear();
-   }
    if (me->has_timer_sub)
       unsubscribe_timer();
    me->pending_update.reset();
@@ -785,6 +869,16 @@ OrderKey AlgoWrapperOrder::index_and_launch_child(ChildOrder &&child, Price p,
    return k;
 }
 
+size_t AlgoWrapperOrder::working_children_count() const {
+   auto working_children_count = 0;
+   for (auto &c : me->children)
+      if (!c.second.state.has_value() ||
+          c.second.state.value().status == ChildStatus::WORKING) {
+         working_children_count++;
+      }
+   return working_children_count;
+}
+
 bool AlgoWrapperOrder::is_working() const {
    return me->status == Status::WORKING;
 }
@@ -799,6 +893,8 @@ log::CharPtrWrapper AlgoWrapperOrder::state_name(Status st) {
       return {"INACTIVE"};
    case Status::WORKING:
       return {"WORKING"};
+   case Status::SHUTTING_DOWN:
+      return {"SHUTTING_DOWN"};
    }
    bts_assert(false);
    return {""};

src/lib/orderman/orders/algo/AlgoWrapperOrder.h

@@ -40,6 +40,7 @@ class AlgoWrapperOrder : public Order,
    Price price() const override;
    Quantity quantity() const override;
    Quantity total_fills() const override;
+   bool is_shutting_down() const override;
    void subscribe_book(SecurityKey) override;
    void unsubscribe_book(SecurityKey) override;
    void subscribe_child_change(OrderKey) override;
@@ -75,10 +76,13 @@ class AlgoWrapperOrder : public Order,
    void transition(Status);
    void _transition(Status);
    void terminate(api::orderman::OrderStatus, std::string const &error = "");
+   void shutdown(api::orderman::OrderStatus, std::string const &error = "");
+   void try_complete_shutdown();
    void cleanup();
    void update_child_record(api::orderman::OrderState const &, ChildRecord *);
    OrderKey index_and_launch_child(ChildOrder &&, Price, Quantity, bool);
    bool is_working() const ;
+   size_t working_children_count() const;
    log::CharPtrWrapper state_name(Status) ;
    ChildStatus map_child_status(api::orderman::OrderStatus) ;
    bool _is_attaching(api::orderman::OrderDefinition const &) const ;

src/lib/orderman/orders/algo/IAlgoServices.h

@@ -20,6 +20,13 @@ class IAlgoServices : public IBeforeLaunchAlgoServices {
    // Returns the filled quantity for this algo
    virtual Quantity total_fills() const = 0;
 
+   // Returns wheter the algo is in the process of shutting down (due
+   // to a cancel, an error, or just having completed its work). When
+   // algo is shutting down any requests to create, update or cancel
+   // child orders will be ignored.
+   virtual bool is_shutting_down() const = 0;
+   
+   
    // --------------------------------------------------------------
    // Event Subscriptions
    // --------------------------------------------------------------

src/lib/orderman/orders/algo/IBeforeLaunchAlgoServices.cpp

@@ -1,8 +1 @@
 #include "lib/orderman/orders/algo/IBeforeLaunchAlgoServices.h"
-namespace bts {
-namespace orderman {
-
-TimeStamp IBeforeLaunchAlgoServices::get_time() const { return Timer::now(); }
-
-} // namespace orderman
-} // namespace bts

src/lib/orderman/orders/algo/IBeforeLaunchAlgoServices.h

@@ -3,7 +3,6 @@
 #include "lib/api/services/OrderService.pb.h"
 #include "lib/domain_model/SecurityKey.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"
-#include "lib/utils/bts_chrono.h"
 
 namespace bts {
 namespace orderman {
@@ -43,8 +42,6 @@ class IBeforeLaunchAlgoServices {
    virtual ChildTemplate &
    make_child_template(api::orderman::OrderDefinition const &child_def,
                        size_t reserve = 1) = 0;
-   
-   virtual TimeStamp get_time() const;
 };
 
 } // namespace orderman

src/lib/orderman/orders/algo/algo_orders/CoverAlgo.cpp

@@ -10,18 +10,8 @@ namespace orderman {
 
 using namespace api::orderman;
 
-enum class CoverAlgo::Status { WORKING, WAITING_FOR_HEDGES, DONE };
-const char *CoverAlgo::status_name(Status st) {
-   switch (st) {
-   case Status::WORKING:
-      return "WORKING";
-   case Status::WAITING_FOR_HEDGES:
-      return "WAITING_FOR_HEDGES";
-   case Status::DONE:
-      return "Done";
-   }
-   return nullptr;
-}
+enum class CoverAlgo::Status { WORKING, WAITING_FOR_HEDGES, DONE, CANCELING };
+
 struct CoverAlgo::PImpl {
    IAlgoServices *algo;
    OrderKey child_key = null_order_key();
@@ -92,8 +82,11 @@ void CoverAlgo::do_launch_attached(IAlgoServices &algo, OrderKey k,
    on_child_change(k, st);
 }
 
-void CoverAlgo::on_child_change(OrderKey k, ChildState const &st) {
-   if (k == me->child_key) {
+void CoverAlgo::on_covered_change(ChildState const &st) {
+   // First launch hedge if appropriate
+   switch (me->status) {
+   case Status::WORKING:
+   case Status::CANCELING:
       if (st.total_fills.value() > me->hedged_fills.value()) {
          auto hedge_qty =
              Quantity(st.total_fills.value() - me->hedged_fills.value());
@@ -116,6 +109,15 @@ void CoverAlgo::on_child_change(OrderKey k, ChildState const &st) {
              hedge_qty.value(), hedge_price.value());
       }
       algo().update_fill_state(st.total_fills, st.last_price);
+      break;
+   case Status::WAITING_FOR_HEDGES:
+   case Status::DONE:
+      break;
+   }
+
+   // Now analyze child to figure out wheter we should terminate
+   switch (me->status) {
+   case Status::WORKING:
       switch (st.status) {
       case ChildStatus::WORKING:
          break;
@@ -125,36 +127,67 @@ void CoverAlgo::on_child_change(OrderKey k, ChildState const &st) {
              algo().order_key().value());
          algo().set_ladder_visibility(false);
          me->status = Status::WAITING_FOR_HEDGES;
+         terminate_if_hedges_complete();
          break;
       case ChildStatus::CANCELED:
-         LOG(OrderMan, Info, "CoverAlgo {}. Child was canceled, terminating",
+         LOG(OrderMan, Info, "CoverAlgo {}. Child was unexpectadly canceled, terminating",
              algo().order_key().value());
-         me->status = Status::DONE;
+         on_done();
          algo().terminate_error("Unexpected child cancel");
          break;
       }
-   } else {
+      break;
+   case Status::CANCELING:
       switch (st.status) {
       case ChildStatus::WORKING:
          break;
       case ChildStatus::CANCELED:
-         if (me->cancel_on_external_hedge_cancel) {
-            LOG(OrderMan, Info, "CoverAlgo {}. Hedge was canceled, terminating",
-                algo().order_key().value());
-            me->status = Status::DONE;
-            algo().terminate_error("Unexpected child cancel");
-         } else
-            me->hedges.erase(
-                std::remove(me->hedges.begin(), me->hedges.end(), k),
-                me->hedges.end());
-         break;
       case ChildStatus::COMPLETED:
-         me->hedges.erase(std::remove(me->hedges.begin(), me->hedges.end(), k),
-                          me->hedges.end());
+         LOG(OrderMan, Info, "CoverAlgo {}. Shutdown complete",
+             algo().order_key().value());
+         on_done();
+         algo().cancel();
          break;
       }
+      break;
+   case Status::WAITING_FOR_HEDGES:
+   case Status::DONE:
+      break;
    }
-   check_terminate();
+}
+
+void CoverAlgo::on_hedge_change(OrderKey k, ChildState const &st) {
+
+   // First adjust our hedge count or cancel if something went wrong
+   switch (st.status) {
+   case ChildStatus::WORKING:
+      break;
+   case ChildStatus::COMPLETED:
+      me->hedges.erase(std::remove(me->hedges.begin(), me->hedges.end(), k),
+                       me->hedges.end());
+      break;
+   case ChildStatus::CANCELED:
+      if (me->cancel_on_external_hedge_cancel) {
+         LOG(OrderMan, Info, "CoverAlgo {}. Hedge was canceled, terminating",
+             algo().order_key().value());
+         on_done();
+         algo().terminate_error("Unexpected child cancel");
+         return;
+      } else
+         me->hedges.erase(std::remove(me->hedges.begin(), me->hedges.end(), k),
+                          me->hedges.end());
+      break;
+   }
+
+   // Now if we are waiting for hedges to complete, bail
+   terminate_if_hedges_complete();
+}
+
+void CoverAlgo::on_child_change(OrderKey k, ChildState const &st) {
+   if (k == me->child_key)
+      on_covered_change(st);
+   else
+      on_hedge_change(k, st);
 }
 
 void CoverAlgo::do_update(api::orderman::Update const &update) {
@@ -211,7 +244,7 @@ void CoverAlgo::update_definition(
        cover.cancel_on_external_hedge_cancel();
 }
 
-void CoverAlgo::check_terminate() {
+void CoverAlgo::terminate_if_hedges_complete() {
    LOG(OrderMan, Diag,
        "CoverAlgo {} check_terminate. State: {} Hedge count: {}",
        algo().order_key(), log::CharPtrWrapper{status_name(me->status)},
@@ -220,15 +253,61 @@ void CoverAlgo::check_terminate() {
    if (me->status == Status::WAITING_FOR_HEDGES && me->hedges.empty()) {
       LOG(OrderMan, Info, "CoverAlgo {}. Last hedge completed, terminating",
           algo().order_key().value());
+      on_done();
       algo().complete();
    }
 }
 
 IAlgoOrder::OnCancelResponse CoverAlgo::on_cancel() {
+   switch (me->status) {
+   case Status::WORKING: {
+      algo().cancel_child(me->child_key);
+      LOG(OrderMan, Info,
+          "CoverAlgo {} received cancel. Canceling child and waiting for an "
+          "ack",
+          algo().order_key());
+      me->status = Status::CANCELING;
+      return IAlgoOrder::OnCancelResponse::IGNORE;
+   }
+   case Status::CANCELING:
+      return IAlgoOrder::OnCancelResponse::IGNORE;
+   case Status::WAITING_FOR_HEDGES:
+      maybe_detach_hedges();
+      return IAlgoOrder::OnCancelResponse::HONOR;
+   case Status::DONE:
+      bts_assert(false);
+      return IAlgoOrder::OnCancelResponse::HONOR;
+   }
+
+   bts_assert(false);
+   return IAlgoOrder::OnCancelResponse::HONOR;
+}
+
+void CoverAlgo::on_done()
+{
+   me->status = Status::DONE;
+   maybe_detach_hedges();
+}
+
+
+void CoverAlgo::maybe_detach_hedges() {
    if (!me->cancel_hedges_on_cancel)
       for (auto &k : me->hedges)
          algo().detach_child(k);
-   return IAlgoOrder::OnCancelResponse::HONOR;
+}
+
+const char *CoverAlgo::status_name(Status st) {
+   switch (st) {
+   case Status::WORKING:
+      return "WORKING";
+   case Status::WAITING_FOR_HEDGES:
+      return "WAITING_FOR_HEDGES";
+   case Status::DONE:
+      return "DONE";
+   case Status::CANCELING:
+      return "CANCELING";
+   }
+   return nullptr;
 }
 
 } // namespace orderman

src/lib/orderman/orders/algo/algo_orders/CoverAlgo.h

@@ -30,8 +30,13 @@ private:
                                api::orderman::CoverOrderDefinition const &);
    void init_hedge_template(IBeforeLaunchAlgoServices &,
                             api::orderman::CustomOrderDefinition const &);
-   void check_terminate();
+   void terminate_if_hedges_complete();
+
+   void on_hedge_change(OrderKey hedge, ChildState const&) ;
+   void on_covered_change(ChildState const&); 
    const char *status_name(Status st);
+   void maybe_detach_hedges();
+   void on_done();
 };
 
 } // namespace orderman

src/lib/orderman/orders/algo/algo_orders/IcebergAlgo.cpp

@@ -145,7 +145,7 @@ void IcebergAlgo::on_child_change(OrderKey k, ChildState const &st) {
          on_timer();
          return;
       case ChildStatus::CANCELED:
-         algo().terminate_error("Unexpected child cancel");
+         algo().cancel("Unexpected child cancel");
          return;
       }
    }

src/lib/orderman/orders/algo/algo_orders/OffsetAlgo.cpp

@@ -0,0 +1,190 @@
+#include "lib/orderman/orders/algo/IAlgoServices.h"
+#include "lib/orderman/orders/algo/algo_orders/OffsetAlgo.h"
+#include "lib/orderman/order_factory/IOrderFactory.h"
+#include "lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h"
+#include "lib/os/os.h"
+#include "lib/utils/PBUtils.h"
+
+namespace bts {
+namespace orderman {
+
+using namespace api::orderman;
+using namespace std;
+
+enum class OffsetAlgo::Status { WAITING_FOR_MD, DEPLOYED };
+
+struct OffsetAlgo::PImpl {
+   IAlgoServices *algo;
+   Status status = Status::WAITING_FOR_MD;
+   OrderKey child_key = null_order_key();
+   ChildTemplate *child_template = nullptr;
+   Book last_book;
+   uint32_t tick_offset_on_deploy;
+   double child_price = os::nan();
+};
+
+OffsetAlgo::OffsetAlgo() : me(new PImpl()) {}
+OffsetAlgo::~OffsetAlgo() {}
+IAlgoServices &OffsetAlgo::algo() const { return *me->algo; }
+
+OrderOpResult OffsetAlgo::do_before_launch(IBeforeLaunchAlgoServices &serv) {
+   auto &def = serv.definition();
+   bts_assert(def.has_custom_def());
+   bts_assert(def.custom_def().has_offset());
+   // Validate the offset part of the definition
+   auto &offset = def.custom_def().offset();
+   if (!offset.has_child() &&
+       OrderKey(def.attach_target_order_key()) == null_order_key()) {
+      return OrderOpResult{
+          ORDER_ERROR_INVALID_ORDER_DEFINITON,
+          "Offset order must have either a child order definition or an "
+          "order to attach to"};
+   }
+   me->tick_offset_on_deploy = offset.tick_offset_on_deploy();
+   if (!serv.is_attaching()) {
+      // if we are not attaching create a child template so we can
+      // quickly launch our child when the time comes.
+      auto child_def = def;
+      *child_def.mutable_custom_def() = def.custom_def().offset().child();
+      me->child_template = &(serv.make_child_template(child_def));
+   }
+   return {};
+}
+
+void OffsetAlgo::do_launch(IAlgoServices &algo) {
+   me->algo = &algo;
+   bts_assert(me->child_template);
+   algo.subscribe_book(SecurityKey(algo.security().id()));
+}
+
+void OffsetAlgo::do_launch_attached(IAlgoServices &algo, OrderKey k,
+                                    ChildState st) {
+   me->algo = &algo;
+   me->child_key = k;
+   me->child_price = st.price.value();
+   algo.subscribe_book(SecurityKey(algo.security().id()));
+}
+
+void OffsetAlgo::do_update(api::orderman::Update const &update) {
+   LOG(OrderMan, Diag, "OffsetAlgo order {} receiving update: {}",
+       algo().order_key().value(), utils::pb::to_string(update));
+   if (update.has_custom()) {
+      if (!update.custom().has_offset()) {
+         algo().terminate_error("Invalid update");
+         return;
+      }
+      me->tick_offset_on_deploy =
+          update.custom().offset().tick_offset_on_deploy();
+   }
+   switch (me->status) {
+   case Status::WAITING_FOR_MD:
+      if (update.has_custom()) {
+         if (algo().is_attaching()) {
+            // don't change the child's price
+            // (need md to determine initial deploy price)
+            bts_assert(!os::is_nan(me->child_price));
+            algo().update_child(me->child_key, mk_child_update(update, me->child_price));
+         } else {
+            auto &def = algo().definition();
+            auto child_def = def;
+            *child_def.mutable_custom_def() = update.custom().offset().child();
+            me->child_template = &(algo().make_child_template(child_def));
+         }
+      }
+      break;
+   case Status::DEPLOYED:
+      algo().update_child(me->child_key, mk_child_update(update));
+      break;
+   }
+}
+
+void OffsetAlgo::on_book(SecurityKey, Book const &b) {
+   me->last_book = b;
+   // log top of book
+   LOG(OrderMan, Diag, "OffsetAlgo order {} on book. BID: {}@{} ASK: {}@{}",
+       algo().order_key().value(), b.side(api::SIDE_BUY)[0].size(),
+       b.side(api::SIDE_BUY)[0].price(), b.side(api::SIDE_SELL)[0].size(),
+       b.side(api::SIDE_SELL)[0].price());
+   // attempt deploy
+   if (me->status == Status::WAITING_FOR_MD)
+      try_deploy();
+}
+
+Update OffsetAlgo::mk_child_update(const Update &source, const double &price_override) {
+   Update target;
+   if (source.has_custom()) {
+      bts_assert(source.custom().has_offset());
+      *target.mutable_custom() = source.custom().offset().child();
+   }
+   if (!os::is_nan(price_override))
+      target.set_price(price_override);
+   else
+      target.set_price(source.price());
+   target.set_quantity(source.quantity());
+   return target;
+}
+
+void OffsetAlgo::try_deploy() {
+   bts_assert(me->status == Status::WAITING_FOR_MD);
+   auto side = algo().definition().side();
+   auto &top_psz = me->last_book.side(side)[0];
+   if (!top_psz.size()) {
+      // log state (still waiting for md)
+      LOG(OrderMan, Diag,
+          "OffsetAlgo order {} still waiting for md.  Order: {} {}@ {} ticks "
+          "away.",
+          algo().order_key().value(), algo().definition().side(),
+          algo().quantity().value(), me->tick_offset_on_deploy);
+
+   } else {
+      const double &offset = securities::tick_size(algo().security()).value() *
+                             me->tick_offset_on_deploy;
+      me->child_price =
+          securities::worsen_price(side, Price(top_psz.price()), Price(offset)).value();
+      if (algo().is_attaching()) {
+         // log imminent update
+         LOG(OrderMan, Diag,
+             "OffsetAlgo order {} attaching child {}.  Order: {} {}@ {} ({} ticks "
+             "away).",
+             algo().order_key().value(),
+             me->child_key.value(), side,
+             algo().quantity().value(),
+             me->child_price, me->tick_offset_on_deploy);
+         // update
+         Update child_update;
+         child_update.set_quantity(algo().quantity().value());
+         child_update.set_price(me->child_price);
+         algo().update_child(me->child_key, child_update);
+      } else {
+         // log imminent deploy
+         LOG(OrderMan, Diag,
+             "OffsetAlgo order {} deploying child.  Order: {} {}@ {} ({} ticks "
+             "away).",
+             algo().order_key().value(), side,
+             algo().quantity().value(),
+             me->child_price, me->tick_offset_on_deploy);
+         // deploy
+         me->child_key =
+             algo().launch_child(*me->child_template, Price(me->child_price), algo().quantity());
+      }
+      algo().set_ladder_visibility(false);
+      me->status = Status::DEPLOYED;
+   }
+}
+
+void OffsetAlgo::on_child_change(OrderKey, ChildState const &st) {
+   me->child_price = st.price.value();
+   algo().update_fill_state(st.total_fills, st.last_price);
+   switch (st.status) {
+   case ChildStatus::WORKING:
+      return;
+   case ChildStatus::COMPLETED:
+      algo().complete();
+      return;
+   case ChildStatus::CANCELED:
+      algo().cancel("Unexpected child cancel");
+      return;
+   }
+}
+} // namespace orderman
+} // namespace bts

src/lib/orderman/orders/algo/algo_orders/OffsetAlgo.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include "lib/orderman/orders/algo/IAlgoOrder.h"
+
+using Update = bts::api::orderman::Update;
+
+namespace bts {
+namespace orderman {
+
+// Offset sends a child-order/algo N ticks away from the top of the book.
+// If necessary, it will wait for the market to arrive before shooting the
+// child.
+
+class OffsetAlgo : public IAlgoOrder {
+ public:
+   OffsetAlgo();
+   ~OffsetAlgo();
+
+   OrderOpResult do_before_launch(IBeforeLaunchAlgoServices &) override;
+   void do_launch(IAlgoServices &) override;
+   void do_launch_attached(IAlgoServices &, OrderKey, ChildState) override;
+   void do_update(Update const &) override;
+   void on_book(SecurityKey, Book const &) override;
+   void on_child_change(OrderKey, ChildState const &) override;
+
+ private:
+   enum class Status;
+   struct PImpl;
+   std::unique_ptr<PImpl> me;
+
+   static Update mk_child_update(const Update& source_update, const double& price_override = os::nan());
+
+   IAlgoServices &algo() const;
+   void try_deploy();
+};
+
+} // namespace orderman
+} // namespace bts

src/lib/orderman/orders/algo/algo_orders/SplitterAlgo.cpp

@@ -1,259 +0,0 @@
-#include "SplitterAlgo.h"
-#include "lib/api/services/OrderService.pb.h"
-#include "lib/domain_model/Security.h"
-#include "lib/logging/logging.h"
-#include "lib/orderman/order_factory/IOrderFactory.h"
-#include "lib/orderman/orderman_utils/orderman_utils.h"
-#include "lib/orderman/orders/algo/IAlgoServices.h"
-
-#include <algorithm>
-#include <tuple>
-
-namespace bts {
-namespace orderman {
-namespace splitter {
-
-using namespace api::orderman;
-
-struct ChildOrderState {
-   OrderKey key;
-   ChildState state;
-   uint32_t min_qty() const { return state.total_fills.value(); }
-};
-
-struct SplitterAlgo::Impl {
-   IAlgoServices* algo = nullptr;
-   ChildTemplate* child_template = nullptr;
-
-   uint32_t clip = 0;
-   uint32_t exposed_qty = 0;
-   uint32_t target_qty = 0;
-
-   bool is_terminated = false;
-   Quantity total_fills = {};
-   Price last_fill_price = {};
-
-   std::vector<ChildOrderState> orders;
-};
-
-SplitterAlgo::SplitterAlgo() : me(new Impl()) { }
-SplitterAlgo::~SplitterAlgo() {}
-
-OrderOpResult SplitterAlgo::do_before_launch(IBeforeLaunchAlgoServices& ibl) {
-   // Validate and store parameters
-
-   auto& def = ibl.definition();
-   bts_assert(def.has_custom_def());
-   bts_assert(def.custom_def().has_splitter());
-   auto& splitter = def.custom_def().splitter();
-
-   me->clip = splitter.max_order_qty();
-   me->exposed_qty = splitter.exposed_qty();
-   if (me->clip <= 0 || me->exposed_qty <= 0) {
-      return OrderOpResult{ORDER_ERROR_INVALID_ORDER_DEFINITON,
-                           "max order quantity and exposed quantity must be greater than 0"};
-   }
-
-   if (me->exposed_qty < me->target_qty) {
-      return OrderOpResult{ORDER_ERROR_INVALID_ORDER_DEFINITON,
-                           "Exposed quantity must be greater than or equal to target quantity"};
-   }
-
-   update_child_template(ibl);
-
-   me->orders.reserve(me->exposed_qty / me->clip + 1);
-
-   if (ibl.is_attaching()) {
-      return OrderOpResult{ORDER_ERROR_OP_NOT_SUPPORTED,
-                           "Splitter does not support attach."};
-   }
-
-   return {};
-}
-
-void SplitterAlgo::do_launch(IAlgoServices& ias) {
-   me->algo = &ias;
-   me->target_qty = ias.quantity().value();
-   // launches initial child orders
-   sync_orders();
-}
-void SplitterAlgo::do_launch_attached(IAlgoServices &, OrderKey, ChildState) {
-   // rejected in before_launch
-   bts_assert(false);
-}
-void SplitterAlgo::do_update(const api::orderman::Update& update) {
-   const CustomOrderDefinition* new_child_def = nullptr;   
-   if (update.has_custom() && update.custom().has_splitter()) {
-      auto& splitter = update.custom().splitter();
-      if (splitter.max_order_qty() > 0) {
-         me->clip = splitter.max_order_qty();
-      }
-      me->exposed_qty = splitter.exposed_qty();
-
-      if (splitter.has_child()) {
-         update_child_template(algo());
-         new_child_def = &splitter.child();
-      }
-   }
-   me->target_qty = update.quantity();
-
-   if (me->exposed_qty < me->target_qty) {
-      algo().cancel("Exposed quantity must be greater than or equal to target quantity");
-      return;
-   }
-
-   sync_orders(new_child_def);
-}
-void SplitterAlgo::on_child_change(OrderKey key, ChildState const & state) {
-   for (auto& child : me->orders) {
-      if (child.key == key) {
-         if (state.total_fills < child.state.total_fills) {
-            algo().cancel("Unexpected fill reduction in fill quantityt");
-            return;
-         }
-         auto delta = state.total_fills.value() - child.state.total_fills.value();
-         if (delta != 0) {
-            me->total_fills = Quantity { me->total_fills.value() + delta };
-            me->last_fill_price = state.last_price;
-            algo().update_fill_state(me->total_fills, me->last_fill_price);
-         }
-         child.state = state;
-         break;
-      }
-   }
-
-   sync_orders();
-}
-
-// TEST case:
-// total = 300, clip = 100
-// constraints: 50,50,50,50
-void compute_desired_sizes(uint32_t total, uint32_t clip, const std::vector<MinMax>& constraints, std::vector<Quantity>* results) {
-   bts_assert(results);
-   results->clear();
-
-   uint32_t irreducible = 0;
-   for (const auto& mm : constraints) irreducible += mm.min;
-
-   // total > overfills ?
-
-   auto citer = constraints.begin();
-   while(total > 0) {
-      auto current = std::min(total, clip);
-      // fit to constraint
-      if (citer != constraints.end()) {
-         irreducible -= citer->min;
-         if (current > citer->max) current = citer->max;
-         if (current < citer->min) current = citer->min;
-
-         // I can't be higher than (total - irreducible)
-         if (current > (total - irreducible)) current = (total-irreducible);
-         citer++;
-      }
-      results->push_back(Quantity{current});
-      total -= current;
-   }
-}
-
-void get_constraints(const std::vector<ChildOrderState>& orders, std::vector<MinMax>* constraints) {
-   bts_assert(constraints);
-   for (const auto& order : orders) {
-      //Note: to maintain queue position set max to current quantity.  This will cause more orders to be added to the end if size increases
-      constraints->push_back(MinMax {
-         order.state.total_fills.value(),
-         order.state.status == ChildStatus::WORKING ?
-            std::numeric_limits<uint32_t>::max() : order.state.total_fills.value()
-      });
-   }
-}
-
-bool SplitterAlgo::is_complete() const {
-   uint32_t fills = 0;
-   for (const auto& o : me->orders) { fills += o.state.total_fills.value(); }
-   return fills >= me->target_qty;
-}
-
-void SplitterAlgo::terminate() const {
-   algo().complete();
-   //?? Do we need to clean up children?
-   for (const auto& o : me->orders) {
-      if (o.state.status == ChildStatus::WORKING) {
-         algo().cancel_child(o.key);
-      }
-   }
-   me->is_terminated = true;
-}
-   
-void SplitterAlgo::update_child_template(IBeforeLaunchAlgoServices& ibl) {
-   // Create child template
-   auto child_def = ibl.definition();
-   bts_assert(child_def.has_custom_def());
-   bts_assert(child_def.custom_def().has_splitter());
-   auto& splitter = child_def.custom_def().splitter();
-   *child_def.mutable_custom_def() = splitter.child();
-   me->child_template = &ibl.make_child_template(child_def);
-}
-
-void SplitterAlgo::sync_orders(const api::orderman::CustomOrderDefinition* new_child_def) {
-   if (me->is_terminated) return;
-   if (is_complete()) {
-      terminate();
-      return;
-   }
-
-   std::sort(me->orders.begin(), me->orders.end(), [](auto& a, auto b) { return b.min_qty() < a.min_qty(); });
-   std::vector<MinMax> constraints;
-   get_constraints(me->orders, &constraints);
-
-   std::vector<Quantity> desired;
-   compute_desired_sizes(me->exposed_qty, me->clip, constraints, &desired);
-
-   auto oiter = me->orders.begin();
-   // iterate to the current number of orders, don't consider orders that we're adding
-   auto order_count = me->orders.size();
-   auto price = algo().price();
-   for (const auto& qty : desired) {
-      if (order_count == 0) {
-         // Ran out of existing orders, time to add more
-         auto key = algo().launch_child(*me->child_template, price, qty);
-         // Assume the order begins successfully. Hopefully we get back some sort of NACK otherwise
-         me->orders.push_back({ key, {} });
-         me->orders.back().state.price = price;
-         me->orders.back().state.quantity = Quantity { qty };
-         me->orders.back().state.status = ChildStatus::WORKING;
-      }
-      else {
-         // Check the current order's quantity and modify it to match desired qty
-         if (oiter->state.status == ChildStatus::WORKING) {
-            if (oiter->state.price != price ||
-               oiter->state.quantity != qty ||
-               new_child_def != nullptr) {
-               Update update;
-               update.set_quantity(qty.value());
-               update.set_price(price.value());
-               if (new_child_def != nullptr) {
-                  *update.mutable_custom() = *new_child_def;
-               }
-               algo().update_child(oiter->key, update);
-            }
-         }
-         oiter++;
-         order_count -= 1;
-      }
-   }
-   // cancel any orders past desired
-   if (desired.size() < me->orders.size()) {
-      for (size_t x = desired.size(); x < me->orders.size(); ++x) {
-         auto& order= me->orders[x];
-         if (order.state.status == ChildStatus::WORKING) {
-            order.state.status = ChildStatus::CANCELED;
-            algo().cancel_child(order.key);
-         }
-      }
-   }
-
-}
-
-IAlgoServices& SplitterAlgo::algo() const { bts_assert(me->algo); return *me->algo; }
-
-}}} //namespace

src/lib/orderman/orders/algo/algo_orders/SplitterAlgo.h

@@ -1,60 +0,0 @@
-#pragma once 
-#include "lib/orderman/orders/algo/IAlgoOrder.h"
-
-namespace bts { 
-namespace orderman { 
-namespace splitter {
-
-/*
-   SplitterAlgo
-
-   Parameters:
-      max_order_qty 
-         -- Clip size. No single order can exceed this quantity.
-      exposed_qty 
-         -- The total quantity we want visible on the exchange.
-         -- This will typically be higher than the quantity which helps capture fills in pro-rata algorithms
-      price, quantity 
-         -- Common to all orders.
-         -- Splitter is filled when it reaches (or exceeds) quantity not exposed_qty
-   
-   SplitterAlgo acts as a single order to its parent but actually divides the total exposure
-   by the clip size and working several orders at once.
-   - It completes when the quantity is matched or exceeded.
-   - It tries to maximize the size of the orders and it may sacrifice queue position to do that.
-   - It reacts to fills and parameter changes and adjusts it's children appropriately.
-   - See splitter_tests.cpp for examples
-   - Future work: By constraining maximum size we can force this algorithm to maintain queue position.
-
-*/
-class SplitterAlgo : public IAlgoOrder
-{
-   struct Impl;
-   std::unique_ptr<Impl> me;
-
-public:
-   SplitterAlgo();
-   ~SplitterAlgo();
-
-   OrderOpResult do_before_launch(IBeforeLaunchAlgoServices &) override;
-   void do_launch(IAlgoServices &) override;
-   void do_launch_attached(IAlgoServices &, OrderKey, ChildState) override;
-   void do_update(api::orderman::Update const &) override;
-   void on_child_change(OrderKey, ChildState const &) override;
-
-private:
-   IAlgoServices& algo() const;
-
-   void sync_orders(const api::orderman::CustomOrderDefinition* new_child_def = nullptr);
-   bool is_complete() const;
-   void terminate() const;
-
-   void update_child_template(IBeforeLaunchAlgoServices&);
-};
-
-// exposed for unit testing purposes.
-struct MinMax {uint32_t min, max;};
-void compute_desired_sizes(uint32_t , uint32_t ,
-                           const std::vector<MinMax>& , std::vector<Quantity>* );
-
-}}} //namespace 

src/lib/orderman/orders/algo/algo_orders/TopAlgo.cpp

@@ -16,6 +16,7 @@ using namespace api::orderman;
 struct TopAlgo::PImpl {
    IAlgoServices *algo;
    spark::QuantityMeasure top_threshold_measure;
+   uint32_t ticks_off = 0;
    OrderKey child_key = null_order_key();
    ChildTemplate *child_template = nullptr;
    Book last_book;
@@ -41,7 +42,7 @@ OrderOpResult TopAlgo::do_before_launch(IBeforeLaunchAlgoServices &serv) {
    }
 
    me->top_threshold_measure = top.top_threshold();
-
+   me->ticks_off = top.ticks_off();
    if (!serv.is_attaching()) {
       // if we are not attaching create a child template so we can
       // quickly launch our child when the time comes.
@@ -87,13 +88,14 @@ void TopAlgo::do_update(api::orderman::Update const &update) {
       auto &top_update = update.custom().top();
       *child_update.mutable_custom() = top_update.child();
       me->top_threshold_measure = top_update.top_threshold();
+      me->ticks_off = top_update.ticks_off();
    }
    algo().update_child(me->child_key, child_update);
 }
 
 void TopAlgo::on_book(SecurityKey, Book const &b) {
    me->last_book = b;
-   ensure_min_quantity();
+   ensure_order_meets_constraints();
 }
 
 void TopAlgo::on_child_change(OrderKey, ChildState const &st) {
@@ -111,14 +113,14 @@ void TopAlgo::on_child_change(OrderKey, ChildState const &st) {
    }
 }
 
-bool TopAlgo::ensure_min_quantity() {
+bool TopAlgo::ensure_order_meets_constraints() {
    LOG(OrderMan, Diag,
        "Top order {} received book update. Current top of book: {}@{}",
        algo().order_key().value(), top_of_book_quantity().value(),
        top_of_book_price().value());
 
    if (better_than_top()) {
-      LOG(OrderMan, Diag,
+      LOG(OrderMan, Info,
           "Top order {} became better than the top of the book, canceling",
           algo().order_key().value());
       algo().cancel("Order would become BBO");
@@ -135,7 +137,7 @@ bool TopAlgo::ensure_min_quantity() {
           algo().order_key().value(), total_min_quantity,
           top_of_book_quantity().value());
       if (top_of_book_quantity() < Quantity(total_min_quantity)) {
-         LOG(OrderMan, Diag,
+         LOG(OrderMan, Info,
              "Top order {} is canceling because min quantiy is not met",
              algo().order_key().value());
          algo().cancel("Top of book is below threshold");
@@ -147,6 +149,26 @@ bool TopAlgo::ensure_min_quantity() {
           algo().order_key().value(), algo().price().value(),
           top_of_book_price().value());
 
+   if (me->ticks_off > 0) {
+      auto tick_diff = securities::price_diff_to_ticks(
+          algo().price().value(), top_of_book_price().value(),
+          securities::tick_size(algo().security()).value());
+      if (std::abs(tick_diff) <= (int)me->ticks_off) {
+         LOG(OrderMan, Info,
+             "Top order {} price {} is less than {} ticks away from top ({}). "
+             "Canceling",
+             algo().order_key().value(), algo().price().value(),
+             me->ticks_off + 1, top_of_book_price().value());
+         algo().cancel("Order is too close to top of the book");
+         return false;
+      } else
+         LOG(OrderMan, Diag,
+             "Top order {} price {} is {} ticks away from top. Top: {} Ticks "
+             "off: {}",
+             algo().order_key().value(), algo().price().value(),
+             std::abs(tick_diff), top_of_book_price().value(), me->ticks_off);
+   }
+
    return true;
 }
 

src/lib/orderman/orders/algo/algo_orders/TopAlgo.h

@@ -27,7 +27,7 @@ class TopAlgo : public IAlgoOrder {
    bool at_top_of_book() const;
    bool better_than_top() const;
    Quantity min_quantity_at_top() const;
-   bool ensure_min_quantity();
+   bool ensure_order_meets_constraints();
    bool has_running_child() const;
    Price top_of_book_price() const;
    Quantity top_of_book_quantity() const;

src/lib/orderman/orders/algo/algo_orders/tests/AlgoTestDriver.cpp

@@ -1,378 +0,0 @@
-#include "AlgoTestDriver.h"
-#include "lib/md/relay/RelayRecorder.h"
-#include "catch/catch.hpp"
-#include "lib/api/services/OrderService.pb.h"
-#include "lib/orderman/orderman_utils/Book.h"
-#include "lib/logging/logging.h"
-
-#include <optional>
-#include <queue>
-
-namespace bts {
-namespace orderman {
-namespace test {
-using namespace api::orderman;
-
-constexpr bool has_flag(ATDFlags prop, ATDFlags flag) {
-   return ((uint32_t)prop & (uint32_t)flag) == (uint32_t)flag;
-}
-
-const std::map<SecurityKey, api::Security>& get_some_securities() {
-   static std::map<SecurityKey, api::Security> results;
-   if (results.empty()) {
-      md::get_securities_from_replay([&](const api::Security& s) {
-         results.insert(std::make_pair(SecurityKey(s.id()), s));
-      });
-   }
-   return results;
-}
-
-
-struct RecursionCounter {
-   size_t depth = 0;
-   virtual void flush() = 0;
-   virtual ~RecursionCounter() {}
-};
-struct RecursionGuard {
-   RecursionGuard(RecursionCounter& c) : counter(c) {
-      counter.depth += 1;
-      REQUIRE(counter.depth < 100);
-   }
-   ~RecursionGuard() { 
-      counter.depth -= 1;
-      if (counter.depth == 0) { counter.flush(); }
-   }
-private:
-   RecursionCounter& counter;
-};
-
-struct AlgoTestDriverBase::Impl : public RecursionCounter {
-   IAlgoOrder& algo;
-   api::orderman::OrderDefinition definition;
-   Price price;
-   Quantity size;
-   ATDFlags flags;
-   TimeStamp now;
-   api::Security security;
-   OrderKey order_key, parent_key;
-   std::tuple<Quantity, Price> fill_state;
-
-   AlgoState state;
-   std::vector<ChildOrder> orders;
-
-   std::map<SecurityKey, Book> books;
-   std::map<SecurityKey, bool> book_subs;
-   bool sub_timer = false;
-
-   // Callbacks
-   std::vector<std::function<void(OrderKey)>> on_child_launching;
-   std::vector<std::function<void(const ChildOrder&)>> on_child_changing;
-   std::vector<std::function<void(const ChildOrder&, const ChildOrder&)>> on_child_changed;
-
-   Impl(IAlgoOrder& o) : algo(o) {}
-
-   OrderKey next_order() {
-      _next_order.value() += 1;
-      return _next_order;
-   }
-   RecursionGuard recusion_guard() { return {*this}; }
-
-   void flush() override;
-   bool flushing = false;
-   // child changes must be asyncronous
-   std::vector<std::tuple<OrderKey, ChildState>> deferred;
-   void defer_child_change(OrderKey k, ChildState cs) { deferred.push_back(std::make_tuple(k,cs)); }
-
-private:
-   OrderKey _next_order = OrderKey(99);
-};
-
-void AlgoTestDriverBase::Impl::flush() {
-   if(flushing) return;
-   flushing = true;
-   for (const auto& update : deferred) {
-      algo.on_child_change(std::get<OrderKey>(update), std::get<ChildState>(update));
-   }
-   deferred.clear();
-   flushing = false;
-}
-
-AlgoTestDriverBase::AlgoTestDriverBase(
-         IAlgoOrder& algo,
-         const api::orderman::OrderDefinition& def,
-         Price price, Quantity size,
-         ATDFlags flags)
-   : me(new Impl(algo)) {
-
-   me->definition = def;
-   me->price = price;
-   me->size = size;
-   me->flags = flags;
-   me->now = Timer::now();
-   me->state = AlgoState::CREATED;
-
-   auto securities = get_some_securities();
-   REQUIRE(securities.size() > 0);
-   me->security = securities.begin()->second;
-
-   me->order_key = me->next_order();
-   if (has_flag(flags, ATDFlags::HAS_PARENT)) {
-      me->parent_key = me->next_order();
-   }
-}
-
-AlgoTestDriverBase::~AlgoTestDriverBase() {}
-
-OrderOpResult AlgoTestDriverBase::before_launch() {
-   REQUIRE(me->state == AlgoState::CREATED);
-   me->state = AlgoState::READY;
-   return me->algo.do_before_launch(*this);
-}
-
-void AlgoTestDriverBase::launch() {
-   REQUIRE(me->state == AlgoState::READY);
-   me->algo.do_launch(*this);
-   me->state = AlgoState::RUNNING;
-}
-
-void AlgoTestDriverBase::launch_attached(OrderKey key, ChildState state) {
-   REQUIRE(me->state == AlgoState::READY);
-   me->algo.do_launch_attached(*this, key, state);
-   me->state = AlgoState::RUNNING;
-}
-void AlgoTestDriverBase::do_update(const api::orderman::Update& update) {
-   auto guard = me->recusion_guard();
-   REQUIRE(me->state == AlgoState::RUNNING);
-   me->algo.do_update(update);
-}
-
-void AlgoTestDriverBase::update_book(const Book& book) {
-   auto key = book.security_key();
-   me->books[key] = book;
-   auto iter = me->book_subs.find(key);
-   if (iter != me->book_subs.end()) {
-      me->algo.on_book(key, book);
-   }
-}
-void AlgoTestDriverBase::advance_time(Duration d) {
-   me->now += d;
-   if (me->sub_timer) {
-      me->algo.on_timer();
-   }
-}
-
-const std::vector<ChildOrder>& AlgoTestDriverBase::child_orders() const {
-   return me->orders;
-}
-
-AlgoState AlgoTestDriverBase::state() const {
-   return me->state;
-}
-
-void AlgoTestDriverBase::fill(OrderKey k, Price price, Quantity qty) {
-   auto guard = me->recusion_guard();
-   for (auto& child : me->orders) {
-      if (child.key == k) {
-         REQUIRE(child.cstate.quantity.value() - child.cstate.total_fills.value() >= qty.value());
-         for (const auto& cb : me->on_child_changing) cb(child);
-         auto old = child;
-
-         auto total_price = child.cstate.avg_price.value() * child.cstate.total_fills.value();
-         total_price += price.value() * qty.value();
-
-         child.cstate.last_fills = qty;
-         child.cstate.total_fills.value() += qty.value();
-         child.cstate.last_price = price;
-         child.cstate.avg_price = Price { total_price / child.cstate.total_fills.value() };
-
-         if (child.cstate.quantity.value() <= child.cstate.total_fills.value()) {
-            child.cstate.status = ChildStatus::COMPLETED;
-         }
-
-         for (const auto& cb : me->on_child_changed) cb(old, child);
-         // No defer because this is called from outside
-         if (child.subscribed)  me->algo.on_child_change(k, child.cstate);
-         break;
-      }
-   }
-}
-
-const ChildOrder* largest_order(const std::vector<ChildOrder>& orders) {
-   uint32_t sz = 0;
-   const ChildOrder* result = nullptr;
-   for (const auto& order : orders) {
-      auto available =
-         order.cstate.quantity.value() - order.cstate.total_fills.value();
-      if (order.cstate.status == ChildStatus::WORKING &&
-          available > sz) {
-         sz = available;
-         result = &order;
-      }
-   }
-   return result;
-}
-void AlgoTestDriverBase::fill(Price p, Quantity q) {
-   auto to_fill = q.value();
-   while(to_fill > 0) {
-      auto order = largest_order(this->child_orders());
-      REQUIRE(order);
-      auto available = (order->cstate.quantity.value() - order->cstate.total_fills.value());
-      if (available >= to_fill) {
-         this->fill(order->key, p, Quantity {to_fill});
-         return;
-      }
-      else {
-         this->fill(order->key, p, Quantity {available});
-         to_fill -= available;
-      }
-   }
-}
-
-void AlgoTestDriverBase::on_child_launching(std::function<void(OrderKey)>&& cb) {
-   me->on_child_launching.push_back(cb);
-}
-void AlgoTestDriverBase::on_child_changing(std::function<void(const ChildOrder&)>&& cb) {
-   me->on_child_changing.push_back(cb);
-}
-void AlgoTestDriverBase::on_child_changed(std::function<void(const ChildOrder&, const ChildOrder&)>&& cb) {
-   me->on_child_changed.push_back(cb);
-}
-
-//
-// Utility
-
-ChildOrder mkchild(OrderKey key, Price price, Quantity qty, bool sub) {
-   ChildOrder result = {};
-   result.key = key;
-   result.cstate.price = price;
-   result.cstate.quantity = qty;
-   result.subscribed = sub;
-   return result;
-}
-
-//
-// IBeforeLaunch
-
-const api::orderman::OrderDefinition& AlgoTestDriverBase::definition() const { return me->definition; }
-const api::Security& AlgoTestDriverBase::security() const { return me->security; }
-OrderKey AlgoTestDriverBase::order_key() const { return me->order_key; }
-OrderKey AlgoTestDriverBase::parent_order_key() const { return me->parent_key; }
-const api::Security* AlgoTestDriverBase::lookup_security(SecurityKey key) const {
-   auto s = get_some_securities();
-   auto iter = s.find(key);
-   if (iter != s.end()) {
-      return &iter->second;
-   }
-   return nullptr;
-}
-bool AlgoTestDriverBase::is_attaching() const { return has_flag(me->flags, ATDFlags::IS_ATTACHING); }
-ChildTemplate& AlgoTestDriverBase::make_child_template(const api::orderman::OrderDefinition&, size_t) {
-   ChildTemplate* t = nullptr;
-   return *t;
-}
-TimeStamp AlgoTestDriverBase::get_time() const { return me->now; }
-
-//
-// IAlgoServices
-
-Price AlgoTestDriverBase::price() const { return me->price; }
-Quantity AlgoTestDriverBase::quantity() const { return me->size; }
-Quantity AlgoTestDriverBase::total_fills() const {
-   return std::get<Quantity>(me->fill_state);
-}
-
-void AlgoTestDriverBase::subscribe_book(SecurityKey key){
-   me->book_subs[key] = true;
-   auto iter = me->books.find(key);
-   if (iter != me->books.end()) {
-      me->algo.on_book(key, iter->second);
-   }
-}
-void AlgoTestDriverBase::unsubscribe_book(SecurityKey key) {
-   me->book_subs.erase(key);
-}
-void AlgoTestDriverBase::subscribe_child_change(OrderKey k) {
-   for (auto& child : me->orders) {
-      if (child.key == k) {
-         child.subscribed = true;
-      }
-   }
-}
-void AlgoTestDriverBase::unsubscribe_child_change(OrderKey k) {
-   for (auto& child : me->orders) {
-      if (child.key == k) {
-         child.subscribed = false;
-      }
-   }
-}
-void AlgoTestDriverBase::subscribe_timer() { me->sub_timer = true; }
-void AlgoTestDriverBase::unsubscribe_timer() { me->sub_timer = false; }
-
-OrderKey AlgoTestDriverBase::launch_child(ChildTemplate&, Price price, Quantity qty, bool sub) {
-   OrderKey key = me->next_order();
-   me->orders.push_back(
-      mkchild(key, price, qty, sub));
-   for(const auto& cb : me->on_child_launching) cb(key);
-   return key;
-}
-OrderKey AlgoTestDriverBase::launch_child(api::orderman::OrderDefinition const & od, Price price, Quantity qty, bool sub) {
-   return launch_child(make_child_template(od), price, qty, sub);
-}
-void AlgoTestDriverBase::update_child(OrderKey key, api::orderman::Update const & update) {
-   auto guard = me->recusion_guard();
-   for (auto& child : me->orders) {
-      if (child.key == key) {
-         for (const auto& cb : me->on_child_changing) cb(child);
-         auto old = child;
-         if (update.price() != 0) {
-            child.cstate.price = Price(update.price());
-         }
-         if (update.quantity() != 0) {
-            child.cstate.quantity = Quantity(update.quantity());
-         }
-         if (child.cstate.quantity.value() <= child.cstate.total_fills.value()) {
-            child.cstate.status = ChildStatus::COMPLETED;
-         }
-         for (const auto& cb : me->on_child_changed) cb(old, child);
-
-         if (child.subscribed) {
-            me->defer_child_change(key, child.cstate);
-         }
-      }
-   }
-}
-void AlgoTestDriverBase::cancel_child(OrderKey key) {
-   auto guard = me->recusion_guard();
-   for (auto& child : me->orders) {
-      if (child.key == key) {
-         REQUIRE(child.cstate.status != ChildStatus::CANCELED);
-         for (const auto& cb : me->on_child_changing) cb(child);
-         auto before = child;
-         child.cstate.status = ChildStatus::CANCELED;
-         for (const auto& cb : me->on_child_changed) cb(before, child);
-         if(child.subscribed) {
-            me->defer_child_change(key, child.cstate);
-         }
-         break;
-      }
-   }
-}
-void AlgoTestDriverBase::detach_child(OrderKey key) {
-   std::remove_if(me->orders.begin(), me->orders.end(), [key](const auto& order) { return order.key == key; });
-}
-void AlgoTestDriverBase::detach_all_children() {
-   me->orders.clear();
-}
-void AlgoTestDriverBase::cancel(std::string const &) {
-   me->state = AlgoState::CANCELED;
-}
-void AlgoTestDriverBase::terminate_error(std::string const &) {
-   me->state = AlgoState::TERMINATED;
-}
-void AlgoTestDriverBase::complete() {
-   me->state = AlgoState::COMPLETED;
-}
-void AlgoTestDriverBase::set_ladder_visibility(bool) { /* whatever, ok */ }
-void AlgoTestDriverBase::update_fill_state(Quantity q, Price p) { me->fill_state = { q, p }; }
-
-}}}//namespace

src/lib/orderman/orders/algo/algo_orders/tests/AlgoTestDriver.h

@@ -1,140 +0,0 @@
-#pragma once
-#include "lib/orderman/orders/algo/IAlgoOrder.h"
-#include "lib/orderman/orders/algo/IAlgoServices.h"
-#include "lib/md/IntPrice.h"
-#include <memory>
-#include <vector>
-#include <string>
-
-namespace bts {
-namespace orderman {
-namespace test {
-
-const std::map<SecurityKey, api::Security>& get_some_securities();
-
-enum class ATDFlags {
-   NONE = 0,
-   IS_ATTACHING = 1,
-   HAS_PARENT = 2,
-};
-
-enum class AlgoState {
-   CREATED,
-   // before_lauh() ->
-   READY,
-   // launch() ->
-   RUNNING,
-   CANCELED,
-   TERMINATED,
-   COMPLETED
-};
-
-struct ChildOrder {
-   OrderKey key;
-   bool subscribed;
-   ChildState cstate;
-   uint32_t active_qty() const { 
-      return cstate.status == ChildStatus::WORKING ? 
-             cstate.quantity.value() - cstate.total_fills.value() : 0;
-   }
-};
-
-class AlgoTestDriverBase : private IAlgoServices {
-   struct Impl;
-   std::unique_ptr<Impl> me;
-
-   public:
-      AlgoTestDriverBase(
-         IAlgoOrder& algo, 
-         const api::orderman::OrderDefinition& def, 
-         Price price, Quantity size,
-         ATDFlags flags=ATDFlags::NONE);
-      ~AlgoTestDriverBase();
-
-   //
-   // Expose some things from IAlgoServices
-   const api::Security& security() const override;
-   Price price() const override;
-   Quantity quantity() const override;
-   Quantity total_fills() const override;
-   
-   //
-   // Change state
-
-   OrderOpResult before_launch();
-   void launch();
-   void launch_attached(OrderKey, ChildState);
-   void do_update(const api::orderman::Update&);
-
-   void update_book(const Book&);
-   void advance_time(Duration);
-
-   //
-   // Inspection methods
-   const std::vector<ChildOrder>& child_orders() const;
-   AlgoState state() const;
-
-   // Fills for this specific order
-   void fill(OrderKey, Price, Quantity);
-   // Fill this quantity, start with the largest child order
-   void fill(Price, Quantity);
-
-   void on_child_launching(std::function<void(OrderKey)>&&);
-   void on_child_changing(std::function<void(const ChildOrder&)>&&);
-   void on_child_changed(std::function<void(const ChildOrder&, const ChildOrder&)>&&);
-   
-   
-
-private:
-
-   //
-   // IBeforeLaunchAlgoServices
-   const api::orderman::OrderDefinition& definition() const override;
-   // const api::Security& security() const override;
-   OrderKey order_key() const override;
-   OrderKey parent_order_key() const override;
-   const api::Security* lookup_security(SecurityKey) const override;
-   bool is_attaching() const override;
-   ChildTemplate& make_child_template(const api::orderman::OrderDefinition& child_def, size_t reserve = 1) override;
-   TimeStamp get_time() const override;
-   
-   //
-   // IAlgoServices
-   void subscribe_book(SecurityKey)override;
-   void unsubscribe_book(SecurityKey) override;
-   void subscribe_child_change(OrderKey) override;
-   void unsubscribe_child_change(OrderKey) override;
-   void subscribe_timer() override;
-   void unsubscribe_timer() override;
-
-   OrderKey launch_child(ChildTemplate &, Price, Quantity, bool) override;
-   OrderKey launch_child(api::orderman::OrderDefinition const &, Price,
-                         Quantity, bool) override;
-   void update_child(OrderKey , api::orderman::Update const &) override; 
-   void cancel_child(OrderKey) override; 
-   void detach_child(OrderKey) override;
-   void detach_all_children() override;
-
-   void cancel(std::string const &) override;
-   void terminate_error(std::string const &) override;
-   void complete() override;
-   void set_ladder_visibility(bool) override;
-   void update_fill_state(Quantity , Price) override;
-
-};
-
-template <class Algo>
-struct AlgoTestDriver : public AlgoTestDriverBase 
-{
-   // Avoids mutual ownership problem
-   Algo _algo;
-   AlgoTestDriver(
-         const api::orderman::OrderDefinition& def, 
-         const md::IntPrice& price, Quantity size,
-         ATDFlags flags=ATDFlags::NONE)
-         : AlgoTestDriverBase(_algo, def, Price {price.price()}, size, flags), _algo()
-         { }
-   const Algo& algo() const { return _algo; }
-};
-
-}}}//namespace

src/lib/orderman/orders/algo/algo_orders/tests/splitter_tests.cpp

@@ -1,600 +0,0 @@
-
-#include "../SplitterAlgo.h"
-#include "AlgoTestDriver.h"
-#include "catch/catch.hpp"
-#include "lib/orderman/orderman_utils/Book.h"
-#include <ctime>
-#include <optional>
-#include <tuple>
-#include <variant>
-#include "lib/utils/bts_chrono.h"
-
-using namespace std;
-using namespace bts::orderman::splitter;
-using namespace bts::api::orderman;
-
-namespace bts {
-namespace orderman {
-namespace test {
-
-/*
-
-- max/exposed, assert correct # of orders on the market
-- completes on target_qty reached
-- completes on target_qty exceeded
-- send fills, check order state
-- send fills, then change target_qty
-
- */
-
-constexpr auto some_price = md::IntPrice{128000, -3};
-
-using ExpectedCount = utils::ValueWrapper<struct ExpectedCountTag, uint32_t>;
-struct SimpleCase {
-   ExpectedCount expected_orders;
-   Quantity target;
-   SplitterOrderDefinition sod;
-};
-
-SimpleCase mkSimple(ExpectedCount expected, uint32_t max_size, uint32_t exposed,
-                    uint32_t target) {
-   SimpleCase result;
-   result.expected_orders = expected;
-   result.target = Quantity{target};
-   result.sod.set_max_order_qty(max_size);
-   result.sod.set_exposed_qty(exposed);
-   return result;
-}
-
-TEST_CASE("compute_desired_sizes fuzzing", "[algo][splitter]") {
-   srand((unsigned)time(0));
-
-   auto start = Timer::now();
-   
-   while (sec_diff(Timer::now(),start) < 5) {
-      uint32_t total = rand() % 100 + 1;
-      uint32_t clip = rand() % 4 + 1;
-      std::vector<MinMax> constraints(rand() % 10);
-      std::vector<Quantity> result;
-
-      uint32_t all_fills = 0;
-      for (auto &mm : constraints) {
-         uint32_t order_working = rand() % 2;
-         uint32_t fills = rand() % 2 * clip;
-         all_fills += fills;
-         mm = {fills, order_working ? fills + rand() % clip + 1 : fills};
-      }
-      if (all_fills < total) {
-         compute_desired_sizes(total, clip, constraints, &result);
-         uint32_t exposed_qty = 0;
-         for (auto q : result)
-            exposed_qty += q.value();
-
-         bool clip_size_honored = true;
-         for (auto q : result)
-            clip_size_honored = clip_size_honored && (q.value() <= clip);
-          
-         if (exposed_qty != total || !clip_size_honored) {
-            std::cout << "total: " << total << "\n"
-                      << "clip: " << clip << "\n{ ";
-
-            for (auto mm : constraints)
-               std::cout << "{" << mm.min << "," << mm.max << "},";
-            std::cout << " }\n";
-         }
-         REQUIRE(clip_size_honored);
-         REQUIRE(exposed_qty == total);
-      }
-   }
-}
-
-TEST_CASE("initial orders", "[algo][splitter]") {
-   std::vector<SimpleCase> cases;
-   cases.push_back(mkSimple(ExpectedCount{10}, 10, 100, 50));
-   cases.push_back(mkSimple(ExpectedCount{11}, 10, 105, 50));
-   cases.push_back(mkSimple(ExpectedCount{5}, 10, 50, 50));
-   cases.push_back(mkSimple(ExpectedCount{5}, 10, 50, 150));
-
-   for (const auto &tcase : cases) {
-      auto expected = tcase.expected_orders.value();
-      api::orderman::OrderDefinition def;
-      def.set_side(api::SIDE_BUY);
-      *def.mutable_custom_def()->mutable_splitter() = tcase.sod;
-      AlgoTestDriver<SplitterAlgo> state(def, some_price, tcase.target);
-
-      REQUIRE(state.before_launch().ok());
-      state.launch();
-      REQUIRE(state.child_orders().size() == expected);
-      REQUIRE(state.state() == AlgoState::RUNNING);
-   }
-}
-
-TEST_CASE("completes on target", "[algo][splitter]") {
-   std::vector<SimpleCase> cases;
-   cases.push_back(mkSimple(ExpectedCount{3}, 30, 90, 50));
-   cases.push_back(mkSimple(ExpectedCount{10}, 10, 100, 50));
-   cases.push_back(mkSimple(ExpectedCount{11}, 10, 105, 50));
-   cases.push_back(mkSimple(ExpectedCount{5}, 10, 50, 50));
-   cases.push_back(mkSimple(ExpectedCount{5}, 10, 50, 150));
-   cases.push_back(mkSimple(ExpectedCount{4}, 13, 50, 150));
-   cases.push_back(mkSimple(ExpectedCount{12}, 13, 150, 50));
-
-   for (const auto &tcase : cases) {
-      auto expected = tcase.expected_orders.value();
-      api::orderman::OrderDefinition def;
-      def.set_side(api::SIDE_BUY);
-      *def.mutable_custom_def()->mutable_splitter() = tcase.sod;
-      AlgoTestDriver<SplitterAlgo> state(def, some_price, tcase.target);
-
-      REQUIRE(state.before_launch().ok());
-      state.launch();
-      REQUIRE(state.child_orders().size() == expected);
-      REQUIRE(state.state() == AlgoState::RUNNING);
-
-      SECTION("Fill completely") {
-         auto orders = state.child_orders();
-         // Fill the order
-         size_t filled = 0;
-         size_t iterations = 0;
-         while (state.state() == AlgoState::RUNNING) {
-            // loop to allow more orders to be added
-            ++iterations;
-            REQUIRE(iterations < 999);
-            for (const auto &order : orders) {
-               state.fill(order.key, Price(some_price.price()),
-                          order.cstate.quantity);
-               filled += order.cstate.quantity.value();
-               REQUIRE(filled == state.total_fills().value());
-               if (filled >= tcase.target.value()) {
-                  REQUIRE(state.state() == AlgoState::COMPLETED);
-                  break;
-               } else {
-                  REQUIRE(state.state() == AlgoState::RUNNING);
-               }
-            }
-         }
-      }
-      SECTION("fill one by one") {
-         const auto &orders = state.child_orders();
-         // Fill 1 per iteration.
-         // Algo should add more orders if not enough are visible initially
-         size_t filled = 0;
-         size_t iterations = 0;
-         while (state.state() == AlgoState::RUNNING) {
-            ++iterations;
-            REQUIRE(iterations < 999); // not making progress
-            for (const auto &order : orders) {
-               if (order.cstate.status == ChildStatus::WORKING) {
-                  auto price = Price{some_price.price()};
-                  state.fill(order.key, price, Quantity{1});
-                  filled += 1;
-                  REQUIRE(filled == state.total_fills().value());
-                  // Fills should never cause new orders to get added
-                  REQUIRE(state.child_orders().size() == expected);
-                  if (filled >= tcase.target.value()) {
-                     REQUIRE(state.state() == AlgoState::COMPLETED);
-                     break;
-                  } else {
-                     REQUIRE(state.state() == AlgoState::RUNNING);
-                  }
-               }
-            }
-         }
-      }
-   }
-}
-
-TEST_CASE("updates", "[algo][splitter]") {
-   // Start in state 0, update to state 1
-   std::vector<std::tuple<SimpleCase, std::vector<uint32_t>, SimpleCase>> cases;
-
-   // reduce max order size, increase # orders
-   cases.push_back({mkSimple(ExpectedCount{10} /*order count*/, 10 /*max*/,
-                             100 /*exposed*/, 50 /*target*/),
-                    {},
-                    mkSimple(ExpectedCount{20}, 5, 100, 50)});
-   /// reduce max order size, keep # orders constant
-   cases.push_back({mkSimple(ExpectedCount{10}, 10, 100, 50),
-                    {},
-                    mkSimple(ExpectedCount{10}, 5, 50, 50)});
-   cases.push_back({mkSimple(ExpectedCount{10}, 10, 100, 50),
-                    {5, 5},
-                    mkSimple(ExpectedCount{8}, 5, 50, 40)});
-   cases.push_back({mkSimple(ExpectedCount{10}, 10, 100, 50),
-                    {5, 5},
-                    mkSimple(ExpectedCount{18}, 5, 100, 50)});
-
-   for (const auto &tcase : cases) {
-      auto case_init = std::get<0>(tcase);
-      api::orderman::OrderDefinition def;
-      def.set_side(api::SIDE_BUY);
-      *def.mutable_custom_def()->mutable_splitter() = case_init.sod;
-      AlgoTestDriver<SplitterAlgo> state(def, some_price, case_init.target);
-
-      REQUIRE(state.before_launch().ok());
-      state.launch();
-      REQUIRE(state.state() == AlgoState::RUNNING);
-      REQUIRE(state.child_orders().size() == case_init.expected_orders.value());
-      for (const auto &o : state.child_orders()) {
-         if (o.cstate.status == ChildStatus::WORKING) {
-            REQUIRE(o.cstate.quantity.value() <= case_init.sod.exposed_qty());
-         }
-      }
-
-      uint32_t fills = 0;
-      for (auto q : std::get<1>(tcase)) {
-         state.fill({}, Quantity{q});
-         fills += q;
-      }
-
-      REQUIRE(state.total_fills().value() == fills);
-
-      Update update;
-      auto case_next = std::get<2>(tcase);
-      update.set_price(some_price.price());
-      update.set_quantity(case_next.target.value());
-      *def.mutable_custom_def()->mutable_splitter() = case_next.sod;
-      *update.mutable_custom() = def.custom_def();
-
-      state.do_update(update);
-      REQUIRE(state.state() == AlgoState::RUNNING);
-      size_t working = 0;
-      for (const auto &c : state.child_orders())
-         if (c.cstate.status == ChildStatus::WORKING)
-            working += 1;
-      REQUIRE(working == case_next.expected_orders.value());
-      for (const auto &o : state.child_orders()) {
-         if (o.cstate.status == ChildStatus::WORKING) {
-            REQUIRE(o.cstate.quantity.value() <= case_next.sod.exposed_qty());
-         }
-      }
-   }
-}
-TEST_CASE("Reduction with fills", "[algo][splitter]") {
-   // Fills represent a floor on how much a trade can be reduced
-   // IF we have expose:400, clip: 100 we get
-   // 100,100,100,100
-   // Receive fills
-   // 100/50, 100/50, 100/50, 100/50
-   // Change expose to 300
-   // naive: 100, 100, 100 but we *can't* reduce the last to 0
-   // expected: 100, 100, 50, 50
-   api::orderman::OrderDefinition def;
-   def.set_side(api::SIDE_BUY);
-   auto *splitter = def.mutable_custom_def()->mutable_splitter();
-   splitter->set_max_order_qty(100);
-   splitter->set_exposed_qty(400);
-   AlgoTestDriver<SplitterAlgo> state(def, some_price, Quantity{300});
-
-   REQUIRE(state.before_launch().ok());
-   state.launch();
-   // check initial state
-   REQUIRE(state.state() == AlgoState::RUNNING);
-   REQUIRE(state.child_orders().size() == 4);
-   for (const auto &c : state.child_orders()) {
-      REQUIRE(c.cstate.quantity.value() == 100);
-   }
-
-   // Add fills
-   for (const auto &order : state.child_orders()) {
-      state.fill(order.key, {}, Quantity{50});
-   }
-
-   // Reduce exposed quantity
-   Update update;
-   update.set_quantity(300);
-   update.mutable_custom()->mutable_splitter()->set_exposed_qty(300);
-   state.do_update(update);
-
-   REQUIRE(state.state() == AlgoState::RUNNING);
-   REQUIRE(state.child_orders().size() == 4);
-   size_t total = 0;
-   for (const auto &order : state.child_orders()) {
-      total += order.cstate.quantity.value();
-   }
-   REQUIRE(total == 300);
-
-   // need this to be 100,100,50,50 not 100,100,100
-   std::set<uint32_t> actual, expected{100, 100, 50, 50};
-   for (const auto &order : state.child_orders()) {
-      actual.insert(order.cstate.quantity.value());
-   }
-   REQUIRE(actual == expected);
-
-   REQUIRE(state.total_fills().value() == 200);
-}
-
-TEST_CASE("Reduction with fills 2", "[algo][splitter]") {
-   // Fills represent a floor on how much a trade can be reduced
-   // IF we have expose:400, clip: 100 we get
-   // 100,100,100,100
-   // Receive fills
-   // 100/35, 100/45, 100/55, 100/65
-   // Change expose to 300
-   // naive: 100, 100, 100 but we *can't* reduce the last to 0
-   // expected: 100, 100, 65, 35
-   api::orderman::OrderDefinition def;
-   def.set_side(api::SIDE_BUY);
-   auto *splitter = def.mutable_custom_def()->mutable_splitter();
-   splitter->set_max_order_qty(100);
-   splitter->set_exposed_qty(400);
-   AlgoTestDriver<SplitterAlgo> state(def, some_price, Quantity{300});
-
-   REQUIRE(state.before_launch().ok());
-   state.launch();
-   // check initial state
-   REQUIRE(state.state() == AlgoState::RUNNING);
-   REQUIRE(state.child_orders().size() == 4);
-   for (const auto &c : state.child_orders()) {
-      REQUIRE(c.cstate.quantity.value() == 100);
-   }
-
-   // Add fills
-   state.fill({}, Quantity{35});
-   state.fill({}, Quantity{45});
-   state.fill({}, Quantity{55});
-   state.fill({}, Quantity{65});
-
-   // Reduce exposed quantity
-   Update update;
-   update.set_quantity(300);
-   update.mutable_custom()->mutable_splitter()->set_exposed_qty(300);
-   state.do_update(update);
-
-   REQUIRE(state.state() == AlgoState::RUNNING);
-   REQUIRE(state.child_orders().size() == 4);
-   size_t total = 0;
-   for (const auto &order : state.child_orders()) {
-      total += order.cstate.quantity.value();
-   }
-   REQUIRE(total == 300);
-
-   // need this to be 100,100,50,50 not 100,100,100
-   std::set<uint32_t> actual, expected{100, 100, 65, 35};
-   for (const auto &order : state.child_orders()) {
-      actual.insert(order.cstate.quantity.value());
-   }
-   REQUIRE(actual == expected);
-}
-
-TEST_CASE("Reduction with fills 3", "[algo][splitter]") {
-   // Fills represent a floor on how much a trade can be reduced
-   // IF we have expose:400, clip: 100 we get
-   // 100,100,100,100
-   // Receive fills
-   // 100/35, 100/45, 100/55, 100/65
-   // Change expose to 201
-   // naive: 100, 100, 1 but we *can't* reduce the last to 0
-   // expected: 66, 55, 45, 35
-   api::orderman::OrderDefinition def;
-   def.set_side(api::SIDE_BUY);
-   auto *splitter = def.mutable_custom_def()->mutable_splitter();
-   splitter->set_max_order_qty(100);
-   splitter->set_exposed_qty(400);
-   AlgoTestDriver<SplitterAlgo> state(def, some_price, Quantity{300});
-
-   REQUIRE(state.before_launch().ok());
-   state.launch();
-   // check initial state
-   REQUIRE(state.state() == AlgoState::RUNNING);
-   REQUIRE(state.child_orders().size() == 4);
-   for (const auto &c : state.child_orders()) {
-      REQUIRE(c.cstate.quantity.value() == 100);
-   }
-
-   // Add fills
-   state.fill({}, Quantity{35});
-   state.fill({}, Quantity{45});
-   state.fill({}, Quantity{55});
-   state.fill({}, Quantity{65});
-
-   // Reduce exposed quantity
-   Update update;
-   update.set_quantity(201);
-   update.mutable_custom()->mutable_splitter()->set_exposed_qty(201);
-   state.do_update(update);
-
-   REQUIRE(state.state() == AlgoState::RUNNING);
-   REQUIRE(state.child_orders().size() == 4);
-   size_t total = 0;
-   for (const auto &order : state.child_orders()) {
-      total += order.cstate.quantity.value();
-   }
-   REQUIRE(total == 201);
-
-   std::set<uint32_t> actual, expected{66, 55, 45, 35};
-   for (const auto &order : state.child_orders()) {
-      actual.insert(order.cstate.quantity.value());
-   }
-   REQUIRE(actual == expected);
-}
-
-TEST_CASE("reduce to completion", "[algo][splitter]") {
-   // Fills represent a floor on how much a trade can be reduced
-   // IF we have expose:400, clip: 100 we get
-   // 100,100,100,100
-   // Receive fills
-   // 100/100, 100/100, 100/0, 100/00
-   // Change expose to 200
-   // expected: algo transitions to copmleted state canceling 2 orders
-   api::orderman::OrderDefinition def;
-   def.set_side(api::SIDE_BUY);
-   auto *splitter = def.mutable_custom_def()->mutable_splitter();
-   splitter->set_max_order_qty(100);
-   splitter->set_exposed_qty(400);
-   AlgoTestDriver<SplitterAlgo> state(def, some_price, Quantity{300});
-
-   REQUIRE(state.before_launch().ok());
-   state.launch();
-   // check initial state
-   REQUIRE(state.state() == AlgoState::RUNNING);
-   REQUIRE(state.child_orders().size() == 4);
-   for (const auto &c : state.child_orders()) {
-      REQUIRE(c.cstate.quantity.value() == 100);
-   }
-
-   // Add fills
-   state.fill({}, Quantity{100});
-   state.fill({}, Quantity{100});
-
-   // Reduce exposed quantity
-   Update update;
-   update.set_quantity(200);
-   def.mutable_custom_def()->mutable_splitter()->set_exposed_qty(200);
-   state.do_update(update);
-
-   REQUIRE(state.state() == AlgoState::COMPLETED);
-   size_t total = 0;
-   for (const auto &order : state.child_orders()) {
-      if (order.cstate.status == ChildStatus::WORKING) {
-         total += order.cstate.quantity.value();
-      } else {
-         total += order.cstate.total_fills.value();
-      }
-   }
-   REQUIRE(total == 200);
-}
-
-TEST_CASE("reduce to completion 2", "[algo][splitter]") {
-   // Fills represent a floor on how much a trade can be reduced
-   // IF we have expose:400, clip: 100 we get
-   // 100,100,100,100
-   // Receive fills
-   // 100/100, 100/100, 100/0, 100/00
-   // Change expose to 200
-   // expected: algo transitions to copmleted state canceling 2 orders
-   api::orderman::OrderDefinition def;
-   def.set_side(api::SIDE_BUY);
-   auto *splitter = def.mutable_custom_def()->mutable_splitter();
-   splitter->set_max_order_qty(100);
-   splitter->set_exposed_qty(400);
-   AlgoTestDriver<SplitterAlgo> state(def, some_price, Quantity{300});
-
-   REQUIRE(state.before_launch().ok());
-   state.launch();
-   // check initial state
-   REQUIRE(state.state() == AlgoState::RUNNING);
-   REQUIRE(state.child_orders().size() == 4);
-   for (const auto &c : state.child_orders()) {
-      REQUIRE(c.cstate.quantity.value() == 100);
-   }
-
-   // Add fills
-   state.fill({}, Quantity{100});
-   state.fill({}, Quantity{50});
-   state.fill({}, Quantity{50});
-
-   // Reduce exposed quantity
-   Update update;
-   update.set_quantity(200);
-   state.do_update(update);
-
-   REQUIRE(state.state() == AlgoState::COMPLETED);
-}
-
-TEST_CASE("Async Fills", "[algo][splitter]") {
-   // OK, NOW it gets tricky because we're going to test the asyncronous stuff
-
-   // Start with 4 orders size 20 outstanding (80 total exposed)
-   // Change exposure to 64 so one of the four will be reduced to 5
-   // Before the change is confirmed fill the one that is reducing by 7
-   // which would constitute an overfill
-   // Algo must reduce a second order to compensate
-
-   api::orderman::OrderDefinition def;
-   def.set_side(api::SIDE_BUY);
-   auto *splitter = def.mutable_custom_def()->mutable_splitter();
-   splitter->set_max_order_qty(20);
-   splitter->set_exposed_qty(80);
-   AlgoTestDriver<SplitterAlgo> state(def, some_price, Quantity{80});
-
-   REQUIRE(state.before_launch().ok());
-   state.launch();
-   // check initial state
-   REQUIRE(state.state() == AlgoState::RUNNING);
-   REQUIRE(state.child_orders().size() == 4);
-   for (const auto &c : state.child_orders()) {
-      REQUIRE(c.cstate.quantity.value() == 20);
-   }
-
-   // Set up some instant fills
-   bool applied_fills = false;
-   state.on_child_changing([&](const auto &) {
-      if (applied_fills)
-         return;
-      applied_fills = true;
-      // 7 fills on the last order which is has a size reduction in flight
-      state.fill(state.child_orders().rbegin()->key, {}, Quantity{7});
-   });
-
-   Update update;
-   update.set_price(state.price().value());
-   update.set_quantity(65);
-   update.mutable_custom()->mutable_splitter()->set_exposed_qty(65);
-   // This should trigger 1 order to be reduced to size 5 from 20
-   // in the meantime we'll trigger fills of size 7
-   // This results in an overfill for the 5 lot order
-   // And the
-   state.do_update(update);
-
-   // expected result:
-   // 7 fills, exposed_qty 65 = 58 active
-   uint32_t active = 0;
-   for (const auto &o : state.child_orders())
-      active += o.active_qty();
-   REQUIRE(active == 58);
-}
-
-TEST_CASE("Increase then decrease", "[algo][splitter]") {
-   // Start with 3 orders of size 100
-   // Increase the size to 400 so we launch a 4th order
-   // Then instantly send another update to 350
-
-   // NOTE: this test is not very good due to limitations in the test driver
-   // I'm leaving it in because it passes but if it starts failing in the future
-   // it's probably OK to disable it
-
-   api::orderman::OrderDefinition def;
-   def.set_side(api::SIDE_BUY);
-   auto *splitter = def.mutable_custom_def()->mutable_splitter();
-   splitter->set_max_order_qty(100);
-   splitter->set_exposed_qty(300);
-   AlgoTestDriver<SplitterAlgo> state(def, some_price, Quantity{300});
-
-   REQUIRE(state.before_launch().ok());
-   state.launch();
-   // check initial state
-   REQUIRE(state.state() == AlgoState::RUNNING);
-   REQUIRE(state.child_orders().size() == 3);
-   for (const auto &c : state.child_orders()) {
-      REQUIRE(c.cstate.quantity.value() == 100);
-   }
-
-   Update update;
-   update.set_price(state.price().value());
-   update.set_quantity(state.quantity().value());
-   update.mutable_custom()->mutable_splitter()->set_exposed_qty(400);
-
-   // Set up some instant fills
-   bool applied_second_update = false;
-   state.on_child_launching([&, update](const auto &) mutable {
-      if (!applied_second_update) {
-         applied_second_update = true;
-         update.mutable_custom()->mutable_splitter()->set_exposed_qty(350);
-         state.do_update(update);
-      }
-   });
-
-   state.do_update(update);
-
-   // expected result:
-   // 350 exposed, NOT 450
-   uint32_t quantity = 0;
-   for (const auto &o : state.child_orders())
-      quantity += o.active_qty();
-   REQUIRE(quantity == 350);
-}
-
-} // namespace test
-} // namespace orderman
-} // namespace bts

src/lib/orderman/orders/algo/doc/state_machine.org

@@ -0,0 +1,39 @@
+#+TITLE: Algo Wrapper State Machien
+#+OPTIONS: ^:nil
+#+OPTIONS: H:2
+#+OPTIONS: ^:nil H:2 author:nil creator:nil timestamp:nil toc:nil
+#+OPTIONS: html-postamble:nil
+
+
+* State Machine
+#+begin_src dot :file algo_wrapper_sm.png :cmdline -Kdot -Tpng
+digraph G {
+   bgcolor="transparent";
+   edge                       [ fontname=Helvetica, fontsize=12 ];
+
+   start                      [style=invisible];
+   COMPLETED                  [style=filled] ;
+   start -> INACTIVE;
+   INACTIVE -> COMPLETED      [label="qty=0|error"];
+   INACTIVE -> W_ATTACH       [label="ok & attach"];
+   INACTIVE -> WORKING        [label="ok & !attach"]; 
+   W_ATTACH -> COMPLETED      [label=attach_fail];
+   W_ATTACH -> WORKING        [label=attach_ok];
+   WORKING  -> COMPLETED      [label="[wchild=0]cancel|complete"];
+   WORKING  -> COMPLETED      [label=error];
+   WORKING  -> SHUTTING_DOWN  [label="[wchild>0]cancel|complete"];
+   SHUTTING_DOWN -> COMPLETED [label="wchild==0|error|on_detach"];
+}
+#+END_SRC
+
+#+RESULTS:
+[[file:algo_wrapper_sm.png]]
+
+* Invariants
+|               | attach_target | exernal_status                       | pending_update | pending_terminate | working_children | subs |
+|---------------+---------------+--------------------------------------+----------------+-------------------+------------------+------|
+| INACTIVE      |               |                                      |                |                   |                  |    0 |
+| W_ATTACH      |             1 | WORKING                              |                |                 0 |                0 |    0 |
+| WORKING       |               | WORKING                              |              0 |                 0 |                  |      |
+| SHUTTING_DOWN |               | WORKING                              |              0 |                 1 |               >0 |    0 |
+| COMPLETED     |               | CANCELED, COMPLETED, ERROR, REJECTED |              0 |                 0 |                0 |    0 |

src/lib/orderman/orders/book_algos/OffsetOrder.cpp

@@ -1,102 +0,0 @@
-#include "OffsetOrder.h"
-
-#include "lib/orderman/actors/order_manager/OrderBuilder.h"
-#include "lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h"
-#include "lib/utils/PBUtils.h"
-#include <google/protobuf/timestamp.pb.h>
-
-namespace bts {
-namespace orderman {
-
-using namespace api::orderman;
-
-struct OffsetOrder::PImpl 
-{
-   PImpl() {}
-   
-   uint32_t tick_offset_on_deploy;  
-};
-
-
-OffsetOrder::OffsetOrder() : me(new PImpl()) {}
-
-OffsetOrder::~OffsetOrder() {}
-
-void OffsetOrder::setup_child(const api::orderman::OrderDefinition& this_def, const PreActivationOrderServices&, 
-                            api::orderman::OrderState& initial_state, api::orderman::OrderDefinition* out_child)
-{
-   initial_state.mutable_offset();
-   auto const &offset_def = this_def.custom_def().offset();   
-   *out_child->mutable_custom_def() = offset_def.child();
-   
-   me->tick_offset_on_deploy = offset_def.tick_offset_on_deploy();   
-}
-
-
-void OffsetOrder::on_book_update(const bts::orderman::BookUpdateEvent& book)
-{
-   switch (this_state().status()) 
-   {
-   case ORDER_STATUS_WORKING:
-      switch (child_state().status()) 
-      {
-         case ORDER_STATUS_INACTIVE: 
-         {
-            if(definition().side() == api::SIDE_BUY)
-            {
-               const auto& same = book.book.side(api::Side::SIDE_BUY)[0];               
-               double price = same.price() - tick_size() * me->tick_offset_on_deploy;
-               activate_child(Price(price), Quantity(this_state().quantity()));
-               transition(ORDER_STATUS_WORKING);
-            }
-            else
-            {               
-               const auto& same = book.book.side(api::Side::SIDE_SELL)[0];
-               double price = same.price() + tick_size() * me->tick_offset_on_deploy;
-               activate_child(Price(price), Quantity(this_state().quantity()));
-               transition(ORDER_STATUS_WORKING);
-            }
-            report_state();   
-         }
-         break;
-         case ORDER_STATUS_WORKING:
-         {
-            detach();
-         }               
-         break;
-            
-         case ORDER_STATUS_PENDING_WORKING:
-            break;
-
-         // child cannot be in any of these states if we are in the working state
-         case ORDER_STATUS_DISCONNECTED: 
-         case ORDER_STATUS_ERROR:
-         case ORDER_STATUS_REJECTED_BY_SPARK:
-         case ORDER_STATUS_COMPLETED:
-         case ORDER_STATUS_CANCELED:
-         case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_:
-         case OrderStatus_INT_MAX_SENTINEL_DO_NOT_USE_:
-            bts_assert(false);
-            break;
-         }
-      break;
-
-   case ORDER_STATUS_COMPLETED:
-   case ORDER_STATUS_CANCELED:
-   case ORDER_STATUS_ERROR:
-      break;
-
-   case ORDER_STATUS_INACTIVE:
-   case ORDER_STATUS_REJECTED_BY_SPARK:
-   case ORDER_STATUS_PENDING_WORKING:
-   case ORDER_STATUS_DISCONNECTED:
-   case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_:
-   case OrderStatus_INT_MAX_SENTINEL_DO_NOT_USE_:
-      bts_assert(false);
-      break;
-   }   
-}
-
-}
-}
-

src/lib/orderman/orders/book_algos/OffsetOrder.h

@@ -1,29 +0,0 @@
-#pragma once
-
-#include "lib/orderman/orders/book_algos/BookAlgo.h"
-
-namespace bts {
-namespace orderman {
-
-class OrderStateUpdateEvent;
-class BookUpdateEvent;
-
-//OffsetOrder deploys N ticks away from the initial same-side inside market it's mainly for composition / fat finger
-
-class OffsetOrder : public BookAlgo
-{
-public:
-   OffsetOrder();
-   virtual ~OffsetOrder();
-   
-protected:
-   void setup_child(const api::orderman::OrderDefinition& this_def, const PreActivationOrderServices&, 
-                    api::orderman::OrderState&, api::orderman::OrderDefinition*) override;
-   void on_book_update(const bts::orderman::BookUpdateEvent&) override;   
-   
-private:   
-   struct PImpl;
-   std::unique_ptr<PImpl> me;
-};
-
-}}

src/lib/orderman/server/OrderServiceServer.cpp

@@ -1,11 +1,13 @@
 #include "OrderServiceServer.h"
 #include "lib/api/public/ServiceContext.h"
 #include "lib/api/services/AdminService.client.pb.h"
+#include "lib/api/services/MarginService.client.pb.h"
+#include "lib/api/services/PositionService.client.pb.h"
 #include "lib/config/config.h"
 #include "lib/config/protos/jetfix.pb.h"
 #include "lib/config/protos/orderman_cfg.pb.h"
-#include "lib/md/MDConsumer.h"
 #include "lib/md/MDClient.h"
+#include "lib/md/MDConsumer.h"
 #include "lib/mercury/actors/Timer.h"
 #include "lib/mercury/executor/Executor.h"
 #include "lib/metrics/metrics.h"
@@ -45,14 +47,14 @@ struct ActorThread {
       stats.publish(loop_metric);
    }
 
-   void start(trace::PerfLog& log) {
+   void start(trace::PerfLog &log) {
       if (cfg.poll())
-         thread = threading::make_thread(cfg.thread_name(), [this,&log]() {
+         thread = threading::make_thread(cfg.thread_name(), [this, &log]() {
             for (;;)
                executor.do_work(log);
          });
       else {
-         thread = threading::make_thread(cfg.thread_name(), [this,&log]() {
+         thread = threading::make_thread(cfg.thread_name(), [this, &log]() {
             const auto inner_iter = 1000;
             while (!done) {
                auto start = Timer::now();
@@ -90,14 +92,16 @@ ActorThread *find_thread_for_actor(
 }
 
 struct OrderServiceServer::PImpl {
-   PImpl(bts::api::ServiceContext &ctx,
-         md::MDClient& mdc,
+   PImpl(bts::api::ServiceContext &ctx, md::MDClient &mdc,
          DynamicOrderRegistry const &d, metrics::MetricsContainer &stats,
          trace::PerfLog &log)
        : service_bridge(), service_ready(false), known_orders(), known_fills(),
          publish_orders(), publish_fills(),
-         admin_service(ctx.locate<api::admin::AdminService>()), md_consumer(mdc.current_consumer()),
-         stats(stats), dyn_order_registry(d), log(log), in_flight_order_ops(),
+         admin_service(ctx.locate<api::admin::AdminService>()),
+         position_service(ctx.locate<api::position::PositionService>()),
+         margin_service(ctx.locate<api::position::MarginService>()),
+         md_consumer(mdc.current_consumer()), stats(stats),
+         dyn_order_registry(d), log(log), in_flight_order_ops(),
          current_req_op_id(0) {}
 
    ServiceBridge *service_bridge;
@@ -111,7 +115,9 @@ struct OrderServiceServer::PImpl {
    OrderServiceServer::OrdersPublisher publish_orders;
    OrderServiceServer::FillsPublisher publish_fills;
    api::admin::AdminService admin_service;
-   bts::md::MDConsumer& md_consumer;
+   api::position::PositionService position_service;
+   api::position::MarginService margin_service;
+   bts::md::MDConsumer &md_consumer;
    metrics::MetricsContainer &stats;
    DynamicOrderRegistry const &dyn_order_registry;
    trace::PerfLog &log;
@@ -120,12 +126,13 @@ struct OrderServiceServer::PImpl {
    uint32_t current_req_op_id;
 };
 
-OrderServiceServer::OrderServiceServer(
-    bts::api::ServiceContext &ctx,
-    md::MDClient& mdc,
-    DynamicOrderRegistry const &d, metrics::MetricsContainer &stats,
-    trace::PerfLog &log)
-   : api::orderman::OrderServiceBase(ctx), me(new PImpl(ctx, mdc, d, stats,log)) {
+OrderServiceServer::OrderServiceServer(bts::api::ServiceContext &ctx,
+                                       md::MDClient &mdc,
+                                       DynamicOrderRegistry const &d,
+                                       metrics::MetricsContainer &stats,
+                                       trace::PerfLog &log)
+    : api::orderman::OrderServiceBase(ctx),
+      me(new PImpl(ctx, mdc, d, stats, log)) {
    me->publish_orders = make_orders_publisher();
    me->publish_fills = make_fills_publisher();
 
@@ -152,7 +159,7 @@ OrderServiceServer::OrderServiceServer(
    bts_assert_msg(thread,
                   "Could not find a configured thread for OrderManager. See "
                   "the OrdermanCfg key");
-   thread->executor.spawn<OrderManager>(d,me->log, mdc.oracle());
+   thread->executor.spawn<OrderManager>(d, me->log, mdc.oracle());
 
    // Timer
    thread = find_thread_for_actor("Timer", me->actor_threads);
@@ -212,12 +219,10 @@ OrderServiceServer::OrderServiceServer(
        },
        [this, &ctx](auto const &order_op_result) {
           ctx.post([this, order_op_result] {
-                this->on_order_op_result(order_op_result);
+             this->on_order_op_result(order_op_result);
           });
        },
-       me->dyn_order_registry,
-       thread->executor.make_proxy(),mdc.oracle());
-
+       me->dyn_order_registry, thread->executor.make_proxy(), mdc.oracle());
 
    thread->executor.activate(me->service_bridge);
 
@@ -235,6 +240,19 @@ OrderServiceServer::OrderServiceServer(
               bts_assert(update.all().size() == 1);
               me->service_bridge->update_admin_data(update.all()[0]);
            });
+
+   // subscribe to position updates and foward them to the service bridge
+   me->position_service.subscribe_pnl_account_data() |
+       rxcpp::operators::subscribe<api::position::Pnl_account_dataUpdate>(
+           [this](auto const &update) {
+              me->service_bridge->update_account_pnl(update);
+           });
+
+   me->margin_service.subscribe_excess_liquidity_report() |
+       rxcpp::operators::subscribe<api::position::Excess_liquidity_reportUpdate>(
+           [this](auto const &update) {
+              me->service_bridge->update_excess_liquidity_report(update);
+           });
 }
 
 OrderServiceServer::~OrderServiceServer() {
@@ -358,7 +376,7 @@ void OrderServiceServer::get_registered_dynamic_order_types(
 void OrderServiceServer::on_order_op_result(
     OrderOpResultEvent const &op_result) {
    auto it = me->in_flight_order_ops.find(op_result.req_id);
-   bts_assert(it!=me->in_flight_order_ops.end());
+   bts_assert(it != me->in_flight_order_ops.end());
    auto &responder = it->second;
    api::orderman::Response r;
    r.set_order_key(op_result.ok.value());

src/lib/utils/AuthSessionManager.cpp

@@ -86,7 +86,7 @@ bool AuthSessionManager::add_or_replace_session(const api::user::TokenPayload& t
       return true;      
    if(datetime::ll_diff_in_us(datetime::ll_now(), _expiration) < 0)      
    {
-      LOG(Gen,Error, "Your BTS Keycode is expired!");   
+      LOG(User,Error, "Your BTS Keycode is expired!");
       if(msg)
          *msg = "Your BTS Keycode is EXPIRED!  Please contact sales@bluetradesys.com";
       return false;
@@ -105,13 +105,18 @@ bool AuthSessionManager::add_or_replace_session(const api::user::TokenPayload& t
    if(it != _map.end())
    {
       _map[tok.id()] = Session{tok};
+      LOG(User, Warn, "KEYCODE: Replacing session {} ", tok.id());
       return true;
    }
    else if(_map.size() < _nusers)
    {
       _map[tok.id()] = Session{tok};
+      LOG(User, Warn, "KEYCODE: Creating session {} ", tok.id());
       return true;         
    }            
+   if(msg)
+      *msg = "KEYCODES: Not Enough Logins available for session";
+   LOG(User, Error, "KEYCODES: Not Enough Logins available for session {}", tok.id());
    return false;
 }
 

src/lib/utils/AuthToken.cpp

@@ -51,6 +51,35 @@ AuthSessionManager& session_manager()
    return mgr;
 }
 
+bool requires_auth_session(const admin::BTSUser& user)
+{
+   //if the user can trade in any account or see market data, it needs a session
+   if(user.has_user_data() && 
+      user.user_data().has_account_permissions() && 
+      user.user_data().account_permissions().account_permissions_size())
+   {
+      for(const auto& i: user.user_data().account_permissions().account_permissions())
+      {
+         if(i.second.has_trading_permission() && i.second.trading_permission().has_permissions())
+            return true;
+      }
+   }   
+   
+   if(user.has_user_data() && 
+      user.user_data().has_exchange_user_data())
+   {
+      if(user.user_data().exchange_user_data().has_cme_data())
+      {
+         for(const auto& i: user.user_data().exchange_user_data().cme_data().md_permissions())
+         {
+            if(i.enabled())
+               return true;
+         }
+      }
+   }  
+   return false; 
+}
+
 AuthToken generate_token(const admin::BTSUser& user, std::string* msg) 
 {
    api::user::TokenPayload payload;
@@ -58,7 +87,9 @@ AuthToken generate_token(const admin::BTSUser& user, std::string* msg)
    payload.set_id(user.id());
    if(user.name() == "bts") 
       payload.set_wheel(true);
-   session_manager().add_or_replace_session(payload, msg);
+      
+   if(requires_auth_session(user))
+      session_manager().add_or_replace_session(payload, msg);      
    auto token = encrypt(payload.SerializeAsString()); 
    bts_assert_d(token.size() < 255-8);
    AuthToken result;

src/lib/utils/Dispatcher.cpp

@@ -214,6 +214,20 @@ void Dispatcher::post(Action&& action, int64_t delay_ms)
    }
 }
 
+void Dispatcher::post_copy(Action action, int64_t delay_ms)
+{
+   if (delay_ms > 0)
+   {
+      shared_ptr<deadline_timer> timer = make_shared<deadline_timer>(_me->service);
+      timer->expires_from_now(boost::posix_time::milliseconds(delay_ms));
+      timer->async_wait([timer, action] (const boost::system::error_code& err) { if (!err) action(); });
+   }
+   else
+   {
+      _me->service.post(action);
+   }
+}
+
 io_service* Dispatcher::io_service() { return &_me->service; }
 
 //

src/lib/utils/Dispatcher.h

@@ -88,6 +88,7 @@ public:
    void invoke(Action action);
    // Queue an action on the dispatcher thread and return immediately.
    void post(Action&& action, int64_t delay_ms=0);
+   void post_copy(Action action, int64_t delay_ms=0);
 
    boost::asio::io_service* io_service();
 

src/lib/utils/ValueWrapper.h

@@ -24,7 +24,6 @@ public:
    bool operator==(ValueWrapper const&v) const {return _v == v._v;}
    bool operator!=(ValueWrapper const&v) const {return _v != v._v;}
    bool operator<(ValueWrapper const&v) const {return _v < v._v;}
-   bool operator>(ValueWrapper const&v) const {return _v > v._v;}
    ValueT const& value() const {return _v;}
    ValueT & value() {return _v;}
 private:

src/lib/utils/tests/AuthTokenTests.cpp

@@ -5,19 +5,34 @@ namespace bts {
 namespace test {
 
 TEST_CASE("AuthToken", "[security]") {
-   api::admin::BTSUser user;
-   user.set_id(42);
-   user.set_name("user1");
+   api::admin::BTSUser admin_user;
+   admin_user.set_id(42);
+   admin_user.set_name("user1");
+   
+   api::admin::BTSUser trader_user;
+   trader_user.set_id(43);
+   trader_user.set_name("trader1");   
+   trader_user.mutable_user_data()->mutable_exchange_user_data()->mutable_cme_data()->add_md_permissions()->set_enabled(true);
 
-   SECTION("round trip") {
-      auto token = api::auth::generate_token(user);
+   SECTION("round trip admin") {
+      auto token = api::auth::generate_token(admin_user);
+      // We use 1 byte for api header size
+      REQUIRE(token.ByteSize() <= 240);
+      api::user::TokenPayload payload;
+      REQUIRE(!api::auth::decode_token(token, &payload));
+      REQUIRE(payload.id() == admin_user.id());
+      REQUIRE(payload.wheel() == false);
+   }
+   
+   SECTION("round trip trader") {
+      auto token = api::auth::generate_token(trader_user);
       // We use 1 byte for api header size
       REQUIRE(token.ByteSize() <= 240);
       api::user::TokenPayload payload;
       REQUIRE(api::auth::decode_token(token, &payload));
-      REQUIRE(payload.id() == user.id());
+      REQUIRE(payload.id() == trader_user.id());
       REQUIRE(payload.wheel() == false);
-   }
+   }   
 
    SECTION("empty token") {
       api::user::TokenPayload payload;
@@ -28,11 +43,11 @@ TEST_CASE("AuthToken", "[security]") {
    }
 
    SECTION("bts is the big wheel") {
-      user.set_name("bts");
-      auto token = api::auth::generate_token(user);
+      admin_user.set_name("bts");
+      auto token = api::auth::generate_token(admin_user);
       api::user::TokenPayload payload;
       REQUIRE(api::auth::decode_token(token, &payload));
-      REQUIRE(payload.id() == user.id());
+      REQUIRE(payload.id() == admin_user.id());
       REQUIRE(payload.wheel());
    }
 }

src/server/sparkd/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IMarginObserver.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/PositionServiceCore.cpp;position/PositionServiceServer.cpp;position/tests/PositionTests.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp)
+set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/tests/PositionTests.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp)

src/server/sparkd/cmds/QuickTest.cpp

@@ -27,6 +27,7 @@ struct Proxy : public IAlgoServices {
    Price price() const { return Price(0); }
    Quantity quantity() const { return Quantity(0); }
    Quantity total_fills() const { return Quantity(0); }
+   bool is_shutting_down() const {return false;}
    void subscribe_book(SecurityKey){};
    void unsubscribe_book(SecurityKey) {}
    void subscribe_child_change(OrderKey) {}

src/server/sparkd/cmds/Run.cpp

@@ -29,6 +29,7 @@
 #include "server/sparkd/md/MarketDataAuxiliaryServiceServer.h"
 #include "server/sparkd/md/run_md.h"
 #include "server/sparkd/position/PositionServiceServer.h"
+#include "server/sparkd/position/MarginServiceServer.h"
 #include "server/sparkd/support/ErrorReportService.h"
 #include "server/sparkd/support/ShellLatencyServer.h"
 #include "server/sparkd/user/AdminServiceServer.h"
@@ -84,11 +85,10 @@ int Run::execute(std::vector<std::string> const &) {
    md::auxiliary::MarketDataAuxiliaryServiceServer md_aux_server(sc);
    //orderman::AlgoServiceServer algo_server(sc);
    bts::api::orderman::algo::AlgoDefinitionServiceServer algo_definition_server(sc);
+      
    
-   
-   
-   
-   position::PositionServiceServer position_service(sc);
+   position::PositionServiceServer position_service(sc,mdclient);
+   position::MarginServiceServer margin_service(sc, mdclient);
    metrics::ShellLatencyServer shell_latency_service(
        sc, stats, utils::Dispatcher::background());
 

src/server/sparkd/cmds/Span.cpp

@@ -1,15 +1,61 @@
 #include "Span.h"
 
+#include "server/sparkd/position/PositionAggregator.h"
 #include "server/sparkd/position/MarginServiceCore.h"
+#include "server/sparkd/position/Position.h"
 
+#include "lib/orderman/orderman_utils/orderman_common.h"
 
-namespace bts 
-{
-namespace cmd 
-{
+#include "lib/config/config.h"
+#include "lib/api/public/ServiceContext.h"
+#include "lib/api/services/AuthService.client.pb.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/utils/helper_macros.h"
+#include "lib/utils/Dispatcher.h"
+
+#include <google/protobuf/text_format.h>
+#include <future>
+#include <memory>
+#include <stdlib.h>
+#include <condition_variable>
+#include <mutex>
+#include <time.h>
+
+using namespace bts::position;
+using namespace bts::api::position;
+
+namespace bts {
+namespace cmd {
 
 int Span::execute(const std::vector<std::string>&)
 {
+   srand(time(NULL));   
+   auto& disp = utils::Dispatcher::current();
+   MarginServiceCore core(disp);
+   
+   auto acct = std::string("ACC A");   
+   Position p;
+   p.security_desc = "ESU9";
+   p.account = acct;
+   p.position = Quantity(rand() % 100);
+   p.side = api::SIDE_BUY;
+   std::vector<Position> pos;
+   pos.push_back(p);
+   p.position = Quantity(rand() % 200);
+   p.side = api::SIDE_SELL;
+   pos.push_back(p);
+   
+   //core.set_verbose_debugging(true);
+   core.compute_margin(acct, pos, [](const api::position::MarginResult& m)
+      {
+         std::string str;
+         google::protobuf::TextFormat::PrintToString(m, &str);
+         
+         std::cout << std::endl << str << std::endl << std::endl;
+      });
+      
+   std::cout << "Waiting for margin results, press any key to exit" << std::endl;
+   disp.run();
    return 0;
 }
 
@@ -18,7 +64,6 @@ std::string Span::help()
    return "\t test CME CORE Span integration \n";
 } 
 
-}
-}
+}}
 
 

src/server/sparkd/position/IMarginObserver.cpp

@@ -1,9 +0,0 @@
-#include "IMarginObserver.h"
-
-namespace bts 
-{
-namespace position 
-{
-   IMarginObserver::~IMarginObserver() {}
-}
-}

src/server/sparkd/position/IMarginObserver.h

@@ -1,18 +0,0 @@
-#pragma once
-
-#include "lib/orderman/orderman_utils/orderman_common.h"
-
-namespace bts 
-{
-namespace position 
-{
-struct MarginResult;
-class 
-IMarginObserver
-{
-public:
-   virtual ~IMarginObserver();
-   virtual void on_margin_change(const orderman::AccountName&, const MarginResult&) = 0;  
-};
-}
-}

src/server/sparkd/position/MarginResult.h

@@ -1,25 +0,0 @@
-#pragma once
-
-#include "lib/orderman/orderman_utils/orderman_common.h"
-#include "lib/domain_model/Security.pb.h"
-
-namespace bts 
-{
-namespace position 
-{
-struct MarginResult
-{
-   bool valid;
-   std::string settle_indicator;
-   std::string settle_qualifier;
-   std::string portfolio_id;
-   std::string as_of_time;
-   std::string calculation_id;   
-   
-   double opt_value;
-   double nonopt_value;
-   double maint_margin;
-   double initial_margin;
-};  
-}
-}

src/server/sparkd/position/MarginServiceCore.cpp

@@ -1,18 +1,16 @@
 #include "MarginServiceCore.h"
 
-#include "PositionServiceCore.h"
 #include "Position.h"
-#include "MarginResult.h"
-#include "IMarginObserver.h"
-
 #include "lib/config/config.h"
 #include "lib/api/services/AuthService.pb.h"
 #include "lib/utils/bts_assert.h"
 #include "lib/utils/Uuid.h"
+#include "lib/utils/Dispatcher.h"
 #include "lib/utils/bts_chrono.h"
 #include "lib/logging/logging.h"
 #include "lib/utils/helper_macros.h"
 #include "lib/utils/datetime.h"
+#include "lib/config/protos/span.pb.h"
 
 
 #include "curl/curl.h"
@@ -23,46 +21,42 @@
 #include <boost/date_time/posix_time/posix_time.hpp>
 #include <utility>
 
-#define CONSOLE_LOG(...) std::cout , __VA_ARGS__  , std::endl
-
-template <typename T> std::ostream& operator,(std::ostream& out, const T& t) { out << t; return out; }
-std::ostream& operator,(std::ostream& out, std::ostream&(*f)(std::ostream&)) { out << f; return out; }
-#define MARGIN_LOG(subsystem,level,...) \
-   bts::log::_thread_logger.logger->enqueue(log::log_ts(),bts::log::subsystem,level,__FILE__,__LINE__,__VA_ARGS__); \
-CONSOLE_LOG(__VA_ARGS__)
+using bts::api::position::MarginResult;
 
 namespace bts 
 {
 namespace position
 {
    
-using MarginMap = boost::container::flat_map<orderman::AccountName, MarginResult>;
+using MarginMap = boost::container::flat_map<std::string, MarginResult>;
+
 struct MarginServiceCore::PImpl
 {
-   securities::ISecurityDB& db;
-   PositionServiceCore position_service;
-   IMarginObserver* observer;
-   MarginMap last_results, last_valid_results;
-   
+   utils::Dispatcher& dispatcher;
+   MarginMap last_results, last_valid_results;   
    std::string url_base, core_user, core_password, post_margin_path, get_margin_path;   
-   size_t max_iterations;
    bool verbose;
    int throttle;
-   bts::datetime::PTime last_update;
-   PImpl(securities::ISecurityDB& d, IMarginObserver* obs) : 
-      db(d), position_service(d), observer(obs), last_results(), last_valid_results(),
-      url_base("https://cmecorenr.cmegroup.com/MarginServiceApi/1.4/"), core_user("api_bts1"), core_password("BT5u53r!"),
+   PImpl(utils::Dispatcher& disp) : 
+      dispatcher(disp),
+      last_results(), last_valid_results(), //api_bts_span BT5Margin!
+      url_base("https://cmecorenr.cmegroup.com/MarginServiceApi/1.4/"), core_user("api_bts1"), core_password("Summer123"),
       post_margin_path("margins/?complete=true"),
       get_margin_path("margins/"),
-      max_iterations(20),
       verbose(false),
-      throttle(30000),
-      last_update()
+      throttle(30000)
    {
+      if(config::get<bts::config::SpanMargin>().login() != "")
+         core_user = config::get<bts::config::SpanMargin>().login();
+      if(config::get<bts::config::SpanMargin>().password() != "")
+         core_password = config::get<bts::config::SpanMargin>().password();
+      if(config::get<bts::config::SpanMargin>().url() != "")
+         url_base = config::get<bts::config::SpanMargin>().url();
+      
       auto res = curl_global_init(CURL_GLOBAL_DEFAULT);
       if(res != CURLE_OK) 
       {
-         MARGIN_LOG(Gen,Error,"curl_global_init() failed: ", std::string(curl_easy_strerror(res)));
+         LOG(Gen,Error,"curl_global_init() failed: {}", std::string(curl_easy_strerror(res)));
       }
    }
    ~PImpl()
@@ -151,7 +145,7 @@ struct MarginServiceCore::PImpl
    struct curl_slist* setup_url_and_headers(CURL* curl, const std::string& xpath)
    {
       std::string full_url = url_base + xpath;
-      MARGIN_LOG(Gen,Info,"CORE Margin URL: ",full_url);
+      LOG(Gen,Info,"CORE Margin URL: {}",full_url);
       curl_easy_setopt(curl, CURLOPT_URL, full_url.c_str());
       //curl_easy_setopt(curl, CURLOPT_URL, "https://cmecorenr.cmegroup.com/MarginServiceApi/1.4/margins/");
       curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, debug_callback);
@@ -182,19 +176,19 @@ struct MarginServiceCore::PImpl
          curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&reply);
          curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload.c_str());         
          
-         MARGIN_LOG(Gen,Info,"CORE Margin Payload: ", payload);
+         LOG(Gen,Info,"CORE Margin Payload: {}", payload);
          auto res = curl_easy_perform(curl); 
          if(res == CURLE_OK)
          {
             std::string resp(reply.data.begin(), reply.data.end());
-            MARGIN_LOG(Gen,Info,"CORE Margin Post Response: ", resp);
+            LOG(Gen,Info,"CORE Margin Post Response: {}", resp);
             succ = true;
          }         
          else 
          {
             std::string error(curl_easy_strerror(res));
-            MARGIN_LOG(Gen,Error,"POST curl_easy_perform() failed: ", error);
-            std::cout << fmt::format("CORE Margin CURL Error: ", error);            
+            LOG(Gen,Error,"POST curl_easy_perform() failed: {}", error);
+            //std::cout << fmt::format("CORE Margin CURL Error: ", error);            
          }
          curl_easy_cleanup(curl);
          if(header)
@@ -202,7 +196,7 @@ struct MarginServiceCore::PImpl
       }
       else 
       {
-         MARGIN_LOG(Gen,Error,"curl_easy_init() failed\n");
+         LOG(Gen,Error,"curl_easy_init() failed\n");
       }
       return succ; 
    }
@@ -221,12 +215,12 @@ struct MarginServiceCore::PImpl
          if(res == CURLE_OK)
          {
             std::string resp(reply.data.begin(), reply.data.end());
-            MARGIN_LOG(Gen,Info,"CORE Margin GET Response: ", resp);                     
+            LOG(Gen,Info,"CORE Margin GET Response: {}", resp);                     
             succ = true;
          }
          else         
          {
-            MARGIN_LOG(Gen,Error,"GET curl_easy_perform() failed: ", std::string(curl_easy_strerror(res)));            
+            LOG(Gen,Error,"GET curl_easy_perform() failed: {}", std::string(curl_easy_strerror(res)));            
          }
          curl_easy_cleanup(curl);
          if(header)
@@ -234,7 +228,7 @@ struct MarginServiceCore::PImpl
       }
       else 
       {
-         MARGIN_LOG(Gen,Error,"curl_easy_init() failed\n");
+         LOG(Gen,Error,"curl_easy_init() failed\n");
       }
       return succ;
    }
@@ -284,7 +278,7 @@ struct MarginServiceCore::PImpl
    
    void parse_margin_calculation(const CurlMsg& msg, MarginResult& result)
    {
-      result.valid = false;
+      result.set_status(MarginResult::Error);
       std::string str(msg.data.begin(), msg.data.end());
       pugi::xml_document doc;
       doc.load_string(str.c_str());
@@ -293,19 +287,19 @@ struct MarginServiceCore::PImpl
       child = child.child("margin");
       if(child) 
       {
-         result.settle_indicator = child.attribute("settleInd").as_string();
-         result.settle_qualifier = child.attribute("settleQual").as_string();
-         result.portfolio_id = child.attribute("portfolioId").as_string();
-         result.as_of_time = child.attribute("asOfTime").as_string();
-         result.calculation_id = child.attribute("id").as_string();
+         result.set_settle_indicator(child.attribute("settleInd").as_string());
+         result.set_settle_qualifier(child.attribute("settleQual").as_string());
+         result.set_portfolio_id(child.attribute("portfolioId").as_string());
+         result.set_as_of_time(child.attribute("asOfTime").as_string());
+         result.set_calculation_id(child.attribute("id").as_string());
          auto val = child.child("amounts");
-         if(val && result.portfolio_id != "")
+         if(val && result.portfolio_id() != "")
          {
-            result.opt_value = child.attribute("optVal").as_double();
-            result.nonopt_value = child.attribute("nonOptVal").as_double();
-            result.maint_margin = child.attribute("maint").as_double();
-            result.initial_margin = child.attribute("init").as_double();
-            result.valid = true;
+            result.set_opt_value(val.attribute("optVal").as_double());
+            result.set_nonopt_value(val.attribute("nonOptVal").as_double());
+            result.set_maint_margin(val.attribute("maint").as_double());
+            result.set_initial_margin(val.attribute("init").as_double());
+            result.set_status(MarginResult::NewValid);
          }         
       }
    }
@@ -321,9 +315,8 @@ struct MarginServiceCore::PImpl
    };   
    
    bool compute_margin(const std::string& account_name, const std::string& portfolid_id, 
-                       const std::vector<Position>& pos, MarginResult& margin)
-   {
-      margin.valid = false;
+                       const std::vector<Position>& pos, std::function<void(const MarginResult&)> cb)
+   {      
       pugi::xml_document doc;
       pugi::xml_node trades_node;
       create_margin_request(portfolid_id, doc, trades_node);
@@ -331,12 +324,9 @@ struct MarginServiceCore::PImpl
       size_t count = 0;
       for(const auto& i : pos)
       {
-         if(i.security && i.security->has_glbx())
-         {
-            int p = i.position.value() * (i.side == api::SIDE_SELL ? -1 : 1);
-            request += fmt::format("BTS, {}, {}, {} {}", account_name, i.security->glbx().security_desc(), p, "\n");
-            count++;
-         }
+         int p = i.position.value() * (i.side == api::SIDE_SELL ? -1 : 1);
+         request += fmt::format("BTS, {}, {}, {}, {}", account_name, i.security_desc, p, "\n");
+         count++;
       }
       if(count > 0)
       {
@@ -345,78 +335,88 @@ struct MarginServiceCore::PImpl
          xml_string_writer writer(xml);
          doc.save(writer);
          
-         size_t iters = 0;
-         while(iters++ < max_iterations)
-         {
-            CurlMsg reply;
-            sleep_for(5s);
-            post(post_margin_path, xml, reply);      
-            std::string portid_resp, calcid_resp;
-            if(parse_margin_response(reply, portid_resp, calcid_resp))
-            {
-               CurlMsg reply2;
-               sleep_for(2s);
-               if(get(get_margin_path + calcid_resp, reply2))
+         CurlMsg reply;
+         post(post_margin_path, xml, reply);      
+         std::string portid_resp, calcid_resp;
+         if(parse_margin_response(reply, portid_resp, calcid_resp))
+         {            
+           std::function<void()> do_web;
+           do_web = 
+               [=]()
                {
-                  parse_margin_calculation(reply2, margin);
-                  return true;
-               }
-            }                  
+                  MarginResult margin;
+                  margin.set_status(MarginResult::Error);
+                  margin.set_account(account_name);
+                  CurlMsg reply2; bool done = false;
+                  try
+                  {
+                     if(get(get_margin_path + calcid_resp, reply2))
+                     {
+                        parse_margin_calculation(reply2, margin);
+                        cb(margin);
+                        done = true;
+                     }
+                  }
+                  catch(const std::exception& ex)
+                  {
+                     LOG(Gen, Error, "Exception in parse_margin_calculation {}", std::string(ex.what()));
+                  }   
+                  catch(...)
+                  {
+                     LOG(Gen, Error, "Unknown Exception in parse_margin_calculation");
+                  }    
+                  if(!done)
+                     dispatcher.post_copy(do_web, 5000);   
+               };            
+            dispatcher.post_copy(do_web, 5000);
+            return true;
          }
       }
       return false;
    }
 };
 
-MarginServiceCore::MarginServiceCore(securities::ISecurityDB& db, IMarginObserver* obs)
-   : me(new PImpl(db, obs))
+MarginServiceCore::MarginServiceCore(utils::Dispatcher& disp) : me(new PImpl(disp))
 {
 }
 
 MarginServiceCore::~MarginServiceCore() {}
 
-bool MarginServiceCore::pump(const orderman::AccountName& acct)
-{
-   MarginResult result;
-   return compute_margin(acct, result);
-}
 
-bool MarginServiceCore::compute_margin(const orderman::AccountName& acct, MarginResult& result)
+void MarginServiceCore::compute_margin(const std::string& acct, const std::vector<Position>& pos, 
+                                       std::function<void(const MarginResult&)> cb)
 {
-   auto now = bts::datetime::now();
-   auto diff = bts::datetime::ptime_diff_in_ms(now, me->last_update);
-   if(me->last_update.is_not_a_date_time() || abs(diff) > me->throttle)
+   me->dispatcher.post([=] ()
    {
-      query_margin_webapi(acct, result);
-      me->last_update = now;
-      return true;
-   }
-   else
-   {
-      get_last_valid_result(acct, result);
-      return false; //false, we did not get new data
-   }
+      query_margin_webapi(acct, pos, cb);
+   }, me->throttle);   
 }
 
-bool MarginServiceCore::query_margin_webapi(const orderman::AccountName& acct, MarginResult& margin)
-{
-   std::vector<Position> pos;
-   me->position_service.query_all_positions(acct, pos);
-   utils::Uuid guid;
-   bool succ = me->compute_margin(orderman::to_str(acct), guid.to_text_string(), pos, margin);
-   if(me->observer)
-      me->observer->on_margin_change(acct, margin);
-   me->last_results.insert_or_assign(acct, margin);
-   if(succ && margin.valid)
-      me->last_valid_results.insert_or_assign(acct, margin);
-   return succ;
-}
-
-bool MarginServiceCore::query_margin_webapi(const std::vector<Position>& pos, MarginResult& margin)
+bool MarginServiceCore::query_margin_webapi(const std::string& acct, const std::vector<Position>& pos, 
+                                            std::function<void(const MarginResult&)> cb)
 {
    utils::Uuid guid;
-   std::string str = guid.to_text_string();
-   return me->compute_margin(str.substr(0, 3), str, pos, margin);      
+   try
+   {
+      return me->compute_margin(acct, guid.to_text_string(), pos, 
+         [=] (const MarginResult& m)
+         {
+            me->last_results.insert_or_assign(acct, m);
+            if(m.status() == MarginResult::NewValid)
+               me->last_valid_results.insert_or_assign(acct, m);
+            cb(m);
+         });
+   }
+   catch(const std::exception& ex)
+   {
+      LOG(Gen, Error, "Exception in query_margin_webapi {}", std::string(ex.what()));
+      return false;
+   }   
+   catch(...)
+   {
+      LOG(Gen, Error, "Unknown Exception in query_margin_webapi");
+      return false;
+   }
 }
 
 void MarginServiceCore::set_verbose_debugging(bool verb)
@@ -424,7 +424,7 @@ void MarginServiceCore::set_verbose_debugging(bool verb)
    me->verbose = verb;
 }
 
-bool MarginServiceCore::get_last_valid_result(const orderman::AccountName& acct, MarginResult& result)
+bool MarginServiceCore::get_last_valid_result(const std::string& acct, MarginResult& result)
 {
    auto it = me->last_valid_results.find(acct);
    if(it != me->last_valid_results.end())
@@ -435,7 +435,7 @@ bool MarginServiceCore::get_last_valid_result(const orderman::AccountName& acct,
    return false;
 }
 
-bool MarginServiceCore::get_last_result(const orderman::AccountName& acct, MarginResult& result)
+bool MarginServiceCore::get_last_result(const std::string& acct, MarginResult& result)
 {
    auto it = me->last_results.find(acct);
    if(it != me->last_results.end())

src/server/sparkd/position/MarginServiceCore.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "lib/orderman/orderman_utils/orderman_common.h"
+#include "lib/api/services/MarginService.pb.h"
 #include "server/sparkd/position/Position.h"
 #include "lib/domain_model/Security.pb.h"
 
@@ -8,33 +9,33 @@
 
 namespace bts 
 {
+namespace utils { class Dispatcher; }
 namespace securities { class ISecurityDB; }
 namespace position
 {
-struct MarginResult;
 class IMarginObserver;
+
 class MarginServiceCore
 {
    BTS_NO_COPY(MarginServiceCore);
 public:
-   MarginServiceCore(securities::ISecurityDB&, IMarginObserver* obs = nullptr);
+
+   MarginServiceCore(utils::Dispatcher&);
    ~MarginServiceCore();
-   
-   //Call as much as you want; it will update IMarginObserver at most once every 30s 
-   bool pump(const orderman::AccountName&);
-   //Call as much as you want; it will query the webapi at most once every 30s 
-   bool compute_margin(const orderman::AccountName&, MarginResult&);
+   void compute_margin(const std::string&, const std::vector<Position>&, 
+                       std::function<void(const api::position::MarginResult&)>);                       
+   void set_verbose_debugging(bool);
+   bool get_last_valid_result(const std::string&, api::position::MarginResult&);
+   bool get_last_result(const std::string&, api::position::MarginResult&);   
+
    
 private:
    friend class PositionTests;
-   bool get_last_valid_result(const orderman::AccountName&, MarginResult&);
-   bool get_last_result(const orderman::AccountName&, MarginResult&);   
-   //This is a blocking call to a WebAPI.  Very Slow.
-   bool query_margin_webapi(const orderman::AccountName&, MarginResult&);
-   //This is a blocking call to a WebAPI.  Very Slow.
-   bool query_margin_webapi(const std::vector<Position>&, MarginResult&);   
-   void set_verbose_debugging(bool);
-
+   
+   //This is a blocking call to a WebAPI.  Very Slow.   
+   bool query_margin_webapi(const std::string&, const std::vector<Position>&, 
+                            std::function<void(const api::position::MarginResult&)>);
+   
 private:
    struct PImpl;
    std::unique_ptr<PImpl> me;

src/server/sparkd/position/MarginServiceServer.cpp

@@ -0,0 +1,312 @@
+#include "MarginServiceServer.h"
+#include "MarginServiceCore.h"
+#include "PositionAggregator.h"
+#include "Position.h"
+
+#include "lib/api/service_helpers/AdminServiceUtils.h"
+#include "lib/orderman/orderman_utils/orderman_utils.h"
+#include "lib/persistence/SqliteStore.h"
+#include "lib/config/config.h"
+#include "lib/config/protos/span.pb.h"
+#include "lib/domain_model/Security.h"
+#include "lib/logging/logging.h"
+#include "lib/utils/bts_assert.h"
+#include "lib/utils/TimeStamp.h"
+#include "lib/utils/Dispatcher.h"
+#include "lib/os/os.h"
+
+#include <set>
+#include <utility>
+
+using namespace bts::api::orderman;
+using namespace bts::api::position;
+using namespace bts::admin;
+using namespace std;
+using bts::api::position::MarginResult;
+
+namespace bts {
+namespace position {
+
+MarginServiceServer::MarginServiceServer(bts::api::ServiceContext& context, md::MDClient& mdclient)
+   : bts::api::position::MarginServiceBase (context),
+   _admin_service(context.locate<bts::api::admin::AdminService>()),
+   _publisher(make_margin_data_stateful_publisher()),
+   _report_publisher(make_excess_liquidity_report_stateful_publisher()),
+   _dispatcher(utils::Dispatcher::get_named("MarginServiceServer")),
+   _timer(_dispatcher, 30000, [=]() { publish_all();} ),
+   _position(new PositionAggregator(context.locate<bts::api::position::PositionService>(), mdclient)),
+   _margin(new MarginServiceCore(_dispatcher)),
+   //state below
+   _lock(),
+   _last_update(),
+   _admin_data(),
+   _last_report(),
+   _first_time(true)
+{
+   if(!config::get<bts::config::SpanMargin>().enabled())
+      return;
+
+   LOG(Gen, Info, "MarginServiceServer: service started");
+
+   _admin_service.subscribe_admin_data() |
+       rxcpp::operators::subscribe<api::admin::Admin_dataUpdate>(
+           [this](auto const &update) 
+           {
+              std::scoped_lock<std::recursive_mutex> l(_lock);
+              bts_assert(update.all().size() == 1);
+              _admin_data = *update.all()[0];
+              
+              if(_first_time)
+              {
+                  auto accts =  AdminServiceUtils::accounts(_admin_data, false);
+                  for(const auto& i : accts)
+                  {                     
+                     incremental_update_report(i->name(), i->id());
+                  }
+                 _first_time = false;
+              }
+           });   
+      
+   _timer.start();
+}
+
+void MarginServiceServer::publish_all()
+{
+   if(config::get<bts::config::SpanMargin>().enabled())   
+   {
+      std::scoped_lock<std::recursive_mutex> l(_lock);            
+      auto accts =  AdminServiceUtils::accounts(_admin_data, false);
+      for(const auto& i : accts)
+      {         
+         datetime::LLDateTime since = 0;
+         std::string acct = i->name();
+         auto it = _last_update.find(acct);
+         if(it != _last_update.end())
+            since = it->second;
+         std::vector<Position> pos;  AccountPNL pnl;
+         if(_position->query_positions(acct, pos, pnl, &since) && pos.size())
+         {
+            LOG(Gen, Info, "MarginServiceServer: requesting margin for {} ", acct);
+            _margin->compute_margin(acct, pos, [this, since, acct, pnl](const MarginResult& m) 
+               { 
+                  std::scoped_lock<std::recursive_mutex> l(_lock);
+                  publish_margin(m);
+                  publish_excess_liquidity_report(m, pnl);
+                  _last_update[acct] = since;
+               });            
+         }                     
+      }
+   }   
+}
+
+MarginServiceServer::~MarginServiceServer() {}
+
+void MarginServiceServer::get_margin(const api::position::MarginRequest& request, 
+                                     const api::Response<api::position::MarginResult>& reply) 
+{
+   if(!config::get<bts::config::SpanMargin>().enabled())
+   {
+      MarginResult res;
+      res.set_account(request.account());
+      reply(res);
+   }
+   else
+   {
+      std::vector<Position> pos; AccountPNL pnl;
+      _position->query_positions(request.account(), pos, pnl);            
+      _margin->compute_margin(request.account(), pos, [=](const MarginResult& m)
+         {
+            reply(m);
+         });
+      }
+}
+
+void MarginServiceServer::get_excess_liquidity_report(const api::position::MarginRequest& request,
+                                                      const api::Response<api::position::ExcessLiquidityReport>& reply)
+{
+   std::scoped_lock<std::recursive_mutex> l(_lock);
+   ExcessLiquidityReport rep;
+   rep.set_account(request.account());
+   auto it = _last_report.find(request.account());
+   if(it != _last_report.end())
+      rep.CopyFrom(it->second);
+   reply(rep);
+}
+
+
+
+void MarginServiceServer::publish_margin(const api::position::MarginResult& result)
+{
+   std::scoped_lock<std::recursive_mutex> l(_lock);
+   _publisher.add_or_update(result);
+   _publisher.flush();      
+}
+
+void MarginServiceServer::publish_excess_liquidity_report(const api::position::MarginResult& result, const api::position::AccountPNL& pnl)
+{
+   std::scoped_lock<std::recursive_mutex> l(_lock);
+   ExcessLiquidityReport report;
+   report.set_account(result.account());
+   report.set_account_id(pnl.account_id());
+   report.mutable_account_pnl()->CopyFrom(pnl);
+   report.mutable_margin_result()->CopyFrom(result);
+   auto acct = AdminServiceUtils::find_account(_admin_data, result.account(), false);   
+   if(acct && acct->has_margin_limit())
+   {
+      report.set_leverage_factor( acct->margin_limit().leverage_factor_percentage() );      
+      report.set_cash_balance(acct->margin_limit().cash_balance());        
+      report.set_should_enforce_margin(acct->margin_limit().enforce_margin() && 
+                                       config::get<bts::config::SpanMargin>().enabled());
+   }   
+
+   double netliq = report.leverage_factor() * report.cash_balance();
+   if(!bts::os::is_nan(result.maint_margin()))
+      netliq -= result.maint_margin();
+   if(!bts::os::is_nan(pnl.day_pnl()))
+      netliq += pnl.day_pnl();
+   report.set_excess_liquidity(netliq);
+   _last_report[result.account()] = report;
+   _report_publisher.add_or_update(report);
+   _report_publisher.flush();      
+}
+
+//so the admin ui knows what to do
+void MarginServiceServer::incremental_update_report(const std::string& account, uint32_t account_id)
+{   
+   std::scoped_lock<std::recursive_mutex> l(_lock);
+   
+   MarginResult margin;
+   if(!_margin->get_last_valid_result(account, margin))
+   {
+      margin.Clear();
+      margin.set_account(account);
+   }
+   AccountPNL pnl;
+   if(!_position->query_pnl(account, pnl))
+   {
+      pnl.Clear();
+      pnl.set_account_name(account);
+      pnl.set_account_id(account_id);
+   }
+   publish_excess_liquidity_report(margin, pnl);
+   _last_update[account] = 0;
+}
+
+
+void MarginServiceServer::set_cash_factor(const ::bts::api::position::CashFactorUpdate& request,
+                                          const api::Response<::bts::api::ResultMessage>& reply)
+{   
+   if(!config::get<bts::config::SpanMargin>().enabled())
+   {
+      api::ResultMessage resp; resp.set_success(true);
+      reply(resp);
+      return;
+   }
+      
+   std::scoped_lock<std::recursive_mutex> l(_lock);
+   auto acct = AdminServiceUtils::find_account(_admin_data, request.account(), false);
+   if(!acct)
+   {
+      auto msg = "Account " + request.account() +"not found";
+      api::ResultMessage resp;
+      resp.set_success(false);
+      resp.set_error_message(msg);      
+      reply(resp);
+   }
+   else
+   {
+      acct->mutable_margin_limit()->set_leverage_factor_percentage(request.leverage_factor());
+      AccountUpdateAPIArgument update;
+      update.mutable_complete_data()->CopyFrom(*acct);
+      
+      _admin_service.update_account(update, 
+         [this, request, reply, update](const api::Result<bts::api::admin::AdminServiceOperationResult>& admin_result) 
+         {
+            api::ResultMessage resp;
+            resp.set_success(admin_result.success());
+            resp.set_error_message(admin_result.error_message());
+            reply(resp);
+            incremental_update_report(request.account(), update.complete_data().id());
+         });         
+   }
+}
+                        
+void MarginServiceServer::set_cash_balance(const ::bts::api::position::CashBalanceUpdate& request,
+                                           const api::Response<::bts::api::ResultMessage>& reply) 
+{
+   if(!config::get<bts::config::SpanMargin>().enabled())
+   {
+      api::ResultMessage resp; resp.set_success(true);
+      reply(resp);
+      return;
+   }
+      
+   std::scoped_lock<std::recursive_mutex> l(_lock);
+   auto acct = AdminServiceUtils::find_account(_admin_data, request.account(), false);
+   if(!acct)
+   {
+      auto msg = "Account " + request.account() +"not found";
+      api::ResultMessage resp;
+      resp.set_success(false);
+      resp.set_error_message(msg);      
+      reply(resp);
+   }
+   else
+   {
+      acct->mutable_margin_limit()->set_cash_balance(request.cash_balance());
+      AccountUpdateAPIArgument update;
+      update.mutable_complete_data()->CopyFrom(*acct);
+      
+      _admin_service.update_account(update, 
+         [this, request, reply, update](const api::Result<bts::api::admin::AdminServiceOperationResult>& admin_result) 
+         {
+            api::ResultMessage resp;
+            resp.set_success(admin_result.success());
+            resp.set_error_message(admin_result.error_message());
+            reply(resp);
+            incremental_update_report(request.account(), update.complete_data().id());
+         });      
+   }
+}
+
+void MarginServiceServer::set_should_enforce_margin(const ::bts::api::position::EnforceUpdate& request,
+                                                    const api::Response<::bts::api::ResultMessage>& reply) 
+{
+   if(!config::get<bts::config::SpanMargin>().enabled())
+   {
+      api::ResultMessage resp; resp.set_success(true);
+      reply(resp);
+      return;
+   }
+      
+   std::scoped_lock<std::recursive_mutex> l(_lock);
+   auto acct = AdminServiceUtils::find_account(_admin_data, request.account(), false);
+   if(!acct)
+   {
+      auto msg = "Account " + request.account() +"not found";
+      api::ResultMessage resp;
+      resp.set_success(false);
+      resp.set_error_message(msg);      
+      reply(resp);
+   }
+   else
+   {
+      acct->mutable_margin_limit()->set_enforce_margin(request.should_enforce_margin());
+      AccountUpdateAPIArgument update;
+      update.mutable_complete_data()->CopyFrom(*acct);
+      
+      _admin_service.update_account(update, 
+         [this, request, reply, update](const api::Result<bts::api::admin::AdminServiceOperationResult>& admin_result) 
+         {
+            api::ResultMessage resp;
+            resp.set_success(admin_result.success());
+            resp.set_error_message(admin_result.error_message());
+            reply(resp);
+            incremental_update_report(request.account(), update.complete_data().id());
+         });      
+   }   
+}
+
+
+}}
+

src/server/sparkd/position/MarginServiceServer.h

@@ -0,0 +1,72 @@
+#pragma once
+
+#include "lib/orderman/orderman_utils/orderman_common.h"
+#include "lib/md/SecurityOracle.h"
+#include "lib/persistence/IStore.h"
+#include "lib/api/public/ServiceContext.h"
+#include "lib/api/services/MarginService.server.pb.h"
+#include "lib/api/services/AdminService.pb.h"
+#include "lib/api/services/AdminService.client.pb.h"
+#include "lib/api/services/PositionService.pb.h"
+#include "lib/api/services/PositionService.client.pb.h"
+#include "lib/utils/bts_no_copy.h"
+#include "lib/utils/datetime.h"
+#include <map>
+#include <memory>
+#include <mutex>
+
+namespace bts {
+namespace utils { class Dispatcher; }
+namespace position {
+   
+struct Position;
+class PositionAggregator;
+class MarginServiceCore;
+
+class MarginServiceServer : public api::position::MarginServiceBase
+{
+public:
+   MarginServiceServer(bts::api::ServiceContext&, md::MDClient&);
+   ~MarginServiceServer();
+   
+   void get_margin(const api::position::MarginRequest& request,
+                   const api::Response<api::position::MarginResult>& reply) override;
+                   
+   void get_excess_liquidity_report(const api::position::MarginRequest& request,
+                                    const api::Response<api::position::ExcessLiquidityReport>& reply) override;                   
+                   
+   void set_cash_factor(const ::bts::api::position::CashFactorUpdate& request,
+                        const api::Response<::bts::api::ResultMessage>& reply) override;
+                        
+   void set_cash_balance(const ::bts::api::position::CashBalanceUpdate& request,
+                         const api::Response<::bts::api::ResultMessage>& reply) override;
+                         
+   void set_should_enforce_margin(const ::bts::api::position::EnforceUpdate& request,
+                                  const api::Response<::bts::api::ResultMessage>& reply) override;
+                         
+   
+private:
+   void publish_margin(const api::position::MarginResult&);
+   void publish_excess_liquidity_report(const api::position::MarginResult&, const api::position::AccountPNL&);
+   void publish_all();
+   void incremental_update_report(const std::string& account, uint32_t account_id);
+
+private:
+   api::admin::AdminService _admin_service;  
+   MarginServiceBase::Margin_dataStatefulPublisher _publisher;   
+   MarginServiceBase::Excess_liquidity_reportStatefulPublisher _report_publisher;   
+   utils::Dispatcher& _dispatcher;
+   utils::DispatcherTimer _timer;
+   std::unique_ptr<PositionAggregator> _position;
+   std::unique_ptr<MarginServiceCore> _margin;   
+   
+private:
+   //state
+   std::recursive_mutex _lock;
+   std::map<std::string, datetime::LLDateTime> _last_update;
+   bts::api::admin::AdminData _admin_data;
+   std::map<std::string, api::position::ExcessLiquidityReport> _last_report;
+   bool _first_time;
+   
+};
+}}