Pushed variant versions of timstamp and oid everywhere.

It is fairly easy for our sequence number to be behind CME if we
crash. Since we can detect this issue (CME tells us about it) we can
try to fix it automatically by incrementing our own sequence
number. This should allow us to recover automatically in some
scenarios.

.gitignore

@@ -35,3 +35,4 @@ trace_data/
 .ionide/*
 /.ignore
 /build
+.clangd/

.vscode/launch.json

@@ -1,13 +1,41 @@
 {
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "Spark Local",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "${workspaceRoot}/build/ninja_debug/server/sparkd",
+            "args": [
+                "run",
+                "test",
+                "-a",
+                "OutrightVanillaOrder"
+            ],
+            "stopAtEntry": false,
+            "cwd": "${workspaceRoot}/build/ninja_debug/server/",
+            "environment": [
+                { "name": "TZ", "value": "America/Chicago" }
+            ],
+            "externalConsole": false,
+            "MIMode": "gdb",
+            "setupCommands": [
+                {
+                    "description": "Enable pretty-printing for gdb",
+                    "text": "-enable-pretty-printing",
+                    "ignoreFailures": true
+                }
+            ]
+        },
         {
             "name": "SparkVM Launch",
             "type": "cppdbg",
             "request": "launch",
             "program": "/home/bts/spark/build/ninja_debug/server/sparkd",
             "args": [
-                "run"
+                "test",
+                "-a",
+                "[ilink3]"
             ],
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/server/",

CMakeLists.txt

@@ -41,6 +41,7 @@ else ()
                                -Wno-missing-braces -Wno-ignored-qualifiers \
                                --std=c++17 -fdiagnostics-color")
   set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer -DNDEBUG -O3")
+  set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG")
   set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wno-undef")
   add_subdirectory(src/server/sparkd)
   add_subdirectory(src/lib)
@@ -55,7 +56,7 @@ else ()
     COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/web ${CMAKE_BINARY_DIR}/server
     COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
     COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
-    COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.13.0.0 ${CMAKE_BINARY_DIR}/server
+    COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0 ${CMAKE_BINARY_DIR}/server
     COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
     COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
     COMMENT "Creating server runtime layout"

doc/api/content/client/getting-started/install.md

@@ -5,7 +5,7 @@ weight = 10
 
 # C#/.NET
 
-We target [.NET Standard 2.0](https://docs.microsoft.com/en-us/dotnet/standard/net-standard) so you can use .NET Framework 4.6.1+ or .NET Core 2.0+.  Simply add a reference to the [Spark Nuget package](https://nuget.org) to get started.
+We target [.NET Standard 2.0](https://docs.microsoft.com/en-us/dotnet/standard/net-standard) so you can use .NET Framework 4.6.1+ or .NET Core 2.0+.  Simply add a reference to the [Spark Nuget package](https://www.nuget.org/packages/spark_sdk/) to get started.
 
 ```
 dotnet new console -lang c#

doc/api/content/client/getting-started/services.md

@@ -124,4 +124,17 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
+#include "lib/api/public/ServiceContext.h"
+#include "lib/api/services/OrderService.client.pb.h"
+using namespace bts::api;
+ServiceContext ctx;
+auto orders = ctx.locate<orderman::OrderService>();
+auto unsub = orders.subscribe_fills().subscribe([](const auto& update) {
+  std::cout 
+    << "Added " << update.added().size() << " items, "
+    << "Removed " << update.removed().size() << " items, "
+    << "Updated " << update.updated().size() << " items, "
+    << "Total items " << update.all().size() << std::endl;
+});
+std::this_thread::sleep_for(std::chrono::minutes(5));
 ```

doc/api/content/client/services/marketdata.md

@@ -1,3 +1,51 @@
 +++
 title = "Market Data"
-+++
++++
+
+
+Use the MarketDataService to subscribe enumerate all available securities.
+
+```cs
+using System;
+using System.Threading;
+using bts.api;
+using bts.api.core;
+
+namespace sample {
+   class Program {
+      static void Main(string[] args) {
+         using var context = new ServiceContext();
+         var md = context.locate<MarketDataService>();
+         using var sub = md.subscribe_securities().Subscribe(update => {
+            foreach (var security in update.added) {
+               Console.WriteLine($"Added security {security.Id}: {security.Symbol}");
+            }
+         });
+
+         Thread.Sleep(TimeSpan.FromMinutes(5));
+      }
+   }
+}
+```
+
+```cpp
+#include "lib/api/public/ServiceContext.h"
+#include "spark/lib/domain_model/Security.h"
+#include "lib/api/services/MarketDataService.client.pb.h"
+#include <thread>
+
+using namespace bts::api;
+
+int main(int argc, char** argv) {
+   ServiceContext ctx;
+   auto md_service = ctx.locate<bts::api::MarketDataService>();
+   auto sub = md_service.subscribe_securities().subscribe([](const auto& update) {
+      for (const auto& s : update.added()) {
+         std::cout << "Added security: " << s->id() << ": " << s->symbol() << std::endl;
+      }
+   });
+
+   std::this_thread::sleep_for(std::chrono::minutes(5));
+   return 0;
+}
+```

doc/api/content/server/getting-started/download.md

@@ -68,12 +68,21 @@ To update your Spark SDK simply download an unzip a new [spark_sdk.zip](https://
 
 To preserve the old image so you can go back to it later run `docker tag spark_sdk spark_sdk:your_custom_tag` before loading the new SDK image.
 
-<!--
-Then stop, remove and restart spark_sdk.
+### Re-build the Dev Container
+
+If you are using Visual Studio Code you also need to rebuild the dev-container so it picks up the new image.
+
+#### Inside VSCode
+To re-build from inside Visual Studio Code.  Click the 'Remote Explorer' icon on the left hand side. 
+You should see your spark_sdk project listed under the Containers section.  Right click on it and select **rebuild-container**.
+
+#### Command Line
+To re-build from the command line run `docker ps -a` and find the row with the spark_sdk image. 
 ```
-c:\bts> docker stop spark_sdk
-c:\bts> docker rm spark_sdk
-c:\bts> docker run -d -v %cd%:/sdk --name spark_sdk bluetradingsytems/spark_sdk
+CONTAINER ID        IMAGE                          COMMAND                  CREATED              STATUS                      PORTS                                NAMES
+54d96cfa3888        spark_sdk                      "/bin/sh -c 'echo Co…"   About a minute ago   Up About a minute           127.0.0.1:9000-9001->9000-9001/tcp   quizzical_lichterman
+```
+Then run `docker rm 54d96cfa3888` (use your container id instead of 54d96cfa3888)  or `docker rm quizzical_lichterman` (use your container name).
+
 
-``` -->
 

doc/api/templates/index.html

@@ -73,7 +73,7 @@
                             <p>
                                 The Spark client SDK allows you to integrate spark with your existing ecosystem. The Spark Client is built entirely on this simple, powerful API.
                             </p>
-                            <a href="" class="button">Get started</a>
+                            <a href="{{ get_url(path="@/client/_index.md") }}" class="button">Get started</a>
                         </div>
                         
                     </div>

doc/user/content/release_notes/_index.md

@@ -1,9 +1,29 @@
-+++
++++
 title = "Release notes"
 weight = 3
 sort_by = "weight"
 insert_anchor_links = "right"
 +++
+## 2.16.2
+### User Visible Change
+  - Synthetic creation dialog has a "fractional" checkbox next to the tick settings.  Users may uncheck this box to enter integer tick sizes, so the synthetic security's prices will be displayed without decimals, or fractional units.
+## 2.16.1
+### Bug Fixes
+  - OrderBook's algo state parameters (in the panel on the right side) are sorted alphabetically
+## 2.16.0
+### Bug Fixes
+  - Solved an issue that could result in incorrect ladder levels
+  - Solved an issue that could lead to incorrect routing of orders in the event of certain MSGW changes
+### API Changes
+  - Add C# client side API samples for order entry, position monitoring and market data observation
+
+## 2.15.10
+### Bug Fixes
+  - Solved an issue that could cause the user interface to fail when playing some alert sounds
+### API Changes
+  - Added settlement and position information to the API
+  - Added the ability for user algos to report custom state to the user interface
+  - Added the ability for algos to discover all active securities
 ## 2.15.3
 ### Bug Fixes
   - Fixed a bug where some CME currency spreads exhibited incorrect tick values
@@ -708,3 +728,4 @@ insert_anchor_links = "right"
 - Start, Day and Total PnL columns are painted with a red or green background color based on if the pnl value is negative or positive respectively
 - Corrected cash per point values, used in pnl calculations, for a number of products
 - PnL report will now remember any customizations you make to filtering and/or sorting criteria
+

global.json

@@ -0,0 +1,6 @@
+{
+  "sdk": {
+    "version": "2.1.508",
+    "rollForward": "latestMajor"
+  }
+}

src/client/Directory.Build.Props

@@ -22,6 +22,10 @@
     <GenDir>$(BaseOutputPath)\gen\$(MSBuildProjectName)</GenDir>
 </PropertyGroup>
 
+<PropertyGroup Condition="'$(VCTargetsPath)'==''">
+    <VCTargetsPath>C:\Program Files (x86)\Microsoft Visual Studio\2017\Professional\Common7\IDE\VC\VCTargets</VCTargetsPath>
+</PropertyGroup>
+
 <Import Project="$(ComputerSpecificConfig)" Condition="Exists($(ComputerSpecificConfig))" />
 
 <ItemGroup>

src/client/spark/Modules/Spreader/ExecutionStrategyEditor.xaml

@@ -549,24 +549,48 @@
               </u:BindingUpdateGroupCollection>
             </u:Behavior.BindingUpdateGroups>
           </TextBox>
-          <TextBlock
+          <StackPanel
             x:Name="PART_TickFactors_Label"
             Grid.Column="2"
-            VerticalAlignment="Center"
-            Margin="{StaticResource Margin.LHeaderPanelItem}"
-            Text="Ticks"
             IsEnabled="True"
+            Orientation="Horizontal"
             Visibility="{Binding LegTuples.Count, RelativeSource={RelativeSource TemplatedParent}, Converter={StaticResource IntNotZeroVisibilityConverter}}"
-          />
-          <c:TickFactorsPicker
+           >
+            <TextBlock
+              VerticalAlignment="Center"
+              Margin="{StaticResource Margin.LHeaderPanelItem}"
+              Text="Ticks ("
+            />
+            <CheckBox
+              Margin="1,0,0,0"
+              IsChecked="{Binding UseFractionalTicks, RelativeSource={RelativeSource TemplatedParent}}"
+              VerticalAlignment="Center"
+              VerticalContentAlignment="Center"
+              Padding="0"
+            >
+              fractional
+            </CheckBox>
+            <TextBlock VerticalAlignment="Center" Margin="1,0,0,0">)</TextBlock>
+          </StackPanel>
+          <Grid
             x:Name="PART_TickFactors_Picker"
             Grid.Column="3"
-            Margin="{StaticResource Margin.LHeaderPanelItem}"
-            HorizontalAlignment="Stretch"
             IsEnabled="True"
-            SelectedTickFactors="{Binding TickFactors, RelativeSource={RelativeSource TemplatedParent}, Mode=TwoWay}"
+            Margin="{StaticResource Margin.LHeaderPanelItem}"
             Visibility="{Binding LegTuples.Count, RelativeSource={RelativeSource TemplatedParent}, Converter={StaticResource IntNotZeroVisibilityConverter}}"
-          />
+          >
+            <c:TickFactorsPicker
+              HorizontalAlignment="Stretch"
+              SelectedTickFactors="{Binding TickFactors, RelativeSource={RelativeSource TemplatedParent}, Mode=TwoWay}"
+              Visibility="{Binding UseFractionalTicks, RelativeSource={RelativeSource TemplatedParent}, Converter={x:Static cv:BooleanToVisibilityConverter.HiddenInstance}}"
+            />
+            <c:TickUpDownInt
+              HorizontalAlignment="Stretch"
+              Minimum="1"
+              Value="{Binding IntegerTickSize, RelativeSource={RelativeSource TemplatedParent}}"
+              Visibility="{Binding UseFractionalTicks, RelativeSource={RelativeSource TemplatedParent}, Converter={StaticResource FalseVisibilityConverter}}"
+            />
+          </Grid>
           <TextBlock
             x:Name="PART_Pricing_Label"
             Grid.Column="4"

src/client/spark/Modules/Spreader/ExecutionStrategyEditor.xaml.cs

@@ -52,6 +52,26 @@ namespace mbts.ui.spark {
       public TickFactors TickFactors { get => GetValue(TickFactorsProperty) as TickFactors; set => SetValue(TickFactorsProperty, value); }
       #endregion
 
+      #region UseFractionalTicks
+      public static readonly DependencyProperty UseFractionalTicksProperty = WPFHelper.create_dp<bool, ExecutionStrategyEditor>(
+         nameof(UseFractionalTicks), FrameworkPropertyMetadataOptions.BindsTwoWayByDefault, true);
+
+      public bool UseFractionalTicks {
+         get => (bool)GetValue(UseFractionalTicksProperty);
+         set => SetValue(UseFractionalTicksProperty, value);
+      }
+      #endregion
+
+      #region IntegerTickSize
+      public static readonly DependencyProperty IntegerTickSizeProperty = WPFHelper.create_dp<int, ExecutionStrategyEditor>(
+         nameof(IntegerTickSize), FrameworkPropertyMetadataOptions.BindsTwoWayByDefault, 1);
+
+      public int IntegerTickSize {
+         get => (int)GetValue(IntegerTickSizeProperty);
+         set => SetValue(IntegerTickSizeProperty, value);
+      }
+      #endregion
+
       #region IsNew
       public static readonly DependencyProperty IsNewProperty =
          DependencyProperty.Register(

src/client/spark/Modules/Spreader/SyntheticEditorView.xaml

@@ -36,6 +36,8 @@
       ContractPickerModel="{Binding contract_picker_model}"
       HedgePricing="{Binding hedge_pricing}"
       HedgePricingItemsSource="{Binding hedge_pricing_options}"
+      UseFractionalTicks="{Binding use_fractional_ticks}"
+      IntegerTickSize="{Binding integer_tick_size}"
     />
     <c:DialogSkirtOKCancel Grid.Row="1" OKCommand="{Binding commit}" CloseOnOK="False"/>
   </Grid>

src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs

@@ -2,7 +2,6 @@
 using bts.collections;
 using bts.shell;
 using bts.spark;
-using bts.utils;
 using mbts.ui;
 using mbts.ui.shell;
 using mbts.ui.spark;
@@ -30,8 +29,16 @@ namespace mbts.spark {
       readonly Property<bool> _has_working_orders, _can_add_leg;
 
       Security _synthetic;
-      bool _set_name_automatically, _set_ticks_automatically, _set_quantities_automatically, _set_price_mults_automatically, _is_setting_quantities, _is_setting_price_mults;
+      bool
+         _set_name_automatically,
+         _set_ticks_automatically,
+         _set_quantities_automatically,
+         _set_price_mults_automatically,
+         _is_setting_quantities,
+         _is_setting_price_mults,
+         _use_fractional_ticks = true;
       TickFactors _tick_factors;
+      int _integer_tick_size = 1;
       ExecutionStrategyImmutable _strategy;
       HedgePricing _hedge_pricing;
 
@@ -44,6 +51,10 @@ namespace mbts.spark {
          : this(synthetic, strategy, false) {
          ExceptionAssert.InvalidOperation.That(synthetic.Id != 0, "This constructor may only be used for already-existing synthetic securities (id must be non-zero).");
          _tick_factors = TickUtils.get_regional_ticks(synthetic.ContractSpec).low_tick;
+         if (TickUtils.is_integer_tick(synthetic.ContractSpec)) {
+            _use_fractional_ticks = false;
+            _integer_tick_size = DoubleHelper.to_optional_int32(synthetic.ContractSpec.LowTick) ?? 0;
+         }
          title = $"Edit '{synthetic.Synthetic.Name}'";
          icon = Icons.ModifySynthetic;
          _has_working_orders = backing(nameof(has_working_orders), security_has_working_orders.OnDispatcher());
@@ -131,14 +142,14 @@ namespace mbts.spark {
                flash_error_msg("Name already taken", SparkUtils.UI.MessageFlashDuration, nameof(name)).ignore();
                valid = false;
             }
-            if (_tick_factors == null || !_tick_factors.is_valid()) {
-               flash_error_msg("Please select ticks", SparkUtils.UI.MessageFlashDuration, nameof(tick_factors)).ignore();
+            if (use_fractional_ticks ? (_tick_factors == null || !_tick_factors.is_valid()) : integer_tick_size == 0) {
+               flash_error_msg("Please select ticks", SparkUtils.UI.MessageFlashDuration, use_fractional_ticks ? nameof(tick_factors) : nameof(integer_tick_size)).ignore();
                valid = false;
             }
             if (valid) {
                _synthetic.Synthetic.Legs.SetItems(leg_tuples.Select(l => l.leg));
                _synthetic.Symbol = _deduce_symbol(_synthetic);
-               _synthetic.ContractSpec = _tick_factors.to_spec();
+               _synthetic.ContractSpec = use_fractional_ticks ? _tick_factors.to_spec() : TickUtils.create_integer_tick_spec(integer_tick_size);
                uint? id = await model.create_synthetic(new SecurityImmutable(_synthetic));
                if (id.HasValue && id != 0) {
                   if (await model.save_strategy(id.Value, get_strategy())) {
@@ -295,6 +306,20 @@ namespace mbts.spark {
       public bool has_working_orders => _has_working_orders.Value;
       public bool is_new { get; }
       public bool is_new_leg_popup_open { get; set; }
+      public bool use_fractional_ticks {
+         get => _use_fractional_ticks;
+         set {
+            if (Set(ref _use_fractional_ticks, value))
+               _set_ticks_automatically = false;
+         }
+      }
+      public int integer_tick_size {
+         get => _integer_tick_size;
+         set {
+            if (Set(ref _integer_tick_size, value))
+               _set_ticks_automatically = false;
+         }
+      }
       public bool can_add_leg => _can_add_leg.Value;
       public SyntheticContract.Types.PriceCalc price_calc { get => _synthetic.Synthetic.PriceCalc; set => _synthetic.Synthetic.PriceCalc = value; }
       public ObservableCollection<LegSpreaderParamsTuple> leg_tuples { get; } = new ObservableCollection<LegSpreaderParamsTuple>();
@@ -323,6 +348,16 @@ namespace mbts.spark {
          _strategy = strategy;
          hedge_pricing = strategy.HedgePricing;
          leg_tuples.SetItems(_get_legs(_synthetic, strategy));
+         if (synthetic.HasContractSpec) {
+            /// assign the fields <see cref="_use_fractional_ticks"/> and <see cref="_integer_tick_size"/>,
+            /// rather than the properties <see cref="use_fractional_ticks"/> and <see cref="integer_tick_size"/>
+            /// because the setters of the former flip <see cref="_set_ticks_automatically"/> to false if the value changes.
+            /// This behavior causes the ticks to be automatically deduced as legs are added/removed,
+            /// until the user explicitly enters a different value (as the properties are bound to tick input controls in the view).
+            Set(ref _use_fractional_ticks, !TickUtils.is_integer_tick(synthetic.ContractSpec), nameof(use_fractional_ticks));
+            if (!use_fractional_ticks)
+               Set(ref _integer_tick_size, DoubleHelper.to_optional_int32(synthetic.ContractSpec.LowTick) ?? 0, nameof(integer_tick_size));
+         }
          RaisePropertyChanged(nameof(name), nameof(tick_factors));
       }
 

src/client/spark/Modules/Trading/Ladder/View/LadderGridHost.cs

@@ -534,14 +534,44 @@ namespace mbts.spark {
          _size_from_row(row, _price_list, _asks, _vm.security.ContractSpec);
 
       uint _volume_from_row(int row) {
-         if (_price_list != null) {
-            var price = _price_list[row];
-            var vapls = _volume_container.read_new_value();
-            if (vapls != null) {
-               return vapls.SelectFirstOrDefault(v => MathUtils.epsilon_equals(v.Price, price), v => v.Volume);
-            } 
+         const int max_tries = 3;
+         var errors = new List<string>(0);
+         int num_tries = 0;
+         for (; num_tries < max_tries; num_tries++) {
+            try {
+               var rv = impl();
+               if (num_tries > 0) {
+                  log_errors(true);
+               }
+               return rv;
+            } catch (Exception ex) {
+               errors.Add(ex.ToString());
+            }
+         }
+         log_errors(false);
+         return 0;
+
+         void log_errors(bool succeeded) {
+            // Only log distinct errors to avoid useless noise.
+            // It's likely that the exception will always be the same (IndexOutOfRange),
+            // however, this bug is not fully understood, so assume nothing.
+            var error_msg = string.Join("; ", errors.Distinct());
+            if (succeeded) {
+               Log.Ui.error($"Ladder '{security}' got volume for row at index '{row}' after {num_tries + 1} attempts (price_list array length is {_price_list.Length}).  The following exceptions were observed: {error_msg}");
+            } else {
+               Log.Ui.error($"Ladder '{security}' failed to get volume for row at index '{row}' after {max_tries} attempts (price_list array length is {_price_list.Length}): {error_msg}");
+            }
+         }
+
+         uint impl() {
+            if (_price_list.SafeTryGetAt(row, out var price)) {
+               var vapls = _volume_container.read_new_value();
+               if (vapls != null) {
+                  return vapls.SelectFirstOrDefault(v => MathUtils.epsilon_equals(v.Price, price), v => v.Volume);
+               }
+            }
+            return 0u;
          }
-         return 0u;
       }
 
 

src/client/spark/Modules/Trading/ManualFills/ManualFillsVM.cs

@@ -2,6 +2,7 @@ using bts.api;
 using bts.api.core;
 using bts.api.position;
 using bts.collections;
+using bts.utils;
 using mbts;
 using mbts.spark;
 using mbts.ui;

src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs

@@ -36,7 +36,6 @@ namespace mbts.spark {
       public OrderBookRowVM(OrderBookVM owner, OrderSecurityRootPair order_tuple, bool is_expanded, Action<bool> on_selection_changed) {
          _owner = owner;
          this.order_tuple = order_tuple;
-         is_visible = !is_child;
          _is_expanded = is_expanded;
          prospective_quantity = this.quantity;
          prospective_price = this.price;
@@ -113,7 +112,6 @@ namespace mbts.spark {
       // Tree
       public bool is_parent { get => _is_parent; set => Set(ref _is_parent, value); }
       public bool is_child => order.has_parent();
-      public bool is_visible { get; set; }
       public bool is_expanded {
          get => _is_expanded;
          set {

src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs

@@ -51,6 +51,7 @@ namespace mbts.spark {
       readonly OrderParamsAggregator _selected_params_mon;
       readonly CommonValueMonitor<SecurityImmutable?> _selected_security_mon = new CommonValueMonitor<SecurityImmutable?>();
       readonly RightPanelModel _right_panel = new RightPanelModel();
+      readonly CustomStateSynchronizer _custom_state_sync;
       OrderBookFillsGridSettings _cached_fills_settings = OrderBookStartupData.DefaultFillsGrid;
       double
          _raw_right_panel_width = OrderBookStartupData.Default.right_panel_width,
@@ -124,6 +125,8 @@ namespace mbts.spark {
          _row_collection.CollectionChanged += _row_collection_CollectionChanged;
          _current_row = backing<OrderBookRowVM>(nameof(current_row));
          _columns.filter_changed += _on_filter_changed;
+         _custom_state_sync = new CustomStateSynchronizer(this);
+         // commands
          collapse_all_rows_command = new DelegateCommand(_row_collection.collapse_all);
          expand_all_rows_command = new DelegateCommand(_row_collection.expand_all);
          var modify_selected_orders_async_cmd = new AsyncCommand(_modify_selected_orders_async, () => _selected_params_mon.all_active && order_mod_vm.is_dirty);
@@ -188,6 +191,7 @@ namespace mbts.spark {
                security_plurality = p;
                RaisePropertyChanged(nameof(security_plurality));
             }),
+            _current_row.Subscribe(_ => _custom_state_sync.sync()),
             orders_source.OnDispatcher().Subscribe(t => {
                if (_selected_rows.Count == 0) {
                   _cached_orders = t.state;
@@ -207,6 +211,8 @@ namespace mbts.spark {
                      }
                   }
                }
+               _custom_state_sync.sync(true);
+               orders_changed?.Invoke();
             }),
             model.on_orders_can_attach.OnDispatcher().Subscribe(table => {
                _cached_attachable = new MappedDictionary<uint, KeyValuePair<uint, bool>, bool>(table.state, kv => kv.Value);
@@ -375,7 +381,7 @@ namespace mbts.spark {
       }
 
       public event Action<OrderRecordImmutable?> selected_order_changed;
-      public event Action begin_attaching, end_attaching;
+      public event Action begin_attaching, end_attaching, orders_changed;
       public event Action<(IEnumerable, bool)> request_selection;
 
       public ObservableAppendCollection<DIYGrid.ColumnBase> columns { get; }
@@ -395,6 +401,7 @@ namespace mbts.spark {
          }
       }
       public OrderRecordImmutable? current_order => current_row?.order;
+      public ObservableCollection<NamedValueVM> current_algo_custom_state { get; } = new ObservableCollection<NamedValueVM>();
       public AlgoDefinitionVM algo_vm => order_mod_vm.algo_vm;
       public OrderParamsVM order_mod_vm { get; }
       public FillsVMContainer fills_vm { get; } = new FillsVMContainer();
@@ -474,6 +481,28 @@ namespace mbts.spark {
       public IObservable<bool> subscribe_show_fills() => _show_fills.observe_current(true);
       public IObservable<bool> subscribe_show_ladder() => _show_ladder.observe_current(true);
       public IObservable<bool> subscribe_show_order_params() => _show_order_params.observe_current(true);
+      public IObservable<OrderSecurityAlgo?> subscribe_single_selected_order() =>
+         Observable.Create<OrderSecurityAlgo?>(o => {
+            OrderSecurityAlgo? last_fired = null;
+            orders_changed += notify;
+            _current_row.Changed += on_current_row_changed;
+            notify();
+            return Disposable.Create(dispose);
+            // local methods
+            void notify() {
+               var next = current_row == null ? (OrderSecurityAlgo?)null : current_row.order_tuple.order;
+               if (next != last_fired)
+                  o.OnNext(last_fired = next);
+            }
+
+            void on_current_row_changed(OrderBookRowVM oldval, OrderBookRowVM newval) =>
+               notify();
+
+            void dispose() {
+               orders_changed -= notify;
+               _current_row.Changed -= on_current_row_changed;
+            }
+         });
 
       public void row_selection_changed(SelectionChangedEventArgs e) {
          try {
@@ -503,6 +532,7 @@ namespace mbts.spark {
             _can_attach_selected_orders.refresh();
             _can_detach_selected_orders.refresh();
             selected_order_changed?.Invoke(current_order);
+            _custom_state_sync.sync();
          } catch (Exception ex) {
             // [John]I observed a strange null-reference exception in this callback when the launcher was shutting down (see https://bluetradesys.atlassian.net/browse/SPARK-2089)
             Log.Ui.error($"[{nameof(OrderBookVM)}] Error handling selection change: {ex}");
@@ -723,6 +753,45 @@ namespace mbts.spark {
          }
       }
 
+      class CustomStateSynchronizer {
+         readonly OrderBookVM _vm;
+         OrderBookRowVM _synced_row;
+
+         public CustomStateSynchronizer(OrderBookVM vm) =>
+            _vm = vm;
+
+         public void sync(bool force_refresh = false) {
+            var row = _vm.current_row != null &&
+               _vm.selected_rows.Select(r => r.order_tuple.root.order.Key).Distinct().TryGetSingle(out uint droot) &&
+               _vm.current_row.order_tuple.root.order.Key == droot ?
+                  _vm.current_row : null;
+            if (force_refresh || row != _synced_row) {
+               _synced_row = row;
+               _refresh();
+            }
+         }
+
+         void _refresh() {
+            _vm.current_algo_custom_state.Clear();
+            if (_synced_row == null)
+               return;
+            var dcs = _synced_row.order.dynamic_custom_state();
+            // while dcs is not expected to be null, best to play it safe and double check.
+            if (dcs.is_null_or_empty()) {
+               var descendent_rows = _vm._row_collection.get_descendent_rows(_synced_row.order_key);
+               if (_vm._row_collection.filter != null)
+                  descendent_rows = descendent_rows.Where(child => _vm._row_collection.filter(child.order_tuple));
+               foreach (var dr in descendent_rows) {
+                  dcs = dr.order.dynamic_custom_state();
+                  if (!dcs.is_null_or_empty())
+                     break;
+               }
+            }
+            foreach (var p in dcs.empty_if_null().OrderBy(kv => kv.Key))
+               _vm.current_algo_custom_state.Add(new NamedValueVM(p.Key, DynamicOrderHelper.fmt_param(p.Value)));
+         }
+      }
+
       public class OrderParamsVM : DisposableViewModelBase {
          readonly SelectorProperty<AccountTuple, AccountTuple?> _accounts;
          readonly OrderBookVM _owner;

src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs

@@ -1,4 +1,5 @@
-using bts.api;
+using bts;
+using bts.api;
 using bts.api.orderman;
 using bts.shell;
 using bts.shell.fills;
@@ -6,9 +7,11 @@ using bts.shell.orderbook;
 using bts.spark;
 using bts.view;
 using mbts.persist;
+using mbts.spark.utilities;
 using mbts.ui;
 using mbts.ui.extensions;
 using mbts.ui.spark;
+using mbts.ui.utilities;
 using mbts.ui.utility;
 using mbts.utils;
 using System;
@@ -24,7 +27,10 @@ namespace mbts.spark {
       const string _window_type = "OrderBook", _default_title = "Order Book";
       readonly INotificationService _notification_service;
       readonly IWindowSettingsManager<OrderBookGridSettings> _settings_manager;
+      readonly bool _can_edit_specific_algo_params;
       IDisposable _attachment_exit_sub;
+      // initialized in thread_init
+      List<IDisposable> _thread_init_resources;
       OrderBookStartupData _state;
       OrdersModel _model;
       OrderBookVM _vm;
@@ -34,13 +40,14 @@ namespace mbts.spark {
       SparkCommand[] _commands;
 
       public OrderBookWindow(OrdersModel model, OrderBookWindowFactory factory, IBus bus, INotificationService notification_service,
-         OrderBookStartupData state, ImageSource icon, IWindowSettingsManager<OrderBookGridSettings> settings_manager)
+         OrderBookStartupData state, ImageSource icon, IWindowSettingsManager<OrderBookGridSettings> settings_manager, bool can_edit_specific_algo_params)
          : base(_default_title, bus, factory, SparkCommands.ShowOrders.icon) {
          _model = model;
          _factory = factory;
          _notification_service = notification_service;
          _state = state;
          _settings_manager = settings_manager;
+         _can_edit_specific_algo_params = can_edit_specific_algo_params;
          thread_id = config.Configuration.get<bts.config.MultiThreadedWindows>().order_book ? "OrderBook" : null;
       }
 
@@ -64,12 +71,14 @@ namespace mbts.spark {
       public string thread_id { get; }
 
       public override void Dispose() {
-         _commands.OfType<IDisposable>().DisposeAll();
+         Disposable.dispose(_thread_init_resources);
          Disposable.dispose(ref _attachment_exit_sub);
          Disposable.dispose(ref _vm);
       }
 
       public void thread_init() {
+         Log.assert(_thread_init_resources == null, $"{nameof(_thread_init_resources)} must be initialized in thread_init.");
+         _thread_init_resources = new List<IDisposable>();
          _menu_display_vm = new SparkMenuDisplayVM();
          if (_state != null)
             _menu_display_vm.load_from(_state.menu_display);
@@ -147,6 +156,15 @@ namespace mbts.spark {
             can_execute: () => _vm.selected_orders.Any(x => passes_recreate_search_criteria(x))
          );
          recreate_selected_orders_command.group = CommandGroups.Modify;
+         ICommand edit_algo_params_cmd;
+         if (_can_edit_specific_algo_params) {
+            _thread_init_resources.Add(CommandHelper.bind_param_source(
+               new DialogCommand<OrderSecurityAlgo?, Window>(mk_algo_params_dialog, has_algo_params),
+               _vm.subscribe_single_selected_order(),
+               out edit_algo_params_cmd, true));
+         } else {
+            edit_algo_params_cmd = DelegateCommand.empty;
+         }
 
          var attach_selected_orders_command = new SparkCommandCommand(
             "Attach To Selected",
@@ -194,14 +212,22 @@ namespace mbts.spark {
          show_grid_settings_command.group = CommandGroups.Auxiliary;
 
          _commands = new SparkCommand[] {
+            // modify commands
             toggle_show_right_panel_command,
             attach_selected_orders_command,
             detach_selected_orders_command,
             recreate_selected_orders_command,
+            new SparkCommandCommand("Algo Params...", Icons.Algo, edit_algo_params_cmd) {
+               group = CommandGroups.Modify,
+               show_in_toolbar = false,
+               show_in_context_menu = _can_edit_specific_algo_params
+            },
+            // cancel commands
             cancel_selected_orders_command,
             cancel_buy_orders_command,
             cancel_sell_orders_command,
             cancel_all_orders_command,
+
             toggle_collapse_all_command,
             toggle_expand_all_command,
             show_grid_settings_command,
@@ -210,9 +236,12 @@ namespace mbts.spark {
             hidden_cmd(new SparkCommandCommand(OrderBookView.CommandDescriptions.RevertOrders, null, _vm.revert_selected_orders_cmd, OrderBookView.KeyboardShortcuts.RevertOrders)),
             _menu_display_vm.spark_cmd
          };
-         _vm.as_demuxer().subscribe<bool>(nameof(OrderBookVM.show_right_panel))
-            .Subscribe(_ => toggle_show_right_panel_command.title = get_toggle_show_right_panel_cmd_title());
+         _thread_init_resources.AddRange(_commands.OfType<IDisposable>());
+         _thread_init_resources.Add(_vm.as_demuxer()
+            .subscribe<bool>(nameof(OrderBookVM.show_right_panel))
+            .Subscribe(_ => toggle_show_right_panel_command.title = get_toggle_show_right_panel_cmd_title()));
          _vm.selected_order_changed += vm_selected_order_changed;
+         _thread_init_resources.Add(Disposable.Create(() => _vm.selected_order_changed -= vm_selected_order_changed));
          // local methods
          string get_toggle_show_right_panel_cmd_title() =>
             _vm.show_right_panel ? OrderBookView.CommandDescriptions.HideRightPanel : OrderBookView.CommandDescriptions.ShowRightPanel;
@@ -234,6 +263,36 @@ namespace mbts.spark {
             attach_selected_orders_command.raiseCanExecuteChanged();
             detach_selected_orders_command.raiseCanExecuteChanged();
          }
+
+         bool has_algo_params(OrderSecurityAlgo? order) =>
+            order.HasValue && !OrderDataHelper.get_metadata(order.Value.algo.value.CustomCase).NoProps;
+
+         Window mk_algo_params_dialog(OrderSecurityAlgo? order) {
+            if (!has_algo_params(order))
+               return null;
+            var algo_def = AlgoDefinitionVM.create(_model, order.Value.algo, editing_permissions: AlgoEditingPermissions.AllowEditValues);
+            var algo_params_vm = new OrderBookAlgoParamsVM(algo_def, order.Value.order.is_non_terminal());
+            var dialog = new OrderBookAlgoParamsDialog {
+               Content = algo_params_vm
+            };
+            ui.WindowStyle.SetTitleContent(dialog, algo_def);
+            dialog.Closed += (s, e) => {
+               Disposable.dispose(ref algo_params_vm);
+               dialog = null;
+            };
+            algo_params_vm.RequestCommit += commit;
+            algo_params_vm.RequestClose += (s, e) => close_dialog();
+            return dialog;
+
+            void close_dialog() {
+               dialog?.Close();
+               dialog = null;
+            }
+            void commit(CustomOrderDefinitionImmutable value) {
+               _model.update_order(order.Value.order, new_algo: value).ignore();
+               close_dialog();
+            }
+         }
       }
 
       protected override void OnParentViewCreated(SparkWindowView parent_view) {

src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs

@@ -1,6 +1,7 @@
 using bts.api;
 using bts.shell.orderbook;
 using bts.spark;
+using mbts.config;
 using mbts.persist;
 using mbts.spark;
 using mbts.ui;
@@ -15,8 +16,11 @@ namespace bts.shell {
       readonly INotificationServiceDataCache _notification_service_data_cache;
       readonly OrdersModel _model;
       readonly UserSettingsHub _settings_hub;
+      readonly OrderBookConfig _config;
 
       public OrderBookWindowFactory(IShellModule owner, IBus bus, INotificationService notification_service, INotificationServiceDataCache notification_service_data_cache, OrdersModel model, UserSettingsHub settings_hub) : base(owner, "SparkOrderBookWindow") {
+         var raw_config = Configuration.get<bts.config.OrderBookOptions>();
+         _config = new OrderBookConfig(raw_config.can_edit_specific_algo_params);
          _bus = bus;
          _notification_service = notification_service;
          _notification_service_data_cache = notification_service_data_cache;
@@ -29,7 +33,8 @@ namespace bts.shell {
          var settings_manager = new WindowSettingsManager<OrderBookGridSettings>(_settings_hub, state.grid_settings_wrapper(),
             us => us.order_book_grid ?? OrderBookStartupData.DefaultGrid,
             (us, s) => us.with(order_book_grid: s));
-         var w = new OrderBookWindow(_model, this, _bus, _notification_service, state, SparkCommands.ShowOrders.icon, settings_manager.as_non_shared());
+         var w = new OrderBookWindow(_model, this, _bus, _notification_service, state,
+            SparkCommands.ShowOrders.icon, settings_manager.as_non_shared(), _config.can_edit_specific_algo_params);
          return Task.FromResult<IShellWindow>(w);
       }
 
@@ -37,5 +42,12 @@ namespace bts.shell {
          var w = await load(startup_data);
          _bus.post(SparkTags.ShowWindow, w);
       }
+
+      readonly struct OrderBookConfig {
+         public readonly bool can_edit_specific_algo_params;
+
+         public OrderBookConfig(bool can_edit_specific_algo_params) =>
+            this.can_edit_specific_algo_params = can_edit_specific_algo_params;
+      }
    }
 }

src/client/spark/Modules/Trading/OrderBook/View/OrderBookAlgoParamsDialog.xaml

@@ -0,0 +1,40 @@
+<Window
+  x:Class="bts.shell.orderbook.OrderBookAlgoParamsDialog"
+  x:ClassModifier="internal"
+  xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+  xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+  xmlns:sp="clr-namespace:mbts.spark"
+  xmlns:ob="clr-namespace:bts.shell.orderbook"
+  xmlns:usp="clr-namespace:mbts.ui.spark"
+  xmlns:u="clr-namespace:mbts.ui"
+  xmlns:c="clr-namespace:mbts.ui.controls"
+  u:WindowStyle.SizeToContent="WidthAndHeight"
+  Margin="0"
+  Padding="0"
+  Style="{StaticResource BTSWindowTitleTemplate}"
+>
+  <u:WindowStyle.TitleContentTemplate>
+    <DataTemplate DataType="{x:Type sp:AlgoDefinitionVM}">
+      <StackPanel HorizontalAlignment="Left" Orientation="Horizontal">
+        <usp:OrderTypeItem Margin="4,0" DisplayName="{Binding name, StringFormat='{}{0} Params'}" BadgeFill="{Binding badge_fill}"/>
+      </StackPanel>
+    </DataTemplate>
+  </u:WindowStyle.TitleContentTemplate>
+  <Window.ContentTemplate>
+    <DataTemplate DataType="{x:Type ob:OrderBookAlgoParamsVM}">
+      <Grid>
+        <Grid.RowDefinitions>
+          <RowDefinition Height="*"/>
+          <RowDefinition Height="Auto"/>
+        </Grid.RowDefinitions>
+        <usp:AlgoEditorGridView
+          Width="390"
+          Margin="5"
+          AlgoParamRows="{Binding param_rows}"
+          CanUserChangeAlgoRoot="False"
+        />
+        <c:DialogSkirt Grid.Row="1" RightItems="{Binding dialog_skirt}"/>
+      </Grid>
+    </DataTemplate>
+  </Window.ContentTemplate>
+</Window>

src/client/spark/Modules/Trading/OrderBook/View/OrderBookAlgoParamsDialog.xaml.cs

@@ -0,0 +1,49 @@
+using bts.api.orderman;
+using bts.collections;
+using mbts.spark;
+using mbts.ui;
+using mbts.ui.controls;
+using mbts.ui.shell;
+using System;
+using System.Collections.Generic;
+using System.Windows;
+
+namespace bts.shell.orderbook {
+   /// <summary>
+   /// Interaction logic for OrderBookAlgoParamsDialog.xaml
+   /// </summary>
+   partial class OrderBookAlgoParamsDialog : Window {
+      public OrderBookAlgoParamsDialog() =>
+         InitializeComponent();
+   }
+
+   class OrderBookAlgoParamsVM : DisposableViewModelBase, IRequestViewClose {
+      readonly ObservableAggregateCollection<AlgoParamRowVM> _param_rows = new ObservableAggregateCollection<AlgoParamRowVM>();
+      readonly DialogSkirtItem _commit, _cancel;
+
+      public OrderBookAlgoParamsVM(AlgoDefinitionVM algo_definition, bool can_user_edit) {
+         this.algo_definition = algo_definition;
+         _dispose = AlgoParamRowsHelper.connect(algo_definition, _param_rows, false);
+         _commit = new DialogSkirtItem("OK", new DelegateCommand(commit, () => can_user_edit)) { is_default = true };
+         _cancel = new DialogSkirtItem("Cancel", new DelegateCommand(close)) { is_cancel = true };
+         // local methods
+         void commit() {
+            RequestCommit?.Invoke(new CustomOrderDefinitionImmutable(algo_definition.algo_root.created_assigned()));
+            close();
+         }
+         void close() => RequestClose?.Invoke(this, EventArgs.Empty);
+      }
+
+      public event EventHandler RequestClose;
+      public event Action<CustomOrderDefinitionImmutable> RequestCommit;
+
+      public AlgoDefinitionVM algo_definition { get; }
+      public IReadOnlyObservableCollection<AlgoParamRowVM> param_rows => _param_rows;
+      public IEnumerable<DialogSkirtItem> dialog_skirt {
+         get {
+            yield return _commit;
+            yield return _cancel;
+         }
+      }
+   }
+}

src/client/spark/Modules/Trading/OrderBook/View/OrderBookView.xaml

@@ -484,6 +484,7 @@
           <Border Padding="12,12,12,6">
             <Grid>
               <Grid.RowDefinitions>
+                <RowDefinition Height="Auto"/>
                 <RowDefinition Height="Auto"/>
                 <RowDefinition Height="*"/>
               </Grid.RowDefinitions>
@@ -495,7 +496,45 @@
                 AlgoParamRows="{Binding param_rows}"
                 IsReadOnly="{Binding DataContext.can_user_modify_selected_orders, ElementName=Root, Converter={StaticResource NotConverter}}"
               />
-              <StackPanel Grid.Row="1" HorizontalAlignment="Right" VerticalAlignment="Top" Margin="0,8,0,0" Orientation="Horizontal">
+              <c:ExpanderToggleButton
+                x:Name="algo_state_toggle"
+                Grid.Row="1"
+                BorderThickness="0"
+                Content="Algo State"
+                HorizontalContentAlignment="Left"
+                Margin="0,4,0,0"
+                VerticalAlignment="Top"
+                Visibility="{Binding current_algo_custom_state.Count, Converter={StaticResource IntNotZeroVisibilityConverter}}"
+              />
+              <c:DIYGrid Grid.Row="1" Rows="{Binding current_algo_custom_state}" Margin="0,28,0,4">
+                <c:DIYGrid.Visibility>
+                  <MultiBinding>
+                    <MultiBinding.Converter>
+                      <cv:MultiValueConverterPipe>
+                        <cv:BooleanAndConverter/>
+                        <cv:BooleanToVisibilityConverter/>
+                      </cv:MultiValueConverterPipe>
+                    </MultiBinding.Converter>
+                    <Binding Path="current_algo_custom_state.Count" Mode="OneWay" Converter="{StaticResource IntNotZeroConverter}"/>
+                    <Binding Path="IsChecked" ElementName="algo_state_toggle"/>
+                  </MultiBinding>
+                </c:DIYGrid.Visibility>
+                <c:DIYGrid.Columns>
+                  <c:DIYGridColumnCollection>
+                    <c:DIYGridColumn id="Name" Header="Param" Width="176"/>
+                    <c:DIYGridColumn id="Value" Header="Value" Width="176"/>
+                  </c:DIYGridColumnCollection>
+                </c:DIYGrid.Columns>
+                <c:DIYGrid.RowTemplate>
+                  <DataTemplate>
+                    <Grid c:DIYGrid.SyncWithColumnGeometry="True">
+                      <c:TextCell c:DIYGrid.ColumnKey="Name" Text="{Binding name}"/>
+                      <c:TextCell c:DIYGrid.ColumnKey="Value" Text="{Binding value}"/>
+                    </Grid>
+                  </DataTemplate>
+                </c:DIYGrid.RowTemplate>
+              </c:DIYGrid>
+              <StackPanel Grid.Row="2" HorizontalAlignment="Right" VerticalAlignment="Top" Margin="0,8,0,0" Orientation="Horizontal">
                 <Button x:Name="modify_orders_button" Command="{Binding modify_selected_orders_cmd}">
                   <Button.Margin>
                     <Thickness Right="{StaticResource DialogSkirtButtonPadding.Right}"/>

src/client/spark/Notifications/INotifier.cs

@@ -2,6 +2,7 @@
 using bts.api;
 using bts.shell;
 using bts.spark;
+using bts.utils;
 using mbts.config;
 using mbts.persist;
 using mbts.spark;
@@ -114,6 +115,7 @@ namespace mbts.ui {
             void dispose_core() {
                Disposable.dispose(ref _settings_sub);
                Disposable.dispose(ref _toast_controller);
+               Disposable.dispose(ref _sound_player);
                if (_dispatcher != null && !_dispatcher.HasShutdownStarted) {
                   _dispatcher.BeginInvokeShutdown(DispatcherPriority.Background);
                }
@@ -158,15 +160,25 @@ namespace mbts.ui {
             }
          }
 
-         class SoundNotificationPlayer {
-            readonly IReadOnlyDictionary<SoundNotification.SystemSound, NAudioPlayer> _players;
-
-            public static SoundNotificationPlayer create() => new SoundNotificationPlayer(_all_players());
+         class SoundNotificationPlayer : IDisposable {
+            IReadOnlyDictionary<SoundNotification.SystemSound, NAudioPlayer> _players;
 
             SoundNotificationPlayer(IReadOnlyDictionary<SoundNotification.SystemSound, NAudioPlayer> players) =>
                _players = players;
 
+            public static SoundNotificationPlayer create() =>
+               new SoundNotificationPlayer(_all_players());
+
+            public void Dispose() {
+               if (_players != null) {
+                  _players.Values.DisposeAll();
+                  _players = null;
+               }
+            }
+
             public void play_sounds(IEnumerable<SoundNotification> sounds) {
+               if (_players == null)
+                  return;
                foreach (var s in sounds) {
                   switch (s.sound_type_case) {
                      case SoundNotification.SoundTypeOneofCase.System: _players.GetValueOrDefault(s.system_sound.Value)?.play(); break;
@@ -177,7 +189,8 @@ namespace mbts.ui {
             }
 
             static IReadOnlyDictionary<SoundNotification.SystemSound, NAudioPlayer> _all_players() =>
-               EnumHelper.get_values<SoundNotification.SystemSound>().ToReadOnlyDictionary(s => s, NAudioPlayer.create);
+               EnumHelper.get_values<SoundNotification.SystemSound>()
+               .ToReadOnlyDictionaryWhere<SoundNotification.SystemSound, NAudioPlayer>(NAudioPlayer.try_create);
          }
 
          // This class adopts to role of view-controller in the mvc paradigm, rather than view-model in mvvm.

src/client/spark/Notifications/NAudioPlayer.cs

@@ -1,4 +1,5 @@
-using bts.shell;
+using bts;
+using bts.shell;
 using mbts.ui.utility;
 using NAudio.Wave;
 using System;
@@ -6,29 +7,71 @@ using System.IO;
 using System.Threading.Tasks;
 
 namespace mbts.ui {
-   class NAudioPlayer {
+   class NAudioPlayer : IDisposable {
       // number of milliseconds to delay playing sound,
       // during which time all subsequent requests to play the same sound will be ignored.
       const int _delay_ms = 75;
       readonly int _start_position;
+      IDisposable _stream_in;
       WaveOutEvent _wave_out = new WaveOutEvent();
       WaveFileReader _wave_reader;
       bool _is_playing, _is_play_requested;
 
-      public NAudioPlayer(Stream uwave_stream, int start_position = 0) {
-         var ms = new MemoryStream();
-         uwave_stream.CopyTo(ms);
-         ms.Position = 0;
-         _wave_reader = new WaveFileReader(ms);
+      /// <paramref name="stream_in"/> should not be disposed by the caller,
+      /// as NAudioPlayer handles that in its own Dispose method;
+      /// this constructor is private is to prevent such misuse.
+      NAudioPlayer(Stream stream_in, int start_position = 0) {
+         _stream_in = stream_in;
+         _wave_reader = new WaveFileReader(stream_in);
          _wave_reader.Position = _start_position = (start_position < _wave_reader.Length ? start_position : 0);
          _wave_out.Init(_wave_reader);
          _wave_out.PlaybackStopped += _on_playback_stopped;
       }
 
-      public static NAudioPlayer create(SoundNotification.SystemSound system_sound) {
-         var stream = _get_stream(system_sound);
-         var sb = EnumHelper.get_attribute<SoundNotification.SystemSound, SilenceBufferAttribute>(system_sound);
-         return sb != null ? new NAudioPlayer(stream, sb.start_position) : new NAudioPlayer(stream);
+      /// <see cref="_playback_stopped"/> is a proxy/buffer for the _wave_out.PlaybackStopped event.
+      /// It serves a couple purposes:
+      ///   1) The main callback (<see cref="_on_playback_stopped(object, StoppedEventArgs)"/>)
+      ///      is guarenteed to be executed first (though it would be anyways, since that
+      ///      callback is hooked up in the constructor, the proxy event makes it obvious).
+      ///   2) <see cref="_playback_stopped"/> is only raised after playback is truly complete,
+      ///      meaning there are no pending requests to repeat the sound.
+      event Action _playback_stopped;
+
+      public static bool try_create(SoundNotification.SystemSound system_sound, out NAudioPlayer result) {
+         if (_try_get_stream(system_sound, out var stream)) {
+            var sb = EnumHelper.get_attribute<SoundNotification.SystemSound, SilenceBufferAttribute>(system_sound);
+            result = sb != null ? new NAudioPlayer(stream, sb.start_position) : new NAudioPlayer(stream);
+            return true;
+         }
+         result = null;
+         return false;
+      }
+
+      public static void play(SoundNotification.SystemSound system_sound) {
+         if (try_create(system_sound, out var player)) {
+            try {
+               var temp = player;
+               temp._playback_stopped += temp.Dispose;
+               player.play();
+               player = null;
+            } catch (Exception ex) {
+               Log.Ui.warn($"Encountered exception playing the system sound '{system_sound}': {ex}.");
+            } finally {
+               player?.Dispose();
+            }
+         } else {
+            Log.Ui.warn($"Unable to play the system sound '{system_sound}': unexpected sound type (did not find an audio stream that corresponds to this enum value).");
+         }
+      }
+
+      public void Dispose() {
+         if (_wave_out != null) {
+            _wave_out.PlaybackStopped -= _on_playback_stopped;
+            _wave_out.Dispose();
+            _wave_out = null;
+         }
+         Disposable.dispose(ref _wave_reader);
+         Disposable.dispose(ref _stream_in);
       }
 
       public void play() {
@@ -46,6 +89,8 @@ namespace mbts.ui {
          if (_is_play_requested) {
             _is_play_requested = false;
             _play_core();
+         } else {
+            _playback_stopped?.Invoke();
          }
       }
 
@@ -56,18 +101,26 @@ namespace mbts.ui {
          _is_playing = false;
       }
 
-      static Stream _get_stream(SoundNotification.SystemSound system_sound) {
+      static bool _try_get_stream(SoundNotification.SystemSound system_sound, out Stream stream) {
+         UnmanagedMemoryStream um_stream;
          switch (system_sound) {
-            case SoundNotification.SystemSound.GameCorrect: return bts.Properties.Resources.game_sound_correct;
-            case SoundNotification.SystemSound.GameIncorrect: return bts.Properties.Resources.game_sound_wrong;
-            case SoundNotification.SystemSound.GameOverArcade: return bts.Properties.Resources.game_over_arcade;
-            case SoundNotification.SystemSound.Whistle: return bts.Properties.Resources.whistle;
-            case SoundNotification.SystemSound.BellCounter: return bts.Properties.Resources.bell_counter;
-            case SoundNotification.SystemSound.Powerup: return bts.Properties.Resources.powerup;
-            case SoundNotification.SystemSound.CashRegister: return bts.Properties.Resources.cash_register_purchase;
-            case SoundNotification.SystemSound.Reload: return bts.Properties.Resources.reload_1911;
-            default: throw new NotImplementedException($"Unknown {nameof(SoundNotification.SystemSound)}: {system_sound}.");
+            case SoundNotification.SystemSound.GameCorrect: um_stream = bts.Properties.Resources.game_sound_correct; break;
+            case SoundNotification.SystemSound.GameIncorrect: um_stream = bts.Properties.Resources.game_sound_wrong; break;
+            case SoundNotification.SystemSound.GameOverArcade: um_stream = bts.Properties.Resources.game_over_arcade; break;
+            case SoundNotification.SystemSound.Whistle: um_stream = bts.Properties.Resources.whistle; break;
+            case SoundNotification.SystemSound.BellCounter: um_stream = bts.Properties.Resources.bell_counter; break;
+            case SoundNotification.SystemSound.Powerup: um_stream = bts.Properties.Resources.powerup; break;
+            case SoundNotification.SystemSound.CashRegister: um_stream = bts.Properties.Resources.cash_register_purchase; break;
+            case SoundNotification.SystemSound.Reload: um_stream = bts.Properties.Resources.reload_1911; break;
+            default:
+               stream = null;
+               return false;
          }
+         stream = new MemoryStream();
+         um_stream.CopyTo(stream);
+         um_stream.Dispose();
+         stream.Position = 0;
+         return true;
       }
    }
-}
+}

src/client/spark/Orders/OrderTypeVMs/DynamicOrderHelper.cs

@@ -23,6 +23,32 @@ namespace mbts.spark {
 
       public IEnumerable<string> dynamic_types => _dynamic_types.DynamicOrderDesc.Select(d => d.DynamicOrderType);
 
+      public static string fmt_param(DynamicOrderParam dyn_param) {
+         if (dyn_param == null)
+            return null;
+         switch (dyn_param.ValueCase) {
+            case DynamicOrderParam.ValueOneofCase.AsArray:
+               return string.Join(", ", dyn_param.AsArray.Element.Select(fmt_param));
+            case DynamicOrderParam.ValueOneofCase.AsBool:
+               return dyn_param.AsBool.ToString();
+            case DynamicOrderParam.ValueOneofCase.AsDouble:
+               return dyn_param.AsDouble.ToString();
+            case DynamicOrderParam.ValueOneofCase.AsEnumerated:
+               return dyn_param.AsEnumerated.EnumValue;
+            case DynamicOrderParam.ValueOneofCase.AsInt32:
+               return dyn_param.AsInt32.ToString();
+            case DynamicOrderParam.ValueOneofCase.AsQuantity:
+               return dyn_param.AsQuantity.ToString();
+            case DynamicOrderParam.ValueOneofCase.AsString:
+               return dyn_param.AsString;
+            default:
+               // implement additional cases as needed
+               // (as of 4/17/20, this method is only used to format
+               // dynamic orders' CustomState params in the orderbook)
+               return $"({dyn_param.ValueCase})";
+         }
+      }
+
       public void add_dynamic_order_defs(List<CustomOrderDefinitionImmutable> defs, Func<CustomOrderDefinition.CustomOneofCase, CustomOrderDefinition> factory) {
          foreach (var d in _dynamic_types.DynamicOrderDesc) {
             var dorder = factory(CustomOrderDefinition.CustomOneofCase.Dynamic);

src/client/spark/Orders/OrderTypeVMs/GenericOrderHelper.cs

@@ -33,7 +33,7 @@ namespace mbts.spark
          {
             return;
          } 
-         var ev = fd.EnumType.Values.First(x => x.Name == value)?.Number;
+         var ev = fd.EnumType.Values.FirstOrDefault(x => x.Name == value)?.Number;
          if(ev.HasValue)
             fd.Accessor.SetValue(message, ev.Value);
       }

src/client/spark/Orders/OrderTypeVMs/GenericOrderVM.cs

@@ -2,7 +2,6 @@
 using bts.api;
 using bts.api.orderman;
 using bts.collections;
-using bts.spark;
 using Google.Protobuf;
 using Google.Protobuf.Reflection;
 using mbts.collections;
@@ -15,21 +14,19 @@ using System.Collections.ObjectModel;
 using System.ComponentModel;
 using System.Linq;
 using System.Reactive.Linq;
+using System.Reflection;
 using AlgoType = bts.api.orderman.CustomOrderDefinition.CustomOneofCase;
 
 namespace mbts.spark {
-   static partial class OrderTypeVMFactory 
-   {
-      internal class GenericOrderFactory
-      {
-         static public IAlgoVM create_vm(AlgoKey key, AlgoType algo_type, string display_name, AlgoEditingPermissions editing_permissions)
-         {
+   static partial class OrderTypeVMFactory {
+      internal class GenericOrderFactory {
+         static public IAlgoVM create_vm(AlgoKey key, AlgoType algo_type, string display_name, AlgoEditingPermissions editing_permissions) {
             var prop = typeof(CustomOrderDefinition).GetProperty(algo_type.ToString(),
-               System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public);
+               BindingFlags.Instance | BindingFlags.Public);
             var rt = prop.PropertyType;
 
             var propi = typeof(CustomOrderDefinitionImmutable).GetProperty(algo_type.ToString(),
-               System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public);
+               BindingFlags.Instance | BindingFlags.Public);
             var rti = propi.PropertyType;
 
             var vmt = typeof(GenericOrderVM<,>);
@@ -38,38 +35,33 @@ namespace mbts.spark {
             object vm = Activator.CreateInstance(makevm, key, algo_type, display_name, editing_permissions);
             return (IAlgoVM)vm;
          }
-         static public void setup_generic_custom_order(CustomOrderDefinition custom, CustomOrderDefinition.CustomOneofCase type)
-         {
+         static public void setup_generic_custom_order(CustomOrderDefinition custom, CustomOrderDefinition.CustomOneofCase type) {
             var prop = typeof(CustomOrderDefinition).GetProperty(type.ToString(),
-               System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public);
+               BindingFlags.Instance | BindingFlags.Public);
             var rt = prop.PropertyType;
             var rtt = Activator.CreateInstance(rt);
-            
+
             var ci = rt.GetProperties().Where(x => x.PropertyType == typeof(CustomOrderDefinition));
-            foreach(var cid in ci)
+            foreach (var cid in ci)
                cid.SetValue(rtt, new CustomOrderDefinition { Limit = new LimitOrderDefinition() });
             prop.SetValue(custom, rtt);
          }
-      }      
-      
+      }
+
       class GenericOrderVM<TMutable, TImmutable> : OrderTypeVMBase<TMutable, TImmutable>
-         where TMutable : class, IMessage, new() where TImmutable : struct, IImmutableMessage
-      {         
+         where TMutable : class, IMessage, new() where TImmutable : struct, IImmutableMessage {
          readonly ObservableKeyedCollection<GenericOrderParamVM<TMutable>, string> _parameters =
             new ObservableKeyedCollection<GenericOrderParamVM<TMutable>, string>(vm => vm.descriptor.Name);
          readonly AlgoType _algo_type;
 
          public GenericOrderVM(AlgoKey key, AlgoType algo_type, string display_name, AlgoEditingPermissions editing_permissions)
-            : base(key, display_name, editing_permissions) 
-         {
+            : base(key, display_name, editing_permissions) {
             _algo_type = algo_type;
             _parameters = new ObservableKeyedCollection<GenericOrderParamVM<TMutable>, string>(vm => vm.descriptor.Name);
             _resources.AddRange(
-               ObservableAutoWrapper.connect_item_hooks(_parameters, p =>
-               {
+               ObservableAutoWrapper.connect_item_hooks(_parameters, p => {
                   p.PropertyChanged += on_param_prop_changed;
-                  return Disposable.Create(() => 
-                  {
+                  return Disposable.Create(() => {
                      p.PropertyChanged -= on_param_prop_changed;
                      remove_property(p.descriptor.Name);
                      p.Dispose();
@@ -80,84 +72,74 @@ namespace mbts.spark {
          }
 
          public string algo_type => _algo_type.ToString();
-         public override string abbreviation => 
+         public override string abbreviation =>
             model == null ? display_name : model.factory.abbreviate_algo(new OneOf<CustomOrderDefinition.CustomOneofCase, string>(_algo_type));
          public override AlgoDisplayGroup group => AlgoDisplayGroup.API;
          public ObservableCollection<GenericOrderParamVM<TMutable>> parameters => _parameters;
 
-         protected override void on_model_internal() 
-         {
+         protected override void on_model_internal() =>
             _sync_generic_params();
-         }
 
-         void _sync_generic_params()
-         {
-            if (model != null)
-            {               
-               Google.Protobuf.Reflection.MessageDescriptor desc = new TMutable().Descriptor;               
+         void _sync_generic_params() {
+            if (model != null) {
+               MessageDescriptor desc = new TMutable().Descriptor;
                _parameters.SyncWithDictionary(
                   desc.Fields.InDeclarationOrder().Where(_is_value_param_desc).ToDictionary(p => p.Name), _mk_param_vm);
             }
          }
-         static bool _is_value_param_desc(FieldDescriptor desc)
-         {
-            return (desc.FieldType != Google.Protobuf.Reflection.FieldType.Message || 
+
+         static bool _is_value_param_desc(FieldDescriptor desc) =>
+            (desc.FieldType != FieldType.Message ||
                _is_quantity_measure_desc(desc));
-         }
 
-         static bool _is_quantity_measure_desc(FieldDescriptor desc)
-         {
-            return desc.FieldType == Google.Protobuf.Reflection.FieldType.Message &&
+         static bool _is_quantity_measure_desc(FieldDescriptor desc) =>
+            desc.FieldType == FieldType.Message &&
                desc.MessageType.FullName == typeof(bts.spark.QuantityMeasure).FullName;
-         }
 
-         protected override IEnumerable<ChildAlgoData> get_selector_data() 
-         {
-            var v = OrderDataHelper.get_child_algo_data(_algo_type);
-            return v;
-         }
+         protected override IEnumerable<ChildAlgoData> get_selector_data() =>
+            OrderDataHelper.get_child_algo_data(_algo_type);
 
-         GenericOrderParamVM<TMutable> _mk_param_vm(FieldDescriptor descriptor) 
-         {
+         GenericOrderParamVM<TMutable> _mk_param_vm(FieldDescriptor descriptor) {
             ExceptionAssert.Argument.NotNull(descriptor, nameof(descriptor));
-
-            if (descriptor.FieldType == FieldType.Message && 
-                descriptor.MessageType.FullName == typeof(bts.spark.QuantityMeasure).FullName)
-            {
+            if (descriptor.FieldType == FieldType.Message &&
+                descriptor.MessageType.FullName == typeof(bts.spark.QuantityMeasure).FullName) {
                var qm = new GenericOrderQuantityVM<TMutable>(descriptor);
                return attached(qm, GenericOrderHelper.get_quantity_value, GenericOrderHelper.set_quantity_value);
-            }
-            else if(descriptor.FieldType == FieldType.Enum)
-            {
-               var pvm = new GenericOrderEnumeratedVM<TMutable>(descriptor);
-               attached(pvm, GenericOrderHelper.get_enumerated_value, GenericOrderHelper.set_enumerated_value);
-               return pvm;
-            }
-            else if (descriptor.FieldType == FieldType.Bool)
+            } else if (descriptor.FieldType == FieldType.Enum) {
+               return EnumParamFactory.create(this, descriptor);
+            } else if (descriptor.FieldType == FieldType.Bool) {
                return mk<bool?, bool>(descriptor, v => v, v => v.GetValueOrDefault());
-            else if (descriptor.FieldType == FieldType.Double || descriptor.FieldType == FieldType.Float)
+            } else if (descriptor.FieldType == FieldType.Double || descriptor.FieldType == FieldType.Float) {
                return mk<double, double>(descriptor, v => v, v => v, double.NaN);
-            else if (descriptor.FieldType == FieldType.Int32)
+            } else if (descriptor.FieldType == FieldType.Int32) {
                return mk<int?, int>(descriptor, v => v, v => v.GetValueOrDefault());
-            else if (descriptor.FieldType == FieldType.UInt32)
+            } else if (descriptor.FieldType == FieldType.UInt32) {
                return mk<uint?, uint>(descriptor, v => v, v => v.GetValueOrDefault());
-            else if (descriptor.FieldType == FieldType.String)
+            } else if (descriptor.FieldType == FieldType.String) {
                return mk<string, string>(descriptor, v => v, v => v);
-            else
-               return null;
+            } else return null;
          }
-         GenericOrderParamVM<TMutable, TView, TModel> mk<TView, TModel>(FieldDescriptor fdesc, Func<TModel, TView> convert, Func<TView, TModel> convert_back, TView default_value = default)
-         {
-            var pvm = new GenericOrderParamVM<TMutable, TView, TModel>(fdesc, convert, convert_back);
-            return attached<TView, GenericOrderParamVM<TMutable, TView, TModel>>(pvm, get_from_param, assign_to_param, default_value);
 
+         GenericOrderParamVM<TMutable, TView, TModel> mk<TView, TModel>(
+            FieldDescriptor fdesc,
+            Func<TModel, TView> convert,
+            Func<TView, TModel> convert_back,
+            TView default_value = default) {
+            //
+            var pvm = new GenericOrderParamVM<TMutable, TView, TModel>(fdesc, convert, convert_back);
+            return attached(pvm, get_from_param, assign_to_param, default_value);
             TView get_from_param(IMessage param, FieldDescriptor fd) => convert(ConvertUtils.try_convert<TModel>(fdesc.Accessor.GetValue(param)));
             void assign_to_param(IMessage param, FieldDescriptor fd, TView value) => fdesc.Accessor.SetValue(param, convert_back(value));
          }
 
-         TParam attached<T, TParam>(TParam pvm, Func<IMessage, FieldDescriptor, T> selector, Action<IMessage, FieldDescriptor, T> assign_to_param, T default_value = default) where TParam : GenericOrderParamVM<TMutable>, IGenericOrderParamVM<T>
-         {
-            add_property<T>(
+         TParam attached<T, TParam>(
+            TParam pvm,
+            Func<IMessage, FieldDescriptor, T>
+            selector, Action<IMessage, FieldDescriptor, T> assign_to_param,
+            T default_value = default)
+            where TParam : GenericOrderParamVM<TMutable>, IGenericOrderParamVM<T> {
+            //
+            add_property(
                pvm.descriptor.Name,
                mdef => mdef != null ? selector(mdef, pvm.descriptor) : default_value,
                (mdef, v) => assign_to_param(mdef, pvm.descriptor, v),
@@ -166,7 +148,37 @@ namespace mbts.spark {
                pvm.subscribe_value(),
                default_value);
             return pvm;
-         }         
+         }
+
+         static class EnumParamFactory {
+            [ThreadStatic]
+            static Dictionary<Type, IFactory> _factories_backing;
+            static Dictionary<Type, IFactory> _factories =>
+               _factories_backing ?? (_factories_backing = new Dictionary<Type, IFactory>());
+
+            public static GenericOrderParamVM<TMutable> create(GenericOrderVM<TMutable, TImmutable> owner, FieldDescriptor descriptor) {
+               if (descriptor.FieldType != FieldType.Enum)
+                  throw ExceptionBuilder.Argument.unexpected_case(descriptor.FieldType, $"{nameof(descriptor)}.{nameof(FieldDescriptor.FieldType)}");
+               if (!_factories.TryGetValue(descriptor.EnumType.ClrType, out var f)) {
+                  // Impl<T> is a nested type, so parent class's generic type args must be included.
+                  _factories[descriptor.EnumType.ClrType] = f = (IFactory)Activator.CreateInstance(typeof(Impl<>)
+                     .MakeGenericType(typeof(TMutable), typeof(TImmutable), descriptor.EnumType.ClrType));
+               }
+               return f.create(owner, descriptor);
+            }
+
+            interface IFactory {
+               GenericOrderParamVM<TMutable> create(GenericOrderVM<TMutable, TImmutable> owner, FieldDescriptor descriptor);
+            }
+
+            class Impl<T> : IFactory where T : struct {
+               static readonly Func<T, T?> _convert = v => v;
+               static readonly Func<T?, T> _convert_back = v => v ?? default;
+
+               public GenericOrderParamVM<TMutable> create(GenericOrderVM<TMutable, TImmutable> owner, FieldDescriptor descriptor) =>
+                  owner.mk(descriptor, _convert, _convert_back, default);
+            }
+         }
       }
    }
-}
+}

src/client/spark/Orders/OrderUtils.cs

@@ -63,6 +63,11 @@ namespace mbts.spark {
       public static string last_msg(this OrderRecordImmutable order) =>
          order.HasState ? order.State.LastMsg : string.Empty;
 
+      public static IReadOnlyDictionary<string, DynamicOrderParam> dynamic_custom_state(this OrderRecordImmutable order) =>
+         order.HasState && order.State.HasDynamic ?
+            (order.State.Dynamic.CustomState ?? DictionaryHelper.empty<string, DynamicOrderParam>()) :
+            DictionaryHelper.empty<string, DynamicOrderParam>();
+
       public static DateTime created(this OrderRecordImmutable order) =>
          order.HasState && order.State.HasCreated ? order.State.Created.ToDateTime().ToLocalTime() : DateTime.MinValue;
 

src/client/spark/Orders/OrdersModel.cs

@@ -215,8 +215,11 @@ namespace mbts.spark {
             FillTuple.key_fn);
          on_fill_tuples_with_root = fill_order_tuples_with_root_join.Cache();
          // manual fills
-         on_manual_fills_securities = _position_service.subscribei_manual_fills()
-            .Naggle(interval, scheduler)
+         on_manual_fills_securities = admin_model
+            .permission_filter(
+               _position_service.subscribei_manual_fills()
+               .Naggle(interval, scheduler),
+               f => f.AccountId)
             .Filter(f => f.FillType == ManualFill.Types.FillType.Local)
             .join(sstore, mf => mf.SecurityKey)
             .Table().Cache();

src/client/spark/Spark.csproj

@@ -12,6 +12,7 @@
     <RuntimeIdentifiers>win7-x64</RuntimeIdentifiers>
     <PlatformTarget>AnyCPU</PlatformTarget>
     <LanguageTargets>$(MSBuildExtensionsPath)\$(VisualStudioVersion)\Bin\Microsoft.CSharp.targets</LanguageTargets>
+    <LanguageTargets Condition="'$(VisualStudioVersion)'=='16.0'">$(MSBuildToolsPath)\Microsoft.CSharp.targets</LanguageTargets>
     <ApplicationIcon>logo.ico</ApplicationIcon>
     <OutputType>WinExe</OutputType>
     <StartupObject>mbts.ui.shell.App</StartupObject>

src/client/spark/core/GlobalExtensions/DoubleHelper.cs

@@ -63,5 +63,20 @@ namespace mbts {
 
       public static double? nan_nullable(double value) =>
          double.IsNaN(value) ? (double?)null : value;
+
+      public static int? to_optional_int32(double value) {
+         if (double.IsNaN(value) || double.IsInfinity(value))
+            return null;
+         if (value > 0) {
+            value = Math.Floor(value);
+            if (value <= int.MaxValue)
+               return (int)value;
+         } else if (value < 0) {
+            value = Math.Ceiling(value);
+            if (value >= int.MinValue)
+               return (int)value;
+         }
+         return null;
+      }
    }
 }

src/client/spark/view/Controls/DIYGrid/Columns/DIYGridColumn.cs

@@ -0,0 +1,9 @@
+namespace mbts.ui.controls {
+   class DIYGridColumn : DIYGrid.ColumnBase<string> {
+      public DIYGridColumn()
+         : base() {
+         is_visible = true;
+         Width = 60;
+      }
+   }
+}

src/client/spark/view/Controls/DIYGrid/Settings/Animation/SoundAnimationView.xaml.cs

@@ -38,11 +38,7 @@ namespace mbts.ui.controls {
       }
 
       public static ICommand PreviewSoundCommand =>
-         new DelegateCommand<SoundNotification.SystemSound>(s =>  {
-            try {
-               NAudioPlayer.create(s).play();
-            } catch (NAudio.MmException) { }
-         });
+         new DelegateCommand<SoundNotification.SystemSound>(NAudioPlayer.play);
 
       #region SoundTypes
       public static readonly DependencyProperty SoundTypesProperty =

src/client/spark/view/Controls/Order/AlgoContentTemplateSelector.xaml

@@ -1005,6 +1005,8 @@
             ContractPickerModel="{Binding contract_picker_model}"
             HedgePricing="{Binding hedge_pricing}"
             HedgePricingItemsSource="{Binding hedge_pricing_options}"
+            UseFractionalTicks="{Binding use_fractional_ticks}"
+            IntegerTickSize="{Binding integer_tick_size}"
           />
           <c:DialogSkirt Grid.Row="1" Margin="-4" VerticalAlignment="Top" RightItems="{Binding skirt_items}"/>
         </Grid>

src/client/spark/view/Converters/BooleanToVisibilityConverter.cs

@@ -12,6 +12,9 @@ namespace mbts.ui.converters
       public Visibility VisibilityForFalse { get { return _visfalse; } set { _visfalse = value; } }
 
       public static BooleanToVisibilityConverter Instance => new BooleanToVisibilityConverter();
+      public static BooleanToVisibilityConverter HiddenInstance => new BooleanToVisibilityConverter {
+         VisibilityForFalse = Visibility.Hidden
+      };
 
       public object Convert(object value, Type targetType, object parameter, System.Globalization.CultureInfo culture) {
          bool flag = false;

src/client/spark/view/Spark/Core/TickUtils.cs

@@ -49,6 +49,28 @@ namespace mbts.spark {
       public static double parse(this ContractSpecificationImmutable? spec, string text, double display_multiplier, string nan_text) =>
          spec.spec_or_default().parse(text, display_multiplier, nan_text);
 
+      public static bool is_integer_tick(ContractSpecificationImmutable spec) =>
+         spec.MainFraction == 0 &&
+         spec.PriceDisplayFormat == 0 &&
+         spec.LowTick > 0 &&
+         MathUtils.is_int(spec.LowTick) &&
+         spec.HighTick > 0 &&
+         MathUtils.is_int(spec.HighTick);
+
+      public static ContractSpecification create_integer_tick_spec(int low_tick) =>
+         create_integer_tick_spec(low_tick, low_tick);
+
+      public static ContractSpecification create_integer_tick_spec(int low_tick, int high_tick) {
+         var rv = SecurityDataHelper.Defaults.ContractSpecification;
+         rv.MainFraction = 0;
+         rv.SubFraction = 0;
+         rv.PriceDisplayFormat = 0;
+         rv.PriceBreak = 0;
+         rv.LowTick = low_tick;
+         rv.HighTick = high_tick;
+         return rv;
+      }
+
       public static double low_tick(NativePriceSize[] bids, NativePriceSize[] asks) {
          double low = bids[bids.Length - 1].price;
          if (double.IsNaN(low))
@@ -326,7 +348,7 @@ namespace mbts.spark {
          }
 
          static bool _check_tick_size(double tick_size) =>
-            (tick_size > 0 && tick_size <= 0.5) || tick_size == 1;
+            tick_size > 0 && (tick_size <= 0.5 || MathUtils.is_int(tick_size));
 
          struct SpecKey : IEquatable<SpecKey> {
             public readonly uint denominator, display_digits;

src/client/spark/view/Utility/CommandHelper.cs

@@ -1,5 +1,7 @@
 using mbts.ui.converters;
+using System;
 using System.Globalization;
+using System.Reactive.Linq;
 using System.Windows;
 using System.Windows.Data;
 using System.Windows.Input;
@@ -12,6 +14,31 @@ namespace mbts.ui.utilities {
       public static string fmt(string title, KeyGesture key_gesture) =>
          key_gesture == null ? title : $"{title} ({key_gesture.GetDisplayStringForCulture(CultureInfo.CurrentCulture)})";
 
+      public static IDisposable bind_param_source<T>(ICommand command, IObservable<T> param_source, out ICommand bound_cmd, bool dispose_command_on_dispose = false) {
+         bool initialized = false;
+         bool can_execute = false;
+         T param = default;
+         var bcmd = new DelegateCommand(() => command.Execute(param), () => can_execute);
+         bound_cmd = bcmd;
+         var sub = param_source.Subscribe(p => {
+            param = p;
+            if (!initialized) {
+               initialized = true;
+               can_execute = command.CanExecute(p);
+               bcmd.RaiseCanExecuteChanged();
+            } else if (can_execute != command.CanExecute(p)) {
+               can_execute = !can_execute;
+               bcmd.RaiseCanExecuteChanged();
+            }
+         });
+         return dispose_command_on_dispose ?
+            Disposable.Create(() => {
+               Disposable.dispose(ref sub);
+               (command as IDisposable)?.Dispose();
+               command = null;
+            }) : sub;
+      }
+
       #region Paraphrased from https://referencesource.microsoft.com/#PresentationFramework/src/Framework/MS/Internal/Commands/CommandHelpers.cs
 
       public static bool CanExecuteCommandSource(ICommandSource commandSource) {

src/client/spark/view/Utility/EnabledValueVM.cs

@@ -30,6 +30,21 @@ namespace mbts.ui {
       public string name { get; }
    }
 
+   class NamedValueVM : ViewModelBase {
+      string _name;
+      object _value;
+
+      public NamedValueVM() { }
+
+      public NamedValueVM(string name, object value) {
+         _name = name;
+         _value = value;
+      }
+
+      public string name { get => _name; set => Set(ref _name, value); }
+      public object value { get => _value; set => Set(ref _value, value); }
+   }
+
    static class EnabledValueVMExtensions {
       public static T? get_nullable<T>(this EnabledValueVM<T> vm) where T : struct =>
          vm.enabled ? vm.value : (T?)null;

src/client/spark_api/security/SecurityHelper.cs

@@ -83,22 +83,24 @@ namespace bts.api {
          }
 
          public static int Compare(SecurityImmutable x, SecurityImmutable y) {
-            if (x.ContractCase == ContractOneofCase.Future) return y.ContractCase == ContractOneofCase.Future ? Compare(x.Symbol, x.Future, y.Symbol, y.Future) : -1;
-            else if (y.ContractCase == ContractOneofCase.Future) return 1;
+            if (x.ContractCase == ContractOneofCase.Future) {
+               return y.ContractCase == ContractOneofCase.Future ? Compare(x.Symbol, x.Future, y.Symbol, y.Future) : -1;
+            } else if (y.ContractCase == ContractOneofCase.Future) {
+               return 1;
+            }
             var xlegs = get_flat_future_legs(x).ToList();
             var ylegs = get_flat_future_legs(y).ToList();
             int max = Math.Min(xlegs.Count, ylegs.Count);
             for (int i = 0; i < max; i++) {
                int rv = Compare(xlegs[i].symbol, xlegs[i].future, ylegs[i].symbol, ylegs[i].future);
-               if (rv != 0) return rv;
-            }
-            for (int i = 0; i < max; i++) {
-               int rv = xlegs[i].quantity.CompareTo(ylegs[i].quantity);
-               if (rv != 0) return rv;
-            }
-            for (int i = 0; i < max; i++) {
-               int rv = xlegs[i].exchange.CompareTo(ylegs[i].exchange);
-               if (rv != 0) return rv;
+               if (rv != 0)
+                  return rv;
+               rv = xlegs[i].quantity.CompareTo(ylegs[i].quantity);
+               if (rv != 0)
+                  return rv;
+               rv = xlegs[i].exchange.CompareTo(ylegs[i].exchange);
+               if (rv != 0)
+                  return rv;
             }
             return xlegs.Count.CompareTo(ylegs.Count);
          }

src/client/spark_api/utils/Log.cs

@@ -36,6 +36,7 @@ namespace bts {
       Plugin,
       Sdk,
       Http,
+      iLink3
       // Please keep this list in sync with logging.h/logging.cpp
    }
    

src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs

@@ -390,6 +390,15 @@ namespace System.Collections.Generic {
       public static ReadOnlyDictionary<TKey, TElement> ToReadOnlyDictionary<T, TKey, TElement>(this IEnumerable<T> enumerable, Func<T, TKey> key_selector, Func<T, TElement> element_selector, IEqualityComparer<TKey> comparer) =>
          new ReadOnlyDictionary<TKey, TElement>(enumerable.ToDictionary(key_selector, element_selector, comparer));
 
+      public static ReadOnlyDictionary<TKey, TElement> ToReadOnlyDictionaryWhere<TKey, TElement>(this IEnumerable<TKey> keys, TryGetValue<TKey, TElement> value_selector) {
+         var mdict = new Dictionary<TKey, TElement>();
+         foreach (var k in keys) {
+            if (value_selector(k, out var val))
+               mdict[k] = val;
+         }
+         return new ReadOnlyDictionary<TKey, TElement>(mdict);
+      }
+
       public static ILookup<TKey, TElement> ToFlatLookup<T, TKey, TElement>(this IEnumerable<T> source, Func<T, TKey> key_selector, Func<T, IEnumerable<TElement>> selector) =>
          source.SelectMany(v => {
             var k = key_selector(v);

src/client/spark_core/Collections/Extensions/ListExtensions.cs

@@ -103,6 +103,15 @@ namespace bts.collections {
          return array;
       }
 
+      public static bool SafeTryGetAt<T>(this IList<T> list, int index, out T value) {
+         if (list != null && index >= 0 && index < list.Count) {
+            value = list[index];
+            return true;
+         }
+         value = default;
+         return false;
+      }
+
       public static bool TryGetAt<T>(this IList<T> list, int index, out T value) {
          if (index >= 0 && index < list.Count) {
             value = list[index];

src/client/spark_core/utils/Delegates.cs

@@ -1,11 +1,11 @@
 using System;
 using System.Globalization;
 
-namespace mbts.utils {
-   delegate bool TryGetValue<T>(out T value);
-   delegate bool TryGetValue<TKey, TValue>(TKey key, out TValue value);
+namespace bts.utils {
+   public delegate bool TryGetValue<T>(out T value);
+   public delegate bool TryGetValue<TKey, TValue>(TKey key, out TValue value);
 
-   static class LogicalDelegates {
+   public static class LogicalDelegates {
       public static readonly Func<bool, bool, bool>
          And = (a, b) => a && b,
          Or = (a, b) => a || b,
@@ -13,7 +13,7 @@ namespace mbts.utils {
          None = (a, b) => !a && !b;
    }
 
-   static class FormattingDelegates {
+   public static class FormattingDelegates {
       public static class Price {
          public static readonly Func<double, string> Value =
             p => double.IsNaN(p) ? "" : p.ToString("#,0.00");
@@ -26,4 +26,4 @@ namespace mbts.utils {
             p => p.ToString("#,0.00", CultureInfo.CurrentCulture);
       }
    }
-}
+}

src/client/spark_core/utils/FilePathHelper.cs

@@ -0,0 +1,21 @@
+using System;
+using System.IO;
+
+namespace bts.utils {
+   public static class FilePathHelper {
+      public static string GetUniquePathWithDate(string path_root, string ext) =>
+         GetUniquePath($"{path_root}_{DateTime.Now:MM - dd - yyyy_hh - mm}", ext);
+
+      public static string GetUniquePath(string path_root, string ext) {
+         var path = $"{path_root}.{ext}";
+         const int max_copies = 2 ^ 14;
+         int i = 1;
+         while (File.Exists(path)) {
+            path = $"{path_root}({i++}).{ext}";
+            if (i == max_copies)
+               throw new InvalidOperationException("Unable to make path unique by concating a number.");
+         }
+         return path;
+      }
+   }
+}

src/client/spark_core/utils/ObservableExtensions.cs

@@ -209,7 +209,7 @@ namespace System.Reactive.Linq {
       /// <returns>An IDisposable object used to unsubscribe from the source sequence, and dispose the latest value subscription.</returns>
       public static IDisposable SubscribeEach<T>(this IObservable<T> source, Func<T, IDisposable> subscribe_value) {
          IDisposable value_sub = null;
-         IDisposable source_sub = source.Select(v => v).Subscribe(on_source);
+         IDisposable source_sub = source.Subscribe(on_source);
          return Disposable.Create(dispose);
          // local methods
          void on_source(T value) {

src/client/spark_core/utils/UpdateExtensions.cs

@@ -1,208 +1,253 @@
-using bts.api.core;
-using bts.collections;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Reactive.Linq;
-
-namespace bts.utils {
-   public static class UpdateExtensions {
-      public static TableUpdate<T, TKey> with_added<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> added) where TKey : IEquatable<TKey> =>
-         TableUpdate<T, TKey>.update(table, added, Enumerable.Empty<Change<T>>(), Enumerable.Empty<T>(), table.key_extractor);
-
-      public static TableUpdate<T, TKey> with_removed<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> removed) where TKey : IEquatable<TKey> =>
-         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), Enumerable.Empty<Change<T>>(), removed, table.key_extractor);
-
-      public static TableUpdate<T, TKey> with_changed<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> changed) where TKey : IEquatable<TKey> =>
-         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), changed.Select(v => new Change<T>(table.state[table.key_extractor(v)], v)), Enumerable.Empty<T>(), table.key_extractor);
-
-      public static TableUpdate<T, TKey> with_changes<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<Change<T>> changes) where TKey : IEquatable<TKey> =>
-         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), changes, Enumerable.Empty<T>(), table.key_extractor);
-
-      public static IObservable<TableUpdate<TResult, TKey>> RootDependent<T, TResult, TKey>(
-         this IObservable<TableUpdate<T, TKey>> source,
-         Func<T, TKey> key,
-         Func<T, TKey> parent_key,
-         Func<T, T, TResult> with_root,
-         Func<T, TResult> without_root,
-         Func<TResult, TKey> result_key)
-         where TKey : IEquatable<TKey> =>
-         RootDependent(source, key, parent_key, with_root, without_root, result_key, default);
-
-      // TODO: unit tests
-      public static IObservable<TableUpdate<TResult, TKey>> RootDependent<T, TKey, TResult>(
-         this IObservable<TableUpdate<T, TKey>> source,
-         Func<T, TKey> key,
-         Func<T, TKey> parent_key,
-         Func<T, T, TResult> get_result,
-         Func<T, TResult> get_result_no_parent,
-         Func<TResult, TKey> result_key,
-         TKey no_parent_key)
-         where TKey : IEquatable<TKey> =>
-         Observable.Create<TableUpdate<TResult, TKey>>(observer => {
-            var state = DictionaryHelper.empty<TKey, T>();
-            var output = TableUpdate<TResult, TKey>.mk_empty(result_key);
-            var parent_to_children = new SetMap<TKey, T>();
-            return source.Subscribe(update);
-            //
-            void update(TableUpdate<T, TKey> input) {
-               state = input.state;
-               var builder = output.to_builder();
-               // execute all parent_to_children changes first, so descendent-propagation will be efficient (otherwise propagation might be needlessly performed on values that are yet to be removed, or performed with the wrong root, if the real root is yet to be added).
-               foreach (var v in input.removed.Concat(input.changes.Select(ch => ch.Old))) {
-                  var k = input.key_extractor(v);
-                  parent_to_children.remove(parent_key(v), v);
-               }
-               foreach (var v in input.added.Concat(input.changed)) {
-                  var pk = parent_key(v);
-                  if (!pk.Equals(no_parent_key))
-                     parent_to_children.add(pk, v);
-               }
-               foreach (var v in input.removed) {
-                  var k = input.key_extractor(v);
-                  builder.remove(k);
-                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
-                     foreach (var child in children) {
-                        var ck = key(child);
-                        builder.add_or_update(ck, get_result_no_parent(child));
-                        foreach (var d in descendents(ck))
-                           builder.add_or_update(key(d), get_result(d, child));
-                     }
-                  }
-               }
-               foreach (var v in input.changed) {
-                  var k = input.key_extractor(v);
-                  var root = get_root(v);
-                  TResult result;
-                  if (key(root).Equals(k)) {
-                     result = get_result_no_parent(v);
-                     foreach (var d in descendents(k))
-                        builder.add_or_update(key(d), get_result(d, v));
-                  } else {
-                     result = get_result(v, root);
-                  }
-                  builder.add_or_update(k, result);
-               }
-               foreach (var v in input.added) {
-                  var k = input.key_extractor(v);
-                  var pk = parent_key(v);
-                  TResult result;
-                  if (pk.Equals(no_parent_key)) {
-                     result = get_result_no_parent(v);
-                     foreach (var d in descendents(k))
-                        builder.add_or_update(key(d), get_result(d, v));
-                  } else {
-                     parent_to_children.add(pk, v);
-                     var root = get_root(v);
-                     if (key(root).Equals(k)) {
-                        result = get_result_no_parent(v);
-                        foreach (var d in descendents(k))
-                           builder.add_or_update(key(d), get_result(d, v));
-                     } else {
-                        result = get_result(v, root);
-                     }
-                  }
-                  builder.add_or_update(k, result);
-               }
-               observer.OnNext(output = builder.to_table());
-            }
-            T get_root(T value) {
-               var pk = parent_key(value);
-               while (!pk.Equals(no_parent_key) && state.TryGetValue(pk, out var parent)) {
-                  value = parent;
-                  pk = parent_key(value);
-               }
-               return value;
-            }
-            IEnumerable<T> descendents(TKey k) {
-               if (parent_to_children.try_lookup(k, out ISet<T> children)) {
-                  foreach (var child in children) {
-                     yield return child;
-                     foreach (var d in descendents(key(child)))
-                        yield return d;
-                  }
-               }
-            }
-         });
-
-      public static IObservable<TableUpdate<T, TKey>> MergeTable<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, IObservable<TableUpdate<T, TKey>> other) where TKey : IEquatable<TKey> =>
-         source.AggregateLatest(other, _Merge<T, TKey>.source_change_del, _Merge<T, TKey>.other_change_del, () => TableUpdate<T, TKey>.empty, TableUpdate<T, TKey>.empty, TableUpdate<T, TKey>.empty);
-
-      public static IObservable<TableUpdate<KeyValuePair<TKey, TResult>, TKey>> ParentDependent<T, TKey, TResult>(
-         this IObservable<TableUpdate<T, TKey>> source,
-         Func<T, TKey> key,
-         Func<T, TKey> parent_key,
-         Func<T, T, TResult> get_result,
-         Func<T, TResult> get_result_no_parent)
-         where TResult : IEquatable<TResult>
-         where TKey : IEquatable<TKey> =>
-         ParentDependent(source, key, parent_key, get_result, get_result_no_parent, EqualityComparer<TResult>.Default, default);
-
-      public static IObservable<TableUpdate<KeyValuePair<TKey, TResult>, TKey>> ParentDependent<T, TKey, TResult>(
-         this IObservable<TableUpdate<T, TKey>> source,
-         Func<T, TKey> key,
-         Func<T, TKey> parent_key,
-         Func<T, T, TResult> get_result,
-         Func<T, TResult> get_result_no_parent,
-         IEqualityComparer<TResult> result_comparer,
-         TKey no_parent_key)
-         where TKey : IEquatable<TKey> =>
-         Observable.Create<TableUpdate<KeyValuePair<TKey, TResult>, TKey>>(observer => {
-            var value_comparer = EqualityComparer.Create<KeyValuePair<TKey, TResult>, TResult>(kv => kv.Value, result_comparer);
-            var state = DictionaryHelper.empty<TKey, T>();
-            var output = TableUpdate<KeyValuePair<TKey, TResult>, TKey>.mk_empty(kv => kv.Key);
-            var parent_to_children = new SetMap<TKey, T>();
-            return source.Subscribe(update);
-            //
-            void update(TableUpdate<T, TKey> input) {
-               state = input.state;
-               var builder = output.to_builder();
-               foreach (var v in input.removed) {
-                  var k = input.key_extractor(v);
-                  builder.remove(k);
-                  parent_to_children.remove(parent_key(v), v);
-                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
-                     foreach (var child in children)
-                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result_no_parent(child)));
-                  }
-               }
-               foreach (var v in input.changed) {
-                  var k = key(v);
-                  builder.add_or_update(new KeyValuePair<TKey, TResult>(k, lookup_result(v)));
-                  parent_to_children.set(parent_key(v), v);
-                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
-                     foreach (var child in children)
-                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result(child, v)));
-                  }
-               }
-               foreach (var v in input.added) {
-                  var k = key(v);
-                  builder.add_or_update(new KeyValuePair<TKey, TResult>(k, lookup_result(v)));
-                  parent_to_children.add(parent_key(v), v);
-                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
-                     foreach (var child in children)
-                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result(child, v)));
-                  }
-               }
-               var result = builder.to_table_checked(value_comparer);
-               if (result.has_changes())
-                  observer.OnNext(output = result);
-            }
-            TResult lookup_result(T child) {
-               var pk = parent_key(child);
-               return !pk.Equals(no_parent_key) && state.TryGetValue(pk, out T parent) ?  get_result(child, parent) : get_result_no_parent(child);
-            }
-         });
-
-      static class _Merge<T, TKey> where TKey : IEquatable<TKey> {
-         public static readonly Func<TableUpdate<T, TKey>, Change<TableUpdate<T, TKey>>, TableUpdate<T, TKey>, TableUpdate<T, TKey>> source_change_del = _source_changed_impl;
-         public static readonly Func<TableUpdate<T, TKey>, TableUpdate<T, TKey>, Change<TableUpdate<T, TKey>>, TableUpdate<T, TKey>> other_change_del = _other_changed_impl;
-
-         static TableUpdate<T, TKey> _source_changed_impl(TableUpdate<T, TKey> target, Change<TableUpdate<T, TKey>> source_change, TableUpdate<T, TKey> other_table) =>
-            TableUpdate<T, TKey>.update(target, source_change.New.added, source_change.New.changes, source_change.New.removed, source_change.New.key_extractor);
-
-         static TableUpdate<T, TKey> _other_changed_impl(TableUpdate<T, TKey> target, TableUpdate<T, TKey> source_table, Change<TableUpdate<T, TKey>> other_change) =>
-            TableUpdate<T, TKey>.update(target, other_change.New.added, other_change.New.changes, other_change.New.removed, other_change.New.key_extractor);
-      }
-   }
-}
+using bts.api.core;
+using bts.collections;
+using bts.utils.proto;
+using Google.Protobuf;
+using mbts;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reactive.Linq;
+
+namespace bts.utils {
+   public static class UpdateExtensions {
+      public static TableUpdate<T, TKey> with_added<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> added) where TKey : IEquatable<TKey> =>
+         TableUpdate<T, TKey>.update(table, added, Enumerable.Empty<Change<T>>(), Enumerable.Empty<T>(), table.key_extractor);
+
+      public static TableUpdate<T, TKey> with_removed<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> removed) where TKey : IEquatable<TKey> =>
+         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), Enumerable.Empty<Change<T>>(), removed, table.key_extractor);
+
+      public static TableUpdate<T, TKey> with_changed<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> changed) where TKey : IEquatable<TKey> =>
+         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), changed.Select(v => new Change<T>(table.state[table.key_extractor(v)], v)), Enumerable.Empty<T>(), table.key_extractor);
+
+      public static TableUpdate<T, TKey> with_changes<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<Change<T>> changes) where TKey : IEquatable<TKey> =>
+         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), changes, Enumerable.Empty<T>(), table.key_extractor);
+
+      public static IObservable<TableUpdate<TResult, TKey>> RootDependent<T, TResult, TKey>(
+         this IObservable<TableUpdate<T, TKey>> source,
+         Func<T, TKey> key,
+         Func<T, TKey> parent_key,
+         Func<T, T, TResult> with_root,
+         Func<T, TResult> without_root,
+         Func<TResult, TKey> result_key)
+         where TKey : IEquatable<TKey> =>
+         RootDependent(source, key, parent_key, with_root, without_root, result_key, default);
+
+      // TODO: unit tests
+      public static IObservable<TableUpdate<TResult, TKey>> RootDependent<T, TKey, TResult>(
+         this IObservable<TableUpdate<T, TKey>> source,
+         Func<T, TKey> key,
+         Func<T, TKey> parent_key,
+         Func<T, T, TResult> get_result,
+         Func<T, TResult> get_result_no_parent,
+         Func<TResult, TKey> result_key,
+         TKey no_parent_key)
+         where TKey : IEquatable<TKey> =>
+         Observable.Create<TableUpdate<TResult, TKey>>(observer => {
+            var state = DictionaryHelper.empty<TKey, T>();
+            var output = TableUpdate<TResult, TKey>.mk_empty(result_key);
+            var parent_to_children = new SetMap<TKey, T>();
+            return source.Subscribe(update);
+            //
+            void update(TableUpdate<T, TKey> input) {
+               state = input.state;
+               var builder = output.to_builder();
+               // execute all parent_to_children changes first, so descendent-propagation will be efficient (otherwise propagation might be needlessly performed on values that are yet to be removed, or performed with the wrong root, if the real root is yet to be added).
+               foreach (var v in input.removed.Concat(input.changes.Select(ch => ch.Old))) {
+                  var k = input.key_extractor(v);
+                  parent_to_children.remove(parent_key(v), v);
+               }
+               foreach (var v in input.added.Concat(input.changed)) {
+                  var pk = parent_key(v);
+                  if (!pk.Equals(no_parent_key))
+                     parent_to_children.add(pk, v);
+               }
+               foreach (var v in input.removed) {
+                  var k = input.key_extractor(v);
+                  builder.remove(k);
+                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
+                     foreach (var child in children) {
+                        var ck = key(child);
+                        builder.add_or_update(ck, get_result_no_parent(child));
+                        foreach (var d in descendents(ck))
+                           builder.add_or_update(key(d), get_result(d, child));
+                     }
+                  }
+               }
+               foreach (var v in input.changed) {
+                  var k = input.key_extractor(v);
+                  var root = get_root(v);
+                  TResult result;
+                  if (key(root).Equals(k)) {
+                     result = get_result_no_parent(v);
+                     foreach (var d in descendents(k))
+                        builder.add_or_update(key(d), get_result(d, v));
+                  } else {
+                     result = get_result(v, root);
+                  }
+                  builder.add_or_update(k, result);
+               }
+               foreach (var v in input.added) {
+                  var k = input.key_extractor(v);
+                  var pk = parent_key(v);
+                  TResult result;
+                  if (pk.Equals(no_parent_key)) {
+                     result = get_result_no_parent(v);
+                     foreach (var d in descendents(k))
+                        builder.add_or_update(key(d), get_result(d, v));
+                  } else {
+                     parent_to_children.add(pk, v);
+                     var root = get_root(v);
+                     if (key(root).Equals(k)) {
+                        result = get_result_no_parent(v);
+                        foreach (var d in descendents(k))
+                           builder.add_or_update(key(d), get_result(d, v));
+                     } else {
+                        result = get_result(v, root);
+                     }
+                  }
+                  builder.add_or_update(k, result);
+               }
+               observer.OnNext(output = builder.to_table());
+            }
+            T get_root(T value) {
+               var pk = parent_key(value);
+               while (!pk.Equals(no_parent_key) && state.TryGetValue(pk, out var parent)) {
+                  value = parent;
+                  pk = parent_key(value);
+               }
+               return value;
+            }
+            IEnumerable<T> descendents(TKey k) {
+               if (parent_to_children.try_lookup(k, out ISet<T> children)) {
+                  foreach (var child in children) {
+                     yield return child;
+                     foreach (var d in descendents(key(child)))
+                        yield return d;
+                  }
+               }
+            }
+         });
+
+      public static IObservable<TableUpdate<T, TKey>> MergeTable<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, IObservable<TableUpdate<T, TKey>> other) where TKey : IEquatable<TKey> =>
+         source.AggregateLatest(other, _Merge<T, TKey>.source_change_del, _Merge<T, TKey>.other_change_del, () => TableUpdate<T, TKey>.empty, TableUpdate<T, TKey>.empty, TableUpdate<T, TKey>.empty);
+
+      public static IObservable<TableUpdate<KeyValuePair<TKey, TResult>, TKey>> ParentDependent<T, TKey, TResult>(
+         this IObservable<TableUpdate<T, TKey>> source,
+         Func<T, TKey> key,
+         Func<T, TKey> parent_key,
+         Func<T, T, TResult> get_result,
+         Func<T, TResult> get_result_no_parent)
+         where TResult : IEquatable<TResult>
+         where TKey : IEquatable<TKey> =>
+         ParentDependent(source, key, parent_key, get_result, get_result_no_parent, EqualityComparer<TResult>.Default, default);
+
+      public static IObservable<TableUpdate<KeyValuePair<TKey, TResult>, TKey>> ParentDependent<T, TKey, TResult>(
+         this IObservable<TableUpdate<T, TKey>> source,
+         Func<T, TKey> key,
+         Func<T, TKey> parent_key,
+         Func<T, T, TResult> get_result,
+         Func<T, TResult> get_result_no_parent,
+         IEqualityComparer<TResult> result_comparer,
+         TKey no_parent_key)
+         where TKey : IEquatable<TKey> =>
+         Observable.Create<TableUpdate<KeyValuePair<TKey, TResult>, TKey>>(observer => {
+            var value_comparer = EqualityComparer.Create<KeyValuePair<TKey, TResult>, TResult>(kv => kv.Value, result_comparer);
+            var state = DictionaryHelper.empty<TKey, T>();
+            var output = TableUpdate<KeyValuePair<TKey, TResult>, TKey>.mk_empty(kv => kv.Key);
+            var parent_to_children = new SetMap<TKey, T>();
+            return source.Subscribe(update);
+            //
+            void update(TableUpdate<T, TKey> input) {
+               state = input.state;
+               var builder = output.to_builder();
+               foreach (var v in input.removed) {
+                  var k = input.key_extractor(v);
+                  builder.remove(k);
+                  parent_to_children.remove(parent_key(v), v);
+                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
+                     foreach (var child in children)
+                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result_no_parent(child)));
+                  }
+               }
+               foreach (var v in input.changed) {
+                  var k = key(v);
+                  builder.add_or_update(new KeyValuePair<TKey, TResult>(k, lookup_result(v)));
+                  parent_to_children.set(parent_key(v), v);
+                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
+                     foreach (var child in children)
+                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result(child, v)));
+                  }
+               }
+               foreach (var v in input.added) {
+                  var k = key(v);
+                  builder.add_or_update(new KeyValuePair<TKey, TResult>(k, lookup_result(v)));
+                  parent_to_children.add(parent_key(v), v);
+                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
+                     foreach (var child in children)
+                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result(child, v)));
+                  }
+               }
+               var result = builder.to_table_checked(value_comparer);
+               if (result.has_changes())
+                  observer.OnNext(output = result);
+            }
+            TResult lookup_result(T child) {
+               var pk = parent_key(child);
+               return !pk.Equals(no_parent_key) && state.TryGetValue(pk, out T parent) ?  get_result(child, parent) : get_result_no_parent(child);
+            }
+         });
+
+      static class _Merge<T, TKey> where TKey : IEquatable<TKey> {
+         public static readonly Func<TableUpdate<T, TKey>, Change<TableUpdate<T, TKey>>, TableUpdate<T, TKey>, TableUpdate<T, TKey>> source_change_del = _source_changed_impl;
+         public static readonly Func<TableUpdate<T, TKey>, TableUpdate<T, TKey>, Change<TableUpdate<T, TKey>>, TableUpdate<T, TKey>> other_change_del = _other_changed_impl;
+
+         static TableUpdate<T, TKey> _source_changed_impl(TableUpdate<T, TKey> target, Change<TableUpdate<T, TKey>> source_change, TableUpdate<T, TKey> other_table) =>
+            TableUpdate<T, TKey>.update(target, source_change.New.added, source_change.New.changes, source_change.New.removed, source_change.New.key_extractor);
+
+         static TableUpdate<T, TKey> _other_changed_impl(TableUpdate<T, TKey> target, TableUpdate<T, TKey> source_table, Change<TableUpdate<T, TKey>> other_change) =>
+            TableUpdate<T, TKey>.update(target, other_change.New.added, other_change.New.changes, other_change.New.removed, other_change.New.key_extractor);
+      }
+   }
+}
+namespace bts.utils.diagnostics {
+   public static class DiagnosticUpdateExtensions {
+      // Hard to imagine why this util would be needed outside of diagnosig bugs.
+      // The #if DEBUG directive is an extra precaution to avoid accidentally writing
+      // files like this on a client's machine.
+#if DEBUG
+      static readonly string _proto_capture_ext = "proto_capture";
+
+      public static IObservable<TableUpdate<T, TKey>> ProtoBufCapture<T, TKey, TProto>(this IObservable<TableUpdate<T, TKey>> source, Func<T, TProto> to_message)
+         where TKey : IEquatable<TKey>
+         where TProto : class, IMessage<TProto> =>
+         ProtoBufCapture(source,
+            FilePathHelper.GetUniquePathWithDate(Path.Combine(Directory.GetCurrentDirectory(), typeof(TProto).Name), _proto_capture_ext),
+            to_message);
+
+      public static IObservable<TableUpdate<T, TKey>> ProtoBufCapture<T, TKey, TProto>(this IObservable<TableUpdate<T, TKey>> source, string path, Func<T, TProto> to_message)
+         where TKey : IEquatable<TKey>
+         where TProto : class, IMessage<TProto> =>
+         Observable.Create<TableUpdate<T, TKey>>(o => {
+            // write the state on dispose, and/or on error
+            IReadOnlyDictionary<TKey, T> state = DictionaryHelper.empty<TKey, T>();
+            var sub = source.Subscribe(table => {
+               state = table.state;
+               o.OnNext(table);
+            }, ex => {
+               save(true);
+               o.OnError(ex);
+            },
+            o.OnCompleted);
+            return Disposable.Create(() => {
+               save(false);
+               Disposable.dispose(ref sub);
+            });
+            void save(bool has_error) {
+               if (state.Count > 0)
+                  FileStreamHelper.WriteProtos(has_error ? $"{path}(error)" : path, state.Values.Select(to_message).ExceptNull());
+            }
+         });
+#endif
+   }
+}

src/client/spark_test/Resources/ResourceHelper.cs

@@ -0,0 +1,13 @@
+using System.IO;
+using System.Reflection;
+
+namespace bts.resources {
+   class ResourceHelper {
+      const string _default_namespace = "spark_test.Resources";
+
+      public static Stream LoadResource(string name) {
+         var ass = Assembly.GetExecutingAssembly();
+         return ass.GetManifestResourceStream(name) ?? ass.GetManifestResourceStream($"{_default_namespace}.{name}");
+      }
+   }
+}

src/client/spark_test/Security/SecurityTests.cs

@@ -1,13 +1,11 @@
-using System;
+using bts.api;
+using bts.resources;
+using bts.test;
+using bts.utils.proto;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using bts.api;
-using bts.test;
 
-namespace bts.mcore
-{
+namespace bts.mcore {
    [TestGroup]
    class SecurityTests {
       [Test]
@@ -28,5 +26,23 @@ namespace bts.mcore
          Assert.That(dict[b] == 1);
          Assert.That(dict[c] == 2);
       }
+
+      [Test]
+      public void test_security_compare() {
+         const string securities_resource = "KrandallOrderBookCrash.security_proto";
+         // verify that the sample size doesn't shrink for some reason.
+         // if it grows a bit that's fine; the generic comparison test
+         // asserts against collections that are too large.
+         const int min_sample_size = 91;
+
+         var securities = StreamHelper.ReadProtos<Security>(
+            ResourceHelper.LoadResource(securities_resource))
+            .Select(s => new SecurityImmutable(s))
+            .ToList();
+         Assert.IsTrue(securities.Count >= min_sample_size);
+
+         ComparisonTests.transitive_comparison(securities, SecurityHelper.Comparer.InstanceImmutable);
+         ComparisonTests.identity_comparison(securities, SecurityHelper.Comparer.InstanceImmutable);
+      }
    }
 }

src/client/spark_test/Utils/ComparisonTests.cs

@@ -0,0 +1,34 @@
+using bts.test;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace bts.mcore {
+   static class ComparisonTests {
+      public static void transitive_comparison<T>(IEnumerable<T> items, IComparer<T> comparer) {
+         // Verify that the sample size isn't too big.
+         // This test compares each item to every other item
+         // (technically twice; lhs/rhs is flipped the second time),
+         // so performance is O(n^2).
+         const int max_sample_size = 4092;
+         var sorted_items = items.ToList();
+         Assert.That(sorted_items.Count <= max_sample_size);
+         sorted_items.Sort(comparer);
+         for (int i = 0; i < sorted_items.Count - 1; i++) {
+            for (int j = i + 1; j < sorted_items.Count; j++) {
+               assert_comp(i, j);
+            }
+         }
+         void assert_comp(int lhs, int rhs) {
+            int pos_comp = lhs.CompareTo(rhs);
+            // hit both sides for good measure
+            Assert.That(comparer.Compare(sorted_items[lhs], sorted_items[rhs]) == pos_comp);
+            Assert.That(comparer.Compare(sorted_items[rhs], sorted_items[lhs]) == -pos_comp);
+         }
+      }
+
+      public static void identity_comparison<T>(IEnumerable<T> items, IComparer<T> comparer) {
+         foreach (var v in items)
+            Assert.That(comparer.Compare(v, v) == 0);
+      }
+   }
+}

src/client/spark_test/Utils/MarketDataUtilities.cs

@@ -61,7 +61,7 @@ public static async Task<IEnumerable<SecurityImmutable>> get_multiple_securities
          return Enumerable.Empty<SecurityImmutable>();
       }
 
-      public static async Task<IEnumerable<FutureContract>> get_expirations(MarketDataRelay relay, string symbol, SecurityType security_type, MDRequest.Types.Flags flags = MDRequest.Types.Flags.Bbo, Venue v = Venue.Sim) {
+      public static async Task<(bool, List<FutureContract>)> get_expirations(MarketDataRelay relay, string symbol, SecurityType security_type, MDRequest.Types.Flags flags = MDRequest.Types.Flags.Bbo, Venue v = Venue.Sim) {
          var request = new MDRequest() {
             Host = _MD_HOST,
             Channels = {
@@ -89,7 +89,7 @@ public static async Task<IEnumerable<SecurityImmutable>> get_multiple_securities
             }
          }
 
-         return (IEnumerable<FutureContract>) futures;
+         return (result.success, futures);
       }
 
    }

src/client/spark_test/Utils/SpreaderUtilities.cs

@@ -22,8 +22,9 @@ namespace bts.tests
    public class SpreaderUtilities {
 
       public static async Task<(SecurityImmutable s1, SecurityImmutable s2)> get_securities_for_common_setup_one(MarketDataRelay relay, string symbol) {
-         var exps  = (await MarketDataUtilities.get_expirations(relay, symbol, SecurityType.Future)).ToList();
+         var (success, exps) = await MarketDataUtilities.get_expirations(relay, symbol, SecurityType.Future);
 
+         Assert.IsTrue(success, "Could not reach market data service");
          Assert.IsNotNull(exps);
          Assert.IsTrue(exps.Count() > 2, "Need at least two expirations");
 

src/client/spark_test/spark_test.csproj

@@ -14,5 +14,7 @@
   <ItemGroup>
     <Content Include="../spark_test/system_test_scripts/*.sh" CopyToOutputDirectory="PreserveNewest"></Content>
   </ItemGroup>
-
+  <ItemGroup>
+    <EmbeddedResource Include="Resources\KrandallOrderBookCrash.security_proto" />
+  </ItemGroup>
 </Project>

src/data_files/files/cert_secure_logon.txt

@@ -1,12 +1,19 @@
-Session ID           Access ID                      Secret Key                                         Creation Date        Expiration Date      Environment          
-
-M4D                  lLtjH74y1Kh5XUJTLyH2           f1K_tROQp2lTIplVp34WxKa7xDUgfFnIteolfik061o        2019-09-03           2020-09-03           CERT                 
-
-MC1                  VEMlkDuLz74OVHSuacQB           IuHyYT3b-9KUo3WsuXykt-fXsjzp7L-JOrdCSQZLNl0        2019-09-03           2020-09-03           CERT                 
-
-MC5                  77lzSUwPLlM9obaXh2mK           z6aA9qfASDKfZkYTVKFqJd-78i0HZ2Q9ec3CKCYIqHg        2019-09-03           2020-09-03           CERT                 
-
-MC7                  RphzI3rn71oGUEiYnX9e           y6OvvPPyoyj2S8YAnE1gFduXT8v29HIMjO5EngYxSNg        2019-09-03           2020-09-03           CERT                 
-
-MC8                  PYBIm2TI9gFPIxmHbuYl           6o-ZolGH2DdeQOlGtJ5Zqoh_kg7VdbxkmjeOHmigKwU        2019-09-03           2020-09-03           CERT                 
-
+Session ID           Access ID                      Secret Key                                         Creation Date        Expiration Date      Environment          
+
+M4D                  lLtjH74y1Kh5XUJTLyH2           f1K_tROQp2lTIplVp34WxKa7xDUgfFnIteolfik061o        2019-09-03           2020-09-03           CERT                 
+
+MC1                  VEMlkDuLz74OVHSuacQB           IuHyYT3b-9KUo3WsuXykt-fXsjzp7L-JOrdCSQZLNl0        2019-09-03           2020-09-03           CERT                 
+
+MC5                  77lzSUwPLlM9obaXh2mK           z6aA9qfASDKfZkYTVKFqJd-78i0HZ2Q9ec3CKCYIqHg        2019-09-03           2020-09-03           CERT                 
+
+MC7                  RphzI3rn71oGUEiYnX9e           y6OvvPPyoyj2S8YAnE1gFduXT8v29HIMjO5EngYxSNg        2019-09-03           2020-09-03           CERT                 
+
+MC8                  PYBIm2TI9gFPIxmHbuYl           6o-ZolGH2DdeQOlGtJ5Zqoh_kg7VdbxkmjeOHmigKwU        2019-09-03           2020-09-03           CERT                 
+
+NGG                  RYcigz3bSLIIBBF2A2Hq           qB1dm9VkTKeXcdQpFzla8B3_jwTgya_-Fk5iyEJvr2E        2020-06-15           2021-06-15           NEWRELEASE           
+
+NGF                  o4OTmmTLQ7Lv4g6bCm26           HbD3FneMyb9Fu-FkCDLscCczHqFcFE2PyWv978RXGyA        2019-12-20           2020-12-20           NEWRELEASE           
+
+UZE                  EeFdZ1FPepk6U4k3xBZA           2LjEtdLvBLLB1QuDJ4KiJW6M4dF6IHWd_UKmo7p3_Us        2020-06-24           2021-06-24           NEWRELEASE           
+
+VAE                  Y8iqatTewuoHX16gghw8           FNcCB3Ks1qleS56quYzH4V7zAu78YY2bCdLRi9tmU9E        2020-06-24           2021-06-24           NEWRELEASE           

src/lib/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/Product.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityMapper.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/service/MarketDataServiceServer.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;md/vendor/OnixsUtils.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metricsLine truncated
+set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/Product.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityMapper.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/service/MarketDataServiceServer.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;md/vendor/OnixsUtils.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metricsLine truncated

src/lib/api/service_helpers/AdminServiceUtils.cpp

@@ -580,7 +580,9 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(const bt
      rv.set_session(conn_ptr->name());
      rv.set_operator_id(cme_user_data->operator_id());
      rv.set_location(cme_user_data->location());
+
      rv.set_smp_id(cme_user_data->smp_id());
+
      if (cme_user_data->smp_instruction()== "O")
         rv.set_smp_instruction(api::orderman::SelfMatchPreventionInstruction::SELF_MATCH_PREVENTION_INSTRUCTION_CANCEL_RESTING);
      else if (cme_user_data->smp_instruction()== "N")
@@ -685,6 +687,7 @@ bts::api::admin::BTSAccount* AdminServiceUtils::create_account(bts::api::admin::
    new_account_ptr->set_name(account_name);
    new_account_ptr->set_description(description);
    new_account_ptr->set_is_active(is_active);
+   new_account_ptr->mutable_risk_limit_enforcement()->set_ignore_spread_position_limits(true);
    return new_account_ptr;
 }
 void AdminServiceUtils::update_account(bts::api::admin::AdminData& admin_data,

src/lib/api/services/OrderService.proto

@@ -104,6 +104,7 @@ message CustomOrderDefinition
     SplitterOrderDefinition   splitter = 29;
     ScratchOrderDefinition scratch = 31;
     ProRataOrderDefinition prorata = 32;
+    DeleteAndCoverDefinition delete_and_cover = 34;
     
     DynamicOrderDefinition dynamic = 1000;
   }
@@ -144,6 +145,13 @@ message TickOrderDefinition {
   bool detach_on_toward = 5;
   CustomOrderDefinition child = 6;
 }
+enum LeanOnToward
+{
+   NO_CHANGE = 0; //continue to lean on initial inside market
+   CANCEL = 1; //cancel algo and order
+   DETACH = 2; //cancel algo but leave order working
+   REPRICE = 3; //lean on new inside market
+}
 message LeanOrderDefinition {
   spark.QuantityMeasure cross_threshold = 1;
   uint32 max_chase_ticks = 2;
@@ -179,6 +187,21 @@ message CoverOrderDefinition {
   CustomOrderDefinition child = 3;
   bool cancel_on_external_hedge_cancel = 4;
   bool cancel_hedges_on_cancel = 5;
+  bool detach_hedges = 6;
+}
+message DeleteAndCoverDefinition{
+  // cover
+  uint32 hedge_ticks = 1;
+  bool cancel_on_external_hedge_cancel = 2;
+  bool cancel_hedges_on_cancel = 3;
+  bool detach_hedges = 4;
+  // main child = top
+  spark.QuantityMeasure top_threshold = 10;
+  uint32 ticks_off = 11;
+  // hedge child = Lean
+  spark.QuantityMeasure cross_threshold = 20;
+  uint32 max_chase_ticks = 21;
+  LeanOnToward trade_toward_behavior = 22;
 }
 message OffsetOrderDefinition {
   uint32 tick_offset_on_deploy = 1;
@@ -360,7 +383,9 @@ message NativeOrderState {
   RoutingRecord routing_record = 5;
   NativeOrderType native_type = 6;
   string last_transact_time = 7;
+  uint64 last_transact_time_unix_ns = 9;
   double stop_px = 8;
+  uint64 exchange_order_id64 = 10;
 }
 
 // Custom order states
@@ -383,7 +408,9 @@ message OCOOrderState {}
 message PricerOrderState {}
 message ScratchOrderState {}
 message ProRataOrderState {}
-message DynamicOrderState {}
+message DynamicOrderState {
+  map<string,DynamicOrderParam> custom_state = 1;
+}
 message QueueHolderState {}
 message SplitterOrderState {}
 
@@ -427,7 +454,8 @@ enum TimeInForce {
   TIME_IN_FORCE_UNDEFINED = 0;
   TIME_IN_FORCE_DAY = 48;
   TIME_IN_FORCE_GTC = 49;
-  TIME_IN_FORCE_IOC = 51;
+  TIME_IN_FORCE_IOC = 51; // FAK
+  TIME_IN_FORCE_DATE = 52;
 }
 
 enum ManualOrderIndicator {
@@ -644,7 +672,7 @@ message RegisteredDynamicOrderTypes {
 
 message DynamicOrderParam {
   oneof value {
-    uint32 as_int32 = 1;
+    int32 as_int32 = 1;
     uint32 as_uint32 = 2;
     double as_double = 3;
     string as_string = 4;

src/lib/config/defaults/MarketData.conf

@@ -23,6 +23,7 @@ PlatformMap {
     }
 }
 
+
 PublisherNamespace {
 	value {
 		shared_memory_namespace: @System.os_user

src/lib/config/defaults/jetfix.conf

@@ -14,7 +14,7 @@ JetFIXSimConnection {
         end_time: "02:00:00"
         start_time: "02:05:00"
         raw_data: "1JE9X"
-        use_cme_secure_logon: false 
+        use_cme_secure_logon: false
         seconds_to_pause : 1
 	max_connection_attempts: 9999999
       }
@@ -31,9 +31,9 @@ JetFIXSimConnection {
         end_time: "02:00:00"
         start_time: "02:05:00"
         raw_data: "1JE9X"
-        use_cme_secure_logon: false 
+        use_cme_secure_logon: false
         seconds_to_pause : 1
-      }      
+      }
    }
 }
 
@@ -49,7 +49,7 @@ JetFIX {
          socket_connect_host_ip: "63.247.113.154"
          socket_connect_port: 80
          sender_compid: "51251"
-         target_compid: "ICE"         
+         target_compid: "ICE"
          start_time: "22:55:00"
          end_time: "22:05:00"
          sender_subid: "BTS"
@@ -57,12 +57,12 @@ JetFIX {
          raw_data: "Starts123"
          max_order_msgs_per_window: 300
          auto_fix_seq_num: true
-         
+
          seconds_to_replay: 6000
       }
       connection: @JetFIXSimConnection.sim
       #connection: @JetFIXSimConnection.sim_ice
-      
+
       connection {
          name: "cme_mc5_99"
          platform: GLBX
@@ -181,9 +181,9 @@ JetFIX {
          sender_subid: "BTS"
          raw_data: "S4PMA"
          auto_fix_seq_num: true
-      }      
+      }
    }
-}   
+}
 
 JetFIX {
    rule: "CannedCertSession.name='gt_cert'"
@@ -196,7 +196,7 @@ JetFIX {
          socket_connect_host_ip: "63.247.113.154"
          socket_connect_port: 80
          sender_compid: "51251"
-         target_compid: "ICE"         
+         target_compid: "ICE"
          start_time: "22:55:00"
          end_time: "22:05:00"
          sender_subid: "BTS"
@@ -204,7 +204,7 @@ JetFIX {
          raw_data: "Starts123"
          max_order_msgs_per_window: 300
          auto_fix_seq_num: true
-         
+
          seconds_to_replay: 6000
       }
       connection: @JetFIXSimConnection.sim
@@ -515,7 +515,7 @@ JetFIX {
          auto_fix_seq_num: true
       }
    }
-}   
+}
 
 
 JetFIX {
@@ -2391,9 +2391,91 @@ JetFIX {
       }
    }
 }
-        
+
+# NrCert gateways
+# 69.50.112.184,12,OESGW
+# 69.50.112.202,50,CME Eurodollar Options
+# 69.50.112.181,52,CME Globex FX Options II
+# 69.50.112.191,54,CME Equity Options
+# 69.50.112.174,56,NYMEX and COMEX Options
+# 69.50.112.172,58,CBOT Treasury Options
+# 69.50.112.195,60,CBOT Commodity and Equity Options
+# 69.50.112.177,64,CME E-mini S&P 500 Futures
+# 69.50.112.168,68,CME Equity Futures excluding E-mini S&P
+# 69.50.112.183,70,CME Commodity Futures
+# 69.50.112.189,72,CBOT Commodity Futures
+# 69.50.112.197,74,KRX Futures; BMD Futures and Options
+# 69.50.112.204,76,NYMEX and COMEX Metals and Alternative Market Futures
+# 69.50.112.167,78,NYMEX Non-Crude Energy Futures
+# 69.50.112.187,80,NYMEX Crude Energy Futures
+# 69.50.112.179,82,CME Eurodollar Futures
+# 69.50.112.198,84,CBOT Treasury Futures
+# 69.50.112.169,88,CME Globex FX Futures and Options
+# 69.50.112.199,99,AutoCert Segment
+
+macro {
+   name: "canned_004"
+   expression: "[CannedCertSession.name, '004']"
+}
+
+JetFIX {
+   rule: "CannedCertSession.name in ['NGG', 'NGF', 'UZE', 'VAE']"
+   value {
+      # ES
+      connection {
+         name: "cme_canned_64"
+         platform: GLBX
+         socket_connect_host_ip: "69.50.112.177"
+         socket_connect_port: 64530
+         sender_compid: $canned_004
+         target_compid: "CME"
+         target_subid: "64"
+         cme_market_segment_id: 64
+         start_time: "00:30:00"
+         end_time: "23:30:00"
+         start_day: "sunday"
+         end_day: "friday"
+         sender_subid: "BTS"
+         use_ilink3: true
+      }
+      # CL
+      connection {
+         name: "cme_canned_80"
+         platform: GLBX
+         socket_connect_host_ip: "69.50.112.187"
+         socket_connect_port: 64530
+         sender_compid: $canned_004
+         target_compid: "CME"
+         target_subid: "80"
+         cme_market_segment_id: 80
+         start_time: "00:30:00"
+         end_time: "23:30:00"
+         start_day: "sunday"
+         end_day: "friday"
+         sender_subid: "BTS"
+         use_ilink3: true
+      }
+      connection {
+         name: "cme_canned_99"
+         platform: GLBX
+         socket_connect_host_ip: "69.50.112.199"
+         socket_connect_port: 64530
+         sender_compid: $canned_004
+         target_compid: "CME"
+         target_subid: "99"
+         cme_market_segment_id: 99
+         start_time: "00:30:00"
+         end_time: "23:30:00"
+         start_day: "sunday"
+         end_day: "friday"
+         sender_subid: "BTS"
+         use_ilink3: true
+      }
+   }
+}
+
 #If no canned cert specified we fallback to just sim
 CannedCertSession { value { name: "" }}
 JetFIX {value {connection: @JetFIXSimConnection.sim}}
-        
+
 

src/lib/config/defaults/keycode.conf

@@ -1,13 +1,10 @@
-
-
 #nusers: 1
 #allow concurrent : true
-#expires: 2020-Mar-31 00:00:00   637212096000000000
-#generation_time: 2020-Jan-06 18:56:23.406931   5248825356261457214
+#expires: 2021-Jan-01 00:00:00   637450560000000000
+#generation_time: 2020-Apr-03 18:39:31.010863   5248901378137496534
 EncryptedKeyCode {
    value {
-      keycode: "Ixb/z6hwFQmojpm4GwNEpQmsqpYT9OTHbcKtROIqj38GwGqyVn54JMDB04V048quO+NAI78DKJXsUWgjDH0y5HQlwN6Lq48s/bdwKqR1OTBPr6xoTImtCb+l5b+8fUGlg7t3X8gfMdxohHgcUi9k9n7ZrnNvqcjdYKoP0CaTZT8zHuBQx3f4Lqrz32gq9rJ5skRjH6ZqNblNdlsF+kOAqXCkvI5p0QfyKgtCodr33qsW2cbqY5gUD9KwsnmFVa8/zuFBucsoM7dHB47yRaIeOFFu7xGayAdAHA8SwLyoxE5y+adIX1SYnAiqAu1raZpeXgRE1LNQk3c7uhUjt0coag=="
+      keycode: "BJBQWsbdTXWa/iLmOf3ekE8vP/NYkf0Q/nUsp/rJ78Ze3W9pNXICCMabmmsr1xOM7aqRIclPgZd955uSSfvFkdNWFTxwPQ5IfnDGopj3QdhUVxhzgVLaXHN+vC0H/MWrDQ5K4WY5J/6VXgUn1LSN/2HNb5iR2sKXeh3Alamg87YtCWEi+scelffE7HK5ERT/3A3Y/e0pqUFhP6xouIjHRhC2FRBb6OoJQVvFwVDADSuKZNo9B6GhC3H0YdHqjFhGDoky5ecgzeKIX8uWJEQNnQXQJ0iTgQ95+YR5IMU+XWdlVfSrc0fuTmj3YWcBQ8Kl2ItEcobyZzJYega3tu6Ftw=="
    }
 }
 
-

src/lib/config/protos/MarketData.proto

@@ -52,6 +52,11 @@ message PublisherOptions
    optional uint32 heartbeat_boot_seconds = 12 [default = 60]; // 1 minute grace period from shm creation
 
    optional bool skip_options = 13 [default = true];
+
+   optional uint32 hot_initial = 20 [default = 1024];
+   optional uint32 hot_growth = 21 [default = 1024];
+   optional uint32 cold_initial = 22 [default = 8192];
+   optional uint32 cold_growth = 23 [default = 2048];
 }
 
 message ICEPublisherOptions
@@ -193,6 +198,11 @@ message RecordOptions {
    optional bool record_sim = 5 [default=false];
 }
 
+// If starting on Sunday pre-load the selected symbols from disk
+message SecurityPreload {
+   repeated string symbol = 1;
+}
+
 //
 // Replay options
 // 

src/lib/config/protos/UI.proto

@@ -76,6 +76,7 @@ message LadderKnobs {
 message OrderBookOptions {
    optional bool animate_child_order_rows = 1 [default = false];
    optional double status_color_opacity_when_selected = 2 [default = 0.35];
+   optional bool can_edit_specific_algo_params = 3 [default = false];
 }
 
 message OrderTicketOptions {

src/lib/config/protos/jetfix.proto

@@ -9,9 +9,9 @@ message JetFIX {
     enum JetFIXPlatform {
 	   GLBX = 1;
 	   CBOE = 6;
-      SIM = 2;
-      ICE = 3;
-      SIMICE = 4;
+           SIM = 2;
+           ICE = 3;
+           SIMICE = 4;
     }
     optional JetFIXPlatform platform = 15                       [default = GLBX];
     
@@ -87,6 +87,7 @@ message JetFIX {
 
     optional uint32 max_connection_attempts = 346               [default=10];
     optional bool auto_fix_seq_num = 347                        [default=false];
+    optional bool use_ilink3       = 348                        [default=false] ;
    }
 
   repeated Connection connection = 10;
@@ -116,4 +117,4 @@ message CMECannedSecureLogon {
 "MC8                  AbEEtBn63WIAuItPDz1c           n3ndjYb03Hx6wBO3me35HoONyDgO5TZKWk0tDSaX9KA        2018-04-05           2019-04-05           CERT                 \n"
 "T39                  UWggBgHWAe07aqW3ps2I           tbb3it0rUQSSTa0TC9QcRwkrHEMDLWhaJH4VQ0FqGB4        2018-04-05           2019-04-05           NEWRELEASE           \n"
 					  ];
-}
+}

src/lib/config/protos/toggles.proto

@@ -67,4 +67,6 @@ message DynamicAlgos{
    optional bool enable_dorman = 1 [default = false];
    optional bool enable_pyramid = 2 [default = false];
    optional bool enable_tas = 3 [default = false];
+   optional bool enable_tascounter = 4 [default = false];
+   optional bool enable_limit_cover = 5 [default = false];
 }

src/lib/domain_model/Security.h

@@ -41,16 +41,16 @@ inline api::Side other_side(api::Side side) {
    }
 }
 
-inline double market_epislon() { return 1e-10; }
+inline double market_epsilon() { return 1e-10; }
 
 // better == p1 more aggressive than p1 ie for a buy order p1 is higher
 // than p2, and for a sell order p1 is lower than p2
 inline bool is_better(api::Side side, Price p1, Price p2) {
    switch (side) {
    case api::SIDE_BUY:
-      return p1.value() - p2.value() > market_epislon();
+      return p1.value() - p2.value() > market_epsilon();
    case api::SIDE_SELL:
-      return p2.value() - p1.value() > market_epislon();
+      return p2.value() - p1.value() > market_epsilon();
    default:
       return false;
    }
@@ -76,7 +76,7 @@ inline Price worsen_price(api::Side side, Price p, Price offset) {
 }
 
 inline bool is_same(Price p1, Price p2) {
-   return std::abs(p1.value() - p2.value()) < market_epislon();
+   return std::abs(p1.value() - p2.value()) < market_epsilon();
 }
 
 inline bool is_equal_or_better(api::Side side, Price p1, Price p2) {
@@ -212,5 +212,5 @@ inline bool gr_than(double p1, double p2, double tick_size) {
           !test_price_equality(p1, p2, tick_size);
 }
 
-
-}}
+} // namespace securities
+} // namespace bts

src/lib/jetfix/codec/FIXEncodingStream.h

@@ -52,7 +52,7 @@ class FIXEncodingStream {
       auto msg_offset = _buffer.size();
 
       _current_msg = FIXFragment(_buffer);
-      _begin_string_bm = _current_msg.add<fix::BeginString>("FIX.4.2");
+      _begin_string_bm = _current_msg.add<fix::BeginString>(std::string_view {"FIX.4.2"});
       _body_length_bm = _current_msg.add<fix::BodyLength>(0, 6);
 
       char mt_encoding[4];

src/lib/jetfix/codec/FieldDefinitions.h

@@ -85,6 +85,7 @@ typedef StringField<307> UnderlyingSecurityDesc;
 typedef StringField<336> TradingSessionID;
 typedef StringField<35> MsgType;
 typedef StringField<37> OrderID;
+typedef BigIntField<37> OrderIDu64;
 typedef StringField<41> OrigClOrdID;
 typedef StringField<440> ClearingAccount;
 typedef StringField<48> SecurityID;

src/lib/jetfix/codec/Fields.h

@@ -2,6 +2,8 @@
 
 #include <memory>
 #include <boost/date_time/posix_time/posix_time.hpp>
+#include <string_view>
+
 #include "lib/jetfix/codec/ReaderCursor.h"
 #include "lib/utils/meta_programming.h"
 #include "lib/utils/datetime.h"
@@ -203,6 +205,13 @@ public:
    static std::pair<uint32_t,uint32_t> write (char*buffer,std::string const &str)
    {return write (buffer,str.data (),str.size ());}
 
+
+#if __cplusplus >= 201703L
+   static uint32_t max_value_size (std::string_view const&str) {return str.size ();}
+   static std::pair<uint32_t,uint32_t> write (char*buffer,std::string_view const &str)
+   {return write (buffer,str.data (),str.size ());}
+#endif
+
    static std::pair<uint32_t,uint32_t> write (char*buffer,const char*str,uint32_t sz)
    {
       memcpy (buffer,str,sz);

src/lib/jetfix/connection/SocketConnection.cpp

@@ -23,9 +23,9 @@ SocketConnection::SocketConnection(bts::config::JetFIX::Connection const &cfg,
     : _fd(-1), _cfg(cfg), _log(cfg.log_fix_stream()), _in_log(), _out_log(),
       _perf_log(perf_log) {
    if (_log) {
-      _in_log.open(std::string("/tmp/fix_raw_read_log_") + session_name(cfg),
+      _in_log.open(std::string("/tmp/fix_raw_read_log_") + session_name(cfg).as_string,
                    std::ios_base::app);
-      _out_log.open(std::string("/tmp/fix_raw_write_log_") + session_name(cfg),
+      _out_log.open(std::string("/tmp/fix_raw_write_log_") + session_name(cfg).as_string,
                     std::ios_base::app);
    }
 }

src/lib/jetfix/gateway/Session.cpp

@@ -783,32 +783,32 @@ OutFIXPtr make_logon(uint32_t seq_num, uint32_t last_seq_num_processed,
    const char *application = "O2";
 #endif
 
-   if(session_cfg.platform() == bts::config::JetFIX_Connection_JetFIXPlatform_ICE || 
-      session_cfg.platform() == bts::config::JetFIX_Connection_JetFIXPlatform_SIMICE)
-   {
-      buffer.add<ICEApplicationSystemName>(application);
-      buffer.add<ICEApplicationSystemVersion>("1.0");
-      buffer.add<ICEApplicationSystemVendor>("BTS");      
-      
-      buffer.add<UserName>(session_cfg.user_name());      
+   if (session_cfg.platform() ==
+           bts::config::JetFIX_Connection_JetFIXPlatform_ICE ||
+       session_cfg.platform() ==
+           bts::config::JetFIX_Connection_JetFIXPlatform_SIMICE) {
+      buffer.add<ICEApplicationSystemName>(std::string_view{application});
+      buffer.add<ICEApplicationSystemVersion>(std::string_view{"1.0"});
+      buffer.add<ICEApplicationSystemVendor>(std::string_view{"BTS"});
+
+      buffer.add<UserName>(session_cfg.user_name());
       buffer.add<RawDataLength>(session_cfg.raw_data().size());
-      buffer.add<RawData>(session_cfg.raw_data());      
+      buffer.add<RawData>(session_cfg.raw_data());
       buffer.add<EncryptMethod>(0);
       buffer.add<ICEStrategyPreference>(1);
       buffer.add<ICETimeStampPreference>(1);
-   }
-   else
-   {
-      buffer.add<ApplicationSystemName>(application);
-      buffer.add<ApplicationSystemVersion>("1.0");
-      buffer.add<ApplicationSystemVendor>("BTS");
+   } else {
+      buffer.add<ApplicationSystemName>(std::string_view{application});
+      buffer.add<ApplicationSystemVersion>(std::string_view{"1.0"});
+      buffer.add<ApplicationSystemVendor>(std::string_view{"BTS"});
 
       if (session_cfg.has_trading_session_id()) {
          buffer.add<fix::NoTradingSessions>(1);
          buffer.add<fix::TradingSessionID>(session_cfg.trading_session_id());
       }
       if (session_cfg.has_udf_support_indicator())
-         buffer.add<fix::UDFSupportIndicator>(session_cfg.udf_support_indicator());
+         buffer.add<fix::UDFSupportIndicator>(
+             session_cfg.udf_support_indicator());
 
       if (session_cfg.use_cme_secure_logon()) {
          bts_assert(session_cfg.sender_compid().size() >= 3);
@@ -819,25 +819,24 @@ OutFIXPtr make_logon(uint32_t seq_num, uint32_t last_seq_num_processed,
          std::string last_seq_num_str(9, ' ');
          i32_toa(last_seq_num_processed, &last_seq_num_str[0], 9);
          auto ld = glbx::calculate_hmac(
-            session_id, seq_num_str, session_cfg.sender_compid(),
-            session_cfg.sender_subid(), time_str, session_cfg.target_subid(),
-            std::to_string(session_cfg.seconds_between_hbs()),
-            session_cfg.sender_locationid(), last_seq_num_str, application, "1.0",
-            "BTS");
-   #ifndef BTS_EDGE
-         LOG(Jetfix, Info, "Canonical string for secure logon: {}", ld.canonical);
-   #endif
+             session_id, seq_num_str, session_cfg.sender_compid(),
+             session_cfg.sender_subid(), time_str, session_cfg.target_subid(),
+             std::to_string(session_cfg.seconds_between_hbs()),
+             session_cfg.sender_locationid(), last_seq_num_str, application,
+             "1.0", "BTS");
+#ifndef BTS_EDGE
+         LOG(Jetfix, Info, "Canonical string for secure logon: {}",
+             ld.canonical);
+#endif
          buffer.add<EncodedTextLen>(ld.access_key.size());
          buffer.add<EncodedText>(ld.access_key);
-         buffer.add<EncryptedPasswordMethod>("CME-1-SHA-256");
+         buffer.add<EncryptedPasswordMethod>(std::string_view{"CME-1-SHA-256"});
          buffer.add<EncryptedPasswordLen>(ld.signature.size());
          buffer.add<EncryptedPassword>(ld.signature);
-      } 
-      else 
-      {
+      } else {
          buffer.add<RawDataLength>(session_cfg.raw_data().size());
          buffer.add<RawData>(session_cfg.raw_data());
-         if(session_cfg.user_name() != "")
+         if (session_cfg.user_name() != "")
             buffer.add<UserName>(session_cfg.user_name());
       }
    }

src/lib/jetfix/store/FIXFileStore.cpp

@@ -185,7 +185,7 @@ void load_seq_nums(std::string const &path, uint32_t *last_seq_num,
 FIXFileStore::FIXFileStore(config::JetFIX::Connection const &cfg,
                            std::string base_path)
     : me(new PImpl(cfg)) {
-   auto sn = session_name(cfg);
+   auto sn = session_name(cfg).as_string;
    me->active = sn.size() > 0;
    if (me->active) {
 

src/lib/jetfix/store/session_name.cpp

@@ -75,49 +75,55 @@ bool in_session (datetime::PTime const &ref_date,
    return closest_earlier_end<closest_earlier_start;
 }
 
-std::string session_name_from_ptime (datetime::PTime t,
+SessionName session_name_from_ptime (datetime::PTime t,
                                      boost::posix_time::time_duration time_of_day_start,
                                      boost::posix_time::time_duration time_of_day_end)
 {
-   std::string result (10,' ');
+   SessionName result;
+   result.as_string = std::string(10,' ');
    auto start_of_session = t;
    
    if (time_of_day_start > time_of_day_end && 
        start_of_session.time_of_day () <= time_of_day_end)
       start_of_session -= boost::gregorian::date_duration (1);
-      
-   sprintf (&result[0], "%04d-%02d-%02d",
+
+   using namespace boost::posix_time;
+   ptime epoch(boost::gregorian::date(1970,1,1));
+   ptime session_start_date(start_of_session.date());
+   result.as_uint64_t = (session_start_date - epoch).total_milliseconds();
+
+   sprintf (&(result.as_string[0]), "%04d-%02d-%02d",
             (unsigned short) start_of_session.date ().year (),
             start_of_session.date ().month ().as_number (),
             start_of_session.date ().day ().as_number ());
    return result;
 }
 
-std::string session_name (datetime::PTime const &ref_date,
-                          boost::gregorian::greg_weekday weekday_start,
-                          boost::posix_time::time_duration time_of_day_start,
-                          boost::gregorian::greg_weekday weekday_end,
-                          boost::posix_time::time_duration time_of_day_end)
+SessionName session_name (datetime::PTime const &ref_date,
+                         boost::gregorian::greg_weekday weekday_start,
+                         boost::posix_time::time_duration time_of_day_start,
+                         boost::gregorian::greg_weekday weekday_end,
+                         boost::posix_time::time_duration time_of_day_end)
 {
    if (in_session (ref_date,weekday_start,time_of_day_start,weekday_end,time_of_day_end)) 
       return session_name_from_ptime (closest_before (ref_date,weekday_start,time_of_day_start),
                                       boost::posix_time::time_duration(0, 0, 0),
                                       boost::posix_time::time_duration(0, 0, 1));
    else
-      return std::string ();
+      return {};
 }
 
-std::string session_name (datetime::PTime const &ref_date,
+SessionName session_name (datetime::PTime const &ref_date,
                           boost::posix_time::time_duration time_of_day_start,
                           boost::posix_time::time_duration time_of_day_end)
 {
    if (in_session (ref_date,time_of_day_start,time_of_day_end))
       return session_name_from_ptime (ref_date, time_of_day_start, time_of_day_end);
    else
-      return "";
+      return {};
 }
 
-std::string session_name (config::JetFIX::Connection const &session_cfg)
+SessionName session_name (config::JetFIX::Connection const &session_cfg)
 {
    auto now = datetime::local_now ();
    
@@ -134,7 +140,6 @@ std::string session_name (config::JetFIX::Connection const &session_cfg)
                            parse_time (session_cfg.end_time ()));
 }
 
-
 }}
  
 

src/lib/jetfix/store/session_name.h

@@ -1,9 +1,9 @@
 #pragma once
 
+#include "lib/config/protos/jetfix.pb.h"
+#include "lib/utils/datetime.h"
 #include <memory>
 #include <string>
-#include "lib/utils/datetime.h"
-#include "lib/config/protos/jetfix.pb.h"
 
 namespace boost {
 namespace gregorian {
@@ -11,39 +11,45 @@ class greg_weekday;
 }
 namespace posix_time {
 class time_duration;
-}}
-
+}
+} // namespace boost
 
 namespace bts {
 namespace jetfix {
 
-boost::gregorian::greg_weekday parse_weekday (std::string weekday_str);
-bool parse_time (std::string const &time_str,boost::posix_time::time_duration *out);
+boost::gregorian::greg_weekday parse_weekday(std::string weekday_str);
+bool parse_time(std::string const &time_str,
+                boost::posix_time::time_duration *out);
 
-datetime::PTime closest_before (datetime::PTime const &ref_date,
-                                boost::gregorian::greg_weekday weekday,
-                                boost::posix_time::time_duration time_of_day);
+datetime::PTime closest_before(datetime::PTime const &ref_date,
+                               boost::gregorian::greg_weekday weekday,
+                               boost::posix_time::time_duration time_of_day);
 
-bool in_session (datetime::PTime const &ref_date,
-                 boost::gregorian::greg_weekday weekday_start,
-                 boost::posix_time::time_duration time_of_day_start,
-                 boost::gregorian::greg_weekday weekday_end,
-                 boost::posix_time::time_duration time_of_day_end);
+bool in_session(datetime::PTime const &ref_date,
+                boost::gregorian::greg_weekday weekday_start,
+                boost::posix_time::time_duration time_of_day_start,
+                boost::gregorian::greg_weekday weekday_end,
+                boost::posix_time::time_duration time_of_day_end);
 
-bool in_session (datetime::PTime const &ref_date,
-                 boost::posix_time::time_duration time_of_day_start,
-                 boost::posix_time::time_duration time_of_day_end);
+bool in_session(datetime::PTime const &ref_date,
+                boost::posix_time::time_duration time_of_day_start,
+                boost::posix_time::time_duration time_of_day_end);
 
-std::string session_name (datetime::PTime const &ref_date,
-                          boost::gregorian::greg_weekday weekday_start,
-                          boost::posix_time::time_duration time_of_day_start,
-                          boost::gregorian::greg_weekday weekday_end,
-                          boost::posix_time::time_duration time_of_day_end);
+struct SessionName {
+   std::string as_string;
+   uint64_t as_uint64_t;
+   bool empty() const {return as_string.empty();};
+};
 
-std::string session_name (datetime::PTime const &ref_date,
-                          boost::posix_time::time_duration time_of_day_start,
-                          boost::posix_time::time_duration time_of_day_end);
+SessionName session_name(datetime::PTime const &ref_date,
+                         boost::gregorian::greg_weekday weekday_start,
+                         boost::posix_time::time_duration time_of_day_start,
+                         boost::gregorian::greg_weekday weekday_end,
+                         boost::posix_time::time_duration time_of_day_end);
+SessionName session_name(datetime::PTime const &ref_date,
+                         boost::posix_time::time_duration time_of_day_start,
+                         boost::posix_time::time_duration time_of_day_end);
+SessionName session_name(config::JetFIX::Connection const &session_cfg);
 
-std::string session_name (config::JetFIX::Connection const &session_cfg);
-
-}}
+} // namespace jetfix
+} // namespace bts

src/lib/jetfix/tests/fix_fragment_tests.cpp

@@ -81,9 +81,9 @@ TEST_CASE ("JetFIX Field2Tests", "[jetfix]")
       std::string out (20, ' ');
       typedef StringField<123> Field;
 
-      REQUIRE(Field::max_value_size ("hello")==5);
+      REQUIRE(Field::max_value_size (std::string_view{"hello"})==5);
 
-      auto result = Field::write (&out[0],"hello");
+      auto result = Field::write (&out[0],std::string_view{"hello"});
       REQUIRE (result.first==5);
       REQUIRE (result.second=='h'+'e'+'l'+'l'+'o');
       REQUIRE (out.compare (0,5,"hello") == 0);
@@ -112,7 +112,7 @@ TEST_CASE ("JetFIX FIXFragmentTests" , "[jetfix]")
       FIXFragment g (0);
       g.add<CharField<1>> ('a');
       g.add<FloatField<2>> (1.2,-1,6);
-      g.add<StringField<3>> ("hello");
+      g.add<StringField<3>> (std::string_view{"hello"});
       g.add<IntField<4>> (123,3);
       const char*t_s = "20150427-19:05:18.922";
       g.add<TimeField<5>> (datetime::parse_FIX_timestamp (t_s));
@@ -141,14 +141,14 @@ TEST_CASE ("JetFIX FIXFragmentTests" , "[jetfix]")
       FIXFragment g;
       auto c_bm = g.add<CharField<1>> ('a');
       auto f_bm = g.add<FloatField<2>> (1.2,-1,6);
-      auto s_bm = g.add<StringField<3>> ("hello");
+      auto s_bm = g.add<StringField<3>> (std::string_view{"hello"});
       auto i_bm = g.add<IntField<4>> (123,3);
       auto t_bm = g.add<TimeField<5>> (datetime::parse_FIX_timestamp (t_s1));
       auto b_bm = g.add<BoolField<6>> (true);
 
       g.overwrite (c_bm,'b');
       g.overwrite (f_bm,1.3,-1,6);
-      g.overwrite (s_bm,"olleh");
+      g.overwrite (s_bm,std::string_view {"olleh"});
       g.overwrite (i_bm,321,3);
       g.overwrite (t_bm,(datetime::parse_FIX_timestamp (t_s2)));
       g.overwrite (b_bm,false);
@@ -172,11 +172,11 @@ TEST_CASE ("JetFIX FIXFragmentTests" , "[jetfix]")
       FIXFragment g1;;
       g1.add<CharField<1>> ('a');
       g1.add<FloatField<2>> (1.2,-1,6);
-      g1.add<StringField<3>> ("hello");
+      g1.add<StringField<3>> (std::string_view{"hello"});
       auto chk1 = g1.checksum ();
       
       FIXFragment g2;
-      g2.add<StringField<4>> ("goodbye");
+      g2.add<StringField<4>> (std::string_view{"goodbye"});
       g2.add<IntField<5>> (321,3);
       auto chk2 = g2.checksum ();
 

src/lib/jetfix/tests/store_tests.cpp

@@ -6,279 +6,270 @@
 #include "lib/jetfix/common/OutFIXPtr.h"
 #include "lib/jetfix/store/FIXFileStore.h"
 #include "lib/jetfix/store/session_name.h"
+#include "lib/logging/logging.h"
 #include "lib/utils/TempDir.h"
 #include "lib/utils/TimeStamp.h"
 #include "lib/utils/datetime.h"
-#include "lib/logging/logging.h"
 
+#include <boost/date_time.hpp>
 #include <boost/date_time/gregorian/gregorian.hpp>
 #include <boost/filesystem.hpp>
-#include <boost/date_time.hpp>
 
 namespace bts {
 namespace jetfix {
 
-TEST_CASE ("JetFIX SessionNameTests", "[jetfix]")
-{
-   SECTION("greg_weekday_works")
-   {
-      const char*days [] = {"sun","Monday","tuesday","Wedn","Thursday","Friday","sat"};
+TEST_CASE("JetFIX SessionNameTests", "[jetfix]") {
+   SECTION("greg_weekday_works") {
+      const char *days[] = {"sun",      "Monday", "tuesday", "Wedn",
+                            "Thursday", "Friday", "sat"};
 
-      int i =0;
+      int i = 0;
       for (auto d : days)
-         REQUIRE (parse_weekday (d)==boost::gregorian::greg_weekday (i++));
-   }
-   
-   SECTION("closest_before_works")
-   {
-      auto same_day_before = closest_before (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                             boost::gregorian::Friday,
-                                             boost::posix_time::time_duration(16, 59, 59));
-      REQUIRE (datetime::FIX_date_time_string (same_day_before)=="20150717-16:59:59");
-
-      auto same_day_after = closest_before (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                            boost::gregorian::Friday,
-                                            boost::posix_time::time_duration(17,0,1));
-      REQUIRE (datetime::FIX_date_time_string (same_day_after)=="20150710-17:00:01");
-
-      auto another_day_same_week = closest_before (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                                   boost::gregorian::Thursday,
-                                                   boost::posix_time::time_duration(16, 59, 59));
-      REQUIRE (datetime::FIX_date_time_string (another_day_same_week)=="20150716-16:59:59");
-
-      auto another_day_previous_week = closest_before (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                                       boost::gregorian::Saturday,
-                                                       boost::posix_time::time_duration(16, 59, 59));
-      REQUIRE (datetime::FIX_date_time_string (another_day_previous_week)=="20150711-16:59:59");
-
+         REQUIRE(parse_weekday(d) == boost::gregorian::greg_weekday(i++));
    }
 
-   SECTION("in_session_works")
-   {
-      REQUIRE (in_session (datetime::make_ptime (2015,7,17,17,0,0,0),
-                               boost::gregorian::Sunday,
-                               boost::posix_time::time_duration(16, 59, 59),
-                               
-                               boost::gregorian::Friday,
-                               boost::posix_time::time_duration(17, 0, 1)));
+   SECTION("closest_before_works") {
+      auto same_day_before =
+          closest_before(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                         boost::gregorian::Friday,
+                         boost::posix_time::time_duration(16, 59, 59));
+      REQUIRE(datetime::FIX_date_time_string(same_day_before) ==
+              "20150717-16:59:59");
 
-      REQUIRE (!in_session (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                boost::gregorian::Sunday,
-                                boost::posix_time::time_duration(16, 59, 59),
-                               
-                                boost::gregorian::Friday,
-                                boost::posix_time::time_duration(16, 59, 59)));
+      auto same_day_after = closest_before(
+          datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+          boost::gregorian::Friday, boost::posix_time::time_duration(17, 0, 1));
+      REQUIRE(datetime::FIX_date_time_string(same_day_after) ==
+              "20150710-17:00:01");
 
-      REQUIRE (in_session (datetime::make_ptime (2015,7,17,17,0,0,0),
-                               boost::posix_time::time_duration(16, 59, 59),
-                               boost::posix_time::time_duration(17, 59, 59)));
+      auto another_day_same_week =
+          closest_before(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                         boost::gregorian::Thursday,
+                         boost::posix_time::time_duration(16, 59, 59));
+      REQUIRE(datetime::FIX_date_time_string(another_day_same_week) ==
+              "20150716-16:59:59");
 
-      REQUIRE (!in_session (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                boost::posix_time::time_duration(0,0,0),
-                                boost::posix_time::time_duration(16, 0,0)));
-
-      REQUIRE (in_session (datetime::make_ptime (2015,7,17,17,0,0,0),
-                               boost::posix_time::time_duration(17, 50, 59),
-                               boost::posix_time::time_duration(17, 45, 59)));
-
-      REQUIRE (!in_session (datetime::make_ptime (2015,7,17,17,47,0,0),
-                                boost::posix_time::time_duration(17, 50, 59),
-                                boost::posix_time::time_duration(17, 45, 59)));
+      auto another_day_previous_week =
+          closest_before(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                         boost::gregorian::Saturday,
+                         boost::posix_time::time_duration(16, 59, 59));
+      REQUIRE(datetime::FIX_date_time_string(another_day_previous_week) ==
+              "20150711-16:59:59");
    }
 
-   SECTION("session_name_works")
-   {
-      REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                boost::gregorian::Sunday,
-                                boost::posix_time::time_duration(16, 59, 59),
-                               
-                                boost::gregorian::Friday,
-                                boost::posix_time::time_duration(17, 0, 1)) ==
-                  "2015-07-12");
+   SECTION("in_session_works") {
+      REQUIRE(in_session(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                         boost::gregorian::Sunday,
+                         boost::posix_time::time_duration(16, 59, 59),
 
+                         boost::gregorian::Friday,
+                         boost::posix_time::time_duration(17, 0, 1)));
 
-      REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                boost::gregorian::Sunday,
-                                boost::posix_time::time_duration(16, 59, 59),
-                                
-                                boost::gregorian::Friday,
-                                boost::posix_time::time_duration(16, 59, 59)) == "");
+      REQUIRE(!in_session(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                          boost::gregorian::Sunday,
+                          boost::posix_time::time_duration(16, 59, 59),
 
-      REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                boost::posix_time::time_duration(16, 59, 59),
-                                boost::posix_time::time_duration(17, 59, 59)) == 
-                  "2015-07-17");
+                          boost::gregorian::Friday,
+                          boost::posix_time::time_duration(16, 59, 59)));
 
-      REQUIRE (session_name (datetime::make_ptime (2015,7,17,18,0,0,0),
-                                boost::posix_time::time_duration(16, 59, 59),
-                                boost::posix_time::time_duration(17, 59, 59)) == 
-                  "");
+      REQUIRE(in_session(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                         boost::posix_time::time_duration(16, 59, 59),
+                         boost::posix_time::time_duration(17, 59, 59)));
 
-      REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,0,0,0),
-                                boost::posix_time::time_duration(17, 50, 59),
-                                boost::posix_time::time_duration(17, 45, 59)) == 
-                  "2015-07-16");
+      REQUIRE(!in_session(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                          boost::posix_time::time_duration(0, 0, 0),
+                          boost::posix_time::time_duration(16, 0, 0)));
 
-      REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,51,0,0),
-                                boost::posix_time::time_duration(17, 50, 59),
-                                boost::posix_time::time_duration(17, 45, 59)) == 
-                  "2015-07-17");
-
-      REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,47,0,0),
-                                boost::posix_time::time_duration(17, 50, 59),
-                                boost::posix_time::time_duration(17, 45, 59)) == 
-                  "");
+      REQUIRE(in_session(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                         boost::posix_time::time_duration(17, 50, 59),
+                         boost::posix_time::time_duration(17, 45, 59)));
 
+      REQUIRE(!in_session(datetime::make_ptime(2015, 7, 17, 17, 47, 0, 0),
+                          boost::posix_time::time_duration(17, 50, 59),
+                          boost::posix_time::time_duration(17, 45, 59)));
+   }
+
+   SECTION("session_name_works") {
+      REQUIRE(session_name(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                           boost::gregorian::Sunday,
+                           boost::posix_time::time_duration(16, 59, 59),
+
+                           boost::gregorian::Friday,
+                           boost::posix_time::time_duration(17, 0, 1)).as_string ==
+              "2015-07-12");
+
+      REQUIRE(session_name(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                           boost::gregorian::Sunday,
+                           boost::posix_time::time_duration(16, 59, 59),
+
+                           boost::gregorian::Friday,
+                           boost::posix_time::time_duration(16, 59, 59)).as_string == "");
+
+      REQUIRE(session_name(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                           boost::posix_time::time_duration(16, 59, 59),
+                           boost::posix_time::time_duration(17, 59, 59)).as_string ==
+              "2015-07-17");
+
+      REQUIRE(session_name(datetime::make_ptime(2015, 7, 17, 18, 0, 0, 0),
+                           boost::posix_time::time_duration(16, 59, 59),
+                           boost::posix_time::time_duration(17, 59, 59)).as_string == "");
+
+      REQUIRE(session_name(datetime::make_ptime(2015, 7, 17, 17, 0, 0, 0),
+                           boost::posix_time::time_duration(17, 50, 59),
+                           boost::posix_time::time_duration(17, 45, 59)).as_string ==
+              "2015-07-16");
+
+      REQUIRE(session_name(datetime::make_ptime(2015, 7, 17, 17, 51, 0, 0),
+                           boost::posix_time::time_duration(17, 50, 59),
+                           boost::posix_time::time_duration(17, 45, 59)).as_string ==
+              "2015-07-17");
+
+      REQUIRE(session_name(datetime::make_ptime(2015, 7, 17, 17, 47, 0, 0),
+                           boost::posix_time::time_duration(17, 50, 59),
+                           boost::posix_time::time_duration(17, 45, 59)).as_string == "");
    }
-   
 }
 
-
-TEST_CASE ("JetFIX FileStoreTests", "[jetfix]")
-{
-   SECTION("empty_store_works")
-   {
+TEST_CASE("JetFIX FileStoreTests", "[jetfix]") {
+   SECTION("empty_store_works") {
       config::JetFIX::Connection c;
-      c.set_name ("test");
-      c.set_sender_compid ("CME");
-      c.set_sender_subid ("G");
-      c.set_target_compid ("1U7001N");
-      c.set_target_subid ("BLUESTREAM");
+      c.set_name("test");
+      c.set_sender_compid("CME");
+      c.set_sender_subid("G");
+      c.set_target_compid("1U7001N");
+      c.set_target_subid("BLUESTREAM");
 
-      c.set_start_time ("00:00:00");
-      c.set_end_time ("23:59:59");
+      c.set_start_time("00:00:00");
+      c.set_end_time("23:59:59");
 
       utils::TempDir td;
-      
-      FIXFileStore fs (c,td.name ());
-      REQUIRE (fs.session_active ());
-      REQUIRE (fs.replay ().size ()==0);
-      REQUIRE (fs.last_seq_num ()==0);
-      REQUIRE (fs.last_seq_num_processed ()==0);
+
+      FIXFileStore fs(c, td.name());
+      REQUIRE(fs.session_active());
+      REQUIRE(fs.replay().size() == 0);
+      REQUIRE(fs.last_seq_num() == 0);
+      REQUIRE(fs.last_seq_num_processed() == 0);
    }
 
-   SECTION("in_msg_is_stored")
-   {
-      auto td = std::make_shared<utils::TempDir> ();
+   SECTION("in_msg_is_stored") {
+      auto td = std::make_shared<utils::TempDir>();
       config::JetFIX::Connection c;
-      c.set_name ("test");
-      c.set_sender_compid ("CME");
-      c.set_sender_subid ("G");
-      c.set_target_compid ("1U7001N");
-      c.set_target_subid ("BLUESTREAM");
+      c.set_name("test");
+      c.set_sender_compid("CME");
+      c.set_sender_subid("G");
+      c.set_target_compid("1U7001N");
+      c.set_target_subid("BLUESTREAM");
 
-      c.set_start_time ("00:00:00");
-      c.set_end_time ("23:59:59");
+      c.set_start_time("00:00:00");
+      c.set_end_time("23:59:59");
 
-      FIXEncodingStream stream (c);
-      auto &buf = stream.new_msg (MessageTypes::Logon);
-      buf.add<fix::TargetLocationID> ("US,NC");
-      buf.add<fix::EncryptMethod> (0);
+      FIXEncodingStream stream(c);
+      auto &buf = stream.new_msg(MessageTypes::Logon);
+      buf.add<fix::TargetLocationID>(std::string_view{"US,NC"});
+      buf.add<fix::EncryptMethod>(0);
       buf.add<fix::HeartBtInt>(30);
-      buf.add<fix::ApplicationSystemName> ("O2");
-      buf.add<fix::ApplicationSystemVersion> ("1.0");
-      buf.add<fix::ApplicationSystemVendor> ("BTS");
-      stream.close ();
-      stream.stamp (2,3,datetime::FIX_timestamp (datetime::now ()));
-      auto fix = stream.to_str ();
-      
+      buf.add<fix::ApplicationSystemName>(std::string_view{"O2"});
+      buf.add<fix::ApplicationSystemVersion>(std::string_view{"1.0"});
+      buf.add<fix::ApplicationSystemVendor>(std::string_view{"BTS"});
+      stream.close();
+      stream.stamp(2, 3, datetime::FIX_timestamp(datetime::now()));
+      auto fix = stream.to_str();
+
       {
-         FIXFileStore fs (c,td->name ());
-         fs.store (InMsgBuffer (fix));
-         fs.set_last_msg_processed (3);
+         FIXFileStore fs(c, td->name());
+         fs.store(InMsgBuffer(fix));
+         fs.set_last_msg_processed(3);
       }
 
-      FIXFileStore fs (c,td->name ());
-      auto replay = fs.replay ();
-      REQUIRE (replay.size ()==1);
-      REQUIRE (replay[0].type ()==MessageTypes::Logon);
-      REQUIRE (replay[0].to_str ()==fix);
-      REQUIRE (fs.last_seq_num ()==0);
-      REQUIRE (fs.last_seq_num_processed ()==3);
+      FIXFileStore fs(c, td->name());
+      auto replay = fs.replay();
+      REQUIRE(replay.size() == 1);
+      REQUIRE(replay[0].type() == MessageTypes::Logon);
+      REQUIRE(replay[0].to_str() == fix);
+      REQUIRE(fs.last_seq_num() == 0);
+      REQUIRE(fs.last_seq_num_processed() == 3);
    }
 
-   SECTION("out_msg_is_stored")
-   {
-      auto td = std::make_shared<utils::TempDir> ();
+   SECTION("out_msg_is_stored") {
+      auto td = std::make_shared<utils::TempDir>();
       config::JetFIX::Connection c;
-      c.set_name ("test");
-      c.set_sender_compid ("CME");
-      c.set_sender_subid ("G");
-      c.set_target_compid ("1U7001N");
-      c.set_target_subid ("BLUESTREAM");
+      c.set_name("test");
+      c.set_sender_compid("CME");
+      c.set_sender_subid("G");
+      c.set_target_compid("1U7001N");
+      c.set_target_subid("BLUESTREAM");
 
-      c.set_start_time ("00:00:00");
-      c.set_end_time ("23:59:59");
+      c.set_start_time("00:00:00");
+      c.set_end_time("23:59:59");
 
       auto msg_vec = std::make_shared<OutFIX>(c);
-      auto &stream = msg_vec->fix.back ();
-      auto &buf = stream.new_msg (MessageTypes::Logon);
-      buf.add<fix::TargetLocationID> ("US,NC");
-      buf.add<fix::EncryptMethod> (0);
+      auto &stream = msg_vec->fix.back();
+      auto &buf = stream.new_msg(MessageTypes::Logon);
+      buf.add<fix::TargetLocationID>(std::string_view{"US,NC"});
+      buf.add<fix::EncryptMethod>(0);
       buf.add<fix::HeartBtInt>(30);
-      buf.add<fix::ApplicationSystemName> ("O2");
-      buf.add<fix::ApplicationSystemVersion> ("1.0");
-      buf.add<fix::ApplicationSystemVendor> ("BTS");
-      stream.close ();
-      stream.stamp (2,3,datetime::FIX_timestamp (datetime::now ()));
-      
+      buf.add<fix::ApplicationSystemName>(std::string_view{"O2"});
+      buf.add<fix::ApplicationSystemVersion>(std::string_view{"1.0"});
+      buf.add<fix::ApplicationSystemVendor>(std::string_view{"BTS"});
+      stream.close();
+      stream.stamp(2, 3, datetime::FIX_timestamp(datetime::now()));
+
       {
-         FIXFileStore fs (c,td->name ());
-         fs.store (msg_vec);
+         FIXFileStore fs(c, td->name());
+         fs.store(msg_vec);
       }
 
-      FIXFileStore fs (c,td->name ());
-      auto replay = fs.replay ();
-      REQUIRE (replay.size ()==0);
-      REQUIRE (fs.last_seq_num ()==3);
-      REQUIRE (fs.last_seq_num_processed ()==0);
+      FIXFileStore fs(c, td->name());
+      auto replay = fs.replay();
+      REQUIRE(replay.size() == 0);
+      REQUIRE(fs.last_seq_num() == 3);
+      REQUIRE(fs.last_seq_num_processed() == 0);
    }
 
-   SECTION("test_performance")
-   {
-      auto td = std::make_shared<utils::TempDir> ();
+   SECTION("test_performance") {
+      auto td = std::make_shared<utils::TempDir>();
       config::JetFIX::Connection c;
-      c.set_name ("test");
-      c.set_sender_compid ("CME");
-      c.set_sender_subid ("G");
-      c.set_target_compid ("1U7001N");
-      c.set_target_subid ("BLUESTREAM");
+      c.set_name("test");
+      c.set_sender_compid("CME");
+      c.set_sender_subid("G");
+      c.set_target_compid("1U7001N");
+      c.set_target_subid("BLUESTREAM");
 
-      c.set_start_time ("00:00:00");
-      c.set_end_time ("23:59:59");
+      c.set_start_time("00:00:00");
+      c.set_end_time("23:59:59");
 
-      auto start = utils::TimeStamp::now ();
+      auto start = utils::TimeStamp::now();
       {
-         FIXFileStore fs (c,td->name ());
-         for (int i = 0; i < 10000;i++) {
+         FIXFileStore fs(c, td->name());
+         for (int i = 0; i < 10000; i++) {
             auto msg_vec = std::make_shared<std::vector<FIXEncodingStream>>();
-            msg_vec->emplace_back (c);
-            auto &stream = msg_vec->back ();
-            auto &buf = stream.new_msg (MessageTypes::Logon);
-            buf.add<fix::TargetLocationID> ("US,NC");
-            buf.add<fix::EncryptMethod> (0);
+            msg_vec->emplace_back(c);
+            auto &stream = msg_vec->back();
+            auto &buf = stream.new_msg(MessageTypes::Logon);
+            buf.add<fix::TargetLocationID>(std::string_view{"US,NC"});
+            buf.add<fix::EncryptMethod>(0);
             buf.add<fix::HeartBtInt>(30);
-            buf.add<fix::ApplicationSystemName> ("O2");
-            buf.add<fix::ApplicationSystemVersion> ("1.0");
-            buf.add<fix::ApplicationSystemVendor> ("BTS");
-            stream.close ();
-            stream.stamp (i,3,datetime::FIX_timestamp (datetime::now () - boost::gregorian::days (1)));
-            fs.store (InMsgBuffer (stream.to_str ()));
+            buf.add<fix::ApplicationSystemName>(std::string_view{"O2"});
+            buf.add<fix::ApplicationSystemVersion>(std::string_view{"1.0"});
+            buf.add<fix::ApplicationSystemVendor>(std::string_view{"BTS"});
+            stream.close();
+            stream.stamp(i, 3,
+                         datetime::FIX_timestamp(datetime::now() -
+                                                 boost::gregorian::days(1)));
+            fs.store(InMsgBuffer(stream.to_str()));
          }
       }
-      auto stop = utils::TimeStamp::now ();
-      LOG(Testing,Info,"jetfix store gen: {} ms", (stop-start).milliseconds () );
-      
-      start = utils::TimeStamp::now ();
-      FIXFileStore fs (c,td->name ());
-      stop = utils::TimeStamp::now ();
-      LOG(Testing,Info,"jetfix store load: {} ms", (stop-start).milliseconds () );
+      auto stop = utils::TimeStamp::now();
+      LOG(Testing, Info, "jetfix store gen: {} ms",
+          (stop - start).milliseconds());
+
+      start = utils::TimeStamp::now();
+      FIXFileStore fs(c, td->name());
+      stop = utils::TimeStamp::now();
+      LOG(Testing, Info, "jetfix store load: {} ms",
+          (stop - start).milliseconds());
    }
-   
 }
 
-}}
-
-    
-  
-
+} // namespace jetfix
+} // namespace bts

src/lib/logging/logging.cpp

@@ -120,6 +120,7 @@ void configure_logging(bts::config::Logging const&cfg, bts::config::ThreadGroupC
                                 {log::Plugin,"Plugin"},
                                 {log::Sdk,"Sdk"},
                                 {log::Http,"Http"},
+                                {log::iLink3,"ilink3"},
    };
 
    real_cfg.sinks["console"] =

src/lib/logging/logging.h

@@ -46,6 +46,7 @@ constexpr log::Subsystem DataFlow   {Latency.ordinal+1     };
 constexpr log::Subsystem Plugin     {DataFlow.ordinal+1    };
 constexpr log::Subsystem Sdk        {Plugin.ordinal+1      };
 constexpr log::Subsystem Http       {Sdk.ordinal+1   };
+constexpr log::Subsystem iLink3     {Http.ordinal+1   };
 
 struct LoggerHandle {
    LoggerHandle(LoggerHandle const&) = delete;

src/lib/md/ExchangeSecurityKey.h

@@ -0,0 +1,5 @@
+#pragma once
+#include "lib/utils/ValueWrapper.h"
+namespace bts {
+using ExchangeSecurityKey = utils::ValueWrapper<struct ExchangeSecurityKeyTag,uint64_t>;
+}

src/lib/md/MDClient.cpp

@@ -92,7 +92,7 @@ MDConsumer& MDClient::current_consumer() const {
    return _self->_consumer;
 }
 SecurityOracle MDClient::oracle() const {
-   return SecurityOracle(*this);
+   return SecurityOracle(_self->_cache);
 }
 std::set<UserSymbol> MDClient::symbols() const {
    return _self->_symbols;

src/lib/md/MDConsumer.cpp

@@ -792,7 +792,7 @@ public:
             // We can safely send a security key to every _instance because by definition
             // a security cannot appear in 2 instances
             if (unresolved_securities.size() > 250) {
-               LOG(md,Warn,"Failed to resolve legs for {}"
+               LOG(md,Info,"Failed to resolve legs for {}"
                    " spreads. This can happen when a spread is subscribed without "
                    "it's underliers and may lead to higher than expected CPU use.",
                    unresolved_securities.size() );
@@ -897,10 +897,14 @@ public:
       scan_securities(resolve_legs);
    }
 
-   SecurityPtr get_security(SecurityKey key) const {
+   const Security* get_security(SecurityKey key) const {
       return _cache.get_security(key);
    }
 
+   std::vector<api::Security const *> get_securities() const {
+      return _cache.get_securities();
+   }
+
    MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const {
       for (auto& inst : _instances) {
          if ((inst->symbol() == symbol && inst->type() == type && inst->exchange() == exchange) ||
@@ -975,7 +979,8 @@ size_t MDConsumer::walk_security_list(SecurityCallback cb, const string& symbol,
 
 size_t MDConsumer::scan_securities(bool resolve_legs) { return _pimpl->scan_securities(resolve_legs); }
 void MDConsumer::on_security(SecurityCallback cb, bool resolve_legs) { _pimpl->on_security(cb, resolve_legs); }
-const api::Security* MDConsumer::get_security(SecurityKey key) const { return _pimpl->get_security(key).get(); }
+const api::Security* MDConsumer::get_security(SecurityKey key) const { return _pimpl->get_security(key); }
+std::vector<api::Security const*> MDConsumer::get_securities() const {return _pimpl->get_securities();}
 
 MDConsumerChannel MDConsumer::get_channel(const std::string& symbol, SecurityType type, Venue exchange) const {
    return _pimpl->get_channel(symbol, type, exchange);

src/lib/md/MDConsumer.h

@@ -59,6 +59,9 @@ public:
    // You still need to call scan_securities() periodically to discover new securities
    const api::Security* get_security(SecurityKey key) const;
 
+   // Retrieve all cached securities
+   std::vector<api::Security const*> get_securities() const;
+
    MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const;
    MDConsumerChannel get_channel(const UserSymbol&) const;
 

src/lib/md/MDProducer.cpp

@@ -20,8 +20,14 @@ public:
       _symbol(symbol),
       _type(type),
       _exchange(exchange),
-      _hot(hot_segment_name(symbol, type, exchange, platform), 1024, 1024, flags & MDProducerFlags::Shared),
-      _cold(cold_segment_name(symbol, type, exchange, platform), 8192, 2048, flags & MDProducerFlags::Shared),
+      _hot(hot_segment_name(symbol, type, exchange, platform), 
+         config::get<PublisherOptions>().hot_initial(), 
+         config::get<PublisherOptions>().hot_growth(),
+         flags & MDProducerFlags::Shared),
+      _cold(cold_segment_name(symbol, type, exchange, platform), 
+         config::get<PublisherOptions>().cold_initial(), 
+         config::get<PublisherOptions>().cold_growth(), 
+         flags & MDProducerFlags::Shared),
       _trades(_cold.header().trade_buffer, BTS_ARRAYSIZE(_cold.header().trade_buffer), true),
       _rfqs(_cold.header().rfq_buffer, BTS_ARRAYSIZE(_cold.header().rfq_buffer), true)
       {

src/lib/md/SecurityCache.cpp

@@ -1,8 +1,10 @@
 #include "SecurityCache.h"
 #include "SecurityDB.h"
 #include "lib/utils/Dispatcher.h"
+#include "lib/utils/LockBox.h"
 
 #include <unordered_map>
+#include <google/btree/btree_map.h>
 #include <tbb/concurrent_queue.h>
 #include <mutex>
 
@@ -10,8 +12,8 @@ namespace bts {
 namespace md {
 
 struct SecurityCacheImpl {
-   std::mutex cache_mutex;
-   std::unordered_map<SecurityKey, SecurityPtr> cache;
+   utils::LockBox<btree::btree_map<SecurityKey, SecurityPtr>> cache;
+   utils::LockBox<btree::btree_map<ExchangeSecurityKey, SecurityPtr>> ex_cache;
    tbb::concurrent_queue<SecurityPtr> update_queue;
    void enqueue_update_db(const SecurityPtr&);
    void flush();
@@ -42,52 +44,85 @@ void SecurityCacheImpl::flush() {
 SecurityCache::SecurityCache() : _me(std::make_shared<SecurityCacheImpl>()) {}
 SecurityCache::~SecurityCache() {}
 
-SecurityPtr SecurityCache::try_get_security(SecurityKey key) const {
-   std::lock_guard<std::mutex> lock(_me->cache_mutex);
-   auto iter = _me->cache.find(key);
-   if (iter != _me->cache.end()) {
-      return iter->second;
-   }
-   return nullptr;
+const Security* SecurityCache::try_get_security(SecurityKey key) const {
+   return _me->cache.lock([key](btree::btree_map<SecurityKey, SecurityPtr>& t) {
+      auto iter = t.find(key);
+      if (iter != t.end()) {
+         return iter->second.get();
+      }
+      return (const api::Security*)nullptr;
+   });
+}
+const Security* SecurityCache::try_get_security(ExchangeSecurityKey key) const {
+   return _me->ex_cache.lock([key](btree::btree_map<ExchangeSecurityKey,SecurityPtr>& t) {
+      auto iter = t.find(key);
+      if (iter != t.end()) {
+         return iter->second.get();
+      }
+      return (const api::Security*)nullptr;
+   });
 }
 
-SecurityPtr SecurityCache::get_security(SecurityKey key) {
-   std::unique_lock<std::mutex> lock(_me->cache_mutex);
-   auto iter = _me->cache.find(key);
-   if (iter != _me->cache.end()) {
-      return iter->second;
-   }
-   else {
-      // unlock for db read
-      lock.unlock();
+
+const Security* SecurityCache::get_security(SecurityKey key) {
+   const Security* result = try_get_security(key);
+   if (result == nullptr) {
+      // outside lock
       auto security = db::find_security(key);
       if (security) {
          LOG(md,Info, "Loaded security from database: {} {}", security->symbol(), security->id());
          return add_db_security(security);
       }
-      return security;
+      return nullptr;
    }
+   return result;
 }
 
-SecurityPtr SecurityCache::add_db_security(const SecurityPtr& security) {
+std::vector<api::Security const*> SecurityCache::get_securities() {
+   std::vector<api::Security const*> result;
+   result.reserve(100000);
+
+   _me->cache.lock([&result](auto& cache) {
+      for (auto const&p : cache) {
+         result.push_back(p.second.get());
+      }
+   });
+   return result;
+}
+
+const Security* SecurityCache::add_db_security(const SecurityPtr& security) {
    bts_assert(security);
-   std::lock_guard<std::mutex> lock(_me->cache_mutex);
-   SecurityKey key(security->id());
-   auto result = _me->cache.insert(std::make_pair(SecurityKey(security->id()), security));
-   // If result was inserted result.first->second is equal to security
-   // If the value was already present it must be a recently inserted 'live' security and we should use it
-   return result.first->second;
+   if (security->has_glbx()) {
+      _me->ex_cache.lock([&security](auto& cache) {
+         ExchangeSecurityKey key(security->id());
+         cache.insert(std::make_pair(key, security));
+      });
+   }
+   return _me->cache.lock([&security](auto& cache) {
+      SecurityKey key(security->id());
+      auto result = cache.insert(std::make_pair(key, security));
+      // If result was inserted result.first->second is equal to security
+      // If the value was already present it must be a recently inserted 'live' security and we should use it
+      return result.first->second.get();
+   });
 }
 bool SecurityCache::add_live_security(const SecurityPtr& security) {
    bts_assert(security);
-   std::lock_guard<std::mutex> lock(_me->cache_mutex);
-   SecurityKey key(security->id());
-   // Insert but don't overwrite existing
-   auto result = _me->cache.insert(std::make_pair(SecurityKey(security->id()), security));
-   _me->enqueue_update_db(security);
-   return result.second;
+   if (security->has_glbx()) {
+      _me->ex_cache.lock([&security](auto& cache) {
+         ExchangeSecurityKey key(security->id());
+         cache.insert(std::make_pair(key, security));
+      });
+   }
+   return _me->cache.lock([&](auto& cache) {
+      SecurityKey key(security->id());
+      // Insert but don't overwrite existing
+      auto result = cache.insert(std::make_pair(SecurityKey(security->id()), security));
+      _me->enqueue_update_db(security);
+      return result.second;
+   });
 }
 
 void SecurityCache::flush_to_disk() { _me->flush(); }
 
-}}//ns
+}}//ns

src/lib/md/SecurityCache.h

@@ -3,6 +3,7 @@
 #include "lib/domain_model/Security.pb.h"
 #include "MDConsumerHandle.h"
 #include "lib/domain_model/SecurityKey.h"
+#include "lib/md/ExchangeSecurityKey.h"
 #include "SecurityPtr.h"
 #include <memory>
 
@@ -20,17 +21,21 @@ public:
    ~SecurityCache();
 
    //Load security from cache with fallback to database
-   SecurityPtr get_security(SecurityKey key); 
+   const Security* get_security(SecurityKey key); 
 
+   // Load all securities from cache, no fallback to database
+   std::vector<api::Security const*> get_securities();
+   
    // Get the security if it's in the cache otherwise return nullptr. Does not query security database.
-   SecurityPtr try_get_security(SecurityKey key) const;
+   const Security* try_get_security(SecurityKey key) const;
+   const Security* try_get_security(ExchangeSecurityKey key) const;
 
    // returns true if the cache was updated
    bool add_live_security(const SecurityPtr& security);
-   SecurityPtr add_db_security(const SecurityPtr& security);
+   const Security* add_db_security(const SecurityPtr& security);
 
    void flush_to_disk();
 
 };
 
-}}//ns
+}}//ns

src/lib/md/SecurityOracle.cpp

@@ -4,10 +4,18 @@
 namespace bts {
 namespace md {
 
-SecurityOracle::SecurityOracle(const MDClient& client): _client(client) {}
+SecurityOracle::SecurityOracle(SecurityCache& cache): _cache(cache) {}
 SecurityOracle::~SecurityOracle() { }
 const api::Security* SecurityOracle::get_security(SecurityKey key) const {
-   return _client.current_consumer().get_security(key);
+   return _cache.get_security(key);
 }
 
-}}//namespace
+const api::Security* SecurityOracle::get_security(ExchangeSecurityKey key) const {
+   return _cache.try_get_security(key);
+}
+
+std::vector<api::Security const*> SecurityOracle::get_securities() const {
+   return _cache.get_securities();
+}
+
+}}//namespace

src/lib/md/SecurityOracle.h

@@ -1,17 +1,23 @@
 #pragma once
+
 #include "lib/domain_model/SecurityKey.h"
+#include "lib/md/ExchangeSecurityKey.h"
+#include "lib/md/SecurityCache.h"
 #include <memory>
+#include <vector>
 
 namespace bts {
 namespace api {class Security;}
 namespace md {
 class MDClient;
 class SecurityOracle {
-   const MDClient& _client;
+   SecurityCache& _cache;
 public:
-   SecurityOracle(const MDClient&);
+   SecurityOracle(SecurityCache&);
    ~SecurityOracle();
    const api::Security* get_security(SecurityKey key) const;
+   const api::Security* get_security(ExchangeSecurityKey key) const;
+   std::vector<api::Security const*> get_securities() const;
 };
 
-}}//namespace
+}}//namespace

src/lib/md/vendor/OnixsUtils.cpp

@@ -18,16 +18,16 @@ const char * onixs_license[] = {
 "    <name>Blue Trading Systems OF 9420-2</name>",
 "  </customer>",
 "  <period>",
-"    <since>2019-05-02</since>",
-"    <till>2020-05-13</till>",
+"    <since>2020-05-01</since>",
+"    <till>2021-05-12</till>",
 "  </period>",
 "  <products>",
 "    <product id=\"4\" name=\"CME Market Data Handler for C++\" />",
 "    <product id=\"7\" name=\"ICE iMpact Multicast Price Feed Handler for C++\" />",
 "    <product id=\"67\" name=\"Eurex Market and Reference Data Interfaces (EMDI, RDI) Handler for C++\" />",
 "  </products>",
-"  <checksum>5E50A1DACD94D7C1BECD316EEAE5AB02</checksum>",
-"</license>",
+"  <checksum>1C9210495AA3C9C884B7A37A4C7AD8F9</checksum>",
+"</license>"
 };
 
 OnixsTemplLicense onixs_create_temporary_license() {

src/lib/orderman/actors/FillStore.cpp

@@ -10,6 +10,7 @@
 #include "lib/persistence/SqliteStore.h"
 #include "lib/utils/PBUtils.h"
 #include "lib/orderman/actors/SODPosition.h"
+#include "lib/utils/overloaded.h"
 
 #include <boost/date_time/posix_time/posix_time.hpp>
 #include <sys/time.h>
@@ -119,8 +120,21 @@ void FillStore::on_execution_report(
                   fill.set_fill_status(api::orderman::Fill::FILL_STATUS_CORRECTED);
                fill.set_fill_key(fk);
 
-               auto exchange_ts = utils::pb::to_timestamp(
-                   datetime::parse_FIX_timestamp(to_str(er.transact_time)));
+               auto exchange_ts = 
+                  std::visit(overloaded { 
+                     [](const orderman::ExchangeTimeStampStr& ts) { 
+                        return utils::pb::to_timestamp(
+                                 datetime::parse_FIX_timestamp(to_str(ts)));
+                     },
+                     [](ExchangeTimeStamp64 ts) {
+                        google::protobuf::Timestamp gts;
+                        auto seconds = ts.value() / 1E9;
+                        gts.set_seconds(seconds);
+                        gts.set_nanos(ts.value() - (seconds * 1E9));
+                        return gts;
+                     }
+                  }, er.transact_time);
+
                *fill.mutable_timestamp() = exchange_ts;
 
                auto it = me->known_orders.find(er.order_key);

src/lib/orderman/actors/GLBXExecutionGateway.cpp

@@ -222,7 +222,9 @@ void GLBXExecutionGateway::on_exe_report(
           msg.get<fix::ManualOrderIndicator>()
               ? api::orderman::MANUAL_ORDER_INDICATOR_YES
               : api::orderman::MANUAL_ORDER_INDICATOR_YES;
-      str_to(msg.get<fix::OrderID>(), &data->exchange_order_id);
+      str_to(msg.get<fix::OrderID>(), 
+         & std::get<ExchangeOrderID>(
+            data->exchange_order_id));
 
       if (msg.has<fix::Price>())
          data->price = securities::decode_exchange_price(
@@ -261,7 +263,9 @@ void GLBXExecutionGateway::on_exe_report(
                             : Price(0);
       if (msg.has<fix::ExecID>())
          str_to(msg.get<fix::ExecID>(), &data->exchange_fill_id);
-      str_to(msg.get<fix::TransactTimeStr>(), &data->transact_time);
+      orderman::ExchangeTimeStampStr t;
+      str_to(msg.get<fix::TransactTimeStr>(), &t);
+      data->transact_time = t;
 
       if (msg.has<fix::CustOrderHandlingInst>())
          data->cust_order_handling_inst = msg.get<fix::CustOrderHandlingInst>();

src/lib/orderman/actors/GLBXExecutionGateway.h

@@ -18,6 +18,7 @@ class GLBXExecutionGateway : public IExecutionGateway {
    void on_securities(const mercury::Update<NewSecuritiesEvent> &) override;
    void on_admin_data(const mercury::Update<NewAdminDataEvent> &) override;
    void send_fix(jetfix::OutFIXPtr const &fix, trace::CorID trace) override;
+   void send_msgs(std::vector<OrderMsg> const&, trace::CorID) override {bts_assert(false);}
 
    OnExeReport& on_exe_report() override;
    OnGwStatusChange& on_gw_status_change() override;

src/lib/orderman/actors/GLBXiLink3ExecutionGateway.cpp

@@ -0,0 +1,284 @@
+#include "lib/orderman/actors/GLBXiLink3ExecutionGateway.h"
+#include "lib/config/config.h"
+#include "lib/config/protos/orderman_cfg.pb.h"
+#include "lib/domain_model/Security.h"
+#include "lib/jetfix/common/OutFIX.h"
+#include "lib/jetfix/store/session_name.h"
+#include "lib/orderman/actors/GLBXExecutionGateway.h"
+#include "lib/orderman/actors/ilink3/IL3Reactor.h"
+#include "lib/orderman/actors/ilink3/ILinkGW.h"
+#include "lib/orderman/actors/ilink3/SessionStore.h"
+#include "lib/orderman/events/NewSecuritiesEvent.h"
+#include "lib/orderman/orderman_utils/fix_order_codec.h"
+#include "lib/orderman/actors/ilink3/IL3MessageFramer.h"
+
+namespace bts {
+namespace orderman {
+namespace ilink3 {
+
+enum class GLBXiLink3ExecutionGateway::State {
+
+   WAITING_FOR_SECURITIES,
+   PAUSED,
+   CONNECTING,
+   READY
+};
+
+struct GLBXiLink3ExecutionGateway::PImpl {
+   PImpl(config::JetFIX::Connection const &cfg, trace::PerfLog &log,
+         md::SecurityOracle o)
+       : state(), cfg(cfg), reactor(), start_of_pause(), on_exe_report(),
+         on_gw_status_change(), on_cancel_reject(), perf_log(log),
+         sec_oracle(o), securities(), securities_by_descriptor(),
+         gw_name(cfg.name()), session_store(), gw(), connection(nullptr),
+         connection_failures(0) {
+      TCPEndpoint endpoint{cfg.socket_connect_host_ip(),
+                           (ushort)cfg.socket_connect_port()};
+      connection = &reactor.register_connection({endpoint});
+   }
+
+   State state;
+   config::JetFIX::Connection const cfg;
+   IL3Reactor reactor;
+   IL3MessageFramer framer;
+   TimeStamp start_of_pause;
+
+   OnExeReport on_exe_report;
+   OnGwStatusChange on_gw_status_change;
+   orderman::OnCancelReject on_cancel_reject;
+   trace::PerfLog &perf_log;
+   md::SecurityOracle sec_oracle;
+   std::unordered_map<SecurityKey::Value, api::Security const *> securities;
+   std::unordered_map<std::string, api::Security const *>
+       securities_by_descriptor;
+   GatewayName gw_name;
+   std::optional<SessionStore> session_store;
+   std::optional<ILinkGW> gw;
+   IL3Connection *connection;
+   size_t connection_failures;
+};
+
+GLBXiLink3ExecutionGateway::GLBXiLink3ExecutionGateway(
+    config::JetFIX::Connection const &cfg, trace::PerfLog &log,
+    md::SecurityOracle o)
+    : me(new PImpl(cfg, log, o)) {
+   me->state = State::WAITING_FOR_SECURITIES;
+}
+
+GLBXiLink3ExecutionGateway::~GLBXiLink3ExecutionGateway() {}
+
+void GLBXiLink3ExecutionGateway::start() {}
+
+void GLBXiLink3ExecutionGateway::dispatch_incoming_fix_messages() {
+   switch (me->state) {
+   case State::WAITING_FOR_SECURITIES:
+      break;
+   case State::PAUSED:
+      on_time();
+      break;
+   case State::READY:
+   case State::CONNECTING:
+      me->reactor.do_work();
+      me->gw->on_time();
+      break;
+   }
+}
+
+void GLBXiLink3ExecutionGateway::on_time() {
+   bts_assert(me->state == State::PAUSED);
+
+   // If we are still paused bail early, this avoids logging too much
+   // or spending too much CPU
+   if (pause_in_effect())
+      return;
+
+   me->start_of_pause = Timer::now();
+
+   // If we alread exceed the maximum number of failures per run
+   // bail. This is a safety measure to avoid overwhelming the
+   // exchange in the event of a bug or operational problem that
+   // results in repeated connection failures.
+   if (max_failures_reached()) {
+      LOG(iLink3, Warn,
+          "Maximum number of connection failures reached for {}. Reboot to try "
+          "again",
+          me->gw_name.value());
+      return;
+   }
+
+   // If we reach this point we are no longer in the pause window and
+   // we have not exceed our connection failure limit. The only thing
+   // to check now is wheter we are inside the exchange's active
+   // hours.
+   auto current_session = jetfix::session_name(me->cfg);
+   if (current_session.empty()) {
+      LOG(iLink3, Info, "Outside of session in ", me->gw_name.value());
+      return;
+   }
+   // Yep, the stars are all aligned, lets try to connect
+   connect(current_session);
+}
+
+void GLBXiLink3ExecutionGateway::connect(jetfix::SessionName const &sn) {
+   bts_assert(me->state == State::PAUSED);
+   transition(State::CONNECTING);
+   Config c;
+   c.cme_session = me->cfg.sender_compid().substr(0, 3);
+   c.firm = me->cfg.sender_compid().substr(3);
+   c.failover_on_disconnect = false;
+
+   me->session_store.emplace(me->sec_oracle, sn, me->cfg);
+   c.raw_message_logger =
+       [store = &me->session_store.value()](auto d, auto const &msg) {
+          store->async_store_raw_msg(d, msg);
+       };
+
+   me->gw.emplace(*me->connection, me->framer, &me->sec_oracle, me->session_store->starting_session_descriptor(), c, me->cfg);              
+
+   me->gw->subscribe_cancel_reject([this](auto k, auto const &msg) {
+      bts_assert(me->state == State::READY || me->state == State::CONNECTING);
+      me->on_cancel_reject(k, msg);
+   });
+
+   me->gw->subscribe_execution_reports([this](auto er) {
+      bts_assert(me->state == State::READY || me->state == State::CONNECTING);
+      me->on_exe_report(*er, {});
+   });
+
+   me->gw->subscribe_session([this](auto const &s) {
+      me->session_store->async_update_session(s);
+   });
+
+   me->gw->subscribe_state([this](auto gw_state) {
+      LOG(iLink3, Info, "Internal gateway state change in {}. State is now {}",
+          me->gw_name.value(),
+          log::CharPtrWrapper{ilink3::state_string(gw_state)});
+      switch (me->state) {
+      case State::CONNECTING:
+         if (gw_state == ilink3::State::READY) {
+            transition(State::READY);
+            me->on_gw_status_change(true);
+         } else if (gw_state == ilink3::State::ERROR) {
+            me->connection_failures++;
+            pause();
+         }
+         break;
+      case State::READY:
+         if (gw_state == ilink3::State::ERROR) {
+            me->connection_failures++;
+            pause();
+            me->on_gw_status_change(false);
+         }
+         break;
+      default:
+         bts_assert(false);
+         break;
+      }
+   });
+   me->gw->connect();
+}
+
+void GLBXiLink3ExecutionGateway::pause() {
+   me->start_of_pause = Timer::now();
+   me->gw.reset();
+   transition(State::PAUSED);
+}
+
+bool GLBXiLink3ExecutionGateway::pause_in_effect() const {
+   return sec_diff(Timer::now(), me->start_of_pause) <=
+          me->cfg.seconds_to_pause();
+}
+
+bool GLBXiLink3ExecutionGateway::max_failures_reached() const {
+   return me->connection_failures > me->cfg.max_connection_attempts();
+}
+
+void GLBXiLink3ExecutionGateway::on_securities(
+    const mercury::Update<NewSecuritiesEvent> &update) {
+   if (update.events) {
+
+      for (auto const &ev : *(update.events)) {
+         auto sec_ptr = ev.security;
+         me->securities[sec_ptr->id()] = sec_ptr;
+         me->securities_by_descriptor[securities::descriptor(*sec_ptr)] =
+             sec_ptr;
+      }
+
+      if (me->state == State::WAITING_FOR_SECURITIES) {
+         LOG(iLink3, Warn,
+             "Received first set of {} securities. Proceeding "
+             "with initialization of execution gateway {}",
+             update.events->size(), me->gw_name.value());
+         // got our security synopsis, ready to start the GW
+         transition(State::PAUSED);
+      }
+   }
+}
+
+void GLBXiLink3ExecutionGateway::on_admin_data(
+    const mercury::Update<NewAdminDataEvent> &) {}
+
+void GLBXiLink3ExecutionGateway::send_msgs(std::vector<OrderMsg> const &msgs,
+                                           trace::CorID) {
+
+   if (me->state == State::READY) {
+      for (auto &m : msgs) {
+         // NewOrderSingle
+         auto create_ptr = std::get_if<CreateOrderMsg>(&m);
+         if (create_ptr) {
+            me->gw->send_order(*create_ptr);
+            break;
+         }
+         // CancelReplace
+         auto cancel_replace_ptr = std::get_if<CancelReplaceOrderMsg>(&m);
+         if (cancel_replace_ptr) {
+            me->gw->send_cancel_replace(*cancel_replace_ptr);
+            break;
+         }
+         // Cancel
+         auto cancel_ptr = std::get_if<CancelOrderMsg>(&m);
+         if (cancel_ptr) {
+            me->gw->send_cancel(*cancel_ptr);
+            break;
+         }
+      }
+      me->gw->flush();
+   }
+}
+
+OnExeReport &GLBXiLink3ExecutionGateway::on_exe_report() {
+   return me->on_exe_report;
+}
+
+OnGwStatusChange &GLBXiLink3ExecutionGateway::on_gw_status_change() {
+   return me->on_gw_status_change;
+}
+
+orderman::OnCancelReject &GLBXiLink3ExecutionGateway::on_cancel_reject() {
+   return me->on_cancel_reject;
+}
+
+log::CharPtrWrapper GLBXiLink3ExecutionGateway::state_name(State st) {
+   switch (st) {
+   case State::WAITING_FOR_SECURITIES:
+      return {"WAITING_FOR_SECURITIES"};
+   case State::PAUSED:
+      return {"PAUSED"};
+   case State::CONNECTING:
+      return {"CONNECTING"};
+   case State::READY:
+      return {"READY"};
+   }
+   bts_assert(false);
+   return {""};
+}
+
+void GLBXiLink3ExecutionGateway::transition(State new_state) {
+   LOG(iLink3, Info, "Transitioning {} from {} to {}", me->gw_name.value(),
+       state_name(me->state), state_name(new_state));
+   me->state = new_state;
+}
+
+} // namespace ilink3
+} // namespace orderman
+} // namespace bts

src/lib/orderman/actors/GLBXiLink3ExecutionGateway.h

@@ -0,0 +1,48 @@
+#pragma once
+
+#include "IExecutionGateway.h"
+#include "lib/logging/logging.h"
+
+namespace bts {
+namespace jetfix {struct SessionName;}
+namespace orderman {
+namespace ilink3 {
+
+struct Session;
+
+class GLBXiLink3ExecutionGateway : public IExecutionGateway {
+ public:   
+   GLBXiLink3ExecutionGateway(config::JetFIX::Connection const &, trace::PerfLog &,
+                              md::SecurityOracle);
+   virtual ~GLBXiLink3ExecutionGateway();
+
+   void start() override;
+   void dispatch_incoming_fix_messages() override;
+   void on_securities(const mercury::Update<NewSecuritiesEvent> &) override;
+   void on_admin_data(const mercury::Update<NewAdminDataEvent> &) override;
+   void send_fix(jetfix::OutFIXPtr const &, trace::CorID ) override {bts_assert(false);}
+   void send_msgs(std::vector<OrderMsg> const&, trace::CorID) override;
+   
+   OnExeReport& on_exe_report() override;
+   OnGwStatusChange& on_gw_status_change() override;
+   OnCancelReject &on_cancel_reject() override;
+
+private:
+   enum class State;
+   struct PImpl;
+   std::unique_ptr<PImpl> me;
+
+   // Time passing
+   void on_time();
+   bool pause_in_effect() const;
+   bool max_failures_reached()const; 
+   static log::CharPtrWrapper state_name(State);
+   void transition(State);
+
+   void pause();
+   void connect(jetfix::SessionName const&);
+};
+
+} // namespace orderman
+} // namespace bts
+}

src/lib/orderman/actors/ICEExecutionGateway.cpp

@@ -390,7 +390,9 @@ void ICEExecutionGateway::on_exe_report(const jetfix::fix::ICEExecutionReportMsg
       data->manual_order_indicator = api::orderman::MANUAL_ORDER_INDICATOR_NO;
       
       std::string oid_string = get_ice_order_id_string(msg);
-      str_to(oid_string, &data->exchange_order_id);
+      str_to(oid_string, 
+         & std::get<ExchangeOrderID>(
+            data->exchange_order_id));
       data->ice_cl_ord_id = 0;
       if(msg.has<fix::Int_ClOrdID>())
          data->ice_cl_ord_id = msg.get<fix::Int_ClOrdID>();
@@ -438,8 +440,10 @@ void ICEExecutionGateway::on_exe_report(const jetfix::fix::ICEExecutionReportMsg
       if (msg.has<fix::TransactTimeStr>())
       {
          std::string stamp = msg.get<fix::TransactTimeStr>();
-         stamp.resize(sizeof(mercury::TimeStampStr));
-         str_to(stamp, &data->transact_time);
+         stamp.resize(sizeof(orderman::ExchangeTimeStampStr));
+         orderman::ExchangeTimeStampStr t;
+         str_to(stamp, &t);
+         data->transact_time = t;
       }
       data->cfg = &me->cfg;
       data->msg = text_msg;

src/lib/orderman/actors/ICEExecutionGateway.h

@@ -17,6 +17,7 @@ class ICEExecutionGateway : public IExecutionGateway {
    void on_securities(const mercury::Update<NewSecuritiesEvent> &) override;
    void on_admin_data(const mercury::Update<NewAdminDataEvent> &) override;
    void send_fix(const jetfix::OutFIXPtr& fix, trace::CorID trace) override;
+   void send_msgs(std::vector<OrderMsg> const&, trace::CorID) override {bts_assert(false);}
 
    OnExeReport& on_exe_report() override;
    OnGwStatusChange& on_gw_status_change() override;

src/lib/orderman/actors/IExecutionGateway.cpp

@@ -1,17 +1,24 @@
 
 #include "IExecutionGateway.h"
-#include "ICEExecutionGateway.h"
 #include "GLBXExecutionGateway.h"
+#include "ICEExecutionGateway.h"
+#include "lib/utils/ValueWrapper.h"
+#include "lib/orderman/actors/GLBXiLink3ExecutionGateway.h"
+
+#include <variant>
 
 namespace bts {
 namespace orderman {
 
 std::unique_ptr<IExecutionGateway>
-ExecutionGatewayFactory::create_gateway(const config::JetFIX::Connection& conn, trace::PerfLog& pl, md::SecurityOracle o)
-{
-   if(conn.platform() == bts::config::JetFIX_Connection_JetFIXPlatform_ICE || 
-      conn.platform() == bts::config::JetFIX_Connection_JetFIXPlatform_SIMICE)
+ExecutionGatewayFactory::create_gateway(const config::JetFIX::Connection &conn,
+                                        trace::PerfLog &pl,
+                                        md::SecurityOracle o) {
+   if (conn.platform() == bts::config::JetFIX_Connection_JetFIXPlatform_ICE ||
+       conn.platform() == bts::config::JetFIX_Connection_JetFIXPlatform_SIMICE)
       return std::make_unique<ICEExecutionGateway>(conn, pl, o);
+   else if (conn.use_ilink3()) 
+      return std::make_unique<ilink3::GLBXiLink3ExecutionGateway>(conn, pl, o);
    else
       return std::make_unique<GLBXExecutionGateway>(conn, pl, o);
 }