Make shared mem size configurable

Transitive security compare

Approved-by: Eric Harding

.gitignore

@@ -35,3 +35,4 @@ trace_data/
 .ionide/*
 /.ignore
 /build
+.clangd/

.vscode/launch.json

@@ -1,6 +1,30 @@
 {
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "Spark Local",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "${workspaceRoot}/build/ninja_debug/server/sparkd",
+            "args": [
+                "run",
+                // "test",
+                // "-a",
+                // "[ilink3]"
+            ],
+            "stopAtEntry": false,
+            "cwd": "${workspaceRoot}/build/ninja_debug/server/",
+            "environment": [],
+            "externalConsole": false,
+            "MIMode": "gdb",
+            "setupCommands": [
+                {
+                    "description": "Enable pretty-printing for gdb",
+                    "text": "-enable-pretty-printing",
+                    "ignoreFailures": true
+                }
+            ]
+        },
         {
             "name": "SparkVM Launch",
             "type": "cppdbg",

doc/api/content/client/getting-started/services.md

@@ -124,4 +124,17 @@ service OrderService {
 ```
 ### C++ Example
 ```cpp
+#include "lib/api/public/ServiceContext.h"
+#include "lib/api/services/OrderService.client.pb.h"
+using namespace bts::api;
+ServiceContext ctx;
+auto orders = ctx.locate<orderman::OrderService>();
+auto unsub = orders.subscribe_fills().subscribe([](const auto& update) {
+  std::cout 
+    << "Added " << update.added().size() << " items, "
+    << "Removed " << update.removed().size() << " items, "
+    << "Updated " << update.updated().size() << " items, "
+    << "Total items " << update.all().size() << std::endl;
+});
+std::this_thread::sleep_for(std::chrono::minutes(5));
 ```

doc/api/content/client/services/marketdata.md

@@ -1,3 +1,51 @@
 +++
 title = "Market Data"
-+++
++++
+
+
+Use the MarketDataService to subscribe enumerate all available securities.
+
+```cs
+using System;
+using System.Threading;
+using bts.api;
+using bts.api.core;
+
+namespace sample {
+   class Program {
+      static void Main(string[] args) {
+         using var context = new ServiceContext();
+         var md = context.locate<MarketDataService>();
+         using var sub = md.subscribe_securities().Subscribe(update => {
+            foreach (var security in update.added) {
+               Console.WriteLine($"Added security {security.Id}: {security.Symbol}");
+            }
+         });
+
+         Thread.Sleep(TimeSpan.FromMinutes(5));
+      }
+   }
+}
+```
+
+```cpp
+#include "lib/api/public/ServiceContext.h"
+#include "spark/lib/domain_model/Security.h"
+#include "lib/api/services/MarketDataService.client.pb.h"
+#include <thread>
+
+using namespace bts::api;
+
+int main(int argc, char** argv) {
+   ServiceContext ctx;
+   auto md_service = ctx.locate<bts::api::MarketDataService>();
+   auto sub = md_service.subscribe_securities().subscribe([](const auto& update) {
+      for (const auto& s : update.added()) {
+         std::cout << "Added security: " << s->id() << ": " << s->symbol() << std::endl;
+      }
+   });
+
+   std::this_thread::sleep_for(std::chrono::minutes(5));
+   return 0;
+}
+```

doc/api/content/server/getting-started/download.md

@@ -68,12 +68,21 @@ To update your Spark SDK simply download an unzip a new [spark_sdk.zip](https://
 
 To preserve the old image so you can go back to it later run `docker tag spark_sdk spark_sdk:your_custom_tag` before loading the new SDK image.
 
-<!--
-Then stop, remove and restart spark_sdk.
+### Re-build the Dev Container
+
+If you are using Visual Studio Code you also need to rebuild the dev-container so it picks up the new image.
+
+#### Inside VSCode
+To re-build from inside Visual Studio Code.  Click the 'Remote Explorer' icon on the left hand side. 
+You should see your spark_sdk project listed under the Containers section.  Right click on it and select **rebuild-container**.
+
+#### Command Line
+To re-build from the command line run `docker ps -a` and find the row with the spark_sdk image. 
 ```
-c:\bts> docker stop spark_sdk
-c:\bts> docker rm spark_sdk
-c:\bts> docker run -d -v %cd%:/sdk --name spark_sdk bluetradingsytems/spark_sdk
+CONTAINER ID        IMAGE                          COMMAND                  CREATED              STATUS                      PORTS                                NAMES
+54d96cfa3888        spark_sdk                      "/bin/sh -c 'echo Co…"   About a minute ago   Up About a minute           127.0.0.1:9000-9001->9000-9001/tcp   quizzical_lichterman
+```
+Then run `docker rm 54d96cfa3888` (use your container id instead of 54d96cfa3888)  or `docker rm quizzical_lichterman` (use your container name).
+
 
-``` -->
 

doc/user/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources /home/ppinto/Dropbox/bts/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)
+set(sources /home/bts/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)

doc/user/content/release_notes/_index.md

@@ -4,6 +4,13 @@ weight = 3
 sort_by = "weight"
 insert_anchor_links = "right"
 +++
+## 2.15.10
+### Bug Fixes
+  - Solved an issue that could cause the user interface to fail when playing some alert sounds
+### API Changes
+  - Added settlement and position information to the API
+  - Added the ability for user algos to report custom state to the user interface
+  - Added the ability for algos to discover all active securities
 ## 2.15.3
 ### Bug Fixes
   - Fixed a bug where some CME currency spreads exhibited incorrect tick values

src/client/spark/App.xaml.cs

@@ -115,7 +115,7 @@ namespace mbts.ui.shell
       {
          try
          {
-            var path = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) + "\\BTS\\stunnel\\";
+            string path = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) + "\\BTS\\stunnel\\";
             if (File.Exists(path + "config\\stunnel.conf"))
             {
                uint job_error = BTSUINativeMethods.setup_process_job_object();   
@@ -124,6 +124,15 @@ namespace mbts.ui.shell
                   kill_stunnel();
                start_stunnel(path, job_error);
             }
+            path = AppDomain.CurrentDomain.BaseDirectory + "\\..\\stunnel\\";
+            if (File.Exists(path + "config\\stunnel.conf"))
+            {
+               uint job_error = BTSUINativeMethods.setup_process_job_object();
+               //if we can't put this process and stunnel into a job, try to stop stunnel then restart
+               if (job_error != 0)
+                  kill_stunnel();
+               start_stunnel(path, job_error);
+            }
          }
          catch(Exception ex)
          {

src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesRowVM.cs

@@ -1,7 +1,6 @@
 using bts.api;
 using bts.co2;
 using bts.utils;
-using bts.utils.csv;
 using mbts;
 using mbts.spark;
 using mbts.ui.controls;
@@ -45,18 +44,6 @@ namespace bts.shell.time_and_sales {
       public void raise_invalidated() =>
          invalidated?.Invoke();
 
-      public static System.Collections.Specialized.OrderedDictionary csv_ordered_properties =>
-            CsvHelper.create_ordering_maintainer(
-               typeof(TimeAndSalesRowVM),
-               new string[] {
-                  nameof(security),
-                  nameof(time),
-                  nameof(price),
-                  nameof(size),
-                  nameof(aggressor_side),
-                  nameof(security_key),
-               });
-
       public bool Equals(TimeAndSalesRowVM other) =>
          other != null &&
          row_key == other.row_key;

src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesWindow.cs

@@ -2,7 +2,6 @@
 using bts.shell.fills;
 using bts.spark;
 using bts.view;
-using bts.utils.csv;
 using mbts;
 using mbts.co2;
 using mbts.persist;
@@ -115,8 +114,6 @@ namespace bts.shell.time_and_sales {
             show_in_context_menu = _can_pause_updates
          };
          _commands = new SparkCommand[] {
-            // csv export
-            new SparkCommandCommand("Export to csv", Icons.Export, FileDialogCommand.export_csv(() => _vm.rows.ToCsv(TimeAndSalesRowVM.csv_ordered_properties), "TaS")),
             // ladder
             SparkCommands.ShowLadderAutoLookup(SparkCommands.KeyGestures.ShowLadder)
                .complete(current_sec_key_src, sec_keys => sec_keys.Count > 0),

src/client/spark/Modules/Trading/Fills/FillsVM.cs

@@ -5,7 +5,6 @@ using bts.collections;
 using bts.shell.fills;
 using bts.spark;
 using bts.utils;
-using bts.utils.csv;
 using mbts.collections;
 using mbts.persist;
 using mbts.spark.filters;
@@ -331,27 +330,6 @@ namespace mbts.spark {
          public override bool Equals(object obj) => Equals(obj as Row);
          public override int GetHashCode() => _fill.FillKey.GetHashCode() ^ _order.Key.GetHashCode();
 
-         public static System.Collections.Specialized.OrderedDictionary csv_ordered_properties =>
-            CsvHelper.create_ordering_maintainer(typeof(Row),
-               new string[] {
-                  nameof(security),
-                  nameof(order_type),
-                  nameof(root_algo_display_name),
-                  nameof(side),
-                  nameof(fill_time),
-                  nameof(user),
-                  nameof(account),
-                  nameof(exchange),
-                  nameof(time_in_force),
-                  nameof(size),
-                  nameof(price),
-                  nameof(fill_kind),
-                  nameof(fill_status),
-                  nameof(fill_key),
-                  nameof(order_key),
-                  nameof(security_key)
-               });
-
          public event Action invalidated;
          void IRowVM.raise_invalidated() => invalidated?.Invoke();
       }

src/client/spark/Modules/Trading/Fills/FillsWindow.cs

@@ -4,7 +4,6 @@ using bts.shell;
 using bts.shell.fills;
 using bts.spark;
 using bts.view;
-using bts.utils.csv;
 using mbts.persist;
 using mbts.ui;
 using mbts.ui.settings;
@@ -62,7 +61,6 @@ namespace mbts.spark {
          _view = new FillsView { DataContext = _vm };
 
          _commands = new List<SparkCommand> {
-            new SparkCommandCommand("Export to csv", Icons.Export, FileDialogCommand.export_csv(() => _vm.rows.ToCsv(FillsVM.Row.csv_ordered_properties), "Fills")),
             // wrap the grid settings view factory in a dialogCommand here since the vm is destroyed/recreated when the security criteria changes
             SparkCommands.window_settings_command(new DialogCommand(() => {
                var vm = new ShellSettingsDialogVM<GridSettings, SimpleGridSettingsVM>(

src/client/spark/Modules/Trading/Ladder/View/LadderGridHost.cs

@@ -534,8 +534,7 @@ namespace mbts.spark {
          _size_from_row(row, _price_list, _asks, _vm.security.ContractSpec);
 
       uint _volume_from_row(int row) {
-         if (_price_list != null) {
-            var price = _price_list[row];
+         if (_price_list.SafeTryGetAt(row, out var price)) {
             var vapls = _volume_container.read_new_value();
             if (vapls != null) {
                return vapls.SelectFirstOrDefault(v => MathUtils.epsilon_equals(v.Price, price), v => v.Volume);

src/client/spark/Modules/Trading/ManualFills/ManualFillsVM.cs

@@ -2,6 +2,7 @@ using bts.api;
 using bts.api.core;
 using bts.api.position;
 using bts.collections;
+using bts.utils;
 using mbts;
 using mbts.spark;
 using mbts.ui;

src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs

@@ -4,7 +4,6 @@ using bts.api.orderman;
 using bts.collections;
 using bts.spark;
 using bts.utils;
-using bts.utils.csv;
 using mbts.ui;
 using mbts.ui.controls;
 using System;
@@ -12,7 +11,6 @@ using System.Collections.Generic;
 using System.ComponentModel;
 using System.Reactive.Linq;
 using System.Runtime.CompilerServices;
-using System.Linq;
 using System.Windows.Input;
 using System.Windows.Media;
 using Columns = mbts.spark.OrderBookGridView.ColumnKeys;
@@ -38,7 +36,6 @@ namespace mbts.spark {
       public OrderBookRowVM(OrderBookVM owner, OrderSecurityRootPair order_tuple, bool is_expanded, Action<bool> on_selection_changed) {
          _owner = owner;
          this.order_tuple = order_tuple;
-         is_visible = !is_child;
          _is_expanded = is_expanded;
          prospective_quantity = this.quantity;
          prospective_price = this.price;
@@ -62,35 +59,6 @@ namespace mbts.spark {
       public event Action<(OrderSecurityRootPair oldval, OrderSecurityRootPair newval)> updated;
       public event PropertyChangedEventHandler PropertyChanged;
 
-      public static System.Collections.Specialized.OrderedDictionary csv_ordered_properties =>
-            CsvHelper.create_ordering_maintainer(
-               typeof(OrderBookRowVM), 
-               new string[] {
-                  nameof(security_text),
-                  nameof(symbol),
-                  nameof(time_created),
-                  nameof(last_change),
-                  nameof(user),
-                  nameof(account),
-                  nameof(exchange),
-                  nameof(time_in_force),
-                  nameof(order_type),
-                  nameof(root_algo_display_name),
-                  nameof(order_sub_type),
-                  nameof(side),
-                  nameof(quantity),
-                  nameof(filled_quantity),
-                  nameof(price),
-                  nameof(avg_price),
-                  nameof(last_price),
-                  nameof(piq),
-                  nameof(status),
-                  // last message (could contains commas, so skipping)
-                  nameof(security_key),
-                  nameof(order_key),
-                  nameof(parent_order_key),
-               });
-
       // Core
       public OrderSecurityRootPair order_tuple { get; private set; }
       public OrderRecordImmutable order => order_tuple.order.order;
@@ -144,7 +112,6 @@ namespace mbts.spark {
       // Tree
       public bool is_parent { get => _is_parent; set => Set(ref _is_parent, value); }
       public bool is_child => order.has_parent();
-      public bool is_visible { get; set; }
       public bool is_expanded {
          get => _is_expanded;
          set {

src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs

@@ -188,6 +188,56 @@ namespace mbts.spark {
                security_plurality = p;
                RaisePropertyChanged(nameof(security_plurality));
             }),
+            Observable.Create<OrderBookRowVM>(o => {
+               OrderBookRowVM last_fired = null;
+               selected_order_changed += on_selected_order_changed;
+               var sub = _current_row.Subscribe(ch => notify());
+               return Disposable.Create(dispose);
+               // local methods
+               void on_selected_order_changed(OrderRecordImmutable? _) =>
+                  notify();
+
+               void notify() {
+                  var cr = current_row_if_distinct_root_selected();
+                  if (!cr.safe_equals(last_fired))
+                     o.OnNext(last_fired = cr);
+               }
+
+               void dispose() {
+                  Disposable.dispose(ref sub);
+                  selected_order_changed -= on_selected_order_changed;
+               }
+            })
+            .SubscribeEach(r => {
+               if (r == null) {
+                  current_algo_custom_state.Clear();
+                  return Disposable.empty;
+               }
+               r.updated += on_updated;
+               refresh_custom_state();
+               return Disposable.Create(() => r.updated -= on_updated);
+               void on_updated((OrderSecurityRootPair oldval, OrderSecurityRootPair newval) _) =>
+                  refresh_custom_state();
+
+               void refresh_custom_state() {
+                  current_algo_custom_state.Clear();
+                  var dcs = r.order.dynamic_custom_state();
+                  // while dcs is not expected to be null, best to play it safe and double check.
+                  if (dcs.is_null_or_empty()) {
+                     var child_rows = _row_collection.get_descendent_rows(r.order_key);
+                     if (_row_collection.filter != null) {
+                        child_rows = child_rows.Where(cr => _row_collection.filter(cr.order_tuple));
+                     }
+                     foreach (var dr in child_rows) {
+                        dcs = dr.order.dynamic_custom_state();
+                        if (!dcs.is_null_or_empty())
+                           break;
+                     }
+                  }
+                  foreach (var p in dcs.empty_if_null().OrderBy(kv => kv.Key))
+                     current_algo_custom_state.Add(new NamedValueVM(p.Key, DynamicOrderHelper.fmt_param(p.Value)));
+               }
+            }),
             orders_source.OnDispatcher().Subscribe(t => {
                if (_selected_rows.Count == 0) {
                   _cached_orders = t.state;
@@ -352,6 +402,15 @@ namespace mbts.spark {
          }
          void on_end_attaching() => Disposable.dispose(ref attach_vm_hooks);
 
+         // the output grid (custom state) in the right panel
+         // will only display info for the 'current_row'
+         // if all selected rows are of the same root order.
+         OrderBookRowVM current_row_if_distinct_root_selected() =>
+            current_row != null &&
+            selected_rows.Select(r => r.order_tuple.root.order.Key).Distinct().TryGetSingle(out uint droot) &&
+            current_row.order_tuple.root.order.Key == droot ?
+            current_row : null;
+
          IDisposable connect_order_mod_vm() =>
             Disposable.Create(
                _selected_params_mon.subscribe_algos().OnDispatcher().Subscribe(order_mod_vm.algo_vm.load_many),
@@ -395,6 +454,7 @@ namespace mbts.spark {
          }
       }
       public OrderRecordImmutable? current_order => current_row?.order;
+      public ObservableCollection<NamedValueVM> current_algo_custom_state { get; } = new ObservableCollection<NamedValueVM>();
       public AlgoDefinitionVM algo_vm => order_mod_vm.algo_vm;
       public OrderParamsVM order_mod_vm { get; }
       public FillsVMContainer fills_vm { get; } = new FillsVMContainer();

src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs

@@ -4,7 +4,6 @@ using bts.shell;
 using bts.shell.fills;
 using bts.shell.orderbook;
 using bts.spark;
-using bts.utils.csv;
 using bts.view;
 using mbts.persist;
 using mbts.ui;
@@ -195,7 +194,6 @@ namespace mbts.spark {
          show_grid_settings_command.group = CommandGroups.Auxiliary;
 
          _commands = new SparkCommand[] {
-            new SparkCommandCommand("Export to csv", Icons.Export, FileDialogCommand.export_csv(() => _vm.rows.ToCsv(OrderBookRowVM.csv_ordered_properties), "OrderBook")),
             toggle_show_right_panel_command,
             attach_selected_orders_command,
             detach_selected_orders_command,

src/client/spark/Modules/Trading/OrderBook/View/OrderBookView.xaml

@@ -484,6 +484,7 @@
           <Border Padding="12,12,12,6">
             <Grid>
               <Grid.RowDefinitions>
+                <RowDefinition Height="Auto"/>
                 <RowDefinition Height="Auto"/>
                 <RowDefinition Height="*"/>
               </Grid.RowDefinitions>
@@ -495,7 +496,45 @@
                 AlgoParamRows="{Binding param_rows}"
                 IsReadOnly="{Binding DataContext.can_user_modify_selected_orders, ElementName=Root, Converter={StaticResource NotConverter}}"
               />
-              <StackPanel Grid.Row="1" HorizontalAlignment="Right" VerticalAlignment="Top" Margin="0,8,0,0" Orientation="Horizontal">
+              <c:ExpanderToggleButton
+                x:Name="algo_state_toggle"
+                Grid.Row="1"
+                BorderThickness="0"
+                Content="Algo State"
+                HorizontalContentAlignment="Left"
+                Margin="0,4,0,0"
+                VerticalAlignment="Top"
+                Visibility="{Binding current_algo_custom_state.Count, Converter={StaticResource IntNotZeroVisibilityConverter}}"
+              />
+              <c:DIYGrid Grid.Row="1" Rows="{Binding current_algo_custom_state}" Margin="0,28,0,4">
+                <c:DIYGrid.Visibility>
+                  <MultiBinding>
+                    <MultiBinding.Converter>
+                      <cv:MultiValueConverterPipe>
+                        <cv:BooleanAndConverter/>
+                        <cv:BooleanToVisibilityConverter/>
+                      </cv:MultiValueConverterPipe>
+                    </MultiBinding.Converter>
+                    <Binding Path="current_algo_custom_state.Count" Mode="OneWay" Converter="{StaticResource IntNotZeroConverter}"/>
+                    <Binding Path="IsChecked" ElementName="algo_state_toggle"/>
+                  </MultiBinding>
+                </c:DIYGrid.Visibility>
+                <c:DIYGrid.Columns>
+                  <c:DIYGridColumnCollection>
+                    <c:DIYGridColumn id="Name" Header="Param" Width="176"/>
+                    <c:DIYGridColumn id="Value" Header="Value" Width="176"/>
+                  </c:DIYGridColumnCollection>
+                </c:DIYGrid.Columns>
+                <c:DIYGrid.RowTemplate>
+                  <DataTemplate>
+                    <Grid c:DIYGrid.SyncWithColumnGeometry="True">
+                      <c:TextCell c:DIYGrid.ColumnKey="Name" Text="{Binding name}"/>
+                      <c:TextCell c:DIYGrid.ColumnKey="Value" Text="{Binding value}"/>
+                    </Grid>
+                  </DataTemplate>
+                </c:DIYGrid.RowTemplate>
+              </c:DIYGrid>
+              <StackPanel Grid.Row="2" HorizontalAlignment="Right" VerticalAlignment="Top" Margin="0,8,0,0" Orientation="Horizontal">
                 <Button x:Name="modify_orders_button" Command="{Binding modify_selected_orders_cmd}">
                   <Button.Margin>
                     <Thickness Right="{StaticResource DialogSkirtButtonPadding.Right}"/>

src/client/spark/Notifications/INotifier.cs

@@ -2,6 +2,7 @@
 using bts.api;
 using bts.shell;
 using bts.spark;
+using bts.utils;
 using mbts.config;
 using mbts.persist;
 using mbts.spark;
@@ -114,6 +115,7 @@ namespace mbts.ui {
             void dispose_core() {
                Disposable.dispose(ref _settings_sub);
                Disposable.dispose(ref _toast_controller);
+               Disposable.dispose(ref _sound_player);
                if (_dispatcher != null && !_dispatcher.HasShutdownStarted) {
                   _dispatcher.BeginInvokeShutdown(DispatcherPriority.Background);
                }
@@ -158,15 +160,25 @@ namespace mbts.ui {
             }
          }
 
-         class SoundNotificationPlayer {
-            readonly IReadOnlyDictionary<SoundNotification.SystemSound, NAudioPlayer> _players;
-
-            public static SoundNotificationPlayer create() => new SoundNotificationPlayer(_all_players());
+         class SoundNotificationPlayer : IDisposable {
+            IReadOnlyDictionary<SoundNotification.SystemSound, NAudioPlayer> _players;
 
             SoundNotificationPlayer(IReadOnlyDictionary<SoundNotification.SystemSound, NAudioPlayer> players) =>
                _players = players;
 
+            public static SoundNotificationPlayer create() =>
+               new SoundNotificationPlayer(_all_players());
+
+            public void Dispose() {
+               if (_players != null) {
+                  _players.Values.DisposeAll();
+                  _players = null;
+               }
+            }
+
             public void play_sounds(IEnumerable<SoundNotification> sounds) {
+               if (_players == null)
+                  return;
                foreach (var s in sounds) {
                   switch (s.sound_type_case) {
                      case SoundNotification.SoundTypeOneofCase.System: _players.GetValueOrDefault(s.system_sound.Value)?.play(); break;
@@ -177,7 +189,8 @@ namespace mbts.ui {
             }
 
             static IReadOnlyDictionary<SoundNotification.SystemSound, NAudioPlayer> _all_players() =>
-               EnumHelper.get_values<SoundNotification.SystemSound>().ToReadOnlyDictionary(s => s, NAudioPlayer.create);
+               EnumHelper.get_values<SoundNotification.SystemSound>()
+               .ToReadOnlyDictionaryWhere<SoundNotification.SystemSound, NAudioPlayer>(NAudioPlayer.try_create);
          }
 
          // This class adopts to role of view-controller in the mvc paradigm, rather than view-model in mvvm.

src/client/spark/Notifications/NAudioPlayer.cs

@@ -1,4 +1,5 @@
-using bts.shell;
+using bts;
+using bts.shell;
 using mbts.ui.utility;
 using NAudio.Wave;
 using System;
@@ -6,29 +7,71 @@ using System.IO;
 using System.Threading.Tasks;
 
 namespace mbts.ui {
-   class NAudioPlayer {
+   class NAudioPlayer : IDisposable {
       // number of milliseconds to delay playing sound,
       // during which time all subsequent requests to play the same sound will be ignored.
       const int _delay_ms = 75;
       readonly int _start_position;
+      IDisposable _stream_in;
       WaveOutEvent _wave_out = new WaveOutEvent();
       WaveFileReader _wave_reader;
       bool _is_playing, _is_play_requested;
 
-      public NAudioPlayer(Stream uwave_stream, int start_position = 0) {
-         var ms = new MemoryStream();
-         uwave_stream.CopyTo(ms);
-         ms.Position = 0;
-         _wave_reader = new WaveFileReader(ms);
+      /// <paramref name="stream_in"/> should not be disposed by the caller,
+      /// as NAudioPlayer handles that in its own Dispose method;
+      /// this constructor is private is to prevent such misuse.
+      NAudioPlayer(Stream stream_in, int start_position = 0) {
+         _stream_in = stream_in;
+         _wave_reader = new WaveFileReader(stream_in);
          _wave_reader.Position = _start_position = (start_position < _wave_reader.Length ? start_position : 0);
          _wave_out.Init(_wave_reader);
          _wave_out.PlaybackStopped += _on_playback_stopped;
       }
 
-      public static NAudioPlayer create(SoundNotification.SystemSound system_sound) {
-         var stream = _get_stream(system_sound);
-         var sb = EnumHelper.get_attribute<SoundNotification.SystemSound, SilenceBufferAttribute>(system_sound);
-         return sb != null ? new NAudioPlayer(stream, sb.start_position) : new NAudioPlayer(stream);
+      /// <see cref="_playback_stopped"/> is a proxy/buffer for the _wave_out.PlaybackStopped event.
+      /// It serves a couple purposes:
+      ///   1) The main callback (<see cref="_on_playback_stopped(object, StoppedEventArgs)"/>)
+      ///      is guarenteed to be executed first (though it would be anyways, since that
+      ///      callback is hooked up in the constructor, the proxy event makes it obvious).
+      ///   2) <see cref="_playback_stopped"/> is only raised after playback is truly complete,
+      ///      meaning there are no pending requests to repeat the sound.
+      event Action _playback_stopped;
+
+      public static bool try_create(SoundNotification.SystemSound system_sound, out NAudioPlayer result) {
+         if (_try_get_stream(system_sound, out var stream)) {
+            var sb = EnumHelper.get_attribute<SoundNotification.SystemSound, SilenceBufferAttribute>(system_sound);
+            result = sb != null ? new NAudioPlayer(stream, sb.start_position) : new NAudioPlayer(stream);
+            return true;
+         }
+         result = null;
+         return false;
+      }
+
+      public static void play(SoundNotification.SystemSound system_sound) {
+         if (try_create(system_sound, out var player)) {
+            try {
+               var temp = player;
+               temp._playback_stopped += temp.Dispose;
+               player.play();
+               player = null;
+            } catch (Exception ex) {
+               Log.Ui.warn($"Encountered exception playing the system sound '{system_sound}': {ex}.");
+            } finally {
+               player?.Dispose();
+            }
+         } else {
+            Log.Ui.warn($"Unable to play the system sound '{system_sound}': unexpected sound type (did not find an audio stream that corresponds to this enum value).");
+         }
+      }
+
+      public void Dispose() {
+         if (_wave_out != null) {
+            _wave_out.PlaybackStopped -= _on_playback_stopped;
+            _wave_out.Dispose();
+            _wave_out = null;
+         }
+         Disposable.dispose(ref _wave_reader);
+         Disposable.dispose(ref _stream_in);
       }
 
       public void play() {
@@ -46,6 +89,8 @@ namespace mbts.ui {
          if (_is_play_requested) {
             _is_play_requested = false;
             _play_core();
+         } else {
+            _playback_stopped?.Invoke();
          }
       }
 
@@ -56,18 +101,26 @@ namespace mbts.ui {
          _is_playing = false;
       }
 
-      static Stream _get_stream(SoundNotification.SystemSound system_sound) {
+      static bool _try_get_stream(SoundNotification.SystemSound system_sound, out Stream stream) {
+         UnmanagedMemoryStream um_stream;
          switch (system_sound) {
-            case SoundNotification.SystemSound.GameCorrect: return bts.Properties.Resources.game_sound_correct;
-            case SoundNotification.SystemSound.GameIncorrect: return bts.Properties.Resources.game_sound_wrong;
-            case SoundNotification.SystemSound.GameOverArcade: return bts.Properties.Resources.game_over_arcade;
-            case SoundNotification.SystemSound.Whistle: return bts.Properties.Resources.whistle;
-            case SoundNotification.SystemSound.BellCounter: return bts.Properties.Resources.bell_counter;
-            case SoundNotification.SystemSound.Powerup: return bts.Properties.Resources.powerup;
-            case SoundNotification.SystemSound.CashRegister: return bts.Properties.Resources.cash_register_purchase;
-            case SoundNotification.SystemSound.Reload: return bts.Properties.Resources.reload_1911;
-            default: throw new NotImplementedException($"Unknown {nameof(SoundNotification.SystemSound)}: {system_sound}.");
+            case SoundNotification.SystemSound.GameCorrect: um_stream = bts.Properties.Resources.game_sound_correct; break;
+            case SoundNotification.SystemSound.GameIncorrect: um_stream = bts.Properties.Resources.game_sound_wrong; break;
+            case SoundNotification.SystemSound.GameOverArcade: um_stream = bts.Properties.Resources.game_over_arcade; break;
+            case SoundNotification.SystemSound.Whistle: um_stream = bts.Properties.Resources.whistle; break;
+            case SoundNotification.SystemSound.BellCounter: um_stream = bts.Properties.Resources.bell_counter; break;
+            case SoundNotification.SystemSound.Powerup: um_stream = bts.Properties.Resources.powerup; break;
+            case SoundNotification.SystemSound.CashRegister: um_stream = bts.Properties.Resources.cash_register_purchase; break;
+            case SoundNotification.SystemSound.Reload: um_stream = bts.Properties.Resources.reload_1911; break;
+            default:
+               stream = null;
+               return false;
          }
+         stream = new MemoryStream();
+         um_stream.CopyTo(stream);
+         um_stream.Dispose();
+         stream.Position = 0;
+         return true;
       }
    }
-}
+}

src/client/spark/Orders/OrderTypeVMs/DynamicOrderHelper.cs

@@ -23,6 +23,32 @@ namespace mbts.spark {
 
       public IEnumerable<string> dynamic_types => _dynamic_types.DynamicOrderDesc.Select(d => d.DynamicOrderType);
 
+      public static string fmt_param(DynamicOrderParam dyn_param) {
+         if (dyn_param == null)
+            return null;
+         switch (dyn_param.ValueCase) {
+            case DynamicOrderParam.ValueOneofCase.AsArray:
+               return string.Join(", ", dyn_param.AsArray.Element.Select(fmt_param));
+            case DynamicOrderParam.ValueOneofCase.AsBool:
+               return dyn_param.AsBool.ToString();
+            case DynamicOrderParam.ValueOneofCase.AsDouble:
+               return dyn_param.AsDouble.ToString();
+            case DynamicOrderParam.ValueOneofCase.AsEnumerated:
+               return dyn_param.AsEnumerated.EnumValue;
+            case DynamicOrderParam.ValueOneofCase.AsInt32:
+               return dyn_param.AsInt32.ToString();
+            case DynamicOrderParam.ValueOneofCase.AsQuantity:
+               return dyn_param.AsQuantity.ToString();
+            case DynamicOrderParam.ValueOneofCase.AsString:
+               return dyn_param.AsString;
+            default:
+               // implement additional cases as needed
+               // (as of 4/17/20, this method is only used to format
+               // dynamic orders' CustomState params in the orderbook)
+               return $"({dyn_param.ValueCase})";
+         }
+      }
+
       public void add_dynamic_order_defs(List<CustomOrderDefinitionImmutable> defs, Func<CustomOrderDefinition.CustomOneofCase, CustomOrderDefinition> factory) {
          foreach (var d in _dynamic_types.DynamicOrderDesc) {
             var dorder = factory(CustomOrderDefinition.CustomOneofCase.Dynamic);

src/client/spark/Orders/OrderUtils.cs

@@ -63,6 +63,11 @@ namespace mbts.spark {
       public static string last_msg(this OrderRecordImmutable order) =>
          order.HasState ? order.State.LastMsg : string.Empty;
 
+      public static IReadOnlyDictionary<string, DynamicOrderParam> dynamic_custom_state(this OrderRecordImmutable order) =>
+         order.HasState && order.State.HasDynamic ?
+            (order.State.Dynamic.CustomState ?? DictionaryHelper.empty<string, DynamicOrderParam>()) :
+            DictionaryHelper.empty<string, DynamicOrderParam>();
+
       public static DateTime created(this OrderRecordImmutable order) =>
          order.HasState && order.State.HasCreated ? order.State.Created.ToDateTime().ToLocalTime() : DateTime.MinValue;
 

src/client/spark/Orders/OrdersModel.cs

@@ -215,8 +215,11 @@ namespace mbts.spark {
             FillTuple.key_fn);
          on_fill_tuples_with_root = fill_order_tuples_with_root_join.Cache();
          // manual fills
-         on_manual_fills_securities = _position_service.subscribei_manual_fills()
-            .Naggle(interval, scheduler)
+         on_manual_fills_securities = admin_model
+            .permission_filter(
+               _position_service.subscribei_manual_fills()
+               .Naggle(interval, scheduler),
+               f => f.AccountId)
             .Filter(f => f.FillType == ManualFill.Types.FillType.Local)
             .join(sstore, mf => mf.SecurityKey)
             .Table().Cache();

src/client/spark/view/Controls/DIYGrid/Columns/DIYGridColumn.cs

@@ -0,0 +1,9 @@
+namespace mbts.ui.controls {
+   class DIYGridColumn : DIYGrid.ColumnBase<string> {
+      public DIYGridColumn()
+         : base() {
+         is_visible = true;
+         Width = 60;
+      }
+   }
+}

src/client/spark/view/Controls/DIYGrid/Settings/Animation/SoundAnimationView.xaml.cs

@@ -38,11 +38,7 @@ namespace mbts.ui.controls {
       }
 
       public static ICommand PreviewSoundCommand =>
-         new DelegateCommand<SoundNotification.SystemSound>(s =>  {
-            try {
-               NAudioPlayer.create(s).play();
-            } catch (NAudio.MmException) { }
-         });
+         new DelegateCommand<SoundNotification.SystemSound>(NAudioPlayer.play);
 
       #region SoundTypes
       public static readonly DependencyProperty SoundTypesProperty =

src/client/spark/view/Extensions/Command/FileDialogCommand.cs

@@ -64,25 +64,6 @@ namespace mbts.ui {
             }
          }, can_execute == null ? null : new Func<object, bool>(_ => can_execute()));
 
-      public static IDialogCommand export_csv(
-         Func<string> get_file_content,
-         string source_window_name) =>
-         new AnonymousFileDialogCommand<object>(_ =>
-            _create_save_file_dialog(
-               Environment.GetFolderPath(Environment.SpecialFolder.Desktop),
-               $"BTS_Spark_{source_window_name}_{DateTime.Now.ToString("yyyyMMdd_HHmm")}.csv",
-               new[] { ("CSV (comma delimited)", new string[] { "csv" }) }
-            ),
-            (_, file) => {
-               try {
-                  File.WriteAllText(file, get_file_content.Invoke());
-               } catch (Exception ex) {
-                  BTSMessageBox.Show($"Error exporting data to csv: {ex}.", "CSV Export Error", BTSMessageBoxButtons.OK, System.Windows.MessageBoxImage.Error, true, false, true);
-               }
-            },
-            // we are always able to perform an export, even when no rows (it will just be a header)
-            (_) => true);
-
       static TDialog _create_file_dialog<TDialog>(string initial_dir = null, IReadOnlyList<(string name, string[] extensions)> filter_items = null) where TDialog : FileDialog, new() {
          var rv = new TDialog();
          if (!string.IsNullOrEmpty(initial_dir))

src/client/spark/view/Utility/EnabledValueVM.cs

@@ -30,6 +30,21 @@ namespace mbts.ui {
       public string name { get; }
    }
 
+   class NamedValueVM : ViewModelBase {
+      string _name;
+      object _value;
+
+      public NamedValueVM() { }
+
+      public NamedValueVM(string name, object value) {
+         _name = name;
+         _value = value;
+      }
+
+      public string name { get => _name; set => Set(ref _name, value); }
+      public object value { get => _value; set => Set(ref _value, value); }
+   }
+
    static class EnabledValueVMExtensions {
       public static T? get_nullable<T>(this EnabledValueVM<T> vm) where T : struct =>
          vm.enabled ? vm.value : (T?)null;

src/client/spark_api/security/SecurityHelper.cs

@@ -83,22 +83,24 @@ namespace bts.api {
          }
 
          public static int Compare(SecurityImmutable x, SecurityImmutable y) {
-            if (x.ContractCase == ContractOneofCase.Future) return y.ContractCase == ContractOneofCase.Future ? Compare(x.Symbol, x.Future, y.Symbol, y.Future) : -1;
-            else if (y.ContractCase == ContractOneofCase.Future) return 1;
+            if (x.ContractCase == ContractOneofCase.Future) {
+               return y.ContractCase == ContractOneofCase.Future ? Compare(x.Symbol, x.Future, y.Symbol, y.Future) : -1;
+            } else if (y.ContractCase == ContractOneofCase.Future) {
+               return 1;
+            }
             var xlegs = get_flat_future_legs(x).ToList();
             var ylegs = get_flat_future_legs(y).ToList();
             int max = Math.Min(xlegs.Count, ylegs.Count);
             for (int i = 0; i < max; i++) {
                int rv = Compare(xlegs[i].symbol, xlegs[i].future, ylegs[i].symbol, ylegs[i].future);
-               if (rv != 0) return rv;
-            }
-            for (int i = 0; i < max; i++) {
-               int rv = xlegs[i].quantity.CompareTo(ylegs[i].quantity);
-               if (rv != 0) return rv;
-            }
-            for (int i = 0; i < max; i++) {
-               int rv = xlegs[i].exchange.CompareTo(ylegs[i].exchange);
-               if (rv != 0) return rv;
+               if (rv != 0)
+                  return rv;
+               rv = xlegs[i].quantity.CompareTo(ylegs[i].quantity);
+               if (rv != 0)
+                  return rv;
+               rv = xlegs[i].exchange.CompareTo(ylegs[i].exchange);
+               if (rv != 0)
+                  return rv;
             }
             return xlegs.Count.CompareTo(ylegs.Count);
          }

src/client/spark_core/Collections/Extensions/EnumerableExtensions.cs

@@ -390,6 +390,15 @@ namespace System.Collections.Generic {
       public static ReadOnlyDictionary<TKey, TElement> ToReadOnlyDictionary<T, TKey, TElement>(this IEnumerable<T> enumerable, Func<T, TKey> key_selector, Func<T, TElement> element_selector, IEqualityComparer<TKey> comparer) =>
          new ReadOnlyDictionary<TKey, TElement>(enumerable.ToDictionary(key_selector, element_selector, comparer));
 
+      public static ReadOnlyDictionary<TKey, TElement> ToReadOnlyDictionaryWhere<TKey, TElement>(this IEnumerable<TKey> keys, TryGetValue<TKey, TElement> value_selector) {
+         var mdict = new Dictionary<TKey, TElement>();
+         foreach (var k in keys) {
+            if (value_selector(k, out var val))
+               mdict[k] = val;
+         }
+         return new ReadOnlyDictionary<TKey, TElement>(mdict);
+      }
+
       public static ILookup<TKey, TElement> ToFlatLookup<T, TKey, TElement>(this IEnumerable<T> source, Func<T, TKey> key_selector, Func<T, IEnumerable<TElement>> selector) =>
          source.SelectMany(v => {
             var k = key_selector(v);

src/client/spark_core/Collections/Extensions/ListExtensions.cs

@@ -103,6 +103,15 @@ namespace bts.collections {
          return array;
       }
 
+      public static bool SafeTryGetAt<T>(this IList<T> list, int index, out T value) {
+         if (list != null && index >= 0 && index < list.Count) {
+            value = list[index];
+            return true;
+         }
+         value = default;
+         return false;
+      }
+
       public static bool TryGetAt<T>(this IList<T> list, int index, out T value) {
          if (index >= 0 && index < list.Count) {
             value = list[index];

src/client/spark_core/utils/CsvHelper.cs

@@ -1,52 +0,0 @@
-using System;
-using System.IO;
-using System.Collections.Generic;
-using System.Collections.Specialized;
-using System.Linq;
-using System.Reflection;
-
-namespace bts.utils.csv {
-   public static class CsvHelper {
-
-      public static string ToCsv<T>(this IEnumerable<T> items, OrderedDictionary properties = null) where T : class {
-         var output = "";
-         var delimiter = ",";
-         if (properties == null) {
-            properties = new OrderedDictionary();
-            foreach (var pi in typeof(T).GetProperties()) {
-               properties.Add(pi.Name, pi);
-            }
-         }
-
-         using (var sw = new StringWriter()) {
-            var dict_values = properties.Values.OfType<PropertyInfo>();
-            var header = dict_values.Select(n => n.Name).Aggregate((first, second) => first + delimiter + second);
-            sw.WriteLine(header);
-            foreach (var i in items) {
-               var row = dict_values.Select(x => x.GetValue(i, null))
-                  .Select(x => x == null ? "null" : x.ToString())
-                  .Aggregate((first, second) => first + delimiter + second);
-               sw.WriteLine(row);
-            }
-            output = sw.ToString();
-         }
-         return output;
-      }
-
-      public static OrderedDictionary create_ordering_maintainer(Type t, string[] property_names_in_order) {
-         OrderedDictionary od = new OrderedDictionary();
-         var props = t.GetProperties();
-         foreach (var n in property_names_in_order) {
-            var query = props.Where(pi => pi.Name == n);
-            if (query.Any()) {
-               var property_info = query.First();
-               if (!od.Contains(property_info)) {
-                  od.Add(n, property_info);
-               }
-            }
-         }
-         return od;
-      }
-
-   }
-}

src/client/spark_core/utils/Delegates.cs

@@ -1,11 +1,11 @@
 using System;
 using System.Globalization;
 
-namespace mbts.utils {
-   delegate bool TryGetValue<T>(out T value);
-   delegate bool TryGetValue<TKey, TValue>(TKey key, out TValue value);
+namespace bts.utils {
+   public delegate bool TryGetValue<T>(out T value);
+   public delegate bool TryGetValue<TKey, TValue>(TKey key, out TValue value);
 
-   static class LogicalDelegates {
+   public static class LogicalDelegates {
       public static readonly Func<bool, bool, bool>
          And = (a, b) => a && b,
          Or = (a, b) => a || b,
@@ -13,7 +13,7 @@ namespace mbts.utils {
          None = (a, b) => !a && !b;
    }
 
-   static class FormattingDelegates {
+   public static class FormattingDelegates {
       public static class Price {
          public static readonly Func<double, string> Value =
             p => double.IsNaN(p) ? "" : p.ToString("#,0.00");
@@ -26,4 +26,4 @@ namespace mbts.utils {
             p => p.ToString("#,0.00", CultureInfo.CurrentCulture);
       }
    }
-}
+}

src/client/spark_core/utils/FilePathHelper.cs

@@ -0,0 +1,21 @@
+using System;
+using System.IO;
+
+namespace bts.utils {
+   public static class FilePathHelper {
+      public static string GetUniquePathWithDate(string path_root, string ext) =>
+         GetUniquePath($"{path_root}_{DateTime.Now:MM - dd - yyyy_hh - mm}", ext);
+
+      public static string GetUniquePath(string path_root, string ext) {
+         var path = $"{path_root}.{ext}";
+         const int max_copies = 2 ^ 14;
+         int i = 1;
+         while (File.Exists(path)) {
+            path = $"{path_root}({i++}).{ext}";
+            if (i == max_copies)
+               throw new InvalidOperationException("Unable to make path unique by concating a number.");
+         }
+         return path;
+      }
+   }
+}

src/client/spark_core/utils/ObservableExtensions.cs

@@ -209,7 +209,7 @@ namespace System.Reactive.Linq {
       /// <returns>An IDisposable object used to unsubscribe from the source sequence, and dispose the latest value subscription.</returns>
       public static IDisposable SubscribeEach<T>(this IObservable<T> source, Func<T, IDisposable> subscribe_value) {
          IDisposable value_sub = null;
-         IDisposable source_sub = source.Select(v => v).Subscribe(on_source);
+         IDisposable source_sub = source.Subscribe(on_source);
          return Disposable.Create(dispose);
          // local methods
          void on_source(T value) {

src/client/spark_core/utils/UpdateExtensions.cs

@@ -1,208 +1,253 @@
-using bts.api.core;
-using bts.collections;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Reactive.Linq;
-
-namespace bts.utils {
-   public static class UpdateExtensions {
-      public static TableUpdate<T, TKey> with_added<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> added) where TKey : IEquatable<TKey> =>
-         TableUpdate<T, TKey>.update(table, added, Enumerable.Empty<Change<T>>(), Enumerable.Empty<T>(), table.key_extractor);
-
-      public static TableUpdate<T, TKey> with_removed<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> removed) where TKey : IEquatable<TKey> =>
-         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), Enumerable.Empty<Change<T>>(), removed, table.key_extractor);
-
-      public static TableUpdate<T, TKey> with_changed<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> changed) where TKey : IEquatable<TKey> =>
-         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), changed.Select(v => new Change<T>(table.state[table.key_extractor(v)], v)), Enumerable.Empty<T>(), table.key_extractor);
-
-      public static TableUpdate<T, TKey> with_changes<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<Change<T>> changes) where TKey : IEquatable<TKey> =>
-         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), changes, Enumerable.Empty<T>(), table.key_extractor);
-
-      public static IObservable<TableUpdate<TResult, TKey>> RootDependent<T, TResult, TKey>(
-         this IObservable<TableUpdate<T, TKey>> source,
-         Func<T, TKey> key,
-         Func<T, TKey> parent_key,
-         Func<T, T, TResult> with_root,
-         Func<T, TResult> without_root,
-         Func<TResult, TKey> result_key)
-         where TKey : IEquatable<TKey> =>
-         RootDependent(source, key, parent_key, with_root, without_root, result_key, default);
-
-      // TODO: unit tests
-      public static IObservable<TableUpdate<TResult, TKey>> RootDependent<T, TKey, TResult>(
-         this IObservable<TableUpdate<T, TKey>> source,
-         Func<T, TKey> key,
-         Func<T, TKey> parent_key,
-         Func<T, T, TResult> get_result,
-         Func<T, TResult> get_result_no_parent,
-         Func<TResult, TKey> result_key,
-         TKey no_parent_key)
-         where TKey : IEquatable<TKey> =>
-         Observable.Create<TableUpdate<TResult, TKey>>(observer => {
-            var state = DictionaryHelper.empty<TKey, T>();
-            var output = TableUpdate<TResult, TKey>.mk_empty(result_key);
-            var parent_to_children = new SetMap<TKey, T>();
-            return source.Subscribe(update);
-            //
-            void update(TableUpdate<T, TKey> input) {
-               state = input.state;
-               var builder = output.to_builder();
-               // execute all parent_to_children changes first, so descendent-propagation will be efficient (otherwise propagation might be needlessly performed on values that are yet to be removed, or performed with the wrong root, if the real root is yet to be added).
-               foreach (var v in input.removed.Concat(input.changes.Select(ch => ch.Old))) {
-                  var k = input.key_extractor(v);
-                  parent_to_children.remove(parent_key(v), v);
-               }
-               foreach (var v in input.added.Concat(input.changed)) {
-                  var pk = parent_key(v);
-                  if (!pk.Equals(no_parent_key))
-                     parent_to_children.add(pk, v);
-               }
-               foreach (var v in input.removed) {
-                  var k = input.key_extractor(v);
-                  builder.remove(k);
-                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
-                     foreach (var child in children) {
-                        var ck = key(child);
-                        builder.add_or_update(ck, get_result_no_parent(child));
-                        foreach (var d in descendents(ck))
-                           builder.add_or_update(key(d), get_result(d, child));
-                     }
-                  }
-               }
-               foreach (var v in input.changed) {
-                  var k = input.key_extractor(v);
-                  var root = get_root(v);
-                  TResult result;
-                  if (key(root).Equals(k)) {
-                     result = get_result_no_parent(v);
-                     foreach (var d in descendents(k))
-                        builder.add_or_update(key(d), get_result(d, v));
-                  } else {
-                     result = get_result(v, root);
-                  }
-                  builder.add_or_update(k, result);
-               }
-               foreach (var v in input.added) {
-                  var k = input.key_extractor(v);
-                  var pk = parent_key(v);
-                  TResult result;
-                  if (pk.Equals(no_parent_key)) {
-                     result = get_result_no_parent(v);
-                     foreach (var d in descendents(k))
-                        builder.add_or_update(key(d), get_result(d, v));
-                  } else {
-                     parent_to_children.add(pk, v);
-                     var root = get_root(v);
-                     if (key(root).Equals(k)) {
-                        result = get_result_no_parent(v);
-                        foreach (var d in descendents(k))
-                           builder.add_or_update(key(d), get_result(d, v));
-                     } else {
-                        result = get_result(v, root);
-                     }
-                  }
-                  builder.add_or_update(k, result);
-               }
-               observer.OnNext(output = builder.to_table());
-            }
-            T get_root(T value) {
-               var pk = parent_key(value);
-               while (!pk.Equals(no_parent_key) && state.TryGetValue(pk, out var parent)) {
-                  value = parent;
-                  pk = parent_key(value);
-               }
-               return value;
-            }
-            IEnumerable<T> descendents(TKey k) {
-               if (parent_to_children.try_lookup(k, out ISet<T> children)) {
-                  foreach (var child in children) {
-                     yield return child;
-                     foreach (var d in descendents(key(child)))
-                        yield return d;
-                  }
-               }
-            }
-         });
-
-      public static IObservable<TableUpdate<T, TKey>> MergeTable<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, IObservable<TableUpdate<T, TKey>> other) where TKey : IEquatable<TKey> =>
-         source.AggregateLatest(other, _Merge<T, TKey>.source_change_del, _Merge<T, TKey>.other_change_del, () => TableUpdate<T, TKey>.empty, TableUpdate<T, TKey>.empty, TableUpdate<T, TKey>.empty);
-
-      public static IObservable<TableUpdate<KeyValuePair<TKey, TResult>, TKey>> ParentDependent<T, TKey, TResult>(
-         this IObservable<TableUpdate<T, TKey>> source,
-         Func<T, TKey> key,
-         Func<T, TKey> parent_key,
-         Func<T, T, TResult> get_result,
-         Func<T, TResult> get_result_no_parent)
-         where TResult : IEquatable<TResult>
-         where TKey : IEquatable<TKey> =>
-         ParentDependent(source, key, parent_key, get_result, get_result_no_parent, EqualityComparer<TResult>.Default, default);
-
-      public static IObservable<TableUpdate<KeyValuePair<TKey, TResult>, TKey>> ParentDependent<T, TKey, TResult>(
-         this IObservable<TableUpdate<T, TKey>> source,
-         Func<T, TKey> key,
-         Func<T, TKey> parent_key,
-         Func<T, T, TResult> get_result,
-         Func<T, TResult> get_result_no_parent,
-         IEqualityComparer<TResult> result_comparer,
-         TKey no_parent_key)
-         where TKey : IEquatable<TKey> =>
-         Observable.Create<TableUpdate<KeyValuePair<TKey, TResult>, TKey>>(observer => {
-            var value_comparer = EqualityComparer.Create<KeyValuePair<TKey, TResult>, TResult>(kv => kv.Value, result_comparer);
-            var state = DictionaryHelper.empty<TKey, T>();
-            var output = TableUpdate<KeyValuePair<TKey, TResult>, TKey>.mk_empty(kv => kv.Key);
-            var parent_to_children = new SetMap<TKey, T>();
-            return source.Subscribe(update);
-            //
-            void update(TableUpdate<T, TKey> input) {
-               state = input.state;
-               var builder = output.to_builder();
-               foreach (var v in input.removed) {
-                  var k = input.key_extractor(v);
-                  builder.remove(k);
-                  parent_to_children.remove(parent_key(v), v);
-                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
-                     foreach (var child in children)
-                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result_no_parent(child)));
-                  }
-               }
-               foreach (var v in input.changed) {
-                  var k = key(v);
-                  builder.add_or_update(new KeyValuePair<TKey, TResult>(k, lookup_result(v)));
-                  parent_to_children.set(parent_key(v), v);
-                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
-                     foreach (var child in children)
-                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result(child, v)));
-                  }
-               }
-               foreach (var v in input.added) {
-                  var k = key(v);
-                  builder.add_or_update(new KeyValuePair<TKey, TResult>(k, lookup_result(v)));
-                  parent_to_children.add(parent_key(v), v);
-                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
-                     foreach (var child in children)
-                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result(child, v)));
-                  }
-               }
-               var result = builder.to_table_checked(value_comparer);
-               if (result.has_changes())
-                  observer.OnNext(output = result);
-            }
-            TResult lookup_result(T child) {
-               var pk = parent_key(child);
-               return !pk.Equals(no_parent_key) && state.TryGetValue(pk, out T parent) ?  get_result(child, parent) : get_result_no_parent(child);
-            }
-         });
-
-      static class _Merge<T, TKey> where TKey : IEquatable<TKey> {
-         public static readonly Func<TableUpdate<T, TKey>, Change<TableUpdate<T, TKey>>, TableUpdate<T, TKey>, TableUpdate<T, TKey>> source_change_del = _source_changed_impl;
-         public static readonly Func<TableUpdate<T, TKey>, TableUpdate<T, TKey>, Change<TableUpdate<T, TKey>>, TableUpdate<T, TKey>> other_change_del = _other_changed_impl;
-
-         static TableUpdate<T, TKey> _source_changed_impl(TableUpdate<T, TKey> target, Change<TableUpdate<T, TKey>> source_change, TableUpdate<T, TKey> other_table) =>
-            TableUpdate<T, TKey>.update(target, source_change.New.added, source_change.New.changes, source_change.New.removed, source_change.New.key_extractor);
-
-         static TableUpdate<T, TKey> _other_changed_impl(TableUpdate<T, TKey> target, TableUpdate<T, TKey> source_table, Change<TableUpdate<T, TKey>> other_change) =>
-            TableUpdate<T, TKey>.update(target, other_change.New.added, other_change.New.changes, other_change.New.removed, other_change.New.key_extractor);
-      }
-   }
-}
+using bts.api.core;
+using bts.collections;
+using bts.utils.proto;
+using Google.Protobuf;
+using mbts;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reactive.Linq;
+
+namespace bts.utils {
+   public static class UpdateExtensions {
+      public static TableUpdate<T, TKey> with_added<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> added) where TKey : IEquatable<TKey> =>
+         TableUpdate<T, TKey>.update(table, added, Enumerable.Empty<Change<T>>(), Enumerable.Empty<T>(), table.key_extractor);
+
+      public static TableUpdate<T, TKey> with_removed<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> removed) where TKey : IEquatable<TKey> =>
+         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), Enumerable.Empty<Change<T>>(), removed, table.key_extractor);
+
+      public static TableUpdate<T, TKey> with_changed<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<T> changed) where TKey : IEquatable<TKey> =>
+         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), changed.Select(v => new Change<T>(table.state[table.key_extractor(v)], v)), Enumerable.Empty<T>(), table.key_extractor);
+
+      public static TableUpdate<T, TKey> with_changes<T, TKey>(this TableUpdate<T, TKey> table, IEnumerable<Change<T>> changes) where TKey : IEquatable<TKey> =>
+         TableUpdate<T, TKey>.update(table, Enumerable.Empty<T>(), changes, Enumerable.Empty<T>(), table.key_extractor);
+
+      public static IObservable<TableUpdate<TResult, TKey>> RootDependent<T, TResult, TKey>(
+         this IObservable<TableUpdate<T, TKey>> source,
+         Func<T, TKey> key,
+         Func<T, TKey> parent_key,
+         Func<T, T, TResult> with_root,
+         Func<T, TResult> without_root,
+         Func<TResult, TKey> result_key)
+         where TKey : IEquatable<TKey> =>
+         RootDependent(source, key, parent_key, with_root, without_root, result_key, default);
+
+      // TODO: unit tests
+      public static IObservable<TableUpdate<TResult, TKey>> RootDependent<T, TKey, TResult>(
+         this IObservable<TableUpdate<T, TKey>> source,
+         Func<T, TKey> key,
+         Func<T, TKey> parent_key,
+         Func<T, T, TResult> get_result,
+         Func<T, TResult> get_result_no_parent,
+         Func<TResult, TKey> result_key,
+         TKey no_parent_key)
+         where TKey : IEquatable<TKey> =>
+         Observable.Create<TableUpdate<TResult, TKey>>(observer => {
+            var state = DictionaryHelper.empty<TKey, T>();
+            var output = TableUpdate<TResult, TKey>.mk_empty(result_key);
+            var parent_to_children = new SetMap<TKey, T>();
+            return source.Subscribe(update);
+            //
+            void update(TableUpdate<T, TKey> input) {
+               state = input.state;
+               var builder = output.to_builder();
+               // execute all parent_to_children changes first, so descendent-propagation will be efficient (otherwise propagation might be needlessly performed on values that are yet to be removed, or performed with the wrong root, if the real root is yet to be added).
+               foreach (var v in input.removed.Concat(input.changes.Select(ch => ch.Old))) {
+                  var k = input.key_extractor(v);
+                  parent_to_children.remove(parent_key(v), v);
+               }
+               foreach (var v in input.added.Concat(input.changed)) {
+                  var pk = parent_key(v);
+                  if (!pk.Equals(no_parent_key))
+                     parent_to_children.add(pk, v);
+               }
+               foreach (var v in input.removed) {
+                  var k = input.key_extractor(v);
+                  builder.remove(k);
+                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
+                     foreach (var child in children) {
+                        var ck = key(child);
+                        builder.add_or_update(ck, get_result_no_parent(child));
+                        foreach (var d in descendents(ck))
+                           builder.add_or_update(key(d), get_result(d, child));
+                     }
+                  }
+               }
+               foreach (var v in input.changed) {
+                  var k = input.key_extractor(v);
+                  var root = get_root(v);
+                  TResult result;
+                  if (key(root).Equals(k)) {
+                     result = get_result_no_parent(v);
+                     foreach (var d in descendents(k))
+                        builder.add_or_update(key(d), get_result(d, v));
+                  } else {
+                     result = get_result(v, root);
+                  }
+                  builder.add_or_update(k, result);
+               }
+               foreach (var v in input.added) {
+                  var k = input.key_extractor(v);
+                  var pk = parent_key(v);
+                  TResult result;
+                  if (pk.Equals(no_parent_key)) {
+                     result = get_result_no_parent(v);
+                     foreach (var d in descendents(k))
+                        builder.add_or_update(key(d), get_result(d, v));
+                  } else {
+                     parent_to_children.add(pk, v);
+                     var root = get_root(v);
+                     if (key(root).Equals(k)) {
+                        result = get_result_no_parent(v);
+                        foreach (var d in descendents(k))
+                           builder.add_or_update(key(d), get_result(d, v));
+                     } else {
+                        result = get_result(v, root);
+                     }
+                  }
+                  builder.add_or_update(k, result);
+               }
+               observer.OnNext(output = builder.to_table());
+            }
+            T get_root(T value) {
+               var pk = parent_key(value);
+               while (!pk.Equals(no_parent_key) && state.TryGetValue(pk, out var parent)) {
+                  value = parent;
+                  pk = parent_key(value);
+               }
+               return value;
+            }
+            IEnumerable<T> descendents(TKey k) {
+               if (parent_to_children.try_lookup(k, out ISet<T> children)) {
+                  foreach (var child in children) {
+                     yield return child;
+                     foreach (var d in descendents(key(child)))
+                        yield return d;
+                  }
+               }
+            }
+         });
+
+      public static IObservable<TableUpdate<T, TKey>> MergeTable<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, IObservable<TableUpdate<T, TKey>> other) where TKey : IEquatable<TKey> =>
+         source.AggregateLatest(other, _Merge<T, TKey>.source_change_del, _Merge<T, TKey>.other_change_del, () => TableUpdate<T, TKey>.empty, TableUpdate<T, TKey>.empty, TableUpdate<T, TKey>.empty);
+
+      public static IObservable<TableUpdate<KeyValuePair<TKey, TResult>, TKey>> ParentDependent<T, TKey, TResult>(
+         this IObservable<TableUpdate<T, TKey>> source,
+         Func<T, TKey> key,
+         Func<T, TKey> parent_key,
+         Func<T, T, TResult> get_result,
+         Func<T, TResult> get_result_no_parent)
+         where TResult : IEquatable<TResult>
+         where TKey : IEquatable<TKey> =>
+         ParentDependent(source, key, parent_key, get_result, get_result_no_parent, EqualityComparer<TResult>.Default, default);
+
+      public static IObservable<TableUpdate<KeyValuePair<TKey, TResult>, TKey>> ParentDependent<T, TKey, TResult>(
+         this IObservable<TableUpdate<T, TKey>> source,
+         Func<T, TKey> key,
+         Func<T, TKey> parent_key,
+         Func<T, T, TResult> get_result,
+         Func<T, TResult> get_result_no_parent,
+         IEqualityComparer<TResult> result_comparer,
+         TKey no_parent_key)
+         where TKey : IEquatable<TKey> =>
+         Observable.Create<TableUpdate<KeyValuePair<TKey, TResult>, TKey>>(observer => {
+            var value_comparer = EqualityComparer.Create<KeyValuePair<TKey, TResult>, TResult>(kv => kv.Value, result_comparer);
+            var state = DictionaryHelper.empty<TKey, T>();
+            var output = TableUpdate<KeyValuePair<TKey, TResult>, TKey>.mk_empty(kv => kv.Key);
+            var parent_to_children = new SetMap<TKey, T>();
+            return source.Subscribe(update);
+            //
+            void update(TableUpdate<T, TKey> input) {
+               state = input.state;
+               var builder = output.to_builder();
+               foreach (var v in input.removed) {
+                  var k = input.key_extractor(v);
+                  builder.remove(k);
+                  parent_to_children.remove(parent_key(v), v);
+                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
+                     foreach (var child in children)
+                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result_no_parent(child)));
+                  }
+               }
+               foreach (var v in input.changed) {
+                  var k = key(v);
+                  builder.add_or_update(new KeyValuePair<TKey, TResult>(k, lookup_result(v)));
+                  parent_to_children.set(parent_key(v), v);
+                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
+                     foreach (var child in children)
+                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result(child, v)));
+                  }
+               }
+               foreach (var v in input.added) {
+                  var k = key(v);
+                  builder.add_or_update(new KeyValuePair<TKey, TResult>(k, lookup_result(v)));
+                  parent_to_children.add(parent_key(v), v);
+                  if (parent_to_children.try_lookup(k, out ISet<T> children)) {
+                     foreach (var child in children)
+                        builder.add_or_update(new KeyValuePair<TKey, TResult>(key(child), get_result(child, v)));
+                  }
+               }
+               var result = builder.to_table_checked(value_comparer);
+               if (result.has_changes())
+                  observer.OnNext(output = result);
+            }
+            TResult lookup_result(T child) {
+               var pk = parent_key(child);
+               return !pk.Equals(no_parent_key) && state.TryGetValue(pk, out T parent) ?  get_result(child, parent) : get_result_no_parent(child);
+            }
+         });
+
+      static class _Merge<T, TKey> where TKey : IEquatable<TKey> {
+         public static readonly Func<TableUpdate<T, TKey>, Change<TableUpdate<T, TKey>>, TableUpdate<T, TKey>, TableUpdate<T, TKey>> source_change_del = _source_changed_impl;
+         public static readonly Func<TableUpdate<T, TKey>, TableUpdate<T, TKey>, Change<TableUpdate<T, TKey>>, TableUpdate<T, TKey>> other_change_del = _other_changed_impl;
+
+         static TableUpdate<T, TKey> _source_changed_impl(TableUpdate<T, TKey> target, Change<TableUpdate<T, TKey>> source_change, TableUpdate<T, TKey> other_table) =>
+            TableUpdate<T, TKey>.update(target, source_change.New.added, source_change.New.changes, source_change.New.removed, source_change.New.key_extractor);
+
+         static TableUpdate<T, TKey> _other_changed_impl(TableUpdate<T, TKey> target, TableUpdate<T, TKey> source_table, Change<TableUpdate<T, TKey>> other_change) =>
+            TableUpdate<T, TKey>.update(target, other_change.New.added, other_change.New.changes, other_change.New.removed, other_change.New.key_extractor);
+      }
+   }
+}
+namespace bts.utils.diagnostics {
+   public static class DiagnosticUpdateExtensions {
+      // Hard to imagine why this util would be needed outside of diagnosig bugs.
+      // The #if DEBUG directive is an extra precaution to avoid accidentally writing
+      // files like this on a client's machine.
+#if DEBUG
+      static readonly string _proto_capture_ext = "proto_capture";
+
+      public static IObservable<TableUpdate<T, TKey>> ProtoBufCapture<T, TKey, TProto>(this IObservable<TableUpdate<T, TKey>> source, Func<T, TProto> to_message)
+         where TKey : IEquatable<TKey>
+         where TProto : class, IMessage<TProto> =>
+         ProtoBufCapture(source,
+            FilePathHelper.GetUniquePathWithDate(Path.Combine(Directory.GetCurrentDirectory(), typeof(TProto).Name), _proto_capture_ext),
+            to_message);
+
+      public static IObservable<TableUpdate<T, TKey>> ProtoBufCapture<T, TKey, TProto>(this IObservable<TableUpdate<T, TKey>> source, string path, Func<T, TProto> to_message)
+         where TKey : IEquatable<TKey>
+         where TProto : class, IMessage<TProto> =>
+         Observable.Create<TableUpdate<T, TKey>>(o => {
+            // write the state on dispose, and/or on error
+            IReadOnlyDictionary<TKey, T> state = DictionaryHelper.empty<TKey, T>();
+            var sub = source.Subscribe(table => {
+               state = table.state;
+               o.OnNext(table);
+            }, ex => {
+               save(true);
+               o.OnError(ex);
+            },
+            o.OnCompleted);
+            return Disposable.Create(() => {
+               save(false);
+               Disposable.dispose(ref sub);
+            });
+            void save(bool has_error) {
+               if (state.Count > 0)
+                  FileStreamHelper.WriteProtos(has_error ? $"{path}(error)" : path, state.Values.Select(to_message).ExceptNull());
+            }
+         });
+#endif
+   }
+}

src/client/spark_test/Resources/ResourceHelper.cs

@@ -0,0 +1,13 @@
+using System.IO;
+using System.Reflection;
+
+namespace bts.resources {
+   class ResourceHelper {
+      const string _default_namespace = "spark_test.Resources";
+
+      public static Stream LoadResource(string name) {
+         var ass = Assembly.GetExecutingAssembly();
+         return ass.GetManifestResourceStream(name) ?? ass.GetManifestResourceStream($"{_default_namespace}.{name}");
+      }
+   }
+}

src/client/spark_test/Security/SecurityTests.cs

@@ -1,13 +1,11 @@
-using System;
+using bts.api;
+using bts.resources;
+using bts.test;
+using bts.utils.proto;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using bts.api;
-using bts.test;
 
-namespace bts.mcore
-{
+namespace bts.mcore {
    [TestGroup]
    class SecurityTests {
       [Test]
@@ -28,5 +26,23 @@ namespace bts.mcore
          Assert.That(dict[b] == 1);
          Assert.That(dict[c] == 2);
       }
+
+      [Test]
+      public void test_security_compare() {
+         const string securities_resource = "KrandallOrderBookCrash.security_proto";
+         // verify that the sample size doesn't shrink for some reason.
+         // if it grows a bit that's fine; the generic comparison test
+         // asserts against collections that are too large.
+         const int min_sample_size = 91;
+
+         var securities = StreamHelper.ReadProtos<Security>(
+            ResourceHelper.LoadResource(securities_resource))
+            .Select(s => new SecurityImmutable(s))
+            .ToList();
+         Assert.IsTrue(securities.Count >= min_sample_size);
+
+         ComparisonTests.transitive_comparison(securities, SecurityHelper.Comparer.InstanceImmutable);
+         ComparisonTests.identity_comparison(securities, SecurityHelper.Comparer.InstanceImmutable);
+      }
    }
 }

src/client/spark_test/Utils/ComparisonTests.cs

@@ -0,0 +1,34 @@
+using bts.test;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace bts.mcore {
+   static class ComparisonTests {
+      public static void transitive_comparison<T>(IEnumerable<T> items, IComparer<T> comparer) {
+         // Verify that the sample size isn't too big.
+         // This test compares each item to every other item
+         // (technically twice; lhs/rhs is flipped the second time),
+         // so performance is O(n^2).
+         const int max_sample_size = 4092;
+         var sorted_items = items.ToList();
+         Assert.That(sorted_items.Count <= max_sample_size);
+         sorted_items.Sort(comparer);
+         for (int i = 0; i < sorted_items.Count - 1; i++) {
+            for (int j = i + 1; j < sorted_items.Count; j++) {
+               assert_comp(i, j);
+            }
+         }
+         void assert_comp(int lhs, int rhs) {
+            int pos_comp = lhs.CompareTo(rhs);
+            // hit both sides for good measure
+            Assert.That(comparer.Compare(sorted_items[lhs], sorted_items[rhs]) == pos_comp);
+            Assert.That(comparer.Compare(sorted_items[rhs], sorted_items[lhs]) == -pos_comp);
+         }
+      }
+
+      public static void identity_comparison<T>(IEnumerable<T> items, IComparer<T> comparer) {
+         foreach (var v in items)
+            Assert.That(comparer.Compare(v, v) == 0);
+      }
+   }
+}

src/client/spark_test/Utils/MarketDataUtilities.cs

@@ -61,7 +61,7 @@ public static async Task<IEnumerable<SecurityImmutable>> get_multiple_securities
          return Enumerable.Empty<SecurityImmutable>();
       }
 
-      public static async Task<IEnumerable<FutureContract>> get_expirations(MarketDataRelay relay, string symbol, SecurityType security_type, MDRequest.Types.Flags flags = MDRequest.Types.Flags.Bbo, Venue v = Venue.Sim) {
+      public static async Task<(bool, List<FutureContract>)> get_expirations(MarketDataRelay relay, string symbol, SecurityType security_type, MDRequest.Types.Flags flags = MDRequest.Types.Flags.Bbo, Venue v = Venue.Sim) {
          var request = new MDRequest() {
             Host = _MD_HOST,
             Channels = {
@@ -89,7 +89,7 @@ public static async Task<IEnumerable<SecurityImmutable>> get_multiple_securities
             }
          }
 
-         return (IEnumerable<FutureContract>) futures;
+         return (result.success, futures);
       }
 
    }

src/client/spark_test/Utils/SpreaderUtilities.cs

@@ -22,8 +22,9 @@ namespace bts.tests
    public class SpreaderUtilities {
 
       public static async Task<(SecurityImmutable s1, SecurityImmutable s2)> get_securities_for_common_setup_one(MarketDataRelay relay, string symbol) {
-         var exps  = (await MarketDataUtilities.get_expirations(relay, symbol, SecurityType.Future)).ToList();
+         var (success, exps) = await MarketDataUtilities.get_expirations(relay, symbol, SecurityType.Future);
 
+         Assert.IsTrue(success, "Could not reach market data service");
          Assert.IsNotNull(exps);
          Assert.IsTrue(exps.Count() > 2, "Need at least two expirations");
 

src/client/spark_test/spark_test.csproj

@@ -14,5 +14,7 @@
   <ItemGroup>
     <Content Include="../spark_test/system_test_scripts/*.sh" CopyToOutputDirectory="PreserveNewest"></Content>
   </ItemGroup>
-
+  <ItemGroup>
+    <EmbeddedResource Include="Resources\KrandallOrderBookCrash.security_proto" />
+  </ItemGroup>
 </Project>

src/lib/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/Product.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityMapper.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/service/MarketDataServiceServer.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;md/vendor/OnixsUtils.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metricsLine truncated
+set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/Product.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityMapper.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/service/MarketDataServiceServer.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;md/vendor/OnixsUtils.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metricsLine truncated

src/lib/api/service_helpers/AdminServiceUtils.cpp

@@ -685,6 +685,7 @@ bts::api::admin::BTSAccount* AdminServiceUtils::create_account(bts::api::admin::
    new_account_ptr->set_name(account_name);
    new_account_ptr->set_description(description);
    new_account_ptr->set_is_active(is_active);
+   new_account_ptr->mutable_risk_limit_enforcement()->set_ignore_spread_position_limits(true);
    return new_account_ptr;
 }
 void AdminServiceUtils::update_account(bts::api::admin::AdminData& admin_data,

src/lib/api/services/OrderService.proto

@@ -383,7 +383,9 @@ message OCOOrderState {}
 message PricerOrderState {}
 message ScratchOrderState {}
 message ProRataOrderState {}
-message DynamicOrderState {}
+message DynamicOrderState {
+  map<string,DynamicOrderParam> custom_state = 1;
+}
 message QueueHolderState {}
 message SplitterOrderState {}
 
@@ -644,7 +646,7 @@ message RegisteredDynamicOrderTypes {
 
 message DynamicOrderParam {
   oneof value {
-    uint32 as_int32 = 1;
+    int32 as_int32 = 1;
     uint32 as_uint32 = 2;
     double as_double = 3;
     string as_string = 4;

src/lib/config/defaults/keycode.conf

@@ -1,13 +1,10 @@
-
-
 #nusers: 1
 #allow concurrent : true
-#expires: 2020-Mar-31 00:00:00   637212096000000000
-#generation_time: 2020-Jan-06 18:56:23.406931   5248825356261457214
+#expires: 2021-Jan-01 00:00:00   637450560000000000
+#generation_time: 2020-Apr-03 18:39:31.010863   5248901378137496534
 EncryptedKeyCode {
    value {
-      keycode: "Ixb/z6hwFQmojpm4GwNEpQmsqpYT9OTHbcKtROIqj38GwGqyVn54JMDB04V048quO+NAI78DKJXsUWgjDH0y5HQlwN6Lq48s/bdwKqR1OTBPr6xoTImtCb+l5b+8fUGlg7t3X8gfMdxohHgcUi9k9n7ZrnNvqcjdYKoP0CaTZT8zHuBQx3f4Lqrz32gq9rJ5skRjH6ZqNblNdlsF+kOAqXCkvI5p0QfyKgtCodr33qsW2cbqY5gUD9KwsnmFVa8/zuFBucsoM7dHB47yRaIeOFFu7xGayAdAHA8SwLyoxE5y+adIX1SYnAiqAu1raZpeXgRE1LNQk3c7uhUjt0coag=="
+      keycode: "BJBQWsbdTXWa/iLmOf3ekE8vP/NYkf0Q/nUsp/rJ78Ze3W9pNXICCMabmmsr1xOM7aqRIclPgZd955uSSfvFkdNWFTxwPQ5IfnDGopj3QdhUVxhzgVLaXHN+vC0H/MWrDQ5K4WY5J/6VXgUn1LSN/2HNb5iR2sKXeh3Alamg87YtCWEi+scelffE7HK5ERT/3A3Y/e0pqUFhP6xouIjHRhC2FRBb6OoJQVvFwVDADSuKZNo9B6GhC3H0YdHqjFhGDoky5ecgzeKIX8uWJEQNnQXQJ0iTgQ95+YR5IMU+XWdlVfSrc0fuTmj3YWcBQ8Kl2ItEcobyZzJYega3tu6Ftw=="
    }
 }
 
-

src/lib/config/protos/MarketData.proto

@@ -52,6 +52,11 @@ message PublisherOptions
    optional uint32 heartbeat_boot_seconds = 12 [default = 60]; // 1 minute grace period from shm creation
 
    optional bool skip_options = 13 [default = true];
+
+   optional uint32 hot_initial = 20 [default = 1024];
+   optional uint32 hot_growth = 21 [default = 1024];
+   optional uint32 cold_initial = 22 [default = 8192];
+   optional uint32 cold_growth = 23 [default = 2048];
 }
 
 message ICEPublisherOptions
@@ -193,6 +198,11 @@ message RecordOptions {
    optional bool record_sim = 5 [default=false];
 }
 
+// If starting on Sunday pre-load the selected symbols from disk
+message SecurityPreload {
+   repeated string symbol = 1;
+}
+
 //
 // Replay options
 // 

src/lib/config/protos/toggles.proto

@@ -67,4 +67,5 @@ message DynamicAlgos{
    optional bool enable_dorman = 1 [default = false];
    optional bool enable_pyramid = 2 [default = false];
    optional bool enable_tas = 3 [default = false];
+   optional bool enable_tascounter = 4 [default = false];
 }

src/lib/md/MDConsumer.cpp

@@ -792,7 +792,7 @@ public:
             // We can safely send a security key to every _instance because by definition
             // a security cannot appear in 2 instances
             if (unresolved_securities.size() > 250) {
-               LOG(md,Warn,"Failed to resolve legs for {}"
+               LOG(md,Info,"Failed to resolve legs for {}"
                    " spreads. This can happen when a spread is subscribed without "
                    "it's underliers and may lead to higher than expected CPU use.",
                    unresolved_securities.size() );
@@ -901,6 +901,10 @@ public:
       return _cache.get_security(key);
    }
 
+   std::vector<api::Security const *> get_securities() const {
+      return _cache.get_securities();
+   }
+
    MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const {
       for (auto& inst : _instances) {
          if ((inst->symbol() == symbol && inst->type() == type && inst->exchange() == exchange) ||
@@ -976,6 +980,7 @@ size_t MDConsumer::walk_security_list(SecurityCallback cb, const string& symbol,
 size_t MDConsumer::scan_securities(bool resolve_legs) { return _pimpl->scan_securities(resolve_legs); }
 void MDConsumer::on_security(SecurityCallback cb, bool resolve_legs) { _pimpl->on_security(cb, resolve_legs); }
 const api::Security* MDConsumer::get_security(SecurityKey key) const { return _pimpl->get_security(key).get(); }
+std::vector<api::Security const*> MDConsumer::get_securities() const {return _pimpl->get_securities();}
 
 MDConsumerChannel MDConsumer::get_channel(const std::string& symbol, SecurityType type, Venue exchange) const {
    return _pimpl->get_channel(symbol, type, exchange);

src/lib/md/MDConsumer.h

@@ -59,6 +59,9 @@ public:
    // You still need to call scan_securities() periodically to discover new securities
    const api::Security* get_security(SecurityKey key) const;
 
+   // Retrieve all cached securities
+   std::vector<api::Security const*> get_securities() const;
+
    MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const;
    MDConsumerChannel get_channel(const UserSymbol&) const;
 

src/lib/md/MDProducer.cpp

@@ -20,8 +20,14 @@ public:
       _symbol(symbol),
       _type(type),
       _exchange(exchange),
-      _hot(hot_segment_name(symbol, type, exchange, platform), 1024, 1024, flags & MDProducerFlags::Shared),
-      _cold(cold_segment_name(symbol, type, exchange, platform), 8192, 2048, flags & MDProducerFlags::Shared),
+      _hot(hot_segment_name(symbol, type, exchange, platform), 
+         config::get<PublisherOptions>().hot_initial(), 
+         config::get<PublisherOptions>().hot_growth(),
+         flags & MDProducerFlags::Shared),
+      _cold(cold_segment_name(symbol, type, exchange, platform), 
+         config::get<PublisherOptions>().cold_initial(), 
+         config::get<PublisherOptions>().cold_growth(), 
+         flags & MDProducerFlags::Shared),
       _trades(_cold.header().trade_buffer, BTS_ARRAYSIZE(_cold.header().trade_buffer), true),
       _rfqs(_cold.header().rfq_buffer, BTS_ARRAYSIZE(_cold.header().rfq_buffer), true)
       {

src/lib/md/SecurityCache.cpp

@@ -69,6 +69,17 @@ SecurityPtr SecurityCache::get_security(SecurityKey key) {
    }
 }
 
+std::vector<api::Security const*> SecurityCache::get_securities()
+{
+   std::vector<api::Security const*> result;
+   result.reserve(100000);
+
+   std::unique_lock<std::mutex> lock(_me->cache_mutex);
+   for (auto const&p : _me->cache)
+      result.push_back(p.second.get());
+   return result;
+}
+
 SecurityPtr SecurityCache::add_db_security(const SecurityPtr& security) {
    bts_assert(security);
    std::lock_guard<std::mutex> lock(_me->cache_mutex);
@@ -90,4 +101,4 @@ bool SecurityCache::add_live_security(const SecurityPtr& security) {
 
 void SecurityCache::flush_to_disk() { _me->flush(); }
 
-}}//ns
+}}//ns

src/lib/md/SecurityCache.h

@@ -22,6 +22,9 @@ public:
    //Load security from cache with fallback to database
    SecurityPtr get_security(SecurityKey key); 
 
+   // Load all securities from cache, no fallback to database
+   std::vector<api::Security const*> get_securities();
+   
    // Get the security if it's in the cache otherwise return nullptr. Does not query security database.
    SecurityPtr try_get_security(SecurityKey key) const;
 
@@ -33,4 +36,4 @@ public:
 
 };
 
-}}//ns
+}}//ns

src/lib/md/SecurityOracle.cpp

@@ -10,4 +10,8 @@ const api::Security* SecurityOracle::get_security(SecurityKey key) const {
    return _client.current_consumer().get_security(key);
 }
 
-}}//namespace
+std::vector<api::Security const*> SecurityOracle::get_securities() const {
+   return _client.current_consumer().get_securities();
+}
+
+}}//namespace

src/lib/md/SecurityOracle.h

@@ -1,6 +1,8 @@
 #pragma once
+
 #include "lib/domain_model/SecurityKey.h"
 #include <memory>
+#include <vector>
 
 namespace bts {
 namespace api {class Security;}
@@ -12,6 +14,7 @@ public:
    SecurityOracle(const MDClient&);
    ~SecurityOracle();
    const api::Security* get_security(SecurityKey key) const;
+   std::vector<api::Security const*> get_securities() const;
 };
 
-}}//namespace
+}}//namespace

src/lib/md/vendor/OnixsUtils.cpp

@@ -18,16 +18,16 @@ const char * onixs_license[] = {
 "    <name>Blue Trading Systems OF 9420-2</name>",
 "  </customer>",
 "  <period>",
-"    <since>2019-05-02</since>",
-"    <till>2020-05-13</till>",
+"    <since>2020-05-01</since>",
+"    <till>2021-05-12</till>",
 "  </period>",
 "  <products>",
 "    <product id=\"4\" name=\"CME Market Data Handler for C++\" />",
 "    <product id=\"7\" name=\"ICE iMpact Multicast Price Feed Handler for C++\" />",
 "    <product id=\"67\" name=\"Eurex Market and Reference Data Interfaces (EMDI, RDI) Handler for C++\" />",
 "  </products>",
-"  <checksum>5E50A1DACD94D7C1BECD316EEAE5AB02</checksum>",
-"</license>",
+"  <checksum>1C9210495AA3C9C884B7A37A4C7AD8F9</checksum>",
+"</license>"
 };
 
 OnixsTemplLicense onixs_create_temporary_license() {

src/lib/orderman/actors/OrderStore.cpp

@@ -154,10 +154,13 @@ void OrderStore::store_and_publish_if_complete(
 
 bool OrderStore::_order_cutoff(const api::orderman::OrderRecord &rec) {
    // TODO: move this to orderutils
-   bool working = (rec.state().status() ==
-                   api::orderman::OrderStatus::ORDER_STATUS_WORKING) ||
-                  (rec.state().status() ==
-                   api::orderman::OrderStatus::ORDER_STATUS_PENDING_WORKING);
+   bool working =
+     (rec.state().status() ==
+      api::orderman::OrderStatus::ORDER_STATUS_WORKING) ||
+     (rec.state().status() ==
+      api::orderman::OrderStatus::ORDER_STATUS_DISCONNECTED) ||
+     (rec.state().status() ==
+      api::orderman::OrderStatus::ORDER_STATUS_PENDING_WORKING);
    return me->soft_cleanup_enabled && !working &&
           rec.state().created().seconds() < me->cutoff_time.tv_sec;
 }

src/lib/orderman/actors/SODPosition.cpp

@@ -7,6 +7,7 @@
 #include "lib/utils/PBUtils.h"
 #include "lib/utils/bts_assert.h"
 #include <google/protobuf/timestamp.pb.h>
+#include <google/protobuf/util/time_util.h>
 #include <sstream>
 #include <sys/time.h>
 
@@ -53,7 +54,7 @@ timeval SODPosition::sod_time() {
    // if the server started before the sod for today
    if (rv.tv_sec >= now_epoch)
       rv = subtract_days (rv,1);
-   return rv;   
+   return rv;
 }
 
 timeval SODPosition::get_order_cutoff_time(
@@ -98,6 +99,48 @@ timeval SODPosition::now(){
    return tv;
 }
 
+google::protobuf::Timestamp SODPosition::today_local_time_to_utc_timestamp (
+                                                   uint16_t hours,
+                                                   uint16_t mins,
+                                                   uint16_t secs,
+                                                   bool add_day_if_less,
+                                                   bool subtract_day_if_more){
+   time_t now_epoch;
+   time(&now_epoch); // seconds since epoch (utc) (independent of machine timezone)
+   auto now_ts = google::protobuf::util::TimeUtil::TimeTToTimestamp(now_epoch);
+   struct tm *ptm_local = localtime(&now_epoch);  // in local timezone
+   ptm_local->tm_hour = hours;
+   ptm_local->tm_min = mins;
+   ptm_local->tm_sec = secs;
+   ptm_local->tm_isdst = -1;
+   time_t ovrd_epoch = mktime(ptm_local);  // converts local time to UTC epoch time
+   if(ovrd_epoch < now_epoch){
+      if (add_day_if_less)
+         ovrd_epoch += 24*60*60;
+   }
+   else{
+      if(subtract_day_if_more)
+         ovrd_epoch -= 24*60*60;
+   }
+   auto ovrd_ts = google::protobuf::util::TimeUtil::TimeTToTimestamp(ovrd_epoch);
+   return ovrd_ts;
+}
+
+google::protobuf::Timestamp SODPosition::add_seconds (
+                                       const google::protobuf::Timestamp& ts,
+                                       int secs_diff){
+   auto secs = google::protobuf::util::TimeUtil::TimestampToSeconds(ts);
+   secs += secs_diff;
+   return google::protobuf::util::TimeUtil::SecondsToTimestamp(secs);
+}
+
+google::protobuf::Timestamp SODPosition::add_nanoseconds (
+                                       const google::protobuf::Timestamp& ts,
+                                       long diff){
+   return google::protobuf::util::TimeUtil::NanosecondsToTimestamp(
+      google::protobuf::util::TimeUtil::TimestampToNanoseconds(ts) + diff);
+}
+
 google::protobuf::Timestamp SODPosition::to_timestamp (timeval tv){
    google::protobuf::Timestamp rv;
    rv.set_seconds(tv.tv_sec);

src/lib/orderman/actors/SODPosition.h

@@ -31,6 +31,18 @@ public:
                      const std::map<std::string,std::string>& ordermap);
    static timeval now();
    static google::protobuf::Timestamp to_timestamp (timeval);
+   static google::protobuf::Timestamp today_local_time_to_utc_timestamp (
+            uint16_t hours,
+            uint16_t mins,
+            uint16_t secs,
+            bool add_day_if_less,
+            bool subtract_day_if_more);
+   static google::protobuf::Timestamp add_seconds (
+                                 const google::protobuf::Timestamp& ts,
+                                 int secs);
+   static google::protobuf::Timestamp add_nanoseconds (
+                                 const google::protobuf::Timestamp& ts,
+                                 long diff);
    bool is_sod (const google::protobuf::Timestamp& fill_time) const;
 };
 

src/lib/orderman/actors/order_manager/OrderManager.cpp

@@ -157,6 +157,7 @@ void OrderManager::on_filled_position(
       }
 
       for (auto const &ev : *(update.events)) {
+         me->services_data.position.update(ev);
          me->services_data.limiter.set_filled_position(
              *ev.security, ev.account_id, ev.long_quantity, ev.short_quantity);
       }
@@ -550,6 +551,8 @@ void OrderManager::init_order_services() {
        std::make_unique<ContentSubscription<TimerEvent, NoKey>>(*this, log);
    me->services_data.order_creation_subs =
        std::make_unique<ContentSubscription<CreateOrderReplyEvent>>(*this, log);
+   me->services_data.position_subs =
+      std::make_unique<ContentSubscription<FilledPositionEvent,NoKey>>(*this, log);
 }
 
 void OrderManager::after_event() {

src/lib/orderman/actors/order_manager/OrderServices.cpp

@@ -32,6 +32,7 @@ void OrderServices::init(OrderServicesData &data) {
    _status_updates.init(*data.status_subs);
    _settlement_updates.init(*data.settlement_subs);
    _time_change.init(*data.time_subs);
+   _position_update.init(*data.position_subs);
    if (_sessions.empty())
       for (auto &sd : data.sessions)
          _sessions.emplace_back(*sd);

src/lib/orderman/actors/order_manager/OrderServices.h

@@ -8,6 +8,7 @@
 #include "lib/orderman/actors/order_manager/SessionServices.h"
 #include "lib/orderman/events/TickerUpdateEvent.h"
 #include "lib/orderman/events/StatusUpdateEvent.h"
+#include "lib/orderman/events/FilledPositionEvent.h"
 #include "lib/orderman/governor/Governor.h"
 #include "lib/orderman/orderman_utils/EventStream.h"
 #include "lib/orderman/orders/Order.h"
@@ -62,6 +63,10 @@ class OrderServices {
       _data->log.log(corid(), entry_kind, uint64_1, txt_1, uint32_1);
    }
 
+   std::vector<PositionUpdate> const& current_positions() {
+      return _data->position.current_positions();
+   }
+
    trace::ScopedPerfLogEntry log_scope(const char *entry_kind,
                                        uint64_t uint64_1 = 0,
                                        const char *txt_1 = "",
@@ -123,6 +128,15 @@ class OrderServices {
    void unsubscribe_from_time() { _time_change.unsub(_time_change_sub); }
    // Time
 
+
+   // Position Updates
+   template <class CB> void subscribe_to_position_updates(CB &&cb) {
+      _position_update_sub = _position_update.sub(
+          [this, cb](auto const &ev) { this->on_external_event(ev, cb); });
+   }
+   void unsubscribe_from_position_updates() { _position_update.unsub(_position_update_sub); }
+   // Position Updates
+   
    // Execution Reports
    using ExeReportSubID = EventStream<NewExecutionReportEvent>::SubID;
    template <class CB>
@@ -298,8 +312,10 @@ class OrderServices {
    EventStream<StatusUpdateEvent> _status_updates;
    EventStream<SettlementUpdateEvent> _settlement_updates;
    EventStream<mercury::TimerEvent, mercury::NoKey> _time_change;
+   EventStream<orderman::FilledPositionEvent, mercury::NoKey> _position_update;
    boost::container::small_vector<SessionServices, 20> _sessions;
    EventStream<mercury::TimerEvent, mercury::NoKey>::SubID _time_change_sub;
+   EventStream<orderman::FilledPositionEvent, mercury::NoKey>::SubID _position_update_sub;
    EventStream<CreateOrderReplyEvent> _create_order_reply;
 }; // namespace orderman
 } // namespace orderman

src/lib/orderman/actors/order_manager/OrderServicesData.h

@@ -35,6 +35,7 @@
 #include "lib/orderman/order_router/OrderRouter.h"
 #include "lib/utils/datetime.h"
 #include "lib/utils/Puid.h"
+#include "lib/orderman/actors/order_manager/PositionIndex.h"
 
 #include <boost/date_time/posix_time/posix_time.hpp>
 
@@ -87,6 +88,7 @@ class OrderServicesData {
    OrderManager &owner;
    trace::PerfLog &log;
    securities::MapSecurityDB sec_db;
+   PositionIndex position;
    OrderRouter order_router;
    Limiter limiter;
    OrderFactory order_factory;
@@ -101,6 +103,8 @@ class OrderServicesData {
        time_subs;
    std::unique_ptr<mercury::ContentSubscription<CreateOrderReplyEvent>>
        order_creation_subs;
+   std::unique_ptr<mercury::ContentSubscription<FilledPositionEvent,mercury::NoKey>>
+       position_subs;
 
    mercury::Stream<CreateOrderRequestEvent> create_order_req_pub;
    mercury::Stream<CreateFIXBufferRequestEvent> create_fix_req_pub;

src/lib/orderman/actors/order_manager/PositionIndex.cpp

@@ -0,0 +1,31 @@
+#include "PositionIndex.h"
+
+#include "lib/orderman/events/FilledPositionEvent.h"
+#include "lib/orderman/orders/algo/PositionUpdate.h"
+
+namespace bts {
+namespace orderman {
+
+struct PositionIndex::PImpl {
+   PImpl() : position_idx(), current_positions() {}
+   std::map<std::pair<SecurityKey,AccountID>,size_t> position_idx;
+   std::vector<PositionUpdate> current_positions;
+};
+
+void PositionIndex::update(FilledPositionEvent const&ev) {
+   auto &idx = me->position_idx[{SecurityKey(ev.security->id()),ev.account_id}];
+   if (idx == 0) {
+      me->current_positions.push_back({});
+      idx = me->current_positions.size();
+   }
+   me->current_positions[idx-1] = {ev.security,ev.account_id,ev.long_quantity, ev.short_quantity};
+}
+
+std::vector<PositionUpdate> const& PositionIndex::current_positions() const {
+   return me->current_positions;
+}
+
+PositionIndex::PositionIndex() : me (new PImpl()) {}
+PositionIndex::~PositionIndex() {}
+
+}}

src/lib/orderman/actors/order_manager/PositionIndex.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <memory>
+#include <vector>
+
+namespace bts {
+namespace orderman {
+
+class FilledPositionEvent;
+struct PositionUpdate;
+
+class PositionIndex {
+public:
+   PositionIndex();
+   ~PositionIndex(); 
+   void update(FilledPositionEvent const&);
+   std::vector<PositionUpdate> const &current_positions() const;
+private:
+   struct PImpl;
+   std::unique_ptr<PImpl> me;
+};
+
+}}

src/lib/orderman/events/OrderStateUpdateEvent.h

@@ -11,16 +11,16 @@ namespace bts {
 namespace orderman {
 
 class OrderStateUpdateEvent : public mercury::Event<OrderStateUpdateEvent> {
- public:
+public:
    using SynopsisPolicy = mercury::CacheAll<OrderStateUpdateEvent>;
 
    OrderStateUpdateEvent()
       : key(null_order_key()), parent_key(null_order_key()) {}
 
    OrderStateUpdateEvent(
-       OrderKey k, OrderKey pk,
-       std::shared_ptr<const api::orderman::OrderDefinition> const &def,
-       api::orderman::OrderState const &state) {
+      OrderKey k, OrderKey pk,
+      std::shared_ptr<const api::orderman::OrderDefinition> const &def,
+      api::orderman::OrderState const &state) {
       init(k, pk, def, state);
    }
 
@@ -41,18 +41,21 @@ class OrderStateUpdateEvent : public mercury::Event<OrderStateUpdateEvent> {
       display_status = state.display_status();
 
       custom = state.custom_case();
-
-      auto native_state = get_native_state(state);
-      if (native_state) {
-         exchange_order_id = native_state->exchange_order_id();
-         last_msg_seq_num = native_state->last_msg_seq_num();
-         sub_error_code = native_state->sub_error_code();
-         native_status = native_state->native_status();
-         routing_record = native_state->routing_record();
-         native_type = native_state->native_type();
-         orderman::str_to(native_state->last_transact_time(),
-                          &last_transact_time);
-         stop_px = native_state->stop_px();
+      if (custom == api::orderman::OrderState::kDynamic) {
+         dynamic_order_state = state.dynamic();
+      } else {
+         auto native_state = get_native_state(state);
+         if (native_state) {
+            exchange_order_id = native_state->exchange_order_id();
+            last_msg_seq_num = native_state->last_msg_seq_num();
+            sub_error_code = native_state->sub_error_code();
+            native_status = native_state->native_status();
+            routing_record = native_state->routing_record();
+            native_type = native_state->native_type();
+            orderman::str_to(native_state->last_transact_time(),
+                             &last_transact_time);
+            stop_px = native_state->stop_px();
+         }
       }
    }
 
@@ -70,7 +73,6 @@ class OrderStateUpdateEvent : public mercury::Event<OrderStateUpdateEvent> {
    uint32_t filled_at_attach_time_quantity;
    std::string last_msg;
    api::orderman::OrderState::DisplayStatus display_status;
-   
    api::orderman::OrderState::CustomCase custom;
 
    // native order state
@@ -82,7 +84,10 @@ class OrderStateUpdateEvent : public mercury::Event<OrderStateUpdateEvent> {
    api::orderman::NativeOrderType native_type;
    mercury::TimeStampStr last_transact_time;
    double stop_px;
-   
+
+   // dynamic order state
+   std::optional<api::orderman::DynamicOrderState> dynamic_order_state;
+
    api::orderman::OrderState quick_rebuild_state() const {
       api::orderman::OrderState result;
       result.set_status(status);
@@ -111,11 +116,11 @@ class OrderStateUpdateEvent : public mercury::Event<OrderStateUpdateEvent> {
          break;
       case api::orderman::OrderState::kStopLimit:
          rebuild_native_state(
-             result.mutable_stop_limit()->mutable_native_state());
+            result.mutable_stop_limit()->mutable_native_state());
          break;
       case api::orderman::OrderState::kGlbxIceberg:
          rebuild_native_state(
-             result.mutable_glbx_iceberg()->mutable_native_state());
+            result.mutable_glbx_iceberg()->mutable_native_state());
          break;
       case api::orderman::OrderState::kIceberg:
          result.mutable_iceberg();
@@ -163,7 +168,7 @@ class OrderStateUpdateEvent : public mercury::Event<OrderStateUpdateEvent> {
          result.mutable_prorata();
          break;
       case api::orderman::OrderState::kDynamic:
-         result.mutable_dynamic();
+         *result.mutable_dynamic() = dynamic_order_state.value();
          break;
       case api::orderman::OrderState::kSplitter:
          result.mutable_splitter();
@@ -175,7 +180,7 @@ class OrderStateUpdateEvent : public mercury::Event<OrderStateUpdateEvent> {
       return result;
    }
 
- private:
+private:
    void rebuild_native_state(api::orderman::NativeOrderState *st) const {
       st->set_stop_px(stop_px);
       st->set_exchange_order_id(exchange_order_id);

src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp

@@ -56,11 +56,12 @@ struct AlgoWrapperOrder::PImpl {
    OrderState const &state() const { return _state; }
 
    Map<SecurityKey, OrderServices::BookSubID> book_subs;
+   Map<SecurityKey, OrderServices::SettlementSubID> settlement_subs;
    Map<SecurityKey, OrderServices::TickerSubID> ticker_subs;
    Map<SecurityKey, OrderServices::StatusSubID> status_subs;
    Map<OrderKey, ChildRecord> children;
    bool has_timer_sub = false;
-
+   bool has_position_sub = false;
    bool is_state_dirty = false;
 
    Map<Edge, Action> transition_actions;
@@ -118,6 +119,13 @@ struct AlgoWrapperOrder::PImpl {
                                            : OrderState::HIDDEN);
    }
 
+   void update_custom_state(api::orderman::DynamicOrderState const&cstate) {
+      is_state_dirty = true;
+      bts_assert(definition.has_custom_def());
+      bts_assert(definition.custom_def().has_dynamic());
+      *_state.mutable_dynamic() = cstate;
+   }
+   
    void set_key(OrderKey k) { _my_key = k; }
 
  private:
@@ -382,6 +390,10 @@ api::Security const *AlgoWrapperOrder::lookup_security(SecurityKey k) const {
    return me->security_db->resolve(k);
 }
 
+std::vector<api::Security const *> AlgoWrapperOrder::known_securities() const {
+   return me->security_db->get_securities();
+}
+
 Price AlgoWrapperOrder::price() const { return Price(me->state().price()); }
 
 Quantity AlgoWrapperOrder::quantity() const {
@@ -396,6 +408,10 @@ bool AlgoWrapperOrder::is_shutting_down() const {
    return me->status == Status::SHUTTING_DOWN;
 }
 
+std::vector<PositionUpdate> const&AlgoWrapperOrder::current_positions() const {
+   return me->services->current_positions();
+}
+
 void AlgoWrapperOrder::subscribe_book(SecurityKey k) {
    if (!is_working())
       return;
@@ -416,6 +432,26 @@ void AlgoWrapperOrder::unsubscribe_book(SecurityKey k) {
       me->services->unsubscribe_from_book(it->second);
 }
 
+void AlgoWrapperOrder::subscribe_settlement(SecurityKey k) {
+   if (!is_working())
+      return;
+   bts_assert(me->settlement_subs.find(k) == me->settlement_subs.end());
+
+   me->settlement_subs[k] =
+       me->services->subscribe_to_settlements(k, [this, k](auto const &ev) {
+          ReportStateGuard _(*this);
+          me->algo->on_settlement(k, Price(ev.settlement_price.price()));
+       });
+}
+
+void AlgoWrapperOrder::unsubscribe_settlement(SecurityKey k) {
+   if (!is_working())
+      return;
+   bts_assert(me->settlement_subs.find(k) != me->settlement_subs.end());
+   if (auto it = me->settlement_subs.find(k); it != me->settlement_subs.end())
+      me->services->unsubscribe_from_settlements(it->second);
+}
+
 void AlgoWrapperOrder::subscribe_ticker(SecurityKey k) {
    if (!is_working())
       return;
@@ -497,6 +533,25 @@ void AlgoWrapperOrder::unsubscribe_timer() {
    me->has_timer_sub = false;
 }
 
+void AlgoWrapperOrder::subscribe_position() {
+   if (!is_working())
+      return;
+   bts_assert(!me->has_position_sub);
+   me->has_position_sub = true;
+   me->services->subscribe_to_position_updates([this](auto const &ev) {
+      ReportStateGuard _(*this);
+      me->algo->on_position({ev.security,ev.account_id,ev.long_quantity, ev.short_quantity});
+   });
+}
+
+void AlgoWrapperOrder::unsubscribe_position() {
+   if (!is_working())
+      return;
+   bts_assert(me->has_position_sub);
+   me->services->unsubscribe_from_position_updates();
+   me->has_position_sub = false;
+}
+
 bool AlgoWrapperOrder::is_attaching() const {
    return me->attach_target != null_order_key();
 }
@@ -610,6 +665,12 @@ void AlgoWrapperOrder::set_ladder_visibility(bool is_visible) {
    me->set_ladder_visibility(is_visible);
 }
 
+void AlgoWrapperOrder::update_custom_state(api::orderman::DynamicOrderState const&cstate) {
+   if (!is_working())
+      return;
+   me->update_custom_state(cstate);
+}
+
 void AlgoWrapperOrder::cancel(std::string const &msg) {
    if (!is_working())
       return;
@@ -841,6 +902,8 @@ void AlgoWrapperOrder::check_invariants() {
       bts_assert(is_attaching());
       bts_assert(me->external_status() == ORDER_STATUS_WORKING);
       bts_assert(me->book_subs.empty());
+      bts_assert(me->settlement_subs.empty());
+      bts_assert(me->ticker_subs.empty());
       bts_assert(me->children.empty());
       bts_assert(!me->has_timer_sub);
       bts_assert(!me->pending_terminate.has_value());
@@ -860,6 +923,8 @@ void AlgoWrapperOrder::check_invariants() {
       bts_assert(me->security);
       bts_assert(me->external_status() == ORDER_STATUS_WORKING);
       bts_assert(me->book_subs.empty());
+      bts_assert(me->settlement_subs.empty());
+      bts_assert(me->ticker_subs.empty());
       bts_assert(!me->has_timer_sub);
       bts_assert(!me->pending_update.has_value());
       bts_assert(me->pending_terminate.has_value());
@@ -873,6 +938,8 @@ void AlgoWrapperOrder::check_invariants() {
                  me->external_status() == ORDER_STATUS_ERROR ||
                  me->external_status() == ORDER_STATUS_REJECTED_BY_SPARK);
       bts_assert(me->book_subs.empty());
+      bts_assert(me->settlement_subs.empty());
+      bts_assert(me->ticker_subs.empty());
       bts_assert(!me->has_timer_sub);
       bts_assert(!me->pending_update.has_value());
       bts_assert(!me->pending_terminate.has_value());
@@ -919,10 +986,18 @@ void AlgoWrapperOrder::shutdown(api::orderman::OrderStatus st,
 }
 
 void AlgoWrapperOrder::cleanup() {
-   for (auto const &s : me->book_subs) {
+   for (auto const &s : me->book_subs) 
       me->services->unsubscribe_from_book(s.second);
-      me->book_subs.clear();
-   }
+   me->book_subs.clear();
+
+   for (auto const &s : me->settlement_subs) 
+      me->services->unsubscribe_from_settlements(s.second);
+   me->settlement_subs.clear();
+
+   for (auto const &s : me->ticker_subs) 
+      me->services->unsubscribe_from_ticker(s.second);
+   me->ticker_subs.clear();
+
    for (auto &c : me->children)
       c.second.order.cancel();
    if (me->has_timer_sub)

src/lib/orderman/orders/algo/AlgoWrapperOrder.h

@@ -33,6 +33,8 @@ class AlgoWrapperOrder : public Order,
    OrderKey order_key() const override;
    OrderKey parent_order_key() const override;
    api::Security const *lookup_security(SecurityKey k) const override;
+   std::vector<api::Security const *> known_securities() const override;
+
    bool is_attaching() const override;
    ChildTemplate &make_child_template(api::orderman::OrderDefinition const &,
                                       size_t) override;
@@ -41,8 +43,12 @@ class AlgoWrapperOrder : public Order,
    Quantity quantity() const override;
    Quantity total_fills() const override;
    bool is_shutting_down() const override;
+   std::vector<PositionUpdate> const&current_positions() const override;
+
    void subscribe_book(SecurityKey) override;
    void unsubscribe_book(SecurityKey) override;
+   void subscribe_settlement(SecurityKey) override;
+   void unsubscribe_settlement(SecurityKey) override;
    void subscribe_child_change(OrderKey) override;
    void unsubscribe_child_change(OrderKey) override;
    void subscribe_timer() override;
@@ -51,6 +57,8 @@ class AlgoWrapperOrder : public Order,
    void unsubscribe_ticker(SecurityKey) override;
    void subscribe_status(SecurityKey) override;
    void unsubscribe_status(SecurityKey) override;
+   void subscribe_position() override;
+   void unsubscribe_position() override;
 
    OrderKey launch_child(ChildTemplate &, Price, Quantity,
                          bool subscribe = true) override;
@@ -64,6 +72,7 @@ class AlgoWrapperOrder : public Order,
    void terminate_error(std::string const &msg) override;
    void complete() override;
    void update_fill_state(Quantity total_fills, Price last_price) override;
+   void update_custom_state(api::orderman::DynamicOrderState const&) override;
    void set_ladder_visibility (bool is_visibile) override;
    
    // Private

src/lib/orderman/orders/algo/IAlgoOrder.cpp

@@ -3,6 +3,7 @@
 
 namespace bts {
 namespace orderman {
+#include "lib/orderman/orders/algo/IAlgoOrder.h"
 IAlgoOrder::~IAlgoOrder() {}
 
 OrderOpResult IAlgoOrder::do_before_launch(IBeforeLaunchAlgoServices &) {

src/lib/orderman/orders/algo/IAlgoOrder.h

@@ -2,6 +2,8 @@
 
 #include "lib/orderman/orders/Order.h"
 #include "lib/orderman/events/TickerUpdateEvent.h"
+#include "lib/orderman/orders/algo/PositionUpdate.h"
+
 #include <memory>
 
 namespace bts {
@@ -44,6 +46,12 @@ class IAlgoOrder {
    // The trading status for the specified security has changed
    virtual void on_status(SecurityKey, co2::TradingStatus) {};
 
+   // The settlement price for the specified security has changed
+   virtual void on_settlement(SecurityKey, Price) {};
+   
+   // The current position for the security, account combination
+   virtual void on_position(PositionUpdate const&) {};
+
    // A periodic timer has fired
    virtual void on_timer() {}
 

src/lib/orderman/orders/algo/IAlgoServices.h

@@ -1,10 +1,15 @@
 #pragma once
 
 #include "lib/orderman/orders/algo/IBeforeLaunchAlgoServices.h"
+#include "lib/orderman/orders/algo/PositionUpdate.h"
 
 namespace bts {
 namespace orderman {
 
+
+// Services to an algo *after* it has been launched. See
+// IBeforeLaunchAlgoServices for services available before launch,
+// that is outside of the latency sensitive path.
 class IAlgoServices : public IBeforeLaunchAlgoServices {
  public:
    // --------------------------------------------------------------
@@ -26,6 +31,13 @@ class IAlgoServices : public IBeforeLaunchAlgoServices {
    // cancel child orders will be ignored.
    virtual bool is_shutting_down() const = 0;
 
+   // Returns the current position per security and account. The
+   // returned reference is only valid immediately after the
+   // call. Callers should make a copy if they wish to use it beyond
+   // that. Callers can keep track of changes to the position by
+   // subscribing to the position update events below.
+   virtual std::vector<PositionUpdate> const&current_positions() const = 0;
+
    // --------------------------------------------------------------
    // Event Subscriptions
    // --------------------------------------------------------------
@@ -34,6 +46,10 @@ class IAlgoServices : public IBeforeLaunchAlgoServices {
    virtual void subscribe_book(SecurityKey) = 0;
    virtual void unsubscribe_book(SecurityKey) = 0;
 
+   // Subscribe to settlement prices for the specified security
+   virtual void subscribe_settlement(SecurityKey) = 0;
+   virtual void unsubscribe_settlement(SecurityKey) = 0;
+
    // Subscribe to child order updates for the specified child
    // order. This is rarely needed since launch_child() by default
    // automatically subscribes to new orders.
@@ -52,6 +68,10 @@ class IAlgoServices : public IBeforeLaunchAlgoServices {
    virtual void subscribe_status(SecurityKey) = 0;
    virtual void unsubscribe_status(SecurityKey) = 0;
 
+   // Subscribe to position updates
+   virtual void subscribe_position() = 0;
+   virtual void unsubscribe_position() = 0;
+
    // --------------------------------------------------------------
    // Child management
    // --------------------------------------------------------------
@@ -102,6 +122,9 @@ class IAlgoServices : public IBeforeLaunchAlgoServices {
    // Set wheter this algo instance should be displayed in the Spark UI ladder
    virtual void set_ladder_visibility(bool is_visible) = 0;
 
+   // Report custom algo state for display in the Spark UI
+   virtual void update_custom_state(api::orderman::DynamicOrderState const&) = 0;
+   
    // Update this algos's filled quantity and last price
    virtual void update_fill_state(Quantity total_fills, Price last_price) = 0;
 };

src/lib/orderman/orders/algo/IBeforeLaunchAlgoServices.h

@@ -31,6 +31,9 @@ class IBeforeLaunchAlgoServices {
    // The security identified by k or nullptr if no such security exists
    virtual api::Security const *lookup_security(SecurityKey k) const = 0;
 
+   // Known active securities
+   virtual std::vector<api::Security const*> known_securities() const = 0;
+   
    // Whether this algo is being attached to an existing order
    // If an algo does not support attaching it can return an error 
    // if is_attaching() == true from do_before_launch.

src/lib/orderman/orders/algo/PositionUpdate.cpp

@@ -0,0 +1,15 @@
+#include "PositionUpdate.h"
+#include "lib/orderman/orderman_utils/orderman_common.h"
+
+namespace bts {
+namespace orderman {
+
+std::ostream& operator<<(std::ostream&out,const PositionUpdate &update) {
+   out << "Security:  " << update.security->id()
+       << " Account: " << update.account_id.value()
+       << " Long Qty: " << update.long_quantity.value()
+       << " Short Qty: " << update.short_quantity.value();
+   return out;
+}
+
+}}

src/lib/orderman/orders/algo/PositionUpdate.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "lib/domain_model/common.h"
+#include "lib/orderman/orderman_utils/orderman_common.h"
+
+namespace bts {
+namespace api { class Security;}
+
+namespace orderman {
+
+struct PositionUpdate {
+   api::Security const *security;
+   AccountID account_id;
+   Quantity long_quantity;
+   Quantity short_quantity;
+};
+
+std::ostream& operator<<(std::ostream&,const PositionUpdate &);
+
+}}
+

src/lib/orderman/orders/algo/algo_orders/RegisterDorman.cpp

@@ -8,6 +8,7 @@
 #include "DormanAlgo.h"
 #include "PyramidAlgo.h"
 #include "TASAlgo.h"
+#include "TASCounterAlgo.h"
 
 using namespace bts;
 using namespace api::orderman;
@@ -148,6 +149,45 @@ extern "C" void init_pyramid_plugin(bts::SparkServices& srv) {
    srv.register_dynamic_algo_order(join_desc, []() { return std::make_unique<orderman::PyramidAlgo>(); });
 }
 
+extern "C" void init_tascounter_plugin(bts::SparkServices& srv) {
+   DynamicOrderDescriptor join_desc;
+
+   join_desc.set_dynamic_order_type("TASCounter");
+   // all 6 contracts. They can theoretically be inferred from the front month
+   // since T1 are calendars 1 month apart starting from the front month
+   // ex Mar-May, May-Jul, Jul-Sep being T1 front, second and third respectively
+   // (assuming Mar is the front month)
+   *join_desc.add_param_desc() = param_security_key("T1 Front");
+   *join_desc.add_param_desc() = param_security_key("T1 Second");
+   *join_desc.add_param_desc() = param_security_key("T1 Third");
+   *join_desc.add_param_desc() = param_security_key("T2 Front");
+   *join_desc.add_param_desc() = param_security_key("T2 Second");
+   *join_desc.add_param_desc() = param_security_key("T3 Front");
+   // tas contracts (could have been derived)
+   *join_desc.add_param_desc() = param_security_key("T1 Front TAS");
+   *join_desc.add_param_desc() = param_security_key("T1 Second TAS");
+   *join_desc.add_param_desc() = param_security_key("T1 Third TAS");
+   // automatic orders
+   *join_desc.add_param_desc() = param_uint32("T1 Front Time");
+   *join_desc.add_param_desc() = param_uint32("T1 Front Qty");
+   *join_desc.add_param_desc() = param_bool("T1 Front Order Enabled");
+
+   *join_desc.add_param_desc() = param_uint32("T1 Second Time");
+   *join_desc.add_param_desc() = param_uint32("T1 Second Qty");
+   *join_desc.add_param_desc() = param_bool("T1 Second Order Enabled");
+
+   *join_desc.add_param_desc() = param_uint32("T1 Third Time");
+   *join_desc.add_param_desc() = param_uint32("T1 Third Qty");
+   *join_desc.add_param_desc() = param_bool("T1 Third Order Enabled");
+   // settlement time
+   *join_desc.add_param_desc() = param_uint32("Settlement Start Hour");
+   *join_desc.add_param_desc() = param_uint32("Settlement Start Minutes");
+   *join_desc.add_param_desc() = param_uint32("Settlement Start Seconds");
+   *join_desc.add_param_desc() = param_uint32("Settlement Period (seconds)");
+   
+   srv.register_dynamic_algo_order(join_desc, []() { return std::make_unique<orderman::TASCounterAlgo>(); });
+}
+
 extern "C" void init_tas_plugin(bts::SparkServices& srv) {
    DynamicOrderDescriptor tas_desc;
 
@@ -171,6 +211,8 @@ extern "C" void init_spark_plugin(bts::SparkServices &srv) {
       init_pyramid_plugin(srv);
    if(dyn_algo_config.enable_tas())
       init_tas_plugin(srv);
+   if(dyn_algo_config.enable_tascounter())
+      init_tascounter_plugin(srv);
 }
 
 }}}

src/lib/orderman/orders/algo/algo_orders/TASCounterAlgo.cpp

@@ -0,0 +1,502 @@
+// TODO
+// put settle time data in the input (X)
+// better names (X)
+// at end of settle, order should dettach and complete (X)
+// order going out logic (X)
+// settlement compute logic (X)
+// actual publish (X)
+// no PTime
+// check settle calc (X)
+// check order shooting logic
+
+#include "lib/orderman/orders/algo/algo_orders/TASCounterAlgo.h"
+
+#include "lib/domain_model/Security.h"
+#include "lib/orderman/order_factory/IOrderFactory.h"
+#include "lib/orderman/orderman_utils/Book.h"
+#include "lib/orderman/orderman_utils/orderman_utils.h"
+#include "lib/orderman/orders/algo/IAlgoServices.h"
+#include "lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h"
+#include "lib/os/os.h"
+#include "lib/domain_model/Security.h"
+#include "lib/utils/PBUtils.h"
+#include "lib/utils/datetime.h"
+#include <google/protobuf/timestamp.pb.h>
+#include "lib/orderman/actors/SODPosition.h"
+#include <google/protobuf/util/time_util.h>
+#include <stdint.h>
+
+namespace bts {
+namespace orderman {
+
+using namespace api::orderman;
+using namespace google::protobuf;
+
+enum class TASCounterAlgo::Status { WORKING, DONE };
+
+// book and trade data for each of the 6 contracts as well as the TAS contracts
+// (the vwap of the TAS contracts is not used, but for uniformity is kept in the
+// the same structure)
+struct ContractMarketData{
+   uint32_t tot_vol = 0;
+   double running_sum = 0;
+   Book book;
+   double estimated_settle = 0.0;
+   void on_trade (const TickerUpdateEvent::Trade& t){
+      tot_vol += t.quantity.value();
+      running_sum += t.price.value()*t.quantity.value();
+   }
+   const double vwap(){
+      return tot_vol == 0 ? os::nan() : running_sum/tot_vol;
+   }
+};
+// the input for the t1 front, second and third orders, as well as the template
+// and working order key are kept here
+struct ContractAutoOrder{
+   // inputs
+   uint32_t n_secs = 0;
+   uint32_t qty = 0;
+   bool enabled = false;
+   // dont know yet which side will be sent
+   ChildTemplate* buy_ord = nullptr;
+   ChildTemplate* sell_ord = nullptr;
+   // computed input
+   // the is AlgoData::settle_stop - n_secs
+   Timestamp algo_start;
+   // output
+   std::optional<OrderKey> tas_order = std::nullopt;
+};
+
+struct AlgoData{
+   double tick_size;
+   Timestamp settle_start;
+   Timestamp settle_stop;
+   ContractMarketData t1_front,
+                      t1_second,
+                      t1_third,
+                      t2_front,
+                      t2_second,
+                      t3_front;
+   ContractMarketData t1_front_tas,
+                      t1_second_tas,
+                      t1_third_tas;
+   ContractAutoOrder t1_front_order,
+                     t1_second_order,
+                     t1_third_order;
+   DynamicOrderState out_state;
+};
+
+struct TASCounterAlgo::PImpl {
+   // reporting
+   uint16_t reporting_period_msecs = 500;
+   bts::datetime::PTime last_report_time = bts::datetime::now();
+   // immutable state
+   IAlgoServices* algo_services = nullptr;
+   // mutable state
+   AlgoData data;
+   // to avoid lookups on md and ticker callbacks
+   std::map<SecurityKey,ContractMarketData*> md_map;
+   Status state = Status::WORKING;   
+};
+
+//fwd decls
+void _copy_output(AlgoData&);
+bool _copy_settle_times (AlgoData&,const Map<std::string,DynamicOrderParam>&);
+void _copy_order_data (AlgoData&,const Map<std::string,DynamicOrderParam>&);
+bool _setup_data(IBeforeLaunchAlgoServices&, AlgoData&,
+                 std::map<SecurityKey,ContractMarketData*>&);
+double round_to_nearest(double price, double tick_size);                 
+long long t1_minus_t2_usecs (const Timestamp&, const Timestamp&);
+long long t_minus_now_usecs (const google::protobuf::Timestamp&);
+bool trade_within_time_win (const Timestamp&, const Timestamp&,
+                            const TickerUpdateEvent::Trade& t);
+bool now_within_time_win (const Timestamp&, const Timestamp&);
+
+
+TASCounterAlgo::TASCounterAlgo() : me(new PImpl()) {}
+TASCounterAlgo::~TASCounterAlgo() {}
+// initialization /////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////
+OrderOpResult TASCounterAlgo::do_before_launch(IBeforeLaunchAlgoServices& serv) {
+   if(_setup_data(serv,me->data,me->md_map))
+      return {};
+   return {OrderError::ORDER_ERROR_INVALID_PARAMETER,
+           std::string("Parameters are not well formed")};
+}
+void TASCounterAlgo::do_launch(IAlgoServices& algo_services) {
+   me->algo_services = &algo_services;
+   algo_services.subscribe_timer();
+   for (const auto& kp : me->md_map){
+      algo_services.subscribe_book(kp.first);
+      algo_services.subscribe_ticker(kp.first);      
+   }
+}
+void TASCounterAlgo::do_launch_attached(
+                                 IAlgoServices& algo_services, 
+                                 OrderKey,
+                                 ChildState) {
+   // not allowed
+   me->algo_services = &algo_services;
+}
+// updates ////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////
+void TASCounterAlgo::do_update(const api::orderman::Update& u) {
+   if(me->state == Status::DONE)
+      return;
+   const auto& p = u.custom().dynamic().order_params();
+   // only order data change is allowed
+   _copy_order_data(me->data,p);
+   _check_and_send_order();
+}
+// callbacks //////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////
+void TASCounterAlgo::on_book(SecurityKey k, const Book& b) {
+   if(me->state == Status::DONE)
+      return;
+   const auto& it = me->md_map.find(k);
+   if(it != me->md_map.end()){
+      it->second->book = b;
+      _check_and_send_order();
+   }
+}
+void TASCounterAlgo::on_ticker(SecurityKey k, const TradeVector& v){
+   if(me->state == Status::DONE)
+      return;
+   const auto& it = me->md_map.find(k);   
+   if(it != me->md_map.end()){
+      auto contract_md = it->second;
+      for (const auto& t : v)
+         if (_within_settle()){
+            contract_md->on_trade(t);
+         }
+      _check_and_send_order();      
+   }
+}
+
+// this only gets called on an explicit algo cancel
+// from the outside
+TASCounterAlgo::OnCancelResponse TASCounterAlgo::on_cancel(){
+   return IAlgoOrder::OnCancelResponse::HONOR;
+}
+
+void TASCounterAlgo::on_timer(){
+   if(me->state == Status::DONE)
+      return;      
+   _check_and_send_order();
+   if(bts::datetime::msecs_since(me->last_report_time) >= 
+                                 me->reporting_period_msecs){
+      me->last_report_time = bts::datetime::now();
+      _report_state();
+   }
+   // check for end and exit
+   if (t1_minus_t2_usecs(util::TimeUtil::GetCurrentTime(),
+                         me->data.settle_stop) > 0){
+      std::string msg = "session over";
+      _cxl_and_end(false,msg);
+   }
+}
+
+IAlgoServices& TASCounterAlgo::_algo() const { return *me->algo_services; }
+
+void TASCounterAlgo::_switch_state(TASCounterAlgo::Status new_state){
+   switch (new_state){
+      case (TASCounterAlgo::Status::WORKING):
+         bts_assert(false);
+         break;
+      case (TASCounterAlgo::Status::DONE):
+         bts_assert(me->state == TASCounterAlgo::Status::WORKING);
+         break;
+   }
+   LOG(OrderMan, Info,"TASCounterAlgo::_switch_state key:{} {}->{} ",
+                      _algo().order_key(),
+                      static_cast<int>(me->state), 
+                      static_cast<int>(new_state));
+   me->state = new_state;
+}
+
+void TASCounterAlgo::_cxl_and_end (bool is_error, const std::string& err_msg){
+   if(me->state == Status::DONE)
+      return;
+   if(is_error)
+      _algo().terminate_error(err_msg);
+   else
+      _algo().complete();
+   _switch_state(TASCounterAlgo::Status::DONE);      
+}
+
+void TASCounterAlgo::_compute_settles(){
+   AlgoData& d = me->data;
+   d.t1_front.estimated_settle = d.t1_front.vwap();
+   d.t1_second.estimated_settle = d.t2_front.vwap() - d.t1_front.vwap();
+   d.t1_third.estimated_settle = d.t3_front.vwap() - d.t1_front.vwap() - 
+                                 d.t1_second.vwap();
+   // round to closest value
+   d.t1_front.estimated_settle = round_to_nearest(
+                        d.t1_front.estimated_settle,me->data.tick_size);
+   d.t1_second.estimated_settle = round_to_nearest(
+                        d.t1_second.estimated_settle,me->data.tick_size);
+   d.t1_third.estimated_settle = round_to_nearest(
+                        d.t1_third.estimated_settle,me->data.tick_size);
+}
+
+// the decision to send an order can be caused by
+// 1. change in settings (=> time window, enabling orders etc)
+// 2. book update (=> TAS book can result in order)
+// 3. ticker (=> new estimated settlement)
+// 4. time passing (=> in the window now)
+void TASCounterAlgo::_check_and_send_order(){
+   _compute_settles();
+   auto should_send_tas_order = 
+                     [](const ContractMarketData& c,
+                         const ContractAutoOrder& o,
+                         const ContractMarketData& tas_c,
+                         bool is_buy,
+                         double& out_price,
+                         uint32_t& out_qty) -> bool {
+      // buying the tas at the offer or selling it at the bid                      
+      const auto& tas_opp_top = is_buy ? tas_c.book.ask()[0] : tas_c.book.bid()[0];
+      const auto& c_same_top = is_buy ? c.book.bid()[0] : c.book.ask()[0];
+      if(tas_opp_top.size() > 0 && c_same_top.size() > 0){
+         // delivery price as defined by where the TAS order is traded
+         double tas_delv_price = c.estimated_settle + tas_opp_top.price();
+         // if the delv price for the buy order is <= contract bid or
+         // or the delv price for the sell order is >= contract offer
+         // then we are buying at the bid or selling at the offer 
+         double c_price = c_same_top.price();
+         if((is_buy && tas_delv_price <= c_price) ||
+            (!is_buy && tas_delv_price >= c_price)){
+            out_qty = std::min(o.qty,tas_opp_top.size());
+            out_price = tas_opp_top.price();
+            return true;
+         }
+      }
+      return false;
+   };
+
+   auto _send_order_tn = [&](
+                            const ContractMarketData& c,
+                            ContractAutoOrder& o,
+                            const ContractMarketData& tas_c) -> void {
+      if(o.enabled && o.n_secs > 0 && o.qty > 0 &&
+         !o.tas_order.has_value() &&
+         now_within_time_win(o.algo_start,me->data.settle_stop)){
+            double price;
+            uint32_t qty;
+            if(should_send_tas_order(c,o,tas_c,true,price,qty)){
+               o.tas_order = _algo().launch_child(*o.buy_ord,
+                                                   Price(price),
+                                                   Quantity(qty));
+            }
+            else if(should_send_tas_order(c,o,tas_c,false,price,qty)){
+               o.tas_order = _algo().launch_child(*o.sell_ord,
+                                                   Price(price),
+                                                   Quantity(qty));
+            }
+      }
+   };
+   _send_order_tn(me->data.t1_front,me->data.t1_front_order,me->data.t1_front_tas);
+   _send_order_tn(me->data.t1_second,me->data.t1_second_order,me->data.t1_second_tas);
+   _send_order_tn(me->data.t1_third,me->data.t1_third_order,me->data.t1_third_tas);
+}
+
+void TASCounterAlgo::_report_state(){
+   _compute_settles();
+   _copy_output(me->data);
+   _algo().update_custom_state(me->data.out_state);
+}
+
+bool TASCounterAlgo::_within_settle() const{
+   return now_within_time_win(me->data.settle_start,me->data.settle_stop);
+}
+///////////////////////////////////////////////////////////////////////////////
+// stateless conversion ///////////////////////////////////////////////////////
+
+// stateless helpers
+ChildTemplate* _create_order_template (IBeforeLaunchAlgoServices& serv,
+                                       SecurityKey key,
+                                       api::Side side){
+   OrderDefinition order_def = serv.definition();
+   order_def.set_security_key (key.value());
+   order_def.set_side(side);   
+   order_def.mutable_custom_def()->mutable_limit()->MergeFrom (
+                                       api::orderman::LimitOrderDefinition());
+   return &(serv.make_child_template(order_def));
+}
+// copies output into the dynamic state for publishing
+void _copy_output(AlgoData& d){
+   auto& m = *(d.out_state.mutable_custom_state());
+   // we can safely assume its within int32
+   auto secs_to_settle = static_cast<int>(t_minus_now_usecs(d.settle_start)/1000000);
+   // if its in settle (-ve value), its set to 0
+   secs_to_settle = std::max(secs_to_settle,0);
+   m["Seconds To Settle"].set_as_int32(secs_to_settle);
+   m["VWAP T1-Front"].set_as_double(d.t1_front.vwap());
+   m["VWAP T1-Second"].set_as_double(d.t1_second.vwap());
+   m["VWAP T1-Third"].set_as_double(d.t1_third.vwap());
+   m["VWAP T2-Front"].set_as_double(d.t2_front.vwap());
+   m["VWAP T2-Second"].set_as_double(d.t2_second.vwap());
+   m["VWAP T3-Front"].set_as_double(d.t3_front.vwap());
+   m["T1-Front Projected Settle"].set_as_double(d.t1_front.estimated_settle);
+   m["T1-Second Projected Settle"].set_as_double(d.t1_second.estimated_settle);
+   m["T1-Third Projected Settle"].set_as_double(d.t1_third.estimated_settle);
+}
+// copies the settle time/period part of the input data
+bool _copy_settle_times (AlgoData& d,
+                         const Map<std::string,DynamicOrderParam>& params){
+   auto hh = get_if<uint32_t>(params, "Settlement Start Hour");
+   auto mm = get_if<uint32_t>(params, "Settlement Start Minutes");
+   auto ss = get_if<uint32_t>(params, "Settlement Start Seconds");
+   auto settle_period_secs = get_if<uint32_t>(params, "Settlement Period (seconds)");
+   if (hh.has_value() && mm.has_value() && 
+       ss.has_value() && settle_period_secs.has_value() &&
+       hh.value() <= 23 && mm.value() <= 59 && ss.value() <= 59){
+      d.settle_start = SODPosition::today_local_time_to_utc_timestamp(
+            hh.value(),
+            mm.value(),
+            ss.value(),
+            true,
+            false);
+      d.settle_stop = SODPosition::add_seconds(d.settle_start,
+                                               settle_period_secs.value());
+      auto settle_start_txt = util::TimeUtil::ToString(d.settle_start);
+      auto settle_stop_txt = util::TimeUtil::ToString(d.settle_stop);
+      LOG(OrderMan, Info, "TASCounterAlgo::_copy_settle_times "
+          "settle_start:{} settle_stop:{} ",
+          settle_start_txt,
+          settle_stop_txt);      
+      return true;
+   }
+   return false;
+}
+// copies the order execution part of the input data
+void _copy_order_data (AlgoData& out_d,
+                       const Map<std::string,DynamicOrderParam>& params){   
+   auto copy_o = [&](ContractAutoOrder& out_order,
+                     std::string time_name,
+                     std::string qty_name,
+                     std::string enabled_name
+                     ) -> bool {
+      out_order.enabled = false;
+      auto time = get_if<uint32_t>(params, time_name);
+      auto qty = get_if<uint32_t>(params, qty_name);
+      auto enabled = get_if<bool>(params, enabled_name);
+      if(time.has_value() && qty.has_value() && enabled.has_value()){
+         out_order.n_secs = time.value();
+         out_order.qty = qty.value();
+         out_order.enabled = enabled.value();
+         out_order.algo_start = SODPosition::add_seconds(out_d.settle_stop,
+                                                         -1 * out_order.n_secs);
+         return true;
+      }
+      return false;
+   };
+   copy_o(out_d.t1_front_order,                                 
+          "T1 Front Time",
+          "T1 Front Qty",
+          "T1 Front Order Enabled");
+   copy_o(out_d.t1_second_order,
+          "T1 Second Time",
+          "T1 Second Qty",
+          "T1 Second Order Enabled");
+   copy_o(out_d.t1_third_order,
+          "T1 Third Time",
+          "T1 Third Qty",
+          "T1 Third Order Enabled");
+}
+// copies all of the input params (in the algo) into proper structures
+bool _setup_data(IBeforeLaunchAlgoServices& serv,
+                 AlgoData& out_d,
+                 std::map<SecurityKey,ContractMarketData*>& out_md_map){
+   auto& def = serv.definition();
+   // [ captures ] (parameters) -> returnTypesDeclaration { lambdaStatements; }   
+   auto init_contract_bk = [&out_md_map](auto& out_contract_md, 
+                            const auto& params,
+                            auto param_name) -> void {
+      auto k = get_if<uint32_t>(params, param_name);
+      if(k.has_value()){
+         out_contract_md.book = Book(SecurityKey(k.value()));
+         out_md_map[out_contract_md.book.security_key()] = &out_contract_md;   
+      }
+   };
+   if (def.has_custom_def() && def.custom_def().has_dynamic()){
+      out_d.tick_size = serv.security().contract_spec().low_tick();
+      const auto& p = def.custom_def().dynamic().order_params();
+      // settlement time
+      if(!_copy_settle_times(out_d,p))
+         return false;
+      // main contracts
+      init_contract_bk(out_d.t1_front, p,"T1 Front");
+      init_contract_bk(out_d.t1_second, p,"T1 Second");
+      init_contract_bk(out_d.t1_third, p,"T1 Third");
+      init_contract_bk(out_d.t2_front, p,"T2 Front");
+      init_contract_bk(out_d.t2_second, p,"T2 Second");
+      init_contract_bk(out_d.t3_front, p,"T3 Front");
+      // TAS contracts
+      init_contract_bk(out_d.t1_front_tas, p,"T1 Front TAS");
+      init_contract_bk(out_d.t1_second_tas, p,"T1 Second TAS");
+      init_contract_bk(out_d.t1_third_tas, p,"T1 Third TAS");
+      // auto orders
+      _copy_order_data(out_d,p);      
+      // order template
+      out_d.t1_front_order.buy_ord = _create_order_template(serv,
+                        out_d.t1_front_tas.book.security_key(),api::Side::SIDE_BUY);
+      out_d.t1_front_order.sell_ord = _create_order_template(serv,
+                        out_d.t1_front_tas.book.security_key(),api::Side::SIDE_SELL);
+      out_d.t1_second_order.buy_ord = _create_order_template(serv,
+                        out_d.t1_second_tas.book.security_key(),api::Side::SIDE_BUY);
+      out_d.t1_second_order.sell_ord = _create_order_template(serv,
+                        out_d.t1_second_tas.book.security_key(),api::Side::SIDE_SELL);
+      out_d.t1_third_order.buy_ord = _create_order_template(serv,
+                        out_d.t1_third_tas.book.security_key(),api::Side::SIDE_BUY);
+      out_d.t1_third_order.sell_ord = _create_order_template(serv,
+                        out_d.t1_third_tas.book.security_key(),api::Side::SIDE_SELL);
+      return true;
+   }
+   return false;
+}
+double round_to_nearest(double price, double tick_size){
+   if(os::is_nan(price))
+      return price;
+   auto buy_price = bts::securities::raw_round_to_quote_price(
+                                                         price,
+                                                         tick_size,
+                                                         api::Side::SIDE_BUY);
+   auto sell_price = buy_price + tick_size;
+   return std::abs(price-buy_price) < std::abs(sell_price-price) ? buy_price :
+                                                                   sell_price;
+}
+// time helpers
+// converts std::chrono::system_clock::time_point to google timestamp
+google::protobuf::Timestamp to_pf_ts (const bts::TimeStamp& ts){   
+   auto ts_us = std::chrono::time_point_cast<std::chrono::microseconds>(ts);
+   auto ts_us_val = ts_us.time_since_epoch();
+   auto usecs = ts_us_val.count();
+   return google::protobuf::util::TimeUtil::MicrosecondsToTimestamp(usecs);
+}
+long long t1_minus_t2_usecs (const Timestamp& t1,const Timestamp& t2){
+   return util::TimeUtil::TimestampToMicroseconds(t1) -
+          util::TimeUtil::TimestampToMicroseconds(t2);
+}
+long long t_minus_now_usecs (const Timestamp& t){
+   auto now = util::TimeUtil::GetCurrentTime();
+   return t1_minus_t2_usecs(t,now);
+}
+bool within_time_win (const Timestamp& beg, const Timestamp& end,
+                      const Timestamp& ts){
+   return t1_minus_t2_usecs(ts,beg)>=0 &&
+          t1_minus_t2_usecs(ts,end)<=0;
+}
+// if current time is within time window
+bool now_within_time_win (const Timestamp& beg,
+                      const Timestamp& end){
+   return within_time_win(beg,end,util::TimeUtil::GetCurrentTime());
+}
+bool trade_within_time_win (const Timestamp& beg, const Timestamp& end,
+                            const TickerUpdateEvent::Trade& t){
+   auto ts = to_pf_ts(t.ts);
+   return within_time_win(beg,end,ts);
+}
+
+} // namespace orderman
+
+} // namespace bts

src/lib/orderman/orders/algo/algo_orders/TASCounterAlgo.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "lib/orderman/orders/algo/IAlgoOrder.h"
+
+namespace bts {
+namespace orderman {
+
+class ChildTemplate;
+
+class TASCounterAlgo : public IAlgoOrder {
+public:
+   TASCounterAlgo();
+   ~TASCounterAlgo();
+   struct TASCounterAlgoParams{
+   };
+public:
+   OrderOpResult do_before_launch(IBeforeLaunchAlgoServices &) override;
+   void do_launch(IAlgoServices &) override;
+   void do_launch_attached(IAlgoServices &, OrderKey, ChildState) override;
+   void do_update(api::orderman::Update const &) override;
+   void on_book(SecurityKey, Book const &) override;
+   void on_ticker(SecurityKey, TradeVector const&) override;
+   IAlgoOrder::OnCancelResponse on_cancel() override;
+   void on_timer() override;
+private:
+   enum class Status;
+
+   struct PImpl;
+   std::unique_ptr<PImpl> me;
+private:
+   IAlgoServices& _algo() const ;
+   void _switch_state (Status new_status);
+   void _cxl_and_end (bool is_error, const std::string& err_msg);
+   void _check_and_send_order();
+   void _compute_settles ();
+   void _report_state();
+   bool _within_settle() const;
+};
+
+} // namespace orderman
+} // namespace bts

src/lib/orderman/orders/algo/algo_orders/TopAlgo.h

@@ -19,7 +19,7 @@ class TopAlgo : public IAlgoOrder {
    void do_update(api::orderman::Update const &) override;
    void on_book(SecurityKey, Book const &) override; 
    void on_child_change(OrderKey, ChildState const &) override;
-
+   
  private:
    struct PImpl;
    std::unique_ptr<PImpl> me;

src/lib/orderman/orders/algo/algo_orders/tests/AlgoTestDriver.cpp

@@ -56,16 +56,19 @@ struct AlgoTestDriverBase::Impl : public RecursionCounter {
    api::Security security;
    OrderKey order_key, parent_key;
    std::tuple<Quantity, Price> fill_state;
-
+   std::vector<PositionUpdate> positions;
    AlgoState state;
    std::vector<ChildOrder> orders;
    std::vector<ChildOrder> detached_orders;
 
    std::map<SecurityKey, Book> books;
+   std::map<SecurityKey, Price> settlements;
    std::map<SecurityKey, bool> book_subs;
+   std::map<SecurityKey, bool> settlement_subs;
    std::map<SecurityKey, bool> ticker_subs;
    std::map<SecurityKey, bool> status_subs;
    bool sub_timer = false;
+   bool sub_position = false;
 
    // Callbacks
    std::vector<std::function<void(OrderKey)>> on_child_launching;
@@ -158,6 +161,15 @@ void AlgoTestDriverBase::update_book(const Book& book) {
       me->algo.on_book(key, book);
    }
 }
+
+void AlgoTestDriverBase::update_settlement(SecurityKey key, Price price) {
+   me->settlements[key] = price;
+   auto iter = me->settlement_subs.find(key);
+   if (iter != me->settlement_subs.end()) {
+      me->algo.on_settlement(key, price);
+   }
+}
+
 void AlgoTestDriverBase::update_ticker(SecurityKey key, const TradeVector& data) {
    auto iter = me->ticker_subs.find(key);
    if (iter != me->ticker_subs.end() && iter->second) {
@@ -301,6 +313,15 @@ const api::Security* AlgoTestDriverBase::lookup_security(SecurityKey key) const
    }
    return nullptr;
 }
+std::vector<api::Security const *> AlgoTestDriverBase::known_securities() const {
+   std::vector<api::Security const *> result;
+   auto s = get_some_securities();
+   for (auto const &p : s) {
+      result.push_back(&p.second);
+   }
+   return result;
+}
+
 bool AlgoTestDriverBase::is_attaching() const { return has_flag(me->flags, ATDFlags::IS_ATTACHING); }
 ChildTemplate& AlgoTestDriverBase::make_child_template(const api::orderman::OrderDefinition& def, size_t) {
    REQUIRE(def.custom_def().custom_case() != CustomOrderDefinition::CUSTOM_NOT_SET);
@@ -321,6 +342,10 @@ bool AlgoTestDriverBase::is_shutting_down() const {
    return false;
 }
 
+std::vector<PositionUpdate> const & AlgoTestDriverBase::current_positions() const {
+   return me->positions;
+}
+
 void AlgoTestDriverBase::subscribe_book(SecurityKey key){
    me->book_subs[key] = true;
    auto iter = me->books.find(key);
@@ -332,6 +357,18 @@ void AlgoTestDriverBase::unsubscribe_book(SecurityKey key) {
    me->book_subs.erase(key);
 }
 
+void AlgoTestDriverBase::subscribe_settlement(SecurityKey key){
+   me->settlement_subs[key] = true;
+   auto iter = me->settlements.find(key);
+   if (iter != me->settlements.end()) {
+      me->algo.on_settlement(key, iter->second);
+   }
+}
+
+void AlgoTestDriverBase::unsubscribe_settlement(SecurityKey key) {
+   me->settlement_subs.erase(key);
+}
+
 void AlgoTestDriverBase::subscribe_ticker(SecurityKey key)  {
    me->ticker_subs[key] = true;
 }
@@ -363,6 +400,8 @@ void AlgoTestDriverBase::unsubscribe_child_change(OrderKey k) {
 void AlgoTestDriverBase::subscribe_timer() { me->sub_timer = true; }
 void AlgoTestDriverBase::unsubscribe_timer() { me->sub_timer = false; }
 
+void AlgoTestDriverBase::subscribe_position() { me->sub_position= true; }
+void AlgoTestDriverBase::unsubscribe_position() { me->sub_position = false; }
 
 OrderKey AlgoTestDriverBase::launch_child(ChildTemplate&, Price price, Quantity qty, bool sub) {
    OrderKey key = me->next_order();
@@ -451,5 +490,6 @@ void AlgoTestDriverBase::complete() {
 }
 void AlgoTestDriverBase::set_ladder_visibility(bool) { /* whatever, ok */ }
 void AlgoTestDriverBase::update_fill_state(Quantity q, Price p) { me->fill_state = { q, p }; }
+void AlgoTestDriverBase::update_custom_state(api::orderman::DynamicOrderState const&) {}
 
-}}}//namespace
+}}}//namespace

src/lib/orderman/orders/algo/algo_orders/tests/AlgoTestDriver.h

@@ -60,7 +60,7 @@ class AlgoTestDriverBase : private IAlgoServices {
    Quantity quantity() const override;
    Quantity total_fills() const override;
    bool is_shutting_down() const override;
-   
+   std::vector<PositionUpdate> const & current_positions() const override;
    //
    // Change state
 
@@ -70,6 +70,7 @@ class AlgoTestDriverBase : private IAlgoServices {
    void do_update(const api::orderman::Update&);
 
    void update_book(const Book&);
+   void update_settlement(SecurityKey , Price ) ;
    void update_ticker(SecurityKey, const TradeVector&);
    void update_status(SecurityKey, co2::TradingStatus);
    void advance_time(Duration, uint32_t steps=1);
@@ -104,6 +105,7 @@ private:
    OrderKey order_key() const override;
    OrderKey parent_order_key() const override;
    const api::Security* lookup_security(SecurityKey) const override;
+   std::vector<api::Security const *> known_securities() const override;
    bool is_attaching() const override;
    ChildTemplate& make_child_template(const api::orderman::OrderDefinition& child_def, size_t reserve = 1) override;
    TimeStamp get_time() const override;
@@ -112,6 +114,8 @@ private:
    // IAlgoServices
    void subscribe_book(SecurityKey)override;
    void unsubscribe_book(SecurityKey) override;
+   void subscribe_settlement(SecurityKey)override;
+   void unsubscribe_settlement(SecurityKey) override;
    void subscribe_child_change(OrderKey) override;
    void unsubscribe_child_change(OrderKey) override;
    void subscribe_timer() override;
@@ -120,6 +124,8 @@ private:
    void unsubscribe_ticker(SecurityKey) override;
    void subscribe_status(SecurityKey) override;
    void unsubscribe_status(SecurityKey) override;
+   void subscribe_position() override;
+   void unsubscribe_position() override;
 
    OrderKey launch_child(ChildTemplate &, Price, Quantity, bool) override;
    OrderKey launch_child(api::orderman::OrderDefinition const &, Price,
@@ -134,7 +140,7 @@ private:
    void complete() override;
    void set_ladder_visibility(bool) override;
    void update_fill_state(Quantity , Price) override;
-
+   void update_custom_state(api::orderman::DynamicOrderState const&) override;
 };
 
 template <class Algo>
@@ -151,4 +157,4 @@ struct AlgoTestDriver : public AlgoTestDriverBase
    const Algo& algo() const { return _algo; }
 };
 
-}}}//namespace
+}}}//namespace

src/lib/orderman/securities/ISecurityDB.h

@@ -13,6 +13,7 @@ class ISecurityDB
 public:
    virtual ~ISecurityDB() {};
    virtual api::Security const *resolve(SecurityKey const&k) const = 0;
+   virtual std::vector<api::Security const*> get_securities() const = 0;
    using SecurityMap = std::unordered_map<SecurityKey,api::Security const*>  ;
 };
 

src/lib/orderman/securities/MapSecurityDB.h

@@ -13,6 +13,11 @@ struct MapSecurityDB : public securities::ISecurityDB
    api::Security const *resolve(SecurityKey const&k) const override {
       return _oracle.get_security(k);
    }
+
+   std::vector<api::Security const*> get_securities() const override {
+      return _oracle.get_securities();
+   }
+
    private:
    md::SecurityOracle _oracle;
 };

src/sdk/examples/client_sdk/.vscode/launch.json

@@ -8,7 +8,7 @@
             "name": "(gdb) Launch",
             "type": "cppdbg",
             "request": "launch",
-            "program": "/sparkbin/sparkd",
+            "program": "${workspaceFolder}/client_sdk_example",
             "args": ["run"],
             "stopAtEntry": false,
             "cwd": "/sparkbin",

src/sdk/examples/client_sdk/client_sdk_example.cpp

@@ -2,6 +2,7 @@
 #include "spark/lib/domain_model/Security.h"
 #include "lib/api/services/AdminService.client.pb.h"
 #include "lib/api/services/PositionService.client.pb.h"
+#include "lib/api/services/MarketDataService.client.pb.h"
 
 #include <string>
 #include <vector>
@@ -30,6 +31,17 @@ int main(int argc, char** argv) {
       }
    });
 
+
+   auto md_service = ctx.locate<bts::api::MarketDataService>();
+   auto sub2 = 
+      md_service
+      .subscribe_securities()
+      .subscribe([](const auto& update) {
+      for (const auto& s : update.added()) {
+         std::cout << "Added security: " << s->id() << ": " << s->symbol() << std::endl;
+      }
+   });
+
    std::this_thread::sleep_for(std::chrono::minutes(5));
    return 0;
 }

src/sdk/examples/dynamic_join_order/order_plugin_example.cpp

@@ -36,6 +36,8 @@ class DynamicJoin : public IAlgoOrder {
    struct PImpl;
    std::unique_ptr<PImpl> me;
 
+   std::string status_name (DynamicJoin::Status);
+   void transition_status(DynamicJoin::Status new_status);
    IAlgoServices &algo_services() const;
    void try_to_join();
 };
@@ -60,6 +62,28 @@ struct DynamicJoin::PImpl {
    Book last_book;
 };
 
+std::string DynamicJoin::status_name (DynamicJoin::Status st) {
+   switch (st) {
+   case DynamicJoin::Status::WAITING_FOR_MD: return "WAITING_FOR_MD";
+   case DynamicJoin::Status::WAITING_TO_JOIN: return "WAITING_TO_JOIN";
+   case DynamicJoin::Status::JOINED: return "JOINED";
+   default:
+      bts_assert(false);
+      return "";
+   }
+}
+void DynamicJoin::transition_status(Status new_status) {
+   me->status = new_status;
+   // Every time we change our internal state we report
+   // a description of the new state so that the user
+   // can monitor the algo progression in the UI
+   api::orderman::DynamicOrderState report;
+   api::orderman::DynamicOrderParam status;
+   status.set_as_string(status_name(new_status));
+   (*report.mutable_custom_state())[std::string("Status:")] = status;
+   me->algo_services->update_custom_state(report);
+}
+   
 DynamicJoin::DynamicJoin() : me(new PImpl()) {}
 DynamicJoin::~DynamicJoin() {}
 IAlgoServices &DynamicJoin::algo_services() const { return *me->algo_services; }
@@ -226,7 +250,7 @@ void DynamicJoin::on_book(SecurityKey, Book const &b) {
       if (other_side_top.size() > 0) {
          // First time we get the opposite side market, we will use
          // this price as the target price for the join
-         me->status = Status::WAITING_TO_JOIN;
+         transition_status(Status::WAITING_TO_JOIN);
          me->book_at_inception = b;
          LOG(OrderMan, Diag,
              "JoinOrder order {} receiving first book: {} Target: {}",
@@ -266,7 +290,7 @@ void DynamicJoin::try_to_join() {
       child_update.set_price(my_side_psz.price());
       child_update.set_quantity(algo_services().quantity().value());
       algo_services().update_child(me->child_key, child_update);
-      me->status = Status::JOINED;
+      transition_status(Status::JOINED);
 
       LOG(OrderMan, Diag, "DynamicJoin order {} joined: {}@{}",
           algo_services().order_key().value(),

src/sdk/sdk_spark.conf

@@ -23,6 +23,8 @@ Ports {
    value {
       config_port: 9000
       qbus_port: 9001
+      glbx_sim_fix_port: 9002
+      ice_sim_fix_port: 9003
    }
 }
 

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp

@@ -9,6 +9,7 @@
 #include "lib/utils/Dispatcher.h"
 #include "lib/utils/proto_helper.h"
 #include "lib/md/protos/co2.pb.h"
+#include "lib/md/SecurityDB.h"
 
 #include "lib/config/protos/MarketData.pb.h"
 #include "lib/config/protos/system.pb.h"
@@ -337,6 +338,23 @@ unique_ptr<MDP3Listener> MDP3Main::prepare_listener(onixs::Handler& handler, con
       listener->add_filter_symbol(symbol.symbol());
    }
 
+   // preload
+   if (datetime::is_local_time_sunday()) {
+      auto& preload_symbols = config::get<md::SecurityPreload>();
+      std::set<std::string> preload(preload_symbols.symbol().begin(), preload_symbols.symbol().end());
+      for (auto& symbol : channel.symbols) {
+         if (preload.find(symbol.symbol()) != preload.end()) {
+            std::set<UserSymbol> unit_set { symbol };
+            db::load_securities(unit_set, [&](const api::Security& security) {
+               with_security_like(security, [&](FixedSecurity& fs) {
+                  convert_security(security, fs, true);
+                  listener->add_security(fs, bts::co2::TradingStatus::TRADINGSTATUS_UNKNOWN);
+               });
+            });
+         }
+      }
+   }
+
    handler.registerListener(*listener);
    handler.registerMarketDataListener(*listener);
    handler.registerSecurityListener(*listener);

src/web/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources ../../externals/linux/include/google/protobuf/descriptor.proto;../../externals/linux/include/google/protobuf/timestamp.proto;../lib/api/apicore.proto;../lib/api/apigateway.proto;../lib/api/apioptions.proto;../lib/api/services/AdminService.proto;../lib/api/services/AlgoDefinitionService.proto;../lib/api/services/AlgoService.proto;../lib/api/services/ApiUnitTestService.proto;../lib/api/services/AuthService.proto;../lib/api/services/BasePositionService.proto;../lib/api/services/ErrorReportService.proto;../lib/api/services/HelloService.proto;../lib/api/services/MBOService.proto;../lib/api/services/MarginService.proto;../lib/api/services/MarketDataAuxiliaryService.proto;../lib/api/services/MarketDataRelay.proto;../lib/api/services/MarketDataService.proto;../lib/api/services/MoneyFileService.proto;../lib/api/services/OrderService.proto;../lib/api/services/OutOfProcessURLListener.proto;../lib/api/services/PersistenceService.proto;../lib/api/services/PositionService.proto;../lib/api/services/ShellDiagnosticService.proto;../lib/api/services/ShellLatencyService.proto;../lib/api/services/SyntheticSecurityService.proto;../lib/domain_model/Security.proto;../lib/md/protos/co2.proto;../lib/sparkproto/SparkProtos.proto;api/build.sh;api/node_modules/exit/test/fixtures/create-files.sh;api/node_modules/protobufjs/cli/node_modules/acorn-jsx/node_modules/acorn/bin/update_authors.sh;api/node_modules/watch/release.sh;api/quiet_build.sh;api/src/DynamicService.ts;api/src/ServiceContext.ts;api/src/bts_assert.ts;ui/src/Access.fs;ui/src/Accounts/ClipLimits.fs;ui/src/Accounts/ProductLimits.fs;ui/src/Accounts/SecurityLimits.fs;ui/src/Accounts/State.fs;ui/src/Accounts/Types.fs;ui/src/Accounts/View.fs;ui/src/App.fs;ui/src/Cmd.fs;ui/src/Controls/Button.fs;ui/src/Controls/DataGrid.fs;ui/src/Controls/Fields.fs;ui/src/Controls/Modal.fs;ui/src/Controls/ReactGrid.fs;ui/src/Controls/Table.fs;ui/src/Controls/nbsp.fs;ui/src/Global.fs;ui/src/Imports.fs;ui/src/Lens.fs;ui/src/Login/Login.fs;ui/src/Nav.fs;ui/src/State.fs;ui/src/Types.fs;ui/src/Users/State.fs;ui/src/Users/Types.fs;ui/src/Users/View.fs;ui/src/Util/EnumUtils.fs;ui/src/Util/JSMap.fs;ui/src/Util/StreamUtils.fs;ui/src/obj/Debug/netstandard2.0/ui.AssemblyInfo.fs)
+set(sources ../../externals/linux/include/google/protobuf/descriptor.proto;../../externals/linux/include/google/protobuf/timestamp.proto;../lib/api/apicore.proto;../lib/api/apigateway.proto;../lib/api/apioptions.proto;../lib/api/services/AdminService.proto;../lib/api/services/AlgoDefinitionService.proto;../lib/api/services/AlgoService.proto;../lib/api/services/ApiUnitTestService.proto;../lib/api/services/AuthService.proto;../lib/api/services/BasePositionService.proto;../lib/api/services/ErrorReportService.proto;../lib/api/services/HelloService.proto;../lib/api/services/MBOService.proto;../lib/api/services/MarginService.proto;../lib/api/services/MarketDataAuxiliaryService.proto;../lib/api/services/MarketDataRelay.proto;../lib/api/services/MarketDataService.proto;../lib/api/services/MoneyFileService.proto;../lib/api/services/OrderService.proto;../lib/api/services/OutOfProcessURLListener.proto;../lib/api/services/PersistenceService.proto;../lib/api/services/PositionService.proto;../lib/api/services/ShellDiagnosticService.proto;../lib/api/services/ShellLatencyService.proto;../lib/api/services/SyntheticSecurityService.proto;../lib/domain_model/Security.proto;../lib/md/protos/co2.proto;../lib/sparkproto/SparkProtos.proto;api/build.sh;api/node_modules/exit/test/fixtures/create-files.sh;api/node_modules/object.assign/test.sh;api/node_modules/watch/release.sh;api/quiet_build.sh;api/src/DynamicService.ts;api/src/ServiceContext.ts;api/src/bts_assert.ts;ui/src/Access.fs;ui/src/Accounts/ClipLimits.fs;ui/src/Accounts/ProductLimits.fs;ui/src/Accounts/SecurityLimits.fs;ui/src/Accounts/State.fs;ui/src/Accounts/Types.fs;ui/src/Accounts/View.fs;ui/src/App.fs;ui/src/Cmd.fs;ui/src/Controls/Button.fs;ui/src/Controls/DataGrid.fs;ui/src/Controls/Fields.fs;ui/src/Controls/Modal.fs;ui/src/Controls/ReactGrid.fs;ui/src/Controls/Table.fs;ui/src/Controls/nbsp.fs;ui/src/Global.fs;ui/src/Imports.fs;ui/src/Lens.fs;ui/src/Login/Login.fs;ui/src/Nav.fs;ui/src/State.fs;ui/src/Types.fs;ui/src/Users/State.fs;ui/src/Users/Types.fs;ui/src/Users/View.fs;ui/src/Util/EnumUtils.fs;ui/src/Util/JSMap.fs;ui/src/Util/StreamUtils.fs;ui/src/obj/Debug/netstandard2.0/ui.AssemblyInfo.fs)

src/web/ui/src/Access.fs

@@ -102,7 +102,7 @@ module Accounts =
         }
       riskLimitEnforcement =
         {  productLimitInNetTerms = true
-           ignoreSpreadPositionLimits = false }
+           ignoreSpreadPositionLimits = true }
       accountLossLimit = 
         { enforce = false
           totalLossLimit = 0u }

testing/manual/algos/TASCounter.txt

@@ -0,0 +1,78 @@
+0. This needs sim market data **not** to be merged with replay (or cert), so we can create our own markets.
+
+1. This is a proprietary algo so its hidden behind a config key. Enable it by adding the following to config
+DynamicAlgos{
+   value {
+      enable_tascounter: true
+   }
+}
+
+2. We will run this test on the grain markets, speficially corn, so make sure you have ZC (it could also be run for other
+grain products, but this test is written for ZC)
+
+3. Open up Algo Studio and make a TASCounter algo.
+Concepts : 
+ZC expirations = Mar, May, Jul, Sep, Dec
+T1 contracts are adjacent calendar spreads so if the front month is May 
+T1 Front = May-Jul 
+T1 Second = Jul-Sep
+T1 Third = Sep-Dec 
+
+T2 skips an expiration
+T2 Front = May-Sep
+T2 Second = Jul-Dec
+T2 Third is not used
+
+T3 skips 2 expirations
+T3 Front = May-Dec
+
+Set up those contracts for the algo.
+
+4. The TAS contracts are the corresponding TAS contracts for ZC (ZCT is the TAS product for ZC).
+T1 Front = ZCT May-Jul
+T1 Second = ZCT Jul-Sep
+T1 Third = ZCT Sep-Dec 
+
+(Since we are running SIM, if these contracts are not available, choose any other contracts that are not used above)
+
+5. For T1 Front time, T1 Qty, and T1 Order Enabled enter 60, 10 and false respectively. Do the same for 
+T2 Front time, T2 Qty, and T2 Order Enabled, as well as T3 Front time, T3 Qty, and T3 Order Enabled.
+
+6. This algo runs during the settlement window of the product (ZC in this case). This is an input parameter which will 
+also make it easier for us to run this test. So enter a time (remember if your server is on Central time, make the adjustment)
+3 or 4 mins from now. Enter the settlement period as 180 seconds.
+
+7. Give the algo a name and save it.
+
+8. Bring up ladders and timesales for all 6 non TAS contracts. Bring up ladder for the TAS contracts.
+
+9. The contracts, price and qty are already part of the algo, so we can launch it from any ladder, with any price, or qty. Launch the algo
+
+10. Bring up the orderbook, and make sure the input params are correct.
+
+11. Make sure there is an output params grid under the input params. All other values except for "Seconds To Settle" should be NAN.
+"Seconds To Settle" should downcount toward the settle window, and should go to 0 when in settlement period.
+
+12. Now once you enter the settlement period, enter a few orders and cross them, across all 6 non TAS contracts. Enter them at a few
+price levels. Make sure they show up in the timesales window.
+
+13. Make sure the VWAP for all these 6 contracts is correct. VWAP is simply sum(price*qty) for all trades / sum (qty).
+
+14. Now make sure the Projected Settles for T1 Front, Second and Third is correct. This is how they are calculated
+T1 Front Projected Settle = VWAP (T1 Front)
+T1 Second Projected Settle = VWAP (T2 Front) - VWAP (T1 Front)
+T1 Third Projected Settle = VWAP(T3 Front) - VWAP(T1 Front) - VWAP(T1 Second)
+These values are rounded to the nearest tick (0.25 in this case)
+
+15. The last step it to make the algo actually shoot orders. I will explain what the concept is and that can be used for testing.
+This is a market making algo, so the trader wants to buy at the bid or sell at the offer. The TAS contract is a special contract that
+gives delivery of the actual contract at the price the TAS was traded at. So for example if you bought N TAS contracts on product X 
+at -1.0, it means you will get delivery of X at settlement (of X) +(-1). If you sold the TAS contract at 0, you will go short on X
+at settlement + 0.
+With this in mind, the algo watches the TAS market and crosses an order (ie buys at the offer or sells at the bid) if it gets delivery
+on the actual product such that its a buy at or below the bid or a sell at or above the offer.
+So for example the TAS Market is Bid=0, Ask=0-2. If we sell at Bid we will go short in the main product at Settlement + 0. 
+If our projected settle (from 14.) is >= the offer of the main product we will sell the TAS.
+The algo will buy or sell the TAS contract if the condition above holds AND if its Order Enabled = True and Time is within the window.
+
+Repeat the above for T1 Front, Second and Third.