GMDL use_given_sec_key set to true to enable replay data to play without copying md and keys table

* Unifies log message structure for improved readability
* Grants us runtime control over log levels via config

.devcontainer/devcontainer.json

@@ -1,15 +1,10 @@
 {
   "dockerFile": "Dockerfile",
-  // Creates local dir to mount to HOME in docker 
-  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
   "hostRequirements": {
-    "memory": "4gb",
-    "storage": "6gb"
+    "memory": "8gb",
+    "storage": "10gb"
   },
-  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
   "containerUser": "bts",
-  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
-  "workspaceFolder": "/home/bts/spark",
   "updateRemoteUserUID": true,
   "shutdownAction": "none",
   // Note: If running multiple containers on the same host, each must use a
@@ -17,7 +12,7 @@
   // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
   "runArgs": [
     "--shm-size",
-    "2gb",
+    "8gb",
     "-p",
     "8000:8000",
     "-p",
@@ -29,22 +24,15 @@
     "-p",
     "12345:12345"
   ],
-  "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
-  ],
   "customizations": {
     "vscode": {
-      "settings": {
-        "clangd.arguments": [
-          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
-        ]
-      },
       "extensions": [
         "eamodio.gitlens",
         "llvm-vs-code-extensions.vscode-clangd",
         "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools"
+        "ms-vscode.cpptools",
+        "xaver.clang-format"
       ]
     }
   }
-}
+}

src/client/spark/Daemons/MDDaemon.cs

@@ -133,17 +133,31 @@ namespace bts {
 
             // set 'unit quantity' for each sec
             foreach (var smd in exch_secs.Values) {
-               var meta = _security_metadata.get(smd.security);
-               if (meta.HasValue) {
-                  exch_secs[smd.id] = new(
-                     smd.security,
-                     smd.md,
-                     meta.Value.HasPriceDisplay ? meta.Value.PriceDisplay : null,
-                     meta.Value.HasQuantityDisplay ? meta.Value.QuantityDisplay : null);
-               }
+               var (priceDisplay, qtyDisplay) = GetDisplaySettings(smd.security);
+               exch_secs[smd.id] = new(smd.security, smd.md, priceDisplay, qtyDisplay);
             }
             _bus.post(SparkTags.ServerSecuritiesTag, exch_secs);
          }
+
+         (TickDisplaySettingsImmutable? Price, TickDisplaySettingsImmutable? Quantity) GetDisplaySettings(SecurityImmutable sec) {
+            var meta = _security_metadata.get(sec);
+            var priceDisplay = meta?.PriceDisplay ?? Security.Defaults.price_display_settingsi;
+            var qtyDisplay = meta?.QuantityDisplay ?? Security.Defaults.quantity_display_settingsi;
+            if (sec.HasContractSpec) {
+               var pd = priceDisplay.CloneToMutable();
+               pd.TickFactors = TickUtils.get_tick(sec, 0).to_api_tick_factors();
+               priceDisplay = new(pd);
+               if (sec.ContractSpec.QuantityInfos.TryGetFirst(out var qtyInfo)) {
+                  var qd = qtyDisplay.CloneToMutable();
+                  qd.ShowDecimalPart = qtyInfo.TickBase.is_real_non_zero() && !(qtyInfo.TickUnit / qtyInfo.TickBase).is_int();
+                  qd.TickFactors ??= new();
+                  qd.TickFactors.TickBase = qtyInfo.TickBase;
+                  qd.TickFactors.TickUnit = qtyInfo.TickUnit;
+                  qtyDisplay = new(qd);
+               }
+            }
+            return (priceDisplay, qtyDisplay);
+         }
       }
 
       public void Dispose() =>

src/client/spark/Modules/Market/MarketView/MarketViewFilterStates.cs

@@ -12,8 +12,13 @@ namespace bts.shell.market_view {
    class MarketViewFilterStates {
       [ProtoMember(37)]
       public Venue[] venue { get; set; }
+
+      [ProtoMember(38)]
+      public string[] currency { get; set; }
+
       [ProtoMember(1)]
       public string security { get; set; }
+
       [ProtoMember(2)]
       public DoubleMinMaxState bid_price { get; set; }
 
@@ -128,6 +133,7 @@ namespace bts.shell.market_view {
          var pb = new ColumnPredicateBuilder<MarketViewVM.Row>();
          // security
          pb.add_discrete_values(venue, r => r.venue);
+         pb.add_discrete_values(currency, r => r.currency);
          pb.add_regex(security, r => r.security.ToString());
          pb.add_regex(security_key, r => r.security_key.ToString());
 

src/client/spark/Modules/Market/MarketView/MarketViewVM.cs

@@ -98,7 +98,14 @@ namespace mbts.spark {
          _sec_meta_store = sec_meta_store;
          _settings_hub = settings_hub;
          security_filter = new RegexFilterVM<Row>(r => r.security.ToString(), nameof(Row.security));
-         currency_filter =new RegexFilterVM<Row>(r => r.currency, nameof(Row.currency));
+         currency_filter = DiscreteValuesFilterVM.Create<Row, string>(
+            bus.subscribe(SparkTags.SecuritiesTag)
+               .Sample(TimeSpan.FromSeconds(1))
+               .Select(sec_mds => sec_mds.Values.Select(smd => smd.security.Currency).Distinct().ToList()),
+            r => r.currency,
+            nameof(Row.currency),
+            EqualityComparer<string>.Default,
+            Comparer<string>.Default);
          // rows
          _wrapped_rows = _rows.WrapSelector(r => r.security_md, s => {
             var r = new Row(_bus, s, _orders_model, _position_model);
@@ -124,7 +131,7 @@ namespace mbts.spark {
 
          add_col(ColKey.Venue, r => r.venue.ToString(), nameof(Row.venue), venue_filter, FilterType.DiscreteValues, ColWidth.Exchange);
          add_col(ColKey.Security, r => r.security.ToString(), nameof(Row.security), security_filter, FilterType.Regex, ColWidth.Security);
-         add_col(ColKey.Currency, r => r.currency, nameof(Row.currency), currency_filter, FilterType.Regex, ColWidth.Currency);
+         add_col(ColKey.Currency, r => r.currency, nameof(Row.currency), currency_filter, FilterType.DiscreteValues, ColWidth.Currency);
          add_col(ColKey.Volume, r => r.volume_text, nameof(Row.volume), volume_filter, FilterType.ULongMinMax, ColWidth.Volume);
 
          // last
@@ -391,7 +398,7 @@ namespace mbts.spark {
 
       public IDataGridColumnPersistentFilter<Row, Venue[]> venue_filter { get; } = DiscreteValuesFilterVM.Common.exchange<Row>(r => r.venue, nameof(Row.venue));
       public IDataGridColumnPersistentFilter<Row, string> security_filter { get; }
-      public IDataGridColumnPersistentFilter<Row, string> currency_filter { get; }
+      public IDataGridColumnPersistentFilter<Row, string[]> currency_filter { get; }
       public IDataGridColumnPersistentFilter<Row, DoubleMinMaxState> bid_price_filter { get; } = new DoubleMinMax<Row>(r => r.bid_price, nameof(Row.bid_price));
       public IDataGridColumnPersistentFilter<Row, DoubleMinMaxState> bid_qty_filter { get; } = new DoubleMinMax<Row>(r => r.bid_qty_display, nameof(Row.bid_qty_display));
       public IDataGridColumnPersistentFilter<Row, DoubleMinMaxState> dbid_qty_filter { get; } = new DoubleMinMax<Row>(r => r.dbid_qty_display, nameof(Row.dbid_qty_display));
@@ -538,6 +545,7 @@ namespace mbts.spark {
       public MarketViewFilterStates get_filter_states() =>
          new() {
             venue = venue_filter.get_state(),
+            currency = currency_filter.get_state(),
             security = security_filter.get_state(),
             bid_price = bid_price_filter.get_state(),
             bid_merged_size = bid_qty_filter.get_state(),
@@ -584,6 +592,7 @@ namespace mbts.spark {
 
          using var _ = rows_view.DeferRefresh();
          venue_filter.load_state(filters.venue);
+         currency_filter.load_state(filters.currency);
          security_filter.load_state(filters.security);
          bid_price_filter.load_state(filters.bid_price);
          bid_qty_filter.load_state(filters.bid_merged_size);

src/client/spark/Modules/Trading/PNL/VM/PNLRowVM.cs

@@ -1,378 +1,380 @@
-using bts.api;
-using bts.utils;
-
-using mbts;
-using mbts.spark;
-using mbts.ui;
-using mbts.ui.shell;
-
-using Spark.Core.Collections;
-using Spark.Core.Data;
-
-using System;
-using System.Collections.Generic;
-using System.Collections.Immutable;
-using System.Linq;
-using System.Reactive.Linq;
-
-using ColumnKeys = bts.shell.pnl.PNLView.Columns;
-using PNLKey = bts.AccountSecurityKey;
-
-namespace bts.shell.pnl {
-   class PNLRowVM : DisposableViewModelBase, IEquatable<PNLRowVM> {
-      public static readonly IComparer<PNLRowVM> key_comparer = Comparer<PNLKey>.Default.wrap<PNLKey, PNLRowVM>(r => r.key);
-      static readonly Func<bool, string> _fmt_has_day_trades = hdt => hdt ? "Yes" : "No";
-      //
-      readonly PNLCellIndexBuilder<PNLRowData> _cell_map = new PNLCellIndexBuilder<PNLRowData>();
-      readonly PNLCellEmbeddedAnimationModel<PNLRowData> _animator;
-      readonly ISecurityMetadataStore _sec_meta_store;
-      bool _refresh_normalized_data = true, _is_selected;
-      PNLNormalizedData _normalized_data;
-      api.domain.TickDisplaySettingsImmutable? _quantity_display_backing;
-
-      public PNLRowVM(
-         PNLRowData data,
-         ISecurityMetadataStore sec_meta_store,
-         ImmutableDictionary<ColumnKeys, PNLCellAppearance> default_appearances,
-         IReadOnlyList<IPNLAnimationTrigger<PNLRowData>> triggers,
-         Func<double, string> fmt_fill_price,
-         Func<double, string> fmt_market_price) {
-         //
-         this.data = data;
-         _sec_meta_store = sec_meta_store;
-         _animator = new PNLCellEmbeddedAnimationModel<PNLRowData>(default_appearances, triggers);
-         _animator.on_row(data);
-         pnl_price = _cell_map.add_price_double(ColumnKeys.PNLPrice, d => d.pnl_price);
-         account_cell = _cell_map.add(ColumnKeys.Account, d => d.account);
-         security_type_cell = _cell_map.add(ColumnKeys.SecurityType, d => d.security?.ContractCase ?? Security.ContractOneofCase.None);
-         exchange_cell = _cell_map.add(ColumnKeys.Exchange, d => d.security?.Exchange ?? Venue.Unset);
-         expiration_cell = _cell_map.add(
-            ColumnKeys.Expiration,
-            d => d.expiration,
-            e => SecurityFormatter.format_expiration(e));
-         symbol_cell = _cell_map.add(ColumnKeys.Symbol, d => d.symbol);
-         security_cell = _cell_map.add(
-            ColumnKeys.Security,
-            d => d.security,
-            null,
-            EqualityComparer<SecurityImmutable?>.Default,
-            Comparer<SecurityImmutable?>.Default);
-         security_key = _cell_map.add(ColumnKeys.SecurityKey, d => d.key.security_id);
-         cpp = _cell_map.add(ColumnKeys.CPP, d => d.effective_cpp);
-         // sod
-         sod_pnl = _cell_map.add_pnl_double(ColumnKeys.SODPNL, d => d.sod_pnl);
-         sod_net_pos = _cell_map.add_net_pos(
-            ColumnKeys.SODNetPos, d => d.display_sod_net_pos, fmt_qty);
-         sod_long_pos = _cell_map.add_long_pos(
-            ColumnKeys.SODLongPos, d => d.display_sod_long_pos, fmt_qty);
-         sod_long_avg_price = _cell_map.add_long_price(ColumnKeys.SODLongAvgPrice, d => d.sod_long_avg_price);
-         sod_short_pos = _cell_map.add_short_pos(
-            ColumnKeys.SODShortPos, d => d.display_sod_short_pos, fmt_qty);
-         sod_short_avg_price = _cell_map.add_short_price(ColumnKeys.SODShortAvgPrice, d => d.sod_short_avg_price);
-         sod_delta = _cell_map.add_pnl_double(ColumnKeys.SODDelta, d => d.sod_delta, fmt_greek);
-         sod_gamma = _cell_map.add_pnl_double(ColumnKeys.SODGamma, d => d.sod_gamma, fmt_greek);
-         sod_vega = _cell_map.add_pnl_double(ColumnKeys.SODVega, d => d.sod_vega, fmt_greek);
-         sod_theta = _cell_map.add_pnl_double(ColumnKeys.SODTheta, d => d.sod_theta, fmt_greek);
-
-         // day
-         day_pnl = _cell_map.add_pnl_double(ColumnKeys.DayPNL, d => d.day_pnl);
-         day_net_pos = _cell_map.add_net_pos(
-            ColumnKeys.DayNetPos, d => d.display_day_net_pos, fmt_qty);
-         day_long_pos = _cell_map.add_long_pos(
-            ColumnKeys.DayLongPos, d => d.display_day_long_pos, fmt_qty);
-         day_long_avg_price = _cell_map.add_long_price(ColumnKeys.DayLongAvgPrice, d => d.day_long_avg_price);
-         day_short_pos = _cell_map.add_short_pos(
-            ColumnKeys.DayShortPos, d => d.display_day_short_pos, fmt_qty);
-         day_short_avg_price = _cell_map.add_short_price(ColumnKeys.DayShortAvgPrice, d => d.day_short_avg_price);
-         day_delta = _cell_map.add_pnl_double(ColumnKeys.DayDelta, d => d.day_delta, fmt_greek);
-         day_gamma = _cell_map.add_pnl_double(ColumnKeys.DayGamma, d => d.day_gamma, fmt_greek);
-         day_vega = _cell_map.add_pnl_double(ColumnKeys.DayVega, d => d.day_vega, fmt_greek);
-         day_theta = _cell_map.add_pnl_double(ColumnKeys.DayTheta, d => d.day_theta, fmt_greek);
-         has_day_trades = _cell_map.add(ColumnKeys.HasDayTrades, d => d.has_day_trades, _fmt_has_day_trades);
-         // working
-         working_net_pos = _cell_map.add_net_pos(
-            ColumnKeys.WorkingNetPos, d => d.display_working_net_pos, fmt_qty);
-         var qty_display = _sec_meta_store.get_qty_display(security);
-         working_long_pos = _cell_map.add_working_long(
-            ColumnKeys.WorkingLongPos,
-            d => d.working.long_data,
-            d => d.bid_price,
-            qty_display);
-         working_long_avg_price = _cell_map.add_long_price(ColumnKeys.WorkingLongAvgPrice, d => d.working_long_avg_price);
-         working_short_pos = _cell_map.add_working_short(
-            ColumnKeys.WorkingShortPos, r => r.working.short_data, r => r.ask_price, qty_display);
-         working_short_avg_price = _cell_map.add_short_price(ColumnKeys.WorkingShortAvgPrice, d => d.working_short_avg_price);
-         // total
-         total_pnl = _cell_map.add_pnl_double(ColumnKeys.TotalPNL, d => d.total_pnl);
-         total_net_pos = _cell_map.add_net_pos(
-            ColumnKeys.TotalNetPos, d => d.display_total_net_pos, fmt_qty);
-         total_long_pos = _cell_map.add_long_pos(
-            ColumnKeys.TotalLongPos, d => d.display_total_long_pos, fmt_qty);
-         total_long_avg_price = _cell_map.add_long_price(ColumnKeys.TotalLongAvgPrice, d => d.total_long_avg_price);
-         total_short_pos = _cell_map.add_short_pos(
-            ColumnKeys.TotalShortPos, d => d.display_total_short_pos, fmt_qty);
-         total_short_avg_price = _cell_map.add_short_price(ColumnKeys.TotalShortAvgPrice, d => d.total_short_avg_price);
-         total_delta = _cell_map.add_pnl_double(
-            ColumnKeys.TotalDelta, d => d.total_delta, fmt_greek);
-         total_gamma = _cell_map.add_pnl_double(
-            ColumnKeys.TotalGamma, d => d.total_gamma, fmt_greek);
-         total_vega = _cell_map.add_pnl_double(ColumnKeys.TotalVega, d => d.total_vega, fmt_greek);
-         total_theta = _cell_map.add_pnl_double(
-            ColumnKeys.TotalTheta, d => d.total_theta, fmt_greek);
-
-         // market
-         bid_price = _cell_map.add_price_double(ColumnKeys.BidPrice, d => d.bid_price);
-         ask_price = _cell_map.add_price_double(ColumnKeys.AskPrice, d => d.ask_price);
-         mid_price = _cell_map.add_price_double(ColumnKeys.MidPrice, d => d.mid_price);
-         last_price = _cell_map.add_price_double(ColumnKeys.LastPrice, d => d.last_price);
-         settlement_price = _cell_map.add_price_double(ColumnKeys.SettlementPrice, d => d.settlement_price);
-
-         _set_price_formatting_delegates(fmt_fill_price, fmt_market_price);
-         _cell_map.on_row(data);
-         _cell_map.set_appearances(_animator.cell_appearances);
-         _dispose = _cell_map.animation_model.OnDispatcher().Subscribe(new_animations => {
-            animations = new_animations.Select(kv => new mbts.ui.spark.CellAnimation(kv.Key.column, kv.Key.type, kv.Value.color.to_brush())).ToList();
-            RaisePropertyChanged(nameof(animations));
-         });
-
-         static string fmt_greek(double v) =>
-            v.is_nan() ? "-" : v.ToString("#,0.00");
-
-         string fmt_qty(double v) =>
-            quantity_display.format(v);
-      }
-
-      // events
-      public event Action<PNLNormalizedData> updated;
-      public event Action<bool> is_selected_changed;
-
-      // data
-      public PNLKey key => data.key;
-      public PNLRowData data { get; private set; }
-      public PNLNormalizedData normalized_data {
-         get {
-            if (_refresh_normalized_data) {
-               _refresh_normalized_data = false;
-               // Only count outrights (spread positions should be normalized into outrights)
-               var sec_type = security?.ContractCase ?? Security.ContractOneofCase.None;
-               var (
-                  sod_long,
-                  sod_short,
-                  sod_delta,
-                  sod_gamma,
-                  sod_vega,
-                  sod_theta,
-                  day_long,
-                  day_short,
-                  day_delta,
-                  day_gamma,
-                  day_vega,
-                  day_theta) = sec_type switch {
-                  Security.ContractOneofCase.Future or Security.ContractOneofCase.Option => (
-                     data.sod.long_pos.round_to_decimal(),
-                     data.sod.short_pos.round_to_decimal(),
-                     data.sod.delta.round_to_decimal(),
-                     data.sod.gamma.round_to_decimal(),
-                     data.sod.vega.round_to_decimal(),
-                     data.sod.theta.round_to_decimal(),
-                     data.day.long_pos.round_to_decimal(),
-                     data.day.short_pos.round_to_decimal(),
-                     data.day.delta.round_to_decimal(),
-                     data.day.gamma.round_to_decimal(),
-                     data.day.vega.round_to_decimal(),
-                     data.day.theta.round_to_decimal()),
-                  _ => (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
-               };
-
-               var working_long = data.working_long_pos.round_to_decimal();
-               var working_short = data.working_short_pos.round_to_decimal();
-
-               var nd = new PNLNormalizedData() {
-                  // sod
-                  sod_pnl = sod_pnl.value.round_to_decimal(),
-                  sod_long_pos = sod_long,
-                  sod_short_pos = sod_short,
-                  sod_delta = sod_delta,
-                  sod_gamma = sod_gamma,
-                  sod_vega = sod_vega,
-                  sod_theta = sod_theta,
-                  display_sod_long_pos = sod_long,
-                  display_sod_short_pos = sod_short,
-
-                  // day
-                  day_pnl = day_pnl.value.round_to_decimal(),
-                  day_long_pos = day_long,
-                  day_short_pos = day_short,
-                  day_delta = day_delta,
-                  day_gamma = day_gamma,
-                  day_vega = day_vega,
-                  day_theta = day_theta,
-                  display_day_long_pos = day_long,
-                  display_day_short_pos = day_short,
-
-                  // working
-                  working_long_pos = working_long,
-                  working_short_pos = working_short,
-                  display_working_long_pos = working_long,
-                  display_working_short_pos = working_short
-               };
-               if (_normalized_data != nd) {
-                  _normalized_data = nd;
-                  updated?.Invoke(nd);
-               }
-            }
-            return _normalized_data;
-         }
-      }
-      // security
-      public string account_name => data.account_name;
-      public Venue exchange => security?.Exchange ?? Venue.Unset;
-      public string symbol => security.symbol_or_default();
-      public DateTime expiration => data.expiration;
-      public Security.ContractOneofCase security_type => security?.ContractCase ?? Security.ContractOneofCase.None;
-      public SecurityImmutable? security => data.security;
-      // cells
-      public PNLCellVM<PNLRowData, AccountTuple> account_cell { get; }
-      public PNLCellVM<PNLRowData, Venue> exchange_cell { get; }
-      public PNLCellVM<PNLRowData, DateTime> expiration_cell { get; }
-      public PNLCellVM<PNLRowData, string> symbol_cell { get; }
-      public PNLCellVM<PNLRowData, Security.ContractOneofCase> security_type_cell { get; }
-      public PNLCellVM<PNLRowData, SecurityImmutable?> security_cell { get; }
-      public PNLCellVM<PNLRowData, uint> security_key { get; }
-      public PNLCellVM<PNLRowData, double> cpp { get; }
-      // sod
-      public PNLCellVM<PNLRowData, double> sod_pnl { get; }
-      public PNLCellVM<PNLRowData, double> sod_net_pos { get; }
-      public PNLCellVM<PNLRowData, double> sod_long_pos { get; }
-      public PNLCellVM<PNLRowData, double> sod_long_avg_price { get; }
-      public PNLCellVM<PNLRowData, double> sod_short_pos { get; }
-      public PNLCellVM<PNLRowData, double> sod_short_avg_price { get; }
-      public PNLCellVM<PNLRowData, double> sod_delta { get; }
-      public PNLCellVM<PNLRowData, double> sod_gamma { get; }
-      public PNLCellVM<PNLRowData, double> sod_vega { get; }
-      public PNLCellVM<PNLRowData, double> sod_theta { get; }
-
-      // day
-      public PNLCellVM<PNLRowData, double> day_pnl { get; }
-      public PNLCellVM<PNLRowData, double> day_net_pos { get; }
-      public PNLCellVM<PNLRowData, double> day_long_pos { get; }
-      public PNLCellVM<PNLRowData, double> day_long_avg_price { get; }
-      public PNLCellVM<PNLRowData, double> day_short_pos { get; }
-      public PNLCellVM<PNLRowData, double> day_short_avg_price { get; }
-      public PNLCellVM<PNLRowData, double> day_delta { get; }
-      public PNLCellVM<PNLRowData, double> day_gamma { get; }
-      public PNLCellVM<PNLRowData, double> day_vega { get; }
-      public PNLCellVM<PNLRowData, double> day_theta { get; }
-      public PNLCellVM<PNLRowData, bool> has_day_trades { get; }
-      // working
-      public PNLCellVM<PNLRowData, double> working_net_pos { get; }
-      public PNLCellVM<PNLRowData, WorkingOrderSideData> working_long_pos { get; }
-      public PNLCellVM<PNLRowData, double> working_long_avg_price { get; }
-      public PNLCellVM<PNLRowData, WorkingOrderSideData> working_short_pos { get; }
-      public PNLCellVM<PNLRowData, double> working_short_avg_price { get; }
-      // total
-      public PNLCellVM<PNLRowData, double> total_pnl { get; }
-      public PNLCellVM<PNLRowData, double> total_net_pos { get; }
-      public PNLCellVM<PNLRowData, double> total_long_pos { get; }
-      public PNLCellVM<PNLRowData, double> total_long_avg_price { get; }
-      public PNLCellVM<PNLRowData, double> total_short_pos { get; }
-      public PNLCellVM<PNLRowData, double> total_short_avg_price { get; }
-      public PNLCellVM<PNLRowData, double> total_delta { get; }
-      public PNLCellVM<PNLRowData, double> total_gamma { get; }
-      public PNLCellVM<PNLRowData, double> total_vega { get; }
-      public PNLCellVM<PNLRowData, double> total_theta { get; }
-
-      // (market)
-      public PNLCellVM<PNLRowData, double> bid_price { get; }
-      public PNLCellVM<PNLRowData, double> mid_price { get; }
-      public PNLCellVM<PNLRowData, double> ask_price { get; }
-      public PNLCellVM<PNLRowData, double> last_price { get; }
-      public PNLCellVM<PNLRowData, double> settlement_price { get; }
-      public PNLCellVM<PNLRowData, double> pnl_price { get; }
-
-      // display
-      public api.domain.TickDisplaySettingsImmutable quantity_display {
-         get {
-            if (_quantity_display_backing.HasValue) {
-               return _quantity_display_backing.Value;
-            }
-            if (security.HasValue) {
-               _quantity_display_backing = _sec_meta_store.get_qty_display(security.Value);
-            }
-            return _quantity_display_backing ??= Security.Defaults.quantity_display_settingsi;
-         }
-      }
-
-      public IList<mbts.ui.spark.CellAnimation> animations { get; private set; }
-
-      public bool is_selected {
-         get => _is_selected;
-         set {
-            if (Set(ref _is_selected, value))
-               is_selected_changed?.Invoke(value);
-         }
-      }
-
-      IEnumerable<PNLCellVM<PNLRowData, double>> _market_price_cells {
-         get {
-            yield return pnl_price;
-            yield return last_price;
-            yield return bid_price;
-            yield return mid_price;
-            yield return ask_price;
-            yield return settlement_price;
-         }
-      }
-
-      // includes sod, day, total, and working avg price columns
-      IEnumerable<PNLCellVM<PNLRowData, double>> _fill_price_cells {
-         get {
-            // sod
-            yield return sod_long_avg_price;
-            yield return sod_short_avg_price;
-            // day
-            yield return day_long_avg_price;
-            yield return day_short_avg_price;
-            // total
-            yield return total_long_avg_price;
-            yield return total_short_avg_price;
-            // working
-            yield return working_long_avg_price;
-            yield return working_short_avg_price;
-         }
-      }
-
-      public void update(PNLRowData data) {
-         _quantity_display_backing = null;
-         _refresh_normalized_data = true;
-         this.data = data;
-         _cell_map.on_row(data);
-         _animator.on_row(data);
-         _cell_map.set_appearances(_animator.cell_appearances);
-      }
-
-      public void on_settings(ImmutableDictionary<ColumnKeys, PNLCellAppearance> defaults, IReadOnlyList<IPNLAnimationTrigger<PNLRowData>> triggers, Func<double, string> fmt_fill_price, Func<double, string> fmt_market_price) {
-         _animator.on_settings(defaults, triggers);
-         _cell_map.set_appearances(_animator.cell_appearances);
-         _set_price_formatting_delegates(fmt_fill_price, fmt_market_price);
-      }
-
-      public bool Equals(PNLRowVM other) =>
-         other != null && key == other.key;
-
-      public override int GetHashCode() =>
-         key.GetHashCode();
-
-      public override string ToString() =>
-         $"account: {account_name} exchange: {exchange} symbol: {symbol} security: {security} buy_quantity: {day_long_pos} sell_quantity: {day_short_pos} pnl: {total_pnl}";
-
-      void _set_price_formatting_delegates(Func<double, string> fmt_fill_price, Func<double, string> fmt_market_price) {
-         foreach (var cell in _fill_price_cells) {
-            cell.format = fmt_fill_price;
-         }
-         foreach (var cell in _market_price_cells) {
-            cell.format = fmt_market_price;
-         }
-      }
-   }
-}
+using bts.api;
+using bts.utils;
+
+using mbts;
+using mbts.spark;
+using mbts.ui;
+using mbts.ui.shell;
+
+using Spark.Core.Collections;
+using Spark.Core.Data;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Reactive.Linq;
+
+using ColumnKeys = bts.shell.pnl.PNLView.Columns;
+using PNLKey = bts.AccountSecurityKey;
+
+namespace bts.shell.pnl {
+   class PNLRowVM : DisposableViewModelBase, IEquatable<PNLRowVM> {
+      public static readonly IComparer<PNLRowVM> key_comparer = Comparer<PNLKey>.Default.wrap<PNLKey, PNLRowVM>(r => r.key);
+      static readonly Func<bool, string> _fmt_has_day_trades = hdt => hdt ? "Yes" : "No";
+      //
+      readonly PNLCellIndexBuilder<PNLRowData> _cell_map = new PNLCellIndexBuilder<PNLRowData>();
+      readonly PNLCellEmbeddedAnimationModel<PNLRowData> _animator;
+      readonly ISecurityMetadataStore _sec_meta_store;
+      bool _refresh_normalized_data = true, _is_selected;
+      PNLNormalizedData _normalized_data;
+      api.domain.TickDisplaySettingsImmutable? _quantity_display_backing;
+
+      public PNLRowVM(
+         PNLRowData data,
+         ISecurityMetadataStore sec_meta_store,
+         ImmutableDictionary<ColumnKeys, PNLCellAppearance> default_appearances,
+         IReadOnlyList<IPNLAnimationTrigger<PNLRowData>> triggers,
+         Func<double, string> fmt_fill_price,
+         Func<double, string> fmt_market_price) {
+         //
+         this.data = data;
+         _sec_meta_store = sec_meta_store;
+         _animator = new PNLCellEmbeddedAnimationModel<PNLRowData>(default_appearances, triggers);
+         _animator.on_row(data);
+         pnl_price = _cell_map.add_price_double(ColumnKeys.PNLPrice, d => d.pnl_price);
+         account_cell = _cell_map.add(ColumnKeys.Account, d => d.account);
+         security_type_cell = _cell_map.add(ColumnKeys.SecurityType, d => d.security?.ContractCase ?? Security.ContractOneofCase.None);
+         exchange_cell = _cell_map.add(ColumnKeys.Exchange, d => d.security?.Exchange ?? Venue.Unset);
+         expiration_cell = _cell_map.add(
+            ColumnKeys.Expiration,
+            d => d.expiration,
+            e => e == DateTime.MinValue
+               ? "N/A"
+               : SecurityFormatter.format_expiration(e));
+         symbol_cell = _cell_map.add(ColumnKeys.Symbol, d => d.symbol);
+         security_cell = _cell_map.add(
+            ColumnKeys.Security,
+            d => d.security,
+            null,
+            EqualityComparer<SecurityImmutable?>.Default,
+            Comparer<SecurityImmutable?>.Default);
+         security_key = _cell_map.add(ColumnKeys.SecurityKey, d => d.key.security_id);
+         cpp = _cell_map.add(ColumnKeys.CPP, d => d.effective_cpp);
+         // sod
+         sod_pnl = _cell_map.add_pnl_double(ColumnKeys.SODPNL, d => d.sod_pnl);
+         sod_net_pos = _cell_map.add_net_pos(
+            ColumnKeys.SODNetPos, d => d.display_sod_net_pos, fmt_qty);
+         sod_long_pos = _cell_map.add_long_pos(
+            ColumnKeys.SODLongPos, d => d.display_sod_long_pos, fmt_qty);
+         sod_long_avg_price = _cell_map.add_long_price(ColumnKeys.SODLongAvgPrice, d => d.sod_long_avg_price);
+         sod_short_pos = _cell_map.add_short_pos(
+            ColumnKeys.SODShortPos, d => d.display_sod_short_pos, fmt_qty);
+         sod_short_avg_price = _cell_map.add_short_price(ColumnKeys.SODShortAvgPrice, d => d.sod_short_avg_price);
+         sod_delta = _cell_map.add_pnl_double(ColumnKeys.SODDelta, d => d.sod_delta, fmt_greek);
+         sod_gamma = _cell_map.add_pnl_double(ColumnKeys.SODGamma, d => d.sod_gamma, fmt_greek);
+         sod_vega = _cell_map.add_pnl_double(ColumnKeys.SODVega, d => d.sod_vega, fmt_greek);
+         sod_theta = _cell_map.add_pnl_double(ColumnKeys.SODTheta, d => d.sod_theta, fmt_greek);
+
+         // day
+         day_pnl = _cell_map.add_pnl_double(ColumnKeys.DayPNL, d => d.day_pnl);
+         day_net_pos = _cell_map.add_net_pos(
+            ColumnKeys.DayNetPos, d => d.display_day_net_pos, fmt_qty);
+         day_long_pos = _cell_map.add_long_pos(
+            ColumnKeys.DayLongPos, d => d.display_day_long_pos, fmt_qty);
+         day_long_avg_price = _cell_map.add_long_price(ColumnKeys.DayLongAvgPrice, d => d.day_long_avg_price);
+         day_short_pos = _cell_map.add_short_pos(
+            ColumnKeys.DayShortPos, d => d.display_day_short_pos, fmt_qty);
+         day_short_avg_price = _cell_map.add_short_price(ColumnKeys.DayShortAvgPrice, d => d.day_short_avg_price);
+         day_delta = _cell_map.add_pnl_double(ColumnKeys.DayDelta, d => d.day_delta, fmt_greek);
+         day_gamma = _cell_map.add_pnl_double(ColumnKeys.DayGamma, d => d.day_gamma, fmt_greek);
+         day_vega = _cell_map.add_pnl_double(ColumnKeys.DayVega, d => d.day_vega, fmt_greek);
+         day_theta = _cell_map.add_pnl_double(ColumnKeys.DayTheta, d => d.day_theta, fmt_greek);
+         has_day_trades = _cell_map.add(ColumnKeys.HasDayTrades, d => d.has_day_trades, _fmt_has_day_trades);
+         // working
+         working_net_pos = _cell_map.add_net_pos(
+            ColumnKeys.WorkingNetPos, d => d.display_working_net_pos, fmt_qty);
+         var qty_display = _sec_meta_store.get_qty_display(security);
+         working_long_pos = _cell_map.add_working_long(
+            ColumnKeys.WorkingLongPos,
+            d => d.working.long_data,
+            d => d.bid_price,
+            qty_display);
+         working_long_avg_price = _cell_map.add_long_price(ColumnKeys.WorkingLongAvgPrice, d => d.working_long_avg_price);
+         working_short_pos = _cell_map.add_working_short(
+            ColumnKeys.WorkingShortPos, r => r.working.short_data, r => r.ask_price, qty_display);
+         working_short_avg_price = _cell_map.add_short_price(ColumnKeys.WorkingShortAvgPrice, d => d.working_short_avg_price);
+         // total
+         total_pnl = _cell_map.add_pnl_double(ColumnKeys.TotalPNL, d => d.total_pnl);
+         total_net_pos = _cell_map.add_net_pos(
+            ColumnKeys.TotalNetPos, d => d.display_total_net_pos, fmt_qty);
+         total_long_pos = _cell_map.add_long_pos(
+            ColumnKeys.TotalLongPos, d => d.display_total_long_pos, fmt_qty);
+         total_long_avg_price = _cell_map.add_long_price(ColumnKeys.TotalLongAvgPrice, d => d.total_long_avg_price);
+         total_short_pos = _cell_map.add_short_pos(
+            ColumnKeys.TotalShortPos, d => d.display_total_short_pos, fmt_qty);
+         total_short_avg_price = _cell_map.add_short_price(ColumnKeys.TotalShortAvgPrice, d => d.total_short_avg_price);
+         total_delta = _cell_map.add_pnl_double(
+            ColumnKeys.TotalDelta, d => d.total_delta, fmt_greek);
+         total_gamma = _cell_map.add_pnl_double(
+            ColumnKeys.TotalGamma, d => d.total_gamma, fmt_greek);
+         total_vega = _cell_map.add_pnl_double(ColumnKeys.TotalVega, d => d.total_vega, fmt_greek);
+         total_theta = _cell_map.add_pnl_double(
+            ColumnKeys.TotalTheta, d => d.total_theta, fmt_greek);
+
+         // market
+         bid_price = _cell_map.add_price_double(ColumnKeys.BidPrice, d => d.bid_price);
+         ask_price = _cell_map.add_price_double(ColumnKeys.AskPrice, d => d.ask_price);
+         mid_price = _cell_map.add_price_double(ColumnKeys.MidPrice, d => d.mid_price);
+         last_price = _cell_map.add_price_double(ColumnKeys.LastPrice, d => d.last_price);
+         settlement_price = _cell_map.add_price_double(ColumnKeys.SettlementPrice, d => d.settlement_price);
+
+         _set_price_formatting_delegates(fmt_fill_price, fmt_market_price);
+         _cell_map.on_row(data);
+         _cell_map.set_appearances(_animator.cell_appearances);
+         _dispose = _cell_map.animation_model.OnDispatcher().Subscribe(new_animations => {
+            animations = new_animations.Select(kv => new mbts.ui.spark.CellAnimation(kv.Key.column, kv.Key.type, kv.Value.color.to_brush())).ToList();
+            RaisePropertyChanged(nameof(animations));
+         });
+
+         static string fmt_greek(double v) =>
+            v.is_nan() ? "-" : v.ToString("#,0.00");
+
+         string fmt_qty(double v) =>
+            quantity_display.format(v);
+      }
+
+      // events
+      public event Action<PNLNormalizedData> updated;
+      public event Action<bool> is_selected_changed;
+
+      // data
+      public PNLKey key => data.key;
+      public PNLRowData data { get; private set; }
+      public PNLNormalizedData normalized_data {
+         get {
+            if (_refresh_normalized_data) {
+               _refresh_normalized_data = false;
+               // Only count outrights (spread positions should be normalized into outrights)
+               var sec_type = security?.ContractCase ?? Security.ContractOneofCase.None;
+               var (
+                  sod_long,
+                  sod_short,
+                  sod_delta,
+                  sod_gamma,
+                  sod_vega,
+                  sod_theta,
+                  day_long,
+                  day_short,
+                  day_delta,
+                  day_gamma,
+                  day_vega,
+                  day_theta) = sec_type switch {
+                  Security.ContractOneofCase.Future or Security.ContractOneofCase.Option => (
+                     data.sod.long_pos.round_to_decimal(),
+                     data.sod.short_pos.round_to_decimal(),
+                     data.sod.delta.round_to_decimal(),
+                     data.sod.gamma.round_to_decimal(),
+                     data.sod.vega.round_to_decimal(),
+                     data.sod.theta.round_to_decimal(),
+                     data.day.long_pos.round_to_decimal(),
+                     data.day.short_pos.round_to_decimal(),
+                     data.day.delta.round_to_decimal(),
+                     data.day.gamma.round_to_decimal(),
+                     data.day.vega.round_to_decimal(),
+                     data.day.theta.round_to_decimal()),
+                  _ => (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
+               };
+
+               var working_long = data.working_long_pos.round_to_decimal();
+               var working_short = data.working_short_pos.round_to_decimal();
+
+               var nd = new PNLNormalizedData() {
+                  // sod
+                  sod_pnl = sod_pnl.value.round_to_decimal(),
+                  sod_long_pos = sod_long,
+                  sod_short_pos = sod_short,
+                  sod_delta = sod_delta,
+                  sod_gamma = sod_gamma,
+                  sod_vega = sod_vega,
+                  sod_theta = sod_theta,
+                  display_sod_long_pos = sod_long,
+                  display_sod_short_pos = sod_short,
+
+                  // day
+                  day_pnl = day_pnl.value.round_to_decimal(),
+                  day_long_pos = day_long,
+                  day_short_pos = day_short,
+                  day_delta = day_delta,
+                  day_gamma = day_gamma,
+                  day_vega = day_vega,
+                  day_theta = day_theta,
+                  display_day_long_pos = day_long,
+                  display_day_short_pos = day_short,
+
+                  // working
+                  working_long_pos = working_long,
+                  working_short_pos = working_short,
+                  display_working_long_pos = working_long,
+                  display_working_short_pos = working_short
+               };
+               if (_normalized_data != nd) {
+                  _normalized_data = nd;
+                  updated?.Invoke(nd);
+               }
+            }
+            return _normalized_data;
+         }
+      }
+      // security
+      public string account_name => data.account_name;
+      public Venue exchange => security?.Exchange ?? Venue.Unset;
+      public string symbol => security.symbol_or_default();
+      public DateTime expiration => data.expiration;
+      public Security.ContractOneofCase security_type => security?.ContractCase ?? Security.ContractOneofCase.None;
+      public SecurityImmutable? security => data.security;
+      // cells
+      public PNLCellVM<PNLRowData, AccountTuple> account_cell { get; }
+      public PNLCellVM<PNLRowData, Venue> exchange_cell { get; }
+      public PNLCellVM<PNLRowData, DateTime> expiration_cell { get; }
+      public PNLCellVM<PNLRowData, string> symbol_cell { get; }
+      public PNLCellVM<PNLRowData, Security.ContractOneofCase> security_type_cell { get; }
+      public PNLCellVM<PNLRowData, SecurityImmutable?> security_cell { get; }
+      public PNLCellVM<PNLRowData, uint> security_key { get; }
+      public PNLCellVM<PNLRowData, double> cpp { get; }
+      // sod
+      public PNLCellVM<PNLRowData, double> sod_pnl { get; }
+      public PNLCellVM<PNLRowData, double> sod_net_pos { get; }
+      public PNLCellVM<PNLRowData, double> sod_long_pos { get; }
+      public PNLCellVM<PNLRowData, double> sod_long_avg_price { get; }
+      public PNLCellVM<PNLRowData, double> sod_short_pos { get; }
+      public PNLCellVM<PNLRowData, double> sod_short_avg_price { get; }
+      public PNLCellVM<PNLRowData, double> sod_delta { get; }
+      public PNLCellVM<PNLRowData, double> sod_gamma { get; }
+      public PNLCellVM<PNLRowData, double> sod_vega { get; }
+      public PNLCellVM<PNLRowData, double> sod_theta { get; }
+
+      // day
+      public PNLCellVM<PNLRowData, double> day_pnl { get; }
+      public PNLCellVM<PNLRowData, double> day_net_pos { get; }
+      public PNLCellVM<PNLRowData, double> day_long_pos { get; }
+      public PNLCellVM<PNLRowData, double> day_long_avg_price { get; }
+      public PNLCellVM<PNLRowData, double> day_short_pos { get; }
+      public PNLCellVM<PNLRowData, double> day_short_avg_price { get; }
+      public PNLCellVM<PNLRowData, double> day_delta { get; }
+      public PNLCellVM<PNLRowData, double> day_gamma { get; }
+      public PNLCellVM<PNLRowData, double> day_vega { get; }
+      public PNLCellVM<PNLRowData, double> day_theta { get; }
+      public PNLCellVM<PNLRowData, bool> has_day_trades { get; }
+      // working
+      public PNLCellVM<PNLRowData, double> working_net_pos { get; }
+      public PNLCellVM<PNLRowData, WorkingOrderSideData> working_long_pos { get; }
+      public PNLCellVM<PNLRowData, double> working_long_avg_price { get; }
+      public PNLCellVM<PNLRowData, WorkingOrderSideData> working_short_pos { get; }
+      public PNLCellVM<PNLRowData, double> working_short_avg_price { get; }
+      // total
+      public PNLCellVM<PNLRowData, double> total_pnl { get; }
+      public PNLCellVM<PNLRowData, double> total_net_pos { get; }
+      public PNLCellVM<PNLRowData, double> total_long_pos { get; }
+      public PNLCellVM<PNLRowData, double> total_long_avg_price { get; }
+      public PNLCellVM<PNLRowData, double> total_short_pos { get; }
+      public PNLCellVM<PNLRowData, double> total_short_avg_price { get; }
+      public PNLCellVM<PNLRowData, double> total_delta { get; }
+      public PNLCellVM<PNLRowData, double> total_gamma { get; }
+      public PNLCellVM<PNLRowData, double> total_vega { get; }
+      public PNLCellVM<PNLRowData, double> total_theta { get; }
+
+      // (market)
+      public PNLCellVM<PNLRowData, double> bid_price { get; }
+      public PNLCellVM<PNLRowData, double> mid_price { get; }
+      public PNLCellVM<PNLRowData, double> ask_price { get; }
+      public PNLCellVM<PNLRowData, double> last_price { get; }
+      public PNLCellVM<PNLRowData, double> settlement_price { get; }
+      public PNLCellVM<PNLRowData, double> pnl_price { get; }
+
+      // display
+      public api.domain.TickDisplaySettingsImmutable quantity_display {
+         get {
+            if (_quantity_display_backing.HasValue) {
+               return _quantity_display_backing.Value;
+            }
+            if (security.HasValue) {
+               _quantity_display_backing = _sec_meta_store.get_qty_display(security.Value);
+            }
+            return _quantity_display_backing ??= Security.Defaults.quantity_display_settingsi;
+         }
+      }
+
+      public IList<mbts.ui.spark.CellAnimation> animations { get; private set; }
+
+      public bool is_selected {
+         get => _is_selected;
+         set {
+            if (Set(ref _is_selected, value))
+               is_selected_changed?.Invoke(value);
+         }
+      }
+
+      IEnumerable<PNLCellVM<PNLRowData, double>> _market_price_cells {
+         get {
+            yield return pnl_price;
+            yield return last_price;
+            yield return bid_price;
+            yield return mid_price;
+            yield return ask_price;
+            yield return settlement_price;
+         }
+      }
+
+      // includes sod, day, total, and working avg price columns
+      IEnumerable<PNLCellVM<PNLRowData, double>> _fill_price_cells {
+         get {
+            // sod
+            yield return sod_long_avg_price;
+            yield return sod_short_avg_price;
+            // day
+            yield return day_long_avg_price;
+            yield return day_short_avg_price;
+            // total
+            yield return total_long_avg_price;
+            yield return total_short_avg_price;
+            // working
+            yield return working_long_avg_price;
+            yield return working_short_avg_price;
+         }
+      }
+
+      public void update(PNLRowData data) {
+         _quantity_display_backing = null;
+         _refresh_normalized_data = true;
+         this.data = data;
+         _cell_map.on_row(data);
+         _animator.on_row(data);
+         _cell_map.set_appearances(_animator.cell_appearances);
+      }
+
+      public void on_settings(ImmutableDictionary<ColumnKeys, PNLCellAppearance> defaults, IReadOnlyList<IPNLAnimationTrigger<PNLRowData>> triggers, Func<double, string> fmt_fill_price, Func<double, string> fmt_market_price) {
+         _animator.on_settings(defaults, triggers);
+         _cell_map.set_appearances(_animator.cell_appearances);
+         _set_price_formatting_delegates(fmt_fill_price, fmt_market_price);
+      }
+
+      public bool Equals(PNLRowVM other) =>
+         other != null && key == other.key;
+
+      public override int GetHashCode() =>
+         key.GetHashCode();
+
+      public override string ToString() =>
+         $"account: {account_name} exchange: {exchange} symbol: {symbol} security: {security} buy_quantity: {day_long_pos} sell_quantity: {day_short_pos} pnl: {total_pnl}";
+
+      void _set_price_formatting_delegates(Func<double, string> fmt_fill_price, Func<double, string> fmt_market_price) {
+         foreach (var cell in _fill_price_cells) {
+            cell.format = fmt_fill_price;
+         }
+         foreach (var cell in _market_price_cells) {
+            cell.format = fmt_market_price;
+         }
+      }
+   }
+}

src/client/spark/Orders/OrderTicket.xaml

@@ -1,161 +1,161 @@
-<Window
-  x:Class="mbts.ui.spark.OrderTicket"
-  xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
-  xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
-  xmlns:msys="clr-namespace:bts.api;assembly=spark_api"
-  xmlns:u="clr-namespace:mbts.ui"
-  xmlns:cv="clr-namespace:mbts.ui.converters"
-  xmlns:usp="clr-namespace:mbts.ui.spark"
-  xmlns:sys="clr-namespace:System;assembly=mscorlib"
-  u:WindowStyle.SizeToContent="WidthAndHeight"
-  u:WindowStyle.TitleContent="{Binding bbo, Mode=OneWay}"
-  Style="{StaticResource BTSWindowTitleTemplate}"
->
-  <Window.Resources>
-    <ResourceDictionary>
-      <ResourceDictionary.MergedDictionaries>
-        <ResourceDictionary Source="/view/Resources/Converters.xaml"/>
-      </ResourceDictionary.MergedDictionaries>
-      <sys:Double x:Key="HorizontalSpacing">8.0</sys:Double>
-      <sys:Double x:Key="VerticalSpacing">4.0</sys:Double>
-      <sys:Double x:Key="ButtonHeight">22.0</sys:Double>
-      <Style x:Key="SellButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
-         <Setter Property="Background" Value="{u:BTSColor MarketData.ask}" />
-         <Setter Property="Foreground" Value="White" />
-         <Setter Property="MinHeight" Value="40" />
-         <Setter Property="FontWeight" Value="Bold" />
-         <Setter Property="HorizontalAlignment" Value="Stretch" />
-         <Setter Property="VerticalAlignment" Value="Stretch" />
-         <Setter Property="Content">
-            <Setter.Value>
-              <TextBlock VerticalAlignment="Center" Text="Sell" />
-            </Setter.Value>
-         </Setter>
-         <Setter Property="Template">
-            <Setter.Value>
-               <ControlTemplate TargetType="Button">
-                  <Border 
-                        x:Name="Border"
-                        CornerRadius="0" 
-                        BorderThickness="1"
-                        BorderBrush="{u:BTSColor UI.Button.normal_border}"
-                        Background="{u:BTSColor MarketData.ask}"
-                        Padding="2"
-                        Height="Auto"
-                        SnapsToDevicePixels="True">
-                        <ContentPresenter 
-                           HorizontalAlignment="Center"
-                           VerticalAlignment="Center"
-                           RecognizesAccessKey="True"
-                           Name="button_content" 
-                        />
-                     </Border>
-                  <ControlTemplate.Triggers>
-                        <Trigger Property="IsMouseOver" Value="True">
-                           <Setter TargetName="Border" Property="Background" Value="#99c00000" />
-                           <Setter TargetName="Border" Property="BorderBrush" Value="{u:BTSColor UI.Button.mouseover_border}" />
-                        </Trigger>
-                     </ControlTemplate.Triggers>
-               </ControlTemplate>
-            </Setter.Value>
-         </Setter>
-      </Style>
-      <Style x:Key="BuyButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
-         <Setter Property="Background" Value="{u:BTSColor MarketData.bid}" />
-         <Setter Property="Foreground" Value="White" />
-         <Setter Property="MinHeight" Value="40" />
-         <Setter Property="FontWeight" Value="Bold" />
-         <Setter Property="HorizontalAlignment" Value="Stretch" />
-         <Setter Property="VerticalAlignment" Value="Stretch" />
-         <Setter Property="Content">
-            <Setter.Value>
-              <TextBlock VerticalAlignment="Center" Text="Buy" />
-            </Setter.Value>
-         </Setter>
-         <Setter Property="Template">
-            <Setter.Value>
-               <ControlTemplate TargetType="Button">
-                  <Border 
-                     x:Name="Border"
-                     CornerRadius="0" 
-                     BorderThickness="1"
-                     BorderBrush="{u:BTSColor UI.Button.normal_border}"
-                     Background="{u:BTSColor MarketData.bid}"
-                     Padding="2"
-                     Height="Auto"
-                     SnapsToDevicePixels="True">
-                     <ContentPresenter 
-                        HorizontalAlignment="Center"
-                        VerticalAlignment="Center"
-                        RecognizesAccessKey="True"
-                        Name="button_content" 
-                     />
-                  </Border>
-                  <ControlTemplate.Triggers>
-                     <Trigger Property="IsMouseOver" Value="True">
-                        <Setter TargetName="Border" Property="Background" Value="#99167ed3" />
-                        <Setter TargetName="Border" Property="BorderBrush" Value="{u:BTSColor UI.Button.mouseover_border}" />
-                     </Trigger>
-                  </ControlTemplate.Triggers>
-               </ControlTemplate>
-            </Setter.Value>
-         </Setter>
-      </Style>
-      </ResourceDictionary>
-  </Window.Resources>
-  <Window.InputBindings>
-    <KeyBinding Key="Esc" Command="{x:Static SystemCommands.CloseWindowCommand}"/>
-  </Window.InputBindings>
-  <Grid Margin="10">
-    <Grid.RowDefinitions>
-      <RowDefinition Height="Auto"/>
-      <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}"/>
-      <RowDefinition Height="Auto"/>
-    </Grid.RowDefinitions>
-    <Grid.ColumnDefinitions>
-      <ColumnDefinition MinWidth="120"/>
-      <ColumnDefinition Width="{Binding Source={StaticResource HorizontalSpacing}}"/>
-      <ColumnDefinition MinWidth="120"/>
-    </Grid.ColumnDefinitions>
-    <usp:OrderModificationControl Grid.ColumnSpan="3" />
-      <Button Grid.Row="2"                 Style="{StaticResource BuyButton}"  Command="{Binding send_order}" CommandParameter="{u:Enum Buy, Type=msys:Side}" />
-      <Button Grid.Row="2" Grid.Column="2" Style="{StaticResource SellButton}" Command="{Binding send_order}" CommandParameter="{u:Enum Sell, Type=msys:Side}" />
-   </Grid>
-  <u:WindowStyle.TitleContentTemplate>
-    <DataTemplate>
-      <TextBlock Style="{StaticResource TextBlock.Dark}">
-        <TextBlock.Inlines>
-          <Run FontWeight="Bold" Text="{Binding security, Mode=OneWay}" />
-          <Run Foreground="{u:BTSColor MarketData.bid}" FontSize="10">
-            <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}" StringFormat="({0})">
-              <Binding Path="bid_size" Mode="OneWay"/>
-              <Binding Path="DataContext.qty_display" RelativeSource="{RelativeSource AncestorType=usp:OrderTicket}"/>
-            </MultiBinding>
-          </Run>
-          <Run Foreground="{u:BTSColor MarketData.bid}" FontWeight="Bold">
-            <Run.Text>
-              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
-                <Binding Path="bid"/>
-                <Binding Path="security.ContractSpec"/>
-              </MultiBinding>
-            </Run.Text>
-          </Run>
-          <Run Foreground="{u:BTSColor MarketData.ask}" FontWeight="Bold">
-            <Run.Text>
-              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
-                <Binding Path="ask"/>
-                <Binding Path="security.ContractSpec"/>
-              </MultiBinding>
-            </Run.Text>
-          </Run>
-          <Run Foreground="{u:BTSColor MarketData.ask}" FontSize="10">
-            <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}" StringFormat="({0})">
-              <Binding Path="ask_size" Mode="OneWay"/>
-              <Binding Path="DataContext.qty_display" RelativeSource="{RelativeSource AncestorType=usp:OrderTicket}"/>
-            </MultiBinding>
-          </Run>
-        </TextBlock.Inlines>
-      </TextBlock>
-    </DataTemplate>
-  </u:WindowStyle.TitleContentTemplate>
-</Window>
+<Window
+  x:Class="mbts.ui.spark.OrderTicket"
+  xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+  xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+  xmlns:msys="clr-namespace:bts.api;assembly=spark_api"
+  xmlns:u="clr-namespace:mbts.ui"
+  xmlns:cv="clr-namespace:mbts.ui.converters"
+  xmlns:usp="clr-namespace:mbts.ui.spark"
+  xmlns:sys="clr-namespace:System;assembly=mscorlib"
+  u:WindowStyle.SizeToContent="WidthAndHeight"
+  u:WindowStyle.TitleContent="{Binding bbo, Mode=OneWay}"
+  Style="{StaticResource BTSWindowTitleTemplate}"
+>
+  <Window.Resources>
+    <ResourceDictionary>
+      <ResourceDictionary.MergedDictionaries>
+        <ResourceDictionary Source="/view/Resources/Converters.xaml"/>
+      </ResourceDictionary.MergedDictionaries>
+      <sys:Double x:Key="HorizontalSpacing">8.0</sys:Double>
+      <sys:Double x:Key="VerticalSpacing">4.0</sys:Double>
+      <sys:Double x:Key="ButtonHeight">22.0</sys:Double>
+      <Style x:Key="SellButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
+         <Setter Property="Background" Value="{u:BTSColor MarketData.ask_dark}" />
+         <Setter Property="Foreground" Value="White" />
+         <Setter Property="MinHeight" Value="40" />
+         <Setter Property="FontWeight" Value="Bold" />
+         <Setter Property="HorizontalAlignment" Value="Stretch" />
+         <Setter Property="VerticalAlignment" Value="Stretch" />
+         <Setter Property="Content">
+            <Setter.Value>
+              <TextBlock VerticalAlignment="Center" Text="Sell" />
+            </Setter.Value>
+         </Setter>
+         <Setter Property="Template">
+            <Setter.Value>
+               <ControlTemplate TargetType="Button">
+                  <Border 
+                        x:Name="Border"
+                        CornerRadius="0" 
+                        BorderThickness="1"
+                        BorderBrush="{u:BTSColor UI.Button.normal_border}"
+                        Background="{u:BTSColor MarketData.ask}"
+                        Padding="2"
+                        Height="Auto"
+                        SnapsToDevicePixels="True">
+                        <ContentPresenter 
+                           HorizontalAlignment="Center"
+                           VerticalAlignment="Center"
+                           RecognizesAccessKey="True"
+                           Name="button_content" 
+                        />
+                     </Border>
+                  <ControlTemplate.Triggers>
+                        <Trigger Property="IsMouseOver" Value="True">
+                           <Setter TargetName="Border" Property="Background" Value="#99c00000" />
+                           <Setter TargetName="Border" Property="BorderBrush" Value="{u:BTSColor UI.Button.mouseover_border}" />
+                        </Trigger>
+                     </ControlTemplate.Triggers>
+               </ControlTemplate>
+            </Setter.Value>
+         </Setter>
+      </Style>
+      <Style x:Key="BuyButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
+         <Setter Property="Background" Value="{u:BTSColor MarketData.bid_dark}" />
+         <Setter Property="Foreground" Value="White" />
+         <Setter Property="MinHeight" Value="40" />
+         <Setter Property="FontWeight" Value="Bold" />
+         <Setter Property="HorizontalAlignment" Value="Stretch" />
+         <Setter Property="VerticalAlignment" Value="Stretch" />
+         <Setter Property="Content">
+            <Setter.Value>
+              <TextBlock VerticalAlignment="Center" Text="Buy" />
+            </Setter.Value>
+         </Setter>
+         <Setter Property="Template">
+            <Setter.Value>
+               <ControlTemplate TargetType="Button">
+                  <Border 
+                     x:Name="Border"
+                     CornerRadius="0" 
+                     BorderThickness="1"
+                     BorderBrush="{u:BTSColor UI.Button.normal_border}"
+                     Background="{u:BTSColor MarketData.bid}"
+                     Padding="2"
+                     Height="Auto"
+                     SnapsToDevicePixels="True">
+                     <ContentPresenter 
+                        HorizontalAlignment="Center"
+                        VerticalAlignment="Center"
+                        RecognizesAccessKey="True"
+                        Name="button_content" 
+                     />
+                  </Border>
+                  <ControlTemplate.Triggers>
+                     <Trigger Property="IsMouseOver" Value="True">
+                        <Setter TargetName="Border" Property="Background" Value="#99167ed3" />
+                        <Setter TargetName="Border" Property="BorderBrush" Value="{u:BTSColor UI.Button.mouseover_border}" />
+                     </Trigger>
+                  </ControlTemplate.Triggers>
+               </ControlTemplate>
+            </Setter.Value>
+         </Setter>
+      </Style>
+      </ResourceDictionary>
+  </Window.Resources>
+  <Window.InputBindings>
+    <KeyBinding Key="Esc" Command="{x:Static SystemCommands.CloseWindowCommand}"/>
+  </Window.InputBindings>
+  <Grid Margin="10">
+    <Grid.RowDefinitions>
+      <RowDefinition Height="Auto"/>
+      <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}"/>
+      <RowDefinition Height="Auto"/>
+    </Grid.RowDefinitions>
+    <Grid.ColumnDefinitions>
+      <ColumnDefinition MinWidth="120"/>
+      <ColumnDefinition Width="{Binding Source={StaticResource HorizontalSpacing}}"/>
+      <ColumnDefinition MinWidth="120"/>
+    </Grid.ColumnDefinitions>
+    <usp:OrderModificationControl Grid.ColumnSpan="3" />
+      <Button Grid.Row="2"                 Style="{StaticResource BuyButton}"  Command="{Binding send_order}" CommandParameter="{u:Enum Buy, Type=msys:Side}" />
+      <Button Grid.Row="2" Grid.Column="2" Style="{StaticResource SellButton}" Command="{Binding send_order}" CommandParameter="{u:Enum Sell, Type=msys:Side}" />
+   </Grid>
+  <u:WindowStyle.TitleContentTemplate>
+    <DataTemplate>
+      <TextBlock Style="{StaticResource TextBlock.Dark}">
+        <TextBlock.Inlines>
+          <Run FontWeight="Bold" Text="{Binding security, Mode=OneWay}" />
+          <Run Foreground="{u:BTSColor MarketData.bid}" FontSize="10">
+            <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}" StringFormat="({0})">
+              <Binding Path="bid_size" Mode="OneWay"/>
+              <Binding Path="DataContext.qty_display" RelativeSource="{RelativeSource AncestorType=usp:OrderTicket}"/>
+            </MultiBinding>
+          </Run>
+          <Run Foreground="{u:BTSColor MarketData.bid}" FontWeight="Bold">
+            <Run.Text>
+              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
+                <Binding Path="bid"/>
+                <Binding Path="security.ContractSpec"/>
+              </MultiBinding>
+            </Run.Text>
+          </Run>
+          <Run Foreground="{u:BTSColor MarketData.ask}" FontWeight="Bold">
+            <Run.Text>
+              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
+                <Binding Path="ask"/>
+                <Binding Path="security.ContractSpec"/>
+              </MultiBinding>
+            </Run.Text>
+          </Run>
+          <Run Foreground="{u:BTSColor MarketData.ask}" FontSize="10">
+            <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}" StringFormat="({0})">
+              <Binding Path="ask_size" Mode="OneWay"/>
+              <Binding Path="DataContext.qty_display" RelativeSource="{RelativeSource AncestorType=usp:OrderTicket}"/>
+            </MultiBinding>
+          </Run>
+        </TextBlock.Inlines>
+      </TextBlock>
+    </DataTemplate>
+  </u:WindowStyle.TitleContentTemplate>
+</Window>

src/client/spark/Risk/PNL/PNLTradeLogRow.cs

@@ -4,6 +4,9 @@ using bts.api;
 
 using BCG;
 using mbts.co2;
+using System.Xml.Serialization;
+using ZstdSharp.Unsafe;
+using bts.proto;
 
 namespace mbts.analytics {
    // Pairs ITradeLogRow along with the data (theo, md) that is needed to compute
@@ -18,20 +21,87 @@ namespace mbts.analytics {
 
       public SecurityImmutable security { get { return md.security; } }
 
-      private double _option_pnl(double trade_price, double now_price) {
-         double coin_terms = (now_price - trade_price) * tl_row.size;
-         double usd_terms = coin_terms * driver;
-         return usd_terms;
+      // private double _option_pnl(double trade_price, double now_price) {
+      //    double coin_terms = (now_price - trade_price) * tl_row.size;
+      //    double usd_terms = coin_terms * driver;
+      //    return usd_terms;
+      // }
+
+      private bool _is_notional {
+         get {
+            return false;
+         }
       }
 
-      private double _future_pnl(double trade_price, double now_price) {
-         // the qty is in USD, so this is the initial cash flow
-         double init_asset_value = tl_row.size;
-         double asset_size = init_asset_value / tl_row.price;
-         double now_asset_value = asset_size * now_price;
-         return now_asset_value - init_asset_value;
+      private bool _is_inverse_option_contract {
+         get {
+            return false;
+         }
       }
 
+      private double _cpp {
+         get {
+            return 1.0;
+         }
+      }
+
+      private bool _is_reverse_option_contract {
+         get {
+            // check for Deribit options
+            return false;
+         }
+      }
+
+      private double _option_pnl {
+         get {
+            if (_is_notional) {
+               var now_price = theo;
+               double coin_terms = (now_price - tl_row.price) * tl_row.size;
+               double usd_terms = coin_terms * driver;
+               return usd_terms;
+            } else {
+               return _cpp * tl_row.size * (theo - tl_row.price);
+            }
+         }
+      }
+
+      private double _spot_pnl {
+         get {
+            // both are computed the same way
+            return _future_pnl;
+         }
+      }
+
+      private double _future_pnl {
+         get {
+            if (_is_notional) {
+               // TODO:
+               // let it fall through right now
+               // the qty is in USD, so this is the initial cash flow
+               var now_price = theo;
+               double init_asset_value = tl_row.size;
+               double asset_size = init_asset_value / tl_row.price;
+               double now_asset_value = asset_size * now_price;
+               return now_asset_value - init_asset_value;
+            } else {
+               // TODO: is position pnl, compute it wrt settlement price?
+               return tl_row.size * (theo - tl_row.price);
+            }
+         }
+      }
+
+      // private double _future_pnl(double trade_price, double now_price) {
+      //    if (_is_notional) {
+      //       // the qty is in USD, so this is the initial cash flow
+      //       double init_asset_value = tl_row.size;
+      //       double asset_size = init_asset_value / tl_row.price;
+      //       double now_asset_value = asset_size * now_price;
+      //       return now_asset_value - init_asset_value;
+      //    } else {
+      //       return tl_row.size * (theo - tl_row.price);
+      //    }
+      // }
+
 
       public double driver {
          get {
@@ -46,11 +116,18 @@ namespace mbts.analytics {
       public double theo {
          get {
             switch (tl_row.security_type) {
+               // TODO:
                case SecurityType.Option:
-                  return tl_row.call_put == PutOrCall.Call ?
-                     opt_theo.fwd_normalized_call : opt_theo.fwd_normalized_put;
+                  if (_is_inverse_option_contract) {
+                     return tl_row.call_put == PutOrCall.Call ?
+                        opt_theo.fwd_normalized_call : opt_theo.fwd_normalized_put;
+                  } else {
+                     return tl_row.call_put == PutOrCall.Call ?
+                        opt_theo.call : opt_theo.put;
+                  }
+               // future and options, just get mid market
                case SecurityType.Future:
-                  return fut_theo.theo;
+               //return fut_theo.theo;
                case SecurityType.Spot:
                   var bbo = top_of_book;
                   return (bbo.Item1 + bbo.Item2) / 2;
@@ -62,28 +139,22 @@ namespace mbts.analytics {
       public double pnl {
          get {
             double rv = Double.NaN;
-            if (is_day_trade) {
-               if (tl_row.security_type == SecurityType.Future) {
-                  rv = _future_pnl(tl_row.price, theo);
-               } else if (tl_row.security_type == SecurityType.Option) {
-                  rv = _option_pnl(tl_row.price, theo);
-               } else if (tl_row.security_type == SecurityType.Spot) {
-                  rv = tl_row.size * (theo - tl_row.price);
-               }
-               return rv;
+            switch (tl_row.security_type) {
+               case SecurityType.Option:
+                  rv = _option_pnl;
+                  break;
+               case SecurityType.Future:
+                  rv = _future_pnl;
+                  break;
+               case SecurityType.Spot:
+                  // = tl_row.size * (theo - tl_row.price);
+                  rv = _spot_pnl;
+                  break;
             }
-            // use settle price ?
-            if (tl_row.security_type == SecurityType.Future) {
-               rv = _future_pnl(tl_row.price, theo);
-            } else if (tl_row.security_type == SecurityType.Option) {
-               rv = _option_pnl(tl_row.price, theo);
-            } else if (tl_row.security_type == SecurityType.Spot) {
-               rv = tl_row.size * tl_row.price - settle_price;
-            }
-
             return rv;
          }
       }
+
       public double settle_price {
          get {
             return md.md == null ? Double.NaN : md.md.settlement_price;

src/client/spark/Security/SecurityExpirationHelper.cs

@@ -36,6 +36,10 @@ namespace mbts {
       }
 
       public static bool is_unexpired(bts.api.SecurityTuple st, DateTime exp_date) {
+         if (st.Venue == Venue.Cboe || st.Venue == Venue.Glbx) {
+            // server is already getting teh righ contracts
+            return true;
+         }
          TimeSpan exp_time = get_exp_time(st);
          DateTime full_exp_utc = exp_date.ToUniversalTime().Date.Add(exp_time);
          DateTime now_utc = DateTime.UtcNow;
@@ -43,6 +47,8 @@ namespace mbts {
       }
 
       public static bool is_unexpired(bts.api.SecurityImmutable sec) {
+         if (sec.security_tuple().Venue == Venue.Cboe || sec.security_tuple().Venue == Venue.Glbx)
+            return true;
          var sec_type = sec.security_type();
          switch (sec_type) {
             case bts.api.SecurityType.Future:

src/client/spark/core/FontsAndColors/BTSColors.cs

@@ -216,8 +216,10 @@ namespace mbts.ui {
          public static BTSColor ask_text => BTSCoreColors.red_dark;
          public static BTSColor bid_text => BTSCoreColors.blue;
          public static BTSColor bid => 0xff76b7fa;
+         public static BTSColor bid_dark => 0xff26239a;
          public static BTSColor bid_alt => BTSCoreColors.blue_light.with(0x70);
          public static BTSColor ask => 0xffff6666;
+         public static BTSColor ask_dark => 0xff7a2331;
          public static BTSColor ask_alt => 0x70ffb4b4;
          public static BTSColor last => BTSCoreColors.black;
          public static BTSColor bid_ask_alt_blend => BTSCoreColors.purple_light;

src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs

@@ -26,7 +26,7 @@ namespace mbts.ui.controls {
          public const double
             Account = 62,
             CheckMark = 42,
-            Currency = 54,
+            Currency = 68,
             Exchange = 66,
             FillKey = 68,
             OrderKey = 78,

src/client/spark/view/Controls/Order/OrderTicketResources.xaml

@@ -19,7 +19,7 @@
   <sys:Double x:Key="ButtonHeight">22.0</sys:Double>
 
   <Style x:Key="SellButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
-    <Setter Property="Background" Value="{u:BTSColor MarketData.ask}"/>
+    <Setter Property="Background" Value="{u:BTSBrush MarketData.ask_dark}"/>
     <Setter Property="BorderBrush" Value="{u:BTSColor UI.Button.normal_border}"/>
     <Setter Property="Foreground" Value="{u:BTSColor UI.default_text}" />
     <Setter Property="MinHeight" Value="40"/>
@@ -32,7 +32,7 @@
   </Style>
 
   <Style x:Key="BuyButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
-    <Setter Property="Background" Value="{u:BTSColor MarketData.bid}"/>
+    <Setter Property="Background" Value="{u:BTSBrush MarketData.bid_dark}"/>
     <Setter Property="BorderBrush" Value="{u:BTSColor UI.Button.normal_border}"/>
     <Setter Property="Padding" Value="2"/>
     <Setter Property="Foreground" Value="{u:BTSColor UI.default_text}" />

src/client/spark_api/Orders/OrderRecordTuple.cs

@@ -72,12 +72,14 @@ namespace Spark.Api.Orders {
       // TODO: remove assumption that synthetic quantities are already notional values
       public double notional_value =>
          (security?.HasSynthetic ?? false)
+         && security.Value.Synthetic.PriceCalc == SyntheticContract.Types.PriceCalc.NotionalLeg
             ? order.order.quantity()
             : order.order.notional_value();
 
       // TODO: remove assumption that synthetic quantities are already notional values
       public double filled_notional_value =>
          (security?.HasSynthetic ?? false)
+         && security.Value.Synthetic.PriceCalc == SyntheticContract.Types.PriceCalc.NotionalLeg
             ? order.order.filled_quantity()
             : order.order.filled_notional_value();
 

src/client/spark_core/utils_api/BTS.cs

@@ -1,5 +1,5 @@
 using System;
-using System.Collections.Generic;
+using System.Reflection;
 using System.Text;
 using System.Threading;
 
@@ -10,9 +10,9 @@ namespace bts
       static int _needs_init_spark_lib = 0;
 
       public static void init(string config_source = null) {
-
+         var version = Assembly.GetExecutingAssembly().GetName().Version;
          Console.WriteLine(
-            $"Configuring BTS v2.2.4 with {(config_source is null ? "defaults" : $"'{config_source}'")}");
+            $"Configuring Spark {version} with {(config_source is null ? "defaults" : $"'{config_source}'")}");
          var args = Environment.GetCommandLineArgs();
          utils.BTSNativeMethods.provide_command_line(args, args.Length);
 

src/lib/api/md/DatabentoMarketDataConfig.proto

@@ -15,4 +15,7 @@ message DatabentoMarketDataConfig {
   string dataset = 70;
   uint32 filter_securities_upto_n_months= 80;
   bool load_reference_data = 90;
+  uint32 debug_sec = 100;
+  bool only_securities =  110;
+  string bbo_start_time = 120;
 }

src/lib/api/services/OrderService.cpp

@@ -0,0 +1,174 @@
+#include "OrderService.h"
+#include "lib/api/services/OrderService.pb.h"
+
+namespace bts::api::orderman {
+
+#define PARSE_STRING(name, dest, it)                                           \
+   it = j.find(#name);                                                         \
+   if (it != j.end()) {                                                        \
+      dest.mutable_##name()->clear();                                          \
+      it->get_to(*dest.mutable_##name());                                      \
+   }
+#define PARSE_VALUE(name, dest, it)                                            \
+   it = j.find(#name);                                                         \
+   if (it != j.end()) {                                                        \
+      dest.set_##name(it->get<decltype(dest.name())>());                       \
+   }
+#define PARSE_STRUCT(name, dest, it)                                           \
+   it = j.find(#name);                                                         \
+   if (it != j.end()) {                                                        \
+      it->get_to(*dest.mutable_##name());                                      \
+   }
+#define PARSE_ARRAY(name, dest, it)                                            \
+   it = j.find(#name);                                                         \
+   if (it != j.end() && it->is_array()) {                                      \
+      c.mutable_##name()->Clear();                                             \
+      it->get_to(*c.mutable_##name());                                         \
+   }
+#define PARSE_ONEOF(name, dest, it)                                            \
+   if (!parsed_oneof) {                                                        \
+      it = j.find(#name);                                                      \
+      if (it != j.end()) {                                                     \
+         it->get_to(*dest.mutable_##name());                                   \
+         parsed_oneof = true;                                                  \
+      }                                                                        \
+   }
+#define PARSE_ONEOFS(for_each)                                                 \
+   {                                                                           \
+      bool parsed_oneof = false;                                               \
+      for_each(PARSE_ONEOF)                                                    \
+   }
+
+#define WRITE_VALUE(src, name) j[#name] = src.name()
+#define WRITE_STRUCT(src, name)                                                \
+   if (src.has_##name()) {                                                     \
+      j[#name] = src.name();                                                   \
+   }
+
+#define WRITE_IF_NOT_ZERO(src, name)                                           \
+   if (src.name() != 0) {                                                      \
+      j[#name] = src.name();                                                   \
+   }
+#define WRITE_IF_NOT_NAN(src, name)                                            \
+   if (!std::isnan(src.name())) {                                              \
+      j[#name] = src.name();                                                   \
+   }
+#define WRITE_IF_NOT_EMPTY(src, name)                                          \
+   if (!src.name().empty()) {                                                  \
+      j[#name] = src.name();                                                   \
+   }
+#define WRITE_TIMESTAMP(src, name)                                             \
+   j[#name] = bts::core::TimeStamp::from_nanos(                                \
+                  src.name().seconds() * 1'000'000'000 + src.name().nanos())   \
+                  .to_iso_string()
+
+#define PARSE_TIMESTAMP(name, dest, it)                                        \
+   it = j.find(#name);                                                         \
+   if (it != j.end()) {                                                        \
+      bts::core::TimeStamp tmp_ts =                                            \
+          bts::core::TimeStamp::from_iso_string(it->get<std::string>());       \
+      tmp_ts.to_protobuf(*dest.mutable_##name());                              \
+   }
+
+void to_json(nlohmann::json &j, const Fill &f) {
+   WRITE_IF_NOT_ZERO(f, order_key);
+   WRITE_IF_NOT_ZERO(f, security_key);
+   WRITE_IF_NOT_EMPTY(f, account);
+   WRITE_IF_NOT_NAN(f, last_price);
+   WRITE_IF_NOT_NAN(f, avg_price);
+   WRITE_VALUE(f, side);
+   WRITE_IF_NOT_ZERO(f, last_shares);
+   WRITE_IF_NOT_EMPTY(f, fill_key);
+   WRITE_TIMESTAMP(f, timestamp);
+   WRITE_IF_NOT_ZERO(f, internal_account_id);
+   WRITE_VALUE(f, fill_kind);
+   WRITE_VALUE(f, fill_status);
+}
+
+void from_json(const nlohmann::json &j, Fill &f) {
+   nlohmann::json::const_iterator it;
+   PARSE_VALUE(order_key, f, it);
+   PARSE_VALUE(security_key, f, it);
+   PARSE_STRING(account, f, it);
+   PARSE_VALUE(last_price, f, it);
+   PARSE_VALUE(avg_price, f, it);
+   PARSE_VALUE(side, f, it);
+   PARSE_VALUE(last_shares, f, it);
+   PARSE_STRING(fill_key, f, it);
+   PARSE_TIMESTAMP(timestamp, f, it);
+   PARSE_VALUE(internal_account_id, f, it);
+   PARSE_VALUE(fill_kind, f, it);
+   PARSE_VALUE(fill_status, f, it);
+}
+
+void to_json(nlohmann::json &j, const NativeOrderState &n) {
+   WRITE_VALUE(n, exchange_order_id);
+   WRITE_VALUE(n, last_msg_seq_num);
+   WRITE_VALUE(n, sub_error_code);
+   WRITE_VALUE(n, native_status);
+   WRITE_VALUE(n, routing_record);
+   WRITE_VALUE(n, native_type);
+   WRITE_VALUE(n, last_transact_time);
+   WRITE_VALUE(n, last_transact_time_unix_ns);
+   WRITE_VALUE(n, stop_px);
+   WRITE_VALUE(n, exchange_order_id_u64);
+}
+
+void from_json(const nlohmann::json &j, NativeOrderState &n) {
+   nlohmann::json::const_iterator it;
+   PARSE_STRING(exchange_order_id, n, it);
+   PARSE_VALUE(last_msg_seq_num, n, it);
+   PARSE_VALUE(sub_error_code, n, it);
+   PARSE_VALUE(native_status, n, it);
+   PARSE_STRUCT(routing_record, n, it);
+   PARSE_VALUE(native_type, n, it);
+   PARSE_STRING(last_transact_time, n, it);
+   PARSE_VALUE(last_transact_time_unix_ns, n, it);
+   PARSE_VALUE(stop_px, n, it);
+   PARSE_VALUE(exchange_order_id_u64, n, it);
+}
+
+void to_json(nlohmann::json &j, const RoutingRecord &r) {
+   WRITE_IF_NOT_EMPTY(r, session);
+   WRITE_IF_NOT_EMPTY(r, account);
+   WRITE_IF_NOT_EMPTY(r, operator_id);
+   WRITE_VALUE(r, cti_code);
+   WRITE_VALUE(r, customer_or_firm);
+   WRITE_IF_NOT_EMPTY(r, smp_id);
+   WRITE_VALUE(r, smp_instruction);
+   WRITE_IF_NOT_EMPTY(r, location);
+   WRITE_IF_NOT_EMPTY(r, cust_order_handing_inst);
+   WRITE_IF_NOT_EMPTY(r, sender_compid);
+   WRITE_IF_NOT_EMPTY(r, clearing_firm);
+   WRITE_IF_NOT_EMPTY(r, customer_account_ref_id);
+   WRITE_IF_NOT_EMPTY(r, account_code);
+   WRITE_IF_NOT_EMPTY(r, company_id);
+   WRITE_IF_NOT_EMPTY(r, authorized_trader);
+   WRITE_IF_NOT_EMPTY(r, authorized_group);
+   WRITE_IF_NOT_EMPTY(r, cust_order_handing_inst);
+   WRITE_IF_NOT_EMPTY(r, order_server_id);
+}
+
+void from_json(const nlohmann::json &j, RoutingRecord &r) {
+   nlohmann::json::const_iterator it;
+   PARSE_STRING(session, r, it);
+   PARSE_STRING(account, r, it);
+   PARSE_STRING(operator_id, r, it);
+   PARSE_VALUE(cti_code, r, it);
+   PARSE_VALUE(customer_or_firm, r, it);
+   PARSE_STRING(smp_id, r, it);
+   PARSE_VALUE(smp_instruction, r, it);
+   PARSE_STRING(location, r, it);
+   PARSE_STRING(cust_order_handling_inst, r, it);
+   PARSE_STRING(sender_compid, r, it);
+   PARSE_STRING(clearing_firm, r, it);
+   PARSE_STRING(customer_account_ref_id, r, it);
+   PARSE_STRING(account_code, r, it);
+   PARSE_STRING(company_id, r, it);
+   PARSE_STRING(authorized_trader, r, it);
+   PARSE_STRING(authorized_group, r, it);
+   PARSE_STRING(cust_order_handing_inst, r, it);
+   PARSE_STRING(order_server_id, r, it);
+}
+
+} // namespace bts::api::orderman

src/lib/api/services/OrderService.h

@@ -1,6 +1,10 @@
 #pragma once
 
 #include "lib/api/services/OrderService.pb.h"
+#include "lib/api/services/native_order_type.h"
+#include "lib/api/services/order_status.h"
+#include "lib/api/services/time_in_force.h"
+#include "lib/api/services/trigger_price_type.h"
 #include "lib/core/EnumMacros.h"
 #include "lib/core/FMT_IMPL.h"
 #include "lib/io/JsonReader.h"
@@ -10,22 +14,70 @@
 
 namespace bts::api::orderman {
 
-PB_ENUM_FROM_JSON(OrderStatus)
-PB_ENUM_TO_JSON(OrderStatus)
+PB_ENUM_FROM_JSON(Fill_FillKind)
+PB_ENUM_TO_JSON(Fill_FillKind)
 
-PB_ENUM_FROM_JSON(TriggerPriceType)
-PB_ENUM_TO_JSON(TriggerPriceType)
+PB_ENUM_FROM_JSON(Fill_FillStatus)
+PB_ENUM_TO_JSON(Fill_FillStatus)
+
+void to_json(nlohmann::json &j, const Fill &f);
+void from_json(const nlohmann::json &j, Fill &f);
+
+PB_ENUM_FROM_JSON(OrderSubType)
+PB_ENUM_TO_JSON(OrderSubType)
+
+PB_ENUM_FROM_JSON(SelfMatchPreventionInstruction)
+PB_ENUM_TO_JSON(SelfMatchPreventionInstruction)
+
+PB_ENUM_FROM_JSON(CustomerOrFirm)
+PB_ENUM_TO_JSON(CustomerOrFirm)
+
+void to_json(nlohmann::json &j, const RoutingRecord &r);
+void from_json(const nlohmann::json &j, RoutingRecord &r);
+
+PB_ENUM_FROM_JSON(NativeOrderStatus)
+PB_ENUM_TO_JSON(NativeOrderStatus)
+
+PB_ENUM_FROM_JSON(NativeOrderState_SubError)
+PB_ENUM_TO_JSON(NativeOrderState_SubError)
+
+void to_json(nlohmann::json &j, const NativeOrderState &n);
+void from_json(const nlohmann::json &j, NativeOrderState &n);
 
 } // namespace bts::api::orderman
 
 FMT_IMPL(
-    bts::api::orderman::OrderStatus,
-    s,
+    bts::api::orderman::Fill_FillKind,
+    k,
     "{}",
-    bts::api::orderman::OrderStatus_Name(s));
+    bts::api::orderman::Fill_FillKind_Name(k));
 
 FMT_IMPL(
-    bts::api::orderman::TriggerPriceType,
+    bts::api::orderman::Fill_FillStatus,
+    s,
+    "{}",
+    bts::api::orderman::Fill_FillStatus(s));
+
+FMT_IMPL(
+    bts::api::orderman::OrderSubType,
     t,
     "{}",
-    bts::api::orderman::TriggerPriceType_Name(t));
+    bts::api::orderman::OrderSubType_Name(t));
+
+FMT_IMPL(
+    bts::api::orderman::SelfMatchPreventionInstruction,
+    s,
+    "{}",
+    bts::api::orderman::SelfMatchPreventionInstruction_Name(s));
+
+FMT_IMPL(
+    bts::api::orderman::NativeOrderStatus,
+    s,
+    "{}",
+    bts::api::orderman::NativeOrderStatus_Name(s));
+
+FMT_IMPL(
+    bts::api::orderman::NativeOrderState_SubError,
+    s,
+    "{}",
+    bts::api::orderman::NativeOrderState_SubError_Name(s));

src/lib/api/services/native_order_type.h

@@ -0,0 +1,57 @@
+#pragma once
+
+#include "lib/api/services/OrderService.pb.h"
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+#include "lib/utils/string_utils.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api::orderman {
+
+/**
+ * Parses NativeOrderType from a string. Unlike `bts::api::orderman::NativeOrderType_Parse`,
+ * this implementation is case insensitive, and makes the 'NATIVE_ORDER_TYPE_' prefix optional.
+ */
+bool native_order_type_parse_smart(
+    const std::string &str, NativeOrderType *result);
+
+PB_ENUM_FROM_JSON(NativeOrderType)
+PB_ENUM_TO_JSON(NativeOrderType)
+
+std::ostream &operator<<(std::ostream &out, NativeOrderType t) noexcept;
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline bool native_order_type_parse_smart(
+    const std::string &str, NativeOrderType *result) {
+   if (NativeOrderType_Parse(str, result)) {
+      return true;
+   }
+   std::string s = bts::system::string_utils::copy_to_upper(str);
+   if (NativeOrderType_Parse(s, result)) {
+      return true;
+   }
+   const char *prefix = "NATIVE_ORDER_TYPE_";
+   if (!s.starts_with(prefix)) {
+      s.insert(0, prefix);
+   }
+   return NativeOrderType_Parse(s, result);
+}
+
+inline std::ostream &operator<<(std::ostream &out, NativeOrderType t) noexcept {
+   out << NativeOrderType_Name(t);
+   return out;
+}
+
+} // namespace bts::api::orderman
+
+FMT_IMPL(
+    bts::api::orderman::NativeOrderType,
+    t,
+    "{}",
+    bts::api::orderman::NativeOrderType_Name(t));

src/lib/api/services/order_status.h

@@ -0,0 +1,56 @@
+#pragma once
+
+#include "lib/api/services/OrderService.pb.h"
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+#include "lib/utils/string_utils.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api::orderman {
+
+/**
+ * Parses OrderStatus from a string. Unlike `bts::api::orderman::OrderStatus_Parse`,
+ * this implementation is case insensitive, and makes the 'ORDER_STATUS_' prefix optional.
+ */
+bool order_status_parse_smart(const std::string &str, OrderStatus *result);
+
+PB_ENUM_FROM_JSON(OrderStatus)
+PB_ENUM_TO_JSON(OrderStatus)
+
+std::ostream &operator<<(std::ostream &out, OrderStatus s) noexcept;
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline bool order_status_parse_smart(
+    const std::string &str, OrderStatus *result) {
+   if (OrderStatus_Parse(str, result)) {
+      return true;
+   }
+   std::string s = bts::system::string_utils::copy_to_upper(str);
+   if (OrderStatus_Parse(s, result)) {
+      return true;
+   }
+   const char *prefix = "ORDER_STATUS_";
+   if (!s.starts_with(prefix)) {
+      s.insert(0, prefix);
+   }
+   return OrderStatus_Parse(s, result);
+}
+
+inline std::ostream &operator<<(std::ostream &out, OrderStatus s) noexcept {
+   out << OrderStatus_Name(s);
+   return out;
+}
+
+} // namespace bts::api::orderman
+
+FMT_IMPL(
+    bts::api::orderman::OrderStatus,
+    s,
+    "{}",
+    bts::api::orderman::OrderStatus_Name(s));

src/lib/api/services/time_in_force.h

@@ -0,0 +1,56 @@
+#pragma once
+
+#include "lib/api/services/OrderService.pb.h"
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+#include "lib/utils/string_utils.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api::orderman {
+
+/**
+ * Parses TimeInForce from a string. Unlike `bts::api::orderman::TimeInForce_Parse`,
+ * this implementation is case insensitive, and makes the 'TIME_IN_FORCE_' prefix optional.
+ */
+bool time_in_force_parse_smart(const std::string &str, TimeInForce *result);
+
+PB_ENUM_FROM_JSON(TimeInForce)
+PB_ENUM_TO_JSON(TimeInForce)
+
+std::ostream &operator<<(std::ostream &out, TimeInForce tif) noexcept;
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline bool time_in_force_parse_smart(
+    const std::string &str, TimeInForce *result) {
+   if (TimeInForce_Parse(str, result)) {
+      return true;
+   }
+   std::string s = bts::system::string_utils::copy_to_upper(str);
+   if (TimeInForce_Parse(s, result)) {
+      return true;
+   }
+   const char *prefix = "TIME_IN_FORCE_";
+   if (!s.starts_with(prefix)) {
+      s.insert(0, prefix);
+   }
+   return TimeInForce_Parse(s, result);
+}
+
+inline std::ostream &operator<<(std::ostream &out, TimeInForce tif) noexcept {
+   out << TimeInForce_Name(tif);
+   return out;
+}
+
+} // namespace bts::api::orderman
+
+FMT_IMPL(
+    bts::api::orderman::TimeInForce,
+    v,
+    "{}",
+    bts::api::orderman::TimeInForce_Name(v));

src/lib/api/services/trigger_price_type.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include "lib/api/services/OrderService.pb.h"
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+#include "lib/utils/string_utils.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api::orderman {
+
+/**
+ * Parses TriggerPriceType from a string. Unlike `bts::api::orderman::TriggerPriceType_Parse`,
+ * this implementation is case insensitive, and makes the 'TRIGGER_PRICE_TYPE_' prefix optional.
+ */
+bool trigger_price_type_parse_smart(
+    const std::string &str, TriggerPriceType *result);
+
+PB_ENUM_FROM_JSON(TriggerPriceType)
+PB_ENUM_TO_JSON(TriggerPriceType)
+
+std::ostream &operator<<(std::ostream &out, TriggerPriceType t) noexcept;
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline bool trigger_price_type_parse_smart(
+    const std::string &str, TriggerPriceType *result) {
+   if (TriggerPriceType_Parse(str, result)) {
+      return true;
+   }
+   std::string s = bts::system::string_utils::copy_to_upper(str);
+   if (TriggerPriceType_Parse(s, result)) {
+      return true;
+   }
+   const char *prefix = "TRIGGER_PRICE_TYPE_";
+   if (!s.starts_with(prefix)) {
+      s.insert(0, prefix);
+   }
+   return TriggerPriceType_Parse(s, result);
+}
+
+inline std::ostream &operator<<(
+    std::ostream &out, TriggerPriceType t) noexcept {
+   out << TriggerPriceType_Name(t);
+   return out;
+}
+
+} // namespace bts::api::orderman
+
+FMT_IMPL(
+    bts::api::orderman::TriggerPriceType,
+    t,
+    "{}",
+    bts::api::orderman::TriggerPriceType_Name(t));

src/lib/cmake/GetLinuxClientSources.cmake

@@ -75,6 +75,7 @@ function(get_cpp_sources dest)
     expression/ParserExceptions.cpp
     expression/Value.cpp
     expression/tests/expression_parser_tests.cpp
+    io/JsonReader.cpp
     logging/LoggerHub.cpp
     logging/Sink.cpp
     logging/logging.cpp

src/lib/cmake/GetWindowsClientSources.cmake

@@ -74,6 +74,7 @@ function(get_cpp_sources dest)
     expression/ParserExceptions.cpp
     expression/Value.cpp
     expression/tests/expression_parser_tests.cpp
+    io/JsonReader.cpp
     logging/LoggerHub.cpp
     logging/Sink.cpp
     logging/logging.cpp

src/lib/collections/concurrent_factory.h

@@ -0,0 +1,98 @@
+#pragma once
+
+#include "lib/utils/LockBox.h"
+#include "lib/utils/bts_assert.h"
+#include "lib/utils/type_name.h"
+
+#include <functional>
+#include <memory>
+
+namespace bts::collections {
+
+template <class KeyT, class ValueT>
+class ConcurrentFactory {
+   utils::LockBox<std::unordered_map<KeyT, std::shared_ptr<ValueT>>> store;
+
+ public:
+   std::shared_ptr<ValueT> add(
+       const KeyT &key,
+       std::function<void(std::shared_ptr<ValueT> &)> callback) {
+      return store.lock([&](auto &map) {
+         auto [it, is_new] = map.try_emplace(key, std::make_shared<ValueT>());
+         bts_assert_msgf(
+             is_new, "{}::add. Duplicate key '{}'",
+             utils::cached_type_name<ConcurrentFactory<KeyT, ValueT>>(), key);
+         callback(it->second);
+         return it->second;
+      });
+   }
+
+   std::size_t erase(const KeyT &key) {
+      return store.lock([&](auto &map) {
+         map.erase(key);
+         return map.size();
+      });
+   }
+
+   std::size_t size() const {
+      return store.lock([&](auto &map) { return map.size(); });
+   }
+
+   template <
+       typename CallbackT = std::function<void(std::shared_ptr<ValueT> &)>>
+   std::shared_ptr<const ValueT> get(const KeyT &key, CallbackT callback) {
+      return store.lock([&](auto &map) {
+         auto [it, is_new] = map.try_emplace(key, std::make_shared<ValueT>());
+         bts_assert_msgf(
+             !is_new, "{}::get. Key not found '{}'",
+             utils::cached_type_name<ConcurrentFactory<KeyT, ValueT>>(), key);
+         callback(it->second);
+         return it->second;
+      });
+   }
+
+   template <
+       typename CallbackT =
+           std::function<void(std::shared_ptr<ValueT> &, bool)>>
+   std::shared_ptr<const ValueT> get_or_add(
+       const KeyT &key, CallbackT callback) {
+      return store.lock([&](auto &map) {
+         auto [it, is_new] = map.try_emplace(key);
+         if (is_new) {
+            it->second = std::make_shared<ValueT>();
+         }
+         callback(it->second, is_new);
+         return it->second;
+      });
+   }
+
+   template <
+       typename CallbackT =
+           std::function<void(std::shared_ptr<ValueT> &, bool)>>
+   std::shared_ptr<const ValueT> replace(const KeyT &key, CallbackT callback) {
+      return store.lock([&](auto &map) {
+         auto [it, is_new] = map.try_emplace(key);
+         bts_assert_msgf(
+             !is_new, "{}::replace. Key not found '{}'",
+             utils::cached_type_name<ConcurrentFactory<KeyT, ValueT>>(), key);
+         it->second = std::make_shared<ValueT>();
+         callback(it->second, is_new);
+         return it->second;
+      });
+   }
+
+   template <
+       typename CallbackT =
+           std::function<void(std::shared_ptr<ValueT> &, bool)>>
+   std::shared_ptr<const ValueT> replace_or_add(
+       const KeyT &key, CallbackT callback) {
+      return store.lock([&](auto &map) {
+         auto [it, is_new] = map.try_emplace(key);
+         it->second = std::make_shared<ValueT>();
+         callback(it->second, is_new);
+         return it->second;
+      });
+   }
+};
+
+} // namespace bts::collections

src/lib/config/profiles/default/okx_exe_cfg.conf

@@ -0,0 +1,17 @@
+OkxExeCfg {
+    value{
+        account : "ACC A"
+        ops_id : "op_id_bts"
+        secret_key : "DAF6CBB09475D2520B88D2070788A976"
+        api_key : "4492a9ab-5784-454c-acc7-38dc2a503ed0"
+        passphrase : "Pakistan@123"
+        ws_url : "wss://wspap.okx.com:8443"
+        rest_url : "https://www.okx.com"
+        ws_ping_timeout_seconds : 10 
+        ws_ping_interval_seconds : 25
+        ws_reconnect_interval_seconds : 5
+        ws_reconnect_max_count : 5
+        simulated_trading : true
+        tdmode : "cross"
+    }
+}

src/lib/config/profiles/default/orderman_cfg.conf

@@ -1,34 +1,34 @@
 OrdermanCfg {
-   rule: "DeploymentType.type == 'dev'"
-   value {
-      executor {
-         thread_name: "oman_orders"
-	 poll: false 
-	 hosted_actor: "MDSource"
-	 hosted_actor: "OrderManager"
-	 hosted_actor: "ExecutionGateway"
-      }
-      executor {
-         thread_name: "oman_slow"
-	 poll: false 
-	 hosted_actor: "*"
-      }
-   }
+  rule: "DeploymentType.type == 'dev'"
+  value {
+    executor {
+      thread_name: "oman_orders"
+      poll: false 
+      hosted_actor: "MDSource"
+      hosted_actor: "OrderManager"
+      hosted_actor: "ExecutionGateway"
+    }
+    executor {
+      thread_name: "oman_slow"
+      poll: false 
+	    hosted_actor: "*"
+    }
+  }
 }
 
 OrdermanCfg {
-   value {
-      executor {
-         thread_name: "oman_orders"
-	 poll: true 
-	 hosted_actor: "MDSource"
-	 hosted_actor: "OrderManager"
-	 hosted_actor: "ExecutionGateway"
-      }
-      executor {
-         thread_name: "oman_slow"
-	 poll: false 
-	 hosted_actor: "*"
-      }
-   }
-}
+  value {
+    executor {
+      thread_name: "oman_orders"
+      poll: true 
+      hosted_actor: "MDSource"
+      hosted_actor: "OrderManager"
+      hosted_actor: "ExecutionGateway"
+    }
+    executor {
+      thread_name: "oman_slow"
+	    poll: false 
+      hosted_actor: "*"
+    }
+  }
+}

src/lib/config/protos/okx_exe_cfg.proto

@@ -0,0 +1,18 @@
+syntax = "proto3";
+package bts.config;
+message OkxExeCfg {
+    string account = 1;
+    string ops_id = 2;
+    string secret_key = 3;
+    string api_key = 4;
+    string passphrase = 5;
+    string ws_url = 6;
+    string rest_url = 7;
+    uint32 ws_ping_timeout_seconds = 8;
+    uint32 ws_ping_interval_seconds = 9;
+    uint32 ws_reconnect_interval_seconds = 10;
+    uint32 ws_reconnect_max_count = 11;
+    bool simulated_trading = 12;
+    string tdmode = 13;
+
+}

src/lib/core/EnumMacros.h

@@ -110,6 +110,19 @@
    ENUM_TO_JSON(type)                                                          \
    ENUM_EQUALITY_OPS(type)
 
+#define ENUM_CLASS_DEF_NO_JSON(type, for_all)                                  \
+   class type {                                                                \
+    public:                                                                    \
+      enum Value { for_all(ENUM_DECLARE_VALUE) };                              \
+                                                                               \
+      ENUM_BASIC_MEMBERS(type)                                                 \
+   };                                                                          \
+                                                                               \
+   ENUM_OUTPUT_OPS(type)                                                       \
+   ENUM_NAME_DEF(type, for_all)                                                \
+   ENUM_PARSE_DEF(type, for_all)                                               \
+   ENUM_EQUALITY_OPS(type)
+
 #define ENUM_BITWISE_COMPARISON_OP(enum_type, numeric_type, op)                \
    constexpr enum_type operator op(enum_type lhs, enum_type rhs) noexcept {    \
       return static_cast<enum_type>(static_cast<numeric_type>(lhs)             \
@@ -154,3 +167,4 @@
    inline void to_json(nlohmann::json &j, const enum_type &value) {            \
       j = enum_type##_Name(value);                                             \
    }
+

src/lib/core/TimeStamp.cpp

@@ -1,6 +1,7 @@
 #include "TimeStamp.h"
 #include "lib/utils/datetime.h"
 #include <boost/date_time/posix_time/time_formatters.hpp>
+#include <google/protobuf/timestamp.pb.h>
 
 namespace bts::core {
 
@@ -43,6 +44,12 @@ bool TimeStamp::from_ymd(
    return true;
 }
 
+void TimeStamp::to_protobuf(google::protobuf::Timestamp &ts) const {
+   ts.set_seconds(seconds());
+   bts_assert(nanos() >= seconds() * 1'000'000'000);
+   ts.set_nanos(nanos() - seconds() * 1'000'000'000);
+}
+
 TimeStamp TimeStamp::from_iso_string(const std::string &iso_string) {
    auto ptime = datetime::parse_iso_date_time_string(iso_string);
    return from_ptime(ptime);

src/lib/core/TimeStamp.h

@@ -10,6 +10,7 @@
 #include <boost/date_time/time.hpp>
 #include <cstdint>
 #include <fmt/format.h>
+#include <google/protobuf/timestamp.pb.h>
 #include <limits>
 #include <string>
 
@@ -87,6 +88,7 @@ struct TimeStamp {
    // Gets a TimeStamp representing this date at midnight.
    TimeStamp date() const;
    constexpr boost::posix_time::ptime to_ptime() const;
+   void to_protobuf(google::protobuf::Timestamp &ts) const;
    std::string to_iso_string() const;
    // current epoch ns
    static uint64_t now_ns() {

src/lib/domain_model/PutOrCall.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+#include "lib/utils/string_utils.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api {
+
+/**
+ * Parses PutOrCall from a string. Unlike `bts::api::SecurityType_Parse`, this
+ * implementation is case insensitive, and makes the 'PUTORCALL_' prefix optional.
+ */
+bool put_or_call_parse_smart(const std::string &str, PutOrCall *result);
+
+PB_ENUM_FROM_JSON(PutOrCall)
+PB_ENUM_TO_JSON(PutOrCall)
+
+std::ostream &operator<<(std::ostream &out, const PutOrCall &poc) noexcept;
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline bool put_or_call_parse_smart(const std::string &str, PutOrCall *result) {
+   if (PutOrCall_Parse(str, result)) {
+      return true;
+   }
+   std::string s = bts::system::string_utils::copy_to_upper(str);
+   if (PutOrCall_Parse(s, result)) {
+      return true;
+   }
+   const char *prefix = "PUTORCALL_";
+   if (!s.starts_with(prefix)) {
+      s.insert(0, prefix);
+   }
+   return PutOrCall_Parse(s, result);
+}
+
+inline std::ostream &operator<<(
+    std::ostream &out, const PutOrCall &poc) noexcept {
+   out << PutOrCall_Name(poc);
+   return out;
+}
+
+} // namespace bts::api
+
+FMT_IMPL(bts::api::PutOrCall, poc, "{}", bts::api::PutOrCall_Name(poc));

src/lib/domain_model/Security.cpp

@@ -2,10 +2,13 @@
 #include "lib/config/config.h"
 #include "lib/config/protos/orders_fills_cleanup.pb.h"
 #include "lib/domain_model/Security.pb.h"
+#include "lib/io/JsonMacros.h"
 #include "lib/md/common.h"
+#include "lib/utils/pb_json_macros.h"
 
 #include <boost/date_time/posix_time/posix_time.hpp>
 #include <google/protobuf/text_format.h>
+#include <nlohmann/json.hpp>
 
 namespace bts {
 namespace securities {
@@ -191,3 +194,389 @@ std::string unique_descriptor(const api::Security &security) {
 
 } // namespace securities
 } // namespace bts
+
+namespace bts::api {
+
+using nlohmann::json;
+using std::uint32_t;
+
+void to_json(json &j, const api::ContractQuantityInfo &cqi) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, min_value);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, max_value);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, tick_unit);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, tick_base);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, multiplier);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, max_price);
+}
+
+void from_json(const json &j, api::ContractQuantityInfo &cqi) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, cqi, min_value, it);
+   PB_JSON_READ_VALUE(j, cqi, max_value, it);
+   PB_JSON_READ_VALUE(j, cqi, max_value, it);
+   PB_JSON_READ_VALUE(j, cqi, tick_unit, it);
+   PB_JSON_READ_VALUE(j, cqi, tick_base, it);
+   PB_JSON_READ_VALUE(j, cqi, max_price, it);
+}
+
+void to_json(json &j, const api::ContractSpecification &spec) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, low_tick);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, price_break);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, high_tick);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, cash_per_point);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, price_multiplier);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, main_fraction);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, sub_fraction);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, price_display_format);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, spec, quantity_infos);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, spec, notional_quantity_infos);
+}
+
+void from_json(const json &j, api::ContractSpecification &spec) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, spec, low_tick, it);
+   PB_JSON_READ_VALUE(j, spec, price_break, it);
+   PB_JSON_READ_VALUE(j, spec, high_tick, it);
+   PB_JSON_READ_VALUE(j, spec, cash_per_point, it);
+   PB_JSON_READ_VALUE(j, spec, price_multiplier, it);
+   PB_JSON_READ_VALUE(j, spec, main_fraction, it);
+   PB_JSON_READ_VALUE(j, spec, sub_fraction, it);
+   PB_JSON_READ_VALUE(j, spec, price_display_format, it);
+   PB_JSON_READ_ARRAY(j, spec, quantity_infos, it);
+   PB_JSON_READ_ARRAY(j, spec, notional_quantity_infos, it);
+}
+
+void to_json(json &, const SpotContract &) {}
+void from_json(const json &, SpotContract &) {}
+
+template <class ExpContract>
+static void exp_to_json(json &j, const ExpContract &ec) {
+   j = bts::core::TimeStamp::from_ymd(
+           ec.expiration_year(), ec.expiration_month(), ec.expiration_day())
+           .to_iso_string();
+}
+
+template <class ExpContract>
+static void exp_from_json(const json &j, ExpContract &ec) {
+   const auto dt = bts::core::TimeStamp::from_iso_string(j.get<std::string>());
+   uint32_t y, m, d;
+   dt.to_date(y, m, d);
+   ec.set_expiration_year(y);
+   ec.set_expiration_month(m);
+   ec.set_expiration_day(d);
+}
+
+void to_json(json &j, const FutureContract &f) {
+   exp_to_json(j["expiration"], f);
+}
+
+void from_json(const json &j, FutureContract &f) {
+   const auto it = j.find("expiration");
+   if (it != j.end()) {
+      exp_from_json(*it, f);
+   }
+}
+
+void to_json(json &j, const OptionContract &opt) {
+   exp_to_json(j["expiration"], opt);
+   PB_JSON_WRITE_VALUE(j, opt, strike);
+   PB_JSON_WRITE_VALUE(j, opt, call_put);
+}
+
+void from_json(const json &j, OptionContract &opt) {
+   json::const_iterator it;
+   it = j.find("expiration");
+   if (it != j.end()) {
+      exp_from_json(*it, opt);
+   }
+   PB_JSON_READ_VALUE(j, opt, strike, it);
+   PB_JSON_READ_VALUE(j, opt, call_put, it);
+}
+
+void to_json(json &j, const ContractLeg &leg) {
+   PB_JSON_WRITE_STRUCT(j, leg, security);
+   PB_JSON_WRITE_VALUE(j, leg, quantity);
+   PB_JSON_WRITE_VALUE(j, leg, side);
+   PB_JSON_WRITE_VALUE(j, leg, price_multiplier);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, leg, leg_delta);
+   PB_JSON_WRITE_VALUE(j, leg, price_calc);
+}
+
+void from_json(const json &j, ContractLeg &leg) {
+   json::const_iterator it;
+   PB_JSON_READ_STRUCT(j, leg, security, it);
+   PB_JSON_READ_VALUE(j, leg, quantity, it);
+   PB_JSON_READ_VALUE(j, leg, side, it);
+   PB_JSON_READ_VALUE(j, leg, price_multiplier, it);
+   PB_JSON_READ_VALUE(j, leg, leg_delta, it);
+   PB_JSON_READ_VALUE(j, leg, price_calc, it);
+}
+
+void to_json(json &j, const SpreadContract &spd) {
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, spd, legs);
+}
+
+void from_json(const json &j, SpreadContract &spd) {
+   json::const_iterator it;
+   PB_JSON_READ_ARRAY(j, spd, legs, it);
+}
+
+void to_json(json &j, const SyntheticContract &syn) {
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, syn, name);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, syn, user_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, syn, legs);
+   PB_JSON_WRITE_VALUE(j, syn, price_calc);
+}
+
+void from_json(const json &j, SyntheticContract &syn) {
+   json::const_iterator it;
+   PB_JSON_READ_STRING(j, syn, name, it);
+   PB_JSON_READ_VALUE(j, syn, user_id, it);
+   PB_JSON_READ_ARRAY(j, syn, legs, it);
+   PB_JSON_READ_VALUE(j, syn, price_calc, it);
+}
+
+void to_json(json &j, const GLBXExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, cfi_code);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, cme_symbol);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, market_segment_id);
+}
+
+void from_json(const json &j, GLBXExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+   PB_JSON_READ_STRING(j, data, cfi_code, it);
+   PB_JSON_READ_STRING(j, data, cme_symbol, it);
+   PB_JSON_READ_VALUE(j, data, market_segment_id, it);
+}
+
+void to_json(json &j, const CBOEExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, class_key);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, source);
+}
+
+void from_json(const json &j, CBOEExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+   PB_JSON_READ_VALUE(j, data, class_key, it);
+   PB_JSON_READ_STRING(j, data, source, it);
+}
+
+void to_json(json &j, const CBOECFEExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, class_key);
+}
+
+void from_json(const json &j, CBOECFEExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+   PB_JSON_READ_VALUE(j, data, class_key, it);
+}
+
+void to_json(json &j, const ICEExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, price_denominator);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, deal_denominator);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, settle_denominator);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, market_description);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, cleared_alias);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, product_name);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, hub_alias);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, strip_name);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, mic_code);
+   PB_JSON_WRITE_IF_TRUE(j, data, is_strip);
+}
+
+void from_json(const json &j, ICEExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+   PB_JSON_READ_VALUE(j, data, price_denominator, it);
+   PB_JSON_READ_VALUE(j, data, deal_denominator, it);
+   PB_JSON_READ_VALUE(j, data, settle_denominator, it);
+   PB_JSON_READ_STRING(j, data, market_description, it);
+   PB_JSON_READ_STRING(j, data, cleared_alias, it);
+   PB_JSON_READ_STRING(j, data, product_name, it);
+   PB_JSON_READ_STRING(j, data, hub_alias, it);
+   PB_JSON_READ_STRING(j, data, strip_name, it);
+   PB_JSON_READ_STRING(j, data, mic_code, it);
+   PB_JSON_READ_VALUE(j, data, is_strip, it);
+}
+
+void to_json(json &j, const TAIFEXExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, class_key);
+}
+
+void from_json(const json &j, TAIFEXExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+   PB_JSON_READ_VALUE(j, data, class_key, it);
+}
+
+void to_json(json &j, const DeribitExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+}
+
+void from_json(const json &j, DeribitExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+}
+
+void to_json(json &j, const FTXExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+}
+
+void from_json(const json &j, FTXExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+}
+
+void to_json(json &j, const BinanceExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+}
+
+void from_json(const json &j, BinanceExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+}
+
+void to_json(json &j, const BinanceUsdsFuturesExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+}
+
+void from_json(const json &j, BinanceUsdsFuturesExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+}
+
+void to_json(json &j, const BinanceCoinFuturesExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+}
+
+void from_json(const json &j, BinanceCoinFuturesExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+}
+
+void to_json(json &j, const BinanceUsExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+}
+
+void from_json(const json &j, BinanceUsExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+}
+
+void to_json(json &j, const OkxExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+}
+
+void from_json(const json &j, OkxExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+}
+
+void to_json(json &j, const CoinbaseExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+}
+
+void from_json(const json &j, CoinbaseExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+}
+
+void to_json(json &j, const Security &s) {
+   // clang-format off
+#define CONTRACT_ONEOF(oneof)           \
+   oneof(j, s, Spot, spot)              \
+   oneof(j, s, Future, future)          \
+   oneof(j, s, Option, option)          \
+   oneof(j, s, Spread, spread)          \
+   oneof(j, s, Synthetic, synthetic)
+
+#define EXCHANGE_DATA_ONEOF(oneof)                        \
+   oneof(j, s, Glbx, glbx)                                \
+   oneof(j, s, Cboe, cboe)                                \
+   oneof(j, s, Cfe, cfe)                                  \
+   oneof(j, s, Ice, ice)                                  \
+   oneof(j, s, Deribit, deribit)                          \
+   oneof(j, s, Ftx, ftx)                                  \
+   oneof(j, s, Binance, binance)                          \
+   oneof(j, s, BinanceCoinFutures, binance_coin_futures)  \
+   oneof(j, s, BinanceUsdsFutures, binance_usds_futures)  \
+   oneof(j, s, BinanceUs, binance_us)                     \
+   oneof(j, s, Coinbase, coinbase)                        \
+   oneof(j, s, Okx, okx)                                  \
+   oneof(j, s, Taifex, taifex)
+   // clang-format on
+
+   PB_JSON_WRITE_IF_NOT_ZERO(j, s, id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, s, symbol);
+   PB_JSON_WRITE_VALUE(j, s, exchange);
+   PB_JSON_WRITE_STRUCT(j, s, contract_spec);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, s, currency);
+   PB_JSON_WRITE_ONEOF(j, s, Contract, CONTRACT_ONEOF);
+   PB_JSON_WRITE_ONEOF(j, s, ExchangeData, EXCHANGE_DATA_ONEOF);
+}
+
+void from_json(const json &j, Security &s) {
+   json::const_iterator it;
+
+#define SEC_TYPE_ONEOF(oneof)                                                  \
+   oneof(j, s, future, it);                                                    \
+   oneof(j, s, option, it);                                                    \
+   oneof(j, s, spread, it);                                                    \
+   oneof(j, s, synthetic, it)
+
+#define CONTRACT_CASE_ONEOF(oneof)                                             \
+   oneof(j, s, glbx, it);                                                      \
+   oneof(j, s, cboe, it);                                                      \
+   oneof(j, s, cfe, it);                                                       \
+   oneof(j, s, ice, it);                                                       \
+   oneof(j, s, deribit, it);                                                   \
+   oneof(j, s, ftx, it);                                                       \
+   oneof(j, s, binance, it);                                                   \
+   oneof(j, s, binance_usds_futures, it);                                      \
+   oneof(j, s, binance_coin_futures, it);                                      \
+   oneof(j, s, binance_us, it);                                                \
+   oneof(j, s, coinbase, it);                                                  \
+   oneof(j, s, okx, it);                                                       \
+   oneof(j, s, taifex, it);
+
+   PB_JSON_READ_VALUE(j, s, id, it);
+   PB_JSON_READ_STRING(j, s, symbol, it);
+   PB_JSON_READ_STRING(j, s, currency, it);
+   PB_JSON_READ_STRUCT(j, s, contract_spec, it);
+   PB_JSON_READ_ONEOFS(SEC_TYPE_ONEOF);
+   PB_JSON_READ_ONEOFS(CONTRACT_CASE_ONEOF);
+}
+
+} // namespace bts::api

src/lib/domain_model/Security.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "lib/core/FMT_IMPL.h"
+#include "lib/domain_model/PutOrCall.h"
 #include "lib/domain_model/Security.pb.h"
 #include "lib/domain_model/SecurityType.h"
 #include "lib/domain_model/Side.h"
@@ -9,6 +9,7 @@
 #include "lib/utils/bts_assert.h"
 
 #include <cmath>
+#include <nlohmann/json_fwd.hpp>
 
 namespace boost {
 namespace gregorian {
@@ -16,14 +17,83 @@ class date;
 }
 } // namespace boost
 
-namespace bts {
+namespace bts::api {
 
-namespace api {
 class Security;
 class ICEExchangeData;
-} // namespace api
 
-namespace securities {
+void to_json(nlohmann::json &j, const ContractQuantityInfo &c);
+void from_json(const nlohmann::json &j, ContractQuantityInfo &c);
+
+void to_json(nlohmann::json &j, const ContractSpecification &c);
+void from_json(const nlohmann::json &j, ContractSpecification &c);
+
+void to_json(nlohmann::json &j, const ContractLeg &c);
+void from_json(const nlohmann::json &j, ContractLeg &c);
+
+void to_json(nlohmann::json &j, const SpotContract &c);
+void from_json(const nlohmann::json &j, SpotContract &c);
+
+void to_json(nlohmann::json &j, const FutureContract &c);
+void from_json(const nlohmann::json &j, FutureContract &c);
+
+void to_json(nlohmann::json &j, const OptionContract &c);
+void from_json(const nlohmann::json &j, OptionContract &c);
+
+void to_json(nlohmann::json &j, const SpreadContract &c);
+void from_json(const nlohmann::json &j, SpreadContract &c);
+
+void to_json(nlohmann::json &j, const SyntheticContract_PriceCalc &c);
+void from_json(const nlohmann::json &j, SyntheticContract_PriceCalc &c);
+
+void to_json(nlohmann::json &j, const SyntheticContract &c);
+void from_json(const nlohmann::json &j, SyntheticContract &c);
+
+void to_json(nlohmann::json &j, const GLBXExchangeData &data);
+void from_json(const nlohmann::json &j, GLBXExchangeData &data);
+
+void to_json(nlohmann::json &j, const CBOEExchangeData &data);
+void from_json(const nlohmann::json &j, CBOEExchangeData &data);
+
+void to_json(nlohmann::json &j, const CBOECFEExchangeData &data);
+void from_json(const nlohmann::json &j, CBOECFEExchangeData &data);
+
+void to_json(nlohmann::json &j, const ICEExchangeData &data);
+void from_json(const nlohmann::json &j, ICEExchangeData &data);
+
+void to_json(nlohmann::json &j, const DeribitExchangeData &data);
+void from_json(const nlohmann::json &j, DeribitExchangeData &data);
+
+void to_json(nlohmann::json &j, const FTXExchangeData &data);
+void from_json(const nlohmann::json &j, FTXExchangeData &data);
+
+void to_json(nlohmann::json &j, const BinanceExchangeData &data);
+void from_json(const nlohmann::json &j, BinanceExchangeData &data);
+
+void to_json(nlohmann::json &j, const BinanceUsdsFuturesExchangeData &data);
+void from_json(const nlohmann::json &j, BinanceUsdsFuturesExchangeData &data);
+
+void to_json(nlohmann::json &j, const BinanceCoinFuturesExchangeData &data);
+void from_json(const nlohmann::json &j, BinanceCoinFuturesExchangeData &data);
+
+void to_json(nlohmann::json &j, const BinanceUsExchangeData &data);
+void from_json(const nlohmann::json &j, BinanceUsExchangeData &data);
+
+void to_json(nlohmann::json &j, const OkxExchangeData &data);
+void from_json(const nlohmann::json &j, OkxExchangeData &data);
+
+void to_json(nlohmann::json &j, const CoinbaseExchangeData &data);
+void from_json(const nlohmann::json &j, CoinbaseExchangeData &data);
+
+void to_json(nlohmann::json &j, const Security &s);
+void from_json(const nlohmann::json &j, Security &s);
+
+PB_ENUM_FROM_JSON(SyntheticContract_PriceCalc);
+PB_ENUM_TO_JSON(SyntheticContract_PriceCalc);
+
+} // namespace bts::api
+
+namespace bts::securities {
 
 /**
  * Gets a descriptor string for a security (should be unique within a particular exchange).
@@ -60,7 +130,8 @@ inline api::Side other_side(api::Side side) {
       return side;
    }
 }
-template <class V> class TwoSided {
+template <class V>
+class TwoSided {
  public:
    TwoSided() : buy(), sell() {}
 
@@ -104,7 +175,8 @@ inline bool is_worse(api::Side s, Price p1, Price p2) {
    return is_better(s, p2, p1);
 }
 
-template <class C> void apply_to_both_sides(C c) {
+template <class C>
+void apply_to_both_sides(C c) {
    c(api::SIDE_BUY);
    c(api::SIDE_SELL);
 }
@@ -178,8 +250,8 @@ inline Price encode_exchange_price(api::Security const &sec, Price in) {
       return in;
 }
 
-inline double
-raw_round_to_quote_price(double raw_price, double tick_size, api::Side side) {
+inline double raw_round_to_quote_price(
+    double raw_price, double tick_size, api::Side side) {
    double factor = 1000000000;
    auto i_price = llround(factor * raw_price);
    auto i_tick_size = llround(tick_size * factor);
@@ -222,8 +294,8 @@ inline double checked_round_to_quote_price(
    }
 }
 
-inline Price
-round_to_quote_price(api::Security const &sec, Price in, api::Side side) {
+inline Price round_to_quote_price(
+    api::Security const &sec, Price in, api::Side side) {
    auto p = in.value();
    bts_assert_d(!std::isnan(p));
    if (p < -sec.contract_spec().price_break() ||
@@ -287,8 +359,8 @@ inline const std::string &descriptor(
 // Looking to round md::IntPrice? Look in IntPrice.h
 
 // computes price_2 - price_1 in ticks
-inline int
-price_diff_to_ticks(double price_1, double price_2, double tick_size) {
+inline int price_diff_to_ticks(
+    double price_1, double price_2, double tick_size) {
    return static_cast<int>(std::round((price_2 - price_1) / tick_size));
 }
 // returns the difference (in ticks) bw and order and a market.
@@ -312,8 +384,8 @@ inline int order_diff_market(
 // simple double equality can not be used to compare prices.
 // this function deems 2 prices to be equal if they are within
 // 1 % of the tick size
-inline bool
-test_price_equality(double price_1, double price_2, double tick_size) {
+inline bool test_price_equality(
+    double price_1, double price_2, double tick_size) {
    return std::fabs(price_2 - price_1) < 0.01 * tick_size;
 }
 inline bool gr_than_eq(double p1, double p2, double tick_size) {
@@ -362,7 +434,4 @@ inline bool is_spread_of_strips(const bts::api::Security &sec) {
    return false;
 }
 
-} // namespace securities
-} // namespace bts
-
-FMT_IMPL(bts::api::Security, s, "{}", bts::securities::descriptor(s));
+} // namespace bts::securities

src/lib/io/JsonMacros.h

@@ -3,18 +3,14 @@
 #define _TO_JSON_FIELD(field) j[#field] = t.field;
 #define _FROM_JSON_FIELD(field) bts::io::JsonReader{&j, #field}.read(t.field);
 
-#define SPARK_TO_JSON_IMPL(type, for_each)                                     \
-   inline void to_json(nlohmann::json &j, const type &t) {                     \
-      FOR_EACH(_TO_JSON_FIELD)                                                 \
-   }
+#define SPARK_TO_JSON_IMPL(type, fields)                                       \
+   void to_json(nlohmann::json &j, const type &t) { fields(_TO_JSON_FIELD) }
 
-#define SPARK_FROM_JSON_IMPL(type, for_each)                                   \
-   inline void from_json(const nlohmann::json &j, type &t) {                   \
-      FOR_EACH(_FROM_JSON_FIELD)                                               \
-   }
+#define SPARK_FROM_JSON_IMPL(type, fields)                                     \
+   void from_json(const nlohmann::json &j, type &t) { fields(_FROM_JSON_FIELD) }
 
 #define OSTREAM_JSON_IMPL(type)                                                \
-   inline std::ostream &operator<<(std::ostream &out, const type &t) {         \
+   std::ostream &operator<<(std::ostream &out, const type &t) {                \
       nlohmann::json j;                                                        \
       to_json(j, t);                                                           \
       out << j.dump(2);                                                        \

src/lib/io/JsonReader.h

@@ -94,7 +94,18 @@ class JsonReader {
     * @return True if the read succeeds, otherwise false
     */
    template <class T>
-   bool read_or(const std::string &name, T &result, T default_value) const;
+   bool read_or(const std::string &name, T &result, T default_value = {}) const;
+
+   template <class T>
+   bool read_optional_or(
+       std::optional<T> &result,
+       std::optional<T> default_value = std::nullopt) const;
+
+   template <class T>
+   bool read_optional_or(
+       const std::string &name,
+       std::optional<T> &result,
+       std::optional<T> default_value = std::nullopt) const;
 
  private:
    const nlohmann::json *json_;
@@ -269,6 +280,33 @@ inline bool JsonReader::read_or(
    return false;
 }
 
+template <class T>
+inline bool JsonReader::read_optional_or(
+    std::optional<T> &result, std::optional<T> default_value) const {
+   if (read_optional(result)) {
+      return true;
+   }
+   result = std::move(default_value);
+   return false;
+}
+
+template <class T>
+inline bool JsonReader::read_optional_or(
+    const std::string &name,
+    std::optional<T> &result,
+    std::optional<T> default_value) const {
+   if (json_ && json_->is_object()) {
+      auto it = json_->find(name);
+      if (it != json_->end()) {
+         if (JsonReader{&it.value()}.read_optional(result)) {
+            return true;
+         }
+      }
+   }
+   result = std::move(default_value);
+   return false;
+}
+
 template <class T>
 bool JsonReader::read_flat(
     const std::vector<nlohmann::json> &items, std::vector<T> &result) {

src/lib/io/http/curl_session.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "lib/logging/logging.h"
+
 #include <atomic>
 #include <curl/curl.h>
 #include <curl/easy.h>
@@ -12,6 +14,7 @@ struct CurlSession {
 
    CurlSession() {
       if (++ref_count == 1) {
+         LOG(Http, Info, "Initializing curl (global)");
          curl_global_init(CURL_GLOBAL_ALL);
       }
       curl = curl_easy_init();
@@ -20,6 +23,7 @@ struct CurlSession {
    ~CurlSession() {
       curl_easy_cleanup(curl);
       if (--ref_count == 0) {
+         LOG(Http, Info, "Cleaning up curl (global)");
          curl_global_cleanup();
       }
    }

src/lib/io/http/http_client.cpp

@@ -34,7 +34,31 @@ static size_t write_headers(
     void *ptr, size_t size, size_t nmemb, IHttpResponse *data) {
    return data->write_headers((char *)ptr, size, nmemb);
 }
-
+// static int curl_debug_callback(CURL *handle, curl_infotype type, char *data, size_t size, void *userptr) {
+//    (void)handle;
+//    (void)type;
+//    (void)userptr;
+//     switch (type) {
+//         case CURLINFO_TEXT:
+//             LOG(Http, Info, "CURLINFO_TEXT: {}", std::string(data, size));
+//             break;
+//         case CURLINFO_HEADER_IN:
+//             LOG(Http, Info, "Header In: {}", std::string(data, size));
+//             break;
+//         case CURLINFO_HEADER_OUT:
+//             LOG(Http, Info, "Header Out: {}", std::string(data, size));
+//             break;
+//         case CURLINFO_DATA_IN:
+//             LOG(Http, Info, "Data In: {} bytes, in: {}", size,std::string(data, size));
+//             break;
+//         case CURLINFO_DATA_OUT:
+//             LOG(Http, Info, "Data Out: {} bytes, out: {}", size, std::string(data, size));
+//             break;
+//         default:
+//             break;
+//     }
+//     return 0;
+// }
 void HttpClient::get(
     string url,
     unordered_map<string, string> headers,
@@ -91,7 +115,12 @@ void HttpClient::send(const HttpRequest &req, IHttpResponse *response) {
    CURL *curl = session.curl;
    CurlHeaders curl_headers{req.headers};
    // Prepare req
-   curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
+   //to debug curl requests
+   // curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, curl_debug_callback);
+   // Disable signals to avoid strange stack corruption related to set/longjmp
+   // https://stackoverflow.com/questions/9191668/error-longjmp-causes-uninitialized-stack-frame
+   curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
+   curl_easy_setopt(curl, CURLOPT_FAILONERROR, 0L);
    curl_easy_setopt(curl, CURLOPT_URL, req.url.c_str());
    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers.data());
    if (response != nullptr) {
@@ -108,6 +137,7 @@ void HttpClient::send(const HttpRequest &req, IHttpResponse *response) {
       curl_easy_setopt(curl, CURLOPT_HTTPPOST, 1L);
       curl_easy_setopt(curl, CURLOPT_POSTFIELDS, req.body.c_str());
       curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, req.body.size());
+      // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
       break;
    case io::http::HttpMethod::PUT:
       curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");

src/lib/io/json_formatter.h

@@ -0,0 +1,54 @@
+#pragma once
+
+#include <fmt/format.h>
+#include <nlohmann/json.hpp>
+#include <string>
+
+namespace bts::io {
+
+/**
+ * This wrapper struct provides a means to format arbitrary objects as json
+ * in templated strings (e.g., LOG macros).
+ */
+template <class T>
+struct JsonFormatter {
+   T value;
+
+   std::string to_string() const { return fmt::format("{}", *this); }
+};
+
+template <class T>
+JsonFormatter<T> json_formatter(T value) {
+   return JsonFormatter<T>{std::move(value)};
+}
+
+} // namespace bts::io
+
+template <class T>
+struct fmt::formatter<bts::io::JsonFormatter<T>> {
+   int indent = -1;
+
+   constexpr auto parse(fmt::format_parse_context &ctx) -> decltype(ctx.end()) {
+      const auto end = ctx.end();
+      auto it = ctx.begin();
+      if (it != end) {
+         indent = 0;
+         do {
+            int x = *it - '0';
+            if (x < 0 || x > 9) {
+               return it;
+            }
+            indent = indent * 10 + x;
+         } while (++it != end);
+      }
+      return it;
+   }
+
+   template <typename FormatContext>
+   auto format(const bts::io::JsonFormatter<T> &jf, FormatContext &ctx) const
+       -> decltype(ctx.out()) {
+      nlohmann::json j;
+      to_json(j, jf.value);
+      return fmt::format_to(ctx.out(), "{}", j.dump(indent));
+   }
+};

src/lib/io/websocket/websocket_client.cpp

@@ -3,14 +3,19 @@
 #include "lib/core/TimeSpan.h"
 #include "lib/core/TimeStamp.h"
 #include "lib/io/websocket/websocket_client_config.h"
+#include "lib/io/websocket/websocket_client_logger.h"
 #include "lib/io/websocket/websocket_client_state.h"
 #include "lib/io/websocket/websocket_close_status.h"
 #include "lib/io/websocket/websocket_error_code.h"
 #include "lib/io/websocket/websocketpp_exception_formatter.h"
 #include "lib/logging/logging.h"
+#include "lib/utils/bts_assert.h"
 #include "lib/utils/formatters/exception_formatter.h"
 
+#include <algorithm>
 #include <atomic>
+#include <boost/asio/io_context.hpp>
+#include <chrono>
 #include <exception>
 #include <fmt/format.h>
 #include <mutex>
@@ -18,6 +23,7 @@
 #include <optional>
 #include <stdexcept>
 #include <string>
+#include <system_error>
 #include <thread>
 #include <websocketpp/close.hpp>
 #include <websocketpp/common/connection_hdl.hpp>
@@ -28,11 +34,51 @@
 #include <websocketpp/config/core_client.hpp>
 #include <websocketpp/roles/client_endpoint.hpp>
 
+namespace websocketpp::config {
+
+/// Client config with asio transport and TLS enabled
+struct bts_asio_tls_client : public core_client {
+   typedef bts_asio_tls_client type;
+   typedef core_client base;
+
+   typedef base::concurrency_type concurrency_type;
+
+   typedef base::request_type request_type;
+   typedef base::response_type response_type;
+
+   typedef base::message_type message_type;
+   typedef base::con_msg_manager_type con_msg_manager_type;
+   typedef base::endpoint_msg_manager_type endpoint_msg_manager_type;
+
+   typedef websocketpp::log::WebsocketClientLogger<
+       websocketpp::log::ElevelAdapter>
+       elog_type;
+   typedef websocketpp::log::WebsocketClientLogger<
+       websocketpp::log::AlevelAdapter>
+       alog_type;
+
+   typedef base::rng_type rng_type;
+
+   struct transport_config : public base::transport_config {
+      typedef type::concurrency_type concurrency_type;
+      typedef type::alog_type alog_type;
+      typedef type::elog_type elog_type;
+      typedef type::request_type request_type;
+      typedef type::response_type response_type;
+      typedef transport::asio::tls_socket::endpoint socket_type;
+   };
+
+   typedef transport::asio::endpoint<transport_config> transport_type;
+};
+
+} // namespace websocketpp::config
+
 namespace bts::websocket {
 
 struct WebsocketClientSnapshot {
    int client_id;
    WebsocketClientState state;
+   int n_reconnects;
    int version;
 };
 
@@ -41,9 +87,10 @@ struct WebsocketClientSnapshot {
 FMT_IMPL(
     bts::websocket::WebsocketClientSnapshot,
     t,
-    "id: {}; state: {}; version: {}",
+    "id: {}; state: {}; n_reconnects: {}; version: {}",
     t.client_id,
     t.state,
+    t.n_reconnects,
     t.version);
 
 namespace bts::websocket {
@@ -61,10 +108,11 @@ using std::size_t;
 using std::string;
 using std::thread;
 using std::uint32_t;
-using websocketpp::config::asio_tls_client;
+using std::chrono::nanoseconds;
+using std::this_thread::sleep_for;
 
 using WebsocketppClient =
-    websocketpp::client<websocketpp::config::asio_tls_client>;
+    websocketpp::client<websocketpp::config::bts_asio_tls_client>;
 using ConnectionHdl = websocketpp::connection_hdl;
 using ConnectionPtr = WebsocketppClient::connection_ptr;
 using MessageTypePtr = websocketpp::config::core_client::message_type::ptr;
@@ -85,9 +133,16 @@ using State = WebsocketClientState;
 #define ERROR(...) LOG(Http, Error, __VA_ARGS__)
 #define ERROR_FN(...) LOG(Http, Error, "WebsocketClient::" __VA_ARGS__)
 
+static bts::core::TimeSpan reconnect_interval(std::uint32_t n_attempts) {
+   std::int64_t secs = 5 * n_attempts;
+   secs = std::clamp(secs, 5L, 300L);
+   return bts::core::TimeSpan::from_seconds(secs);
+}
+
 struct WebsocketClient::PImpl {
    WebsocketClientConfig config;
    WebsocketppClient client{};
+   boost::asio::io_context *io_context = nullptr;
    WebsocketppClient::connection_ptr con;
    optional<OpenHandler> open_handler;
    optional<MessageHandler> message_handler;
@@ -97,8 +152,48 @@ struct WebsocketClient::PImpl {
    optional<thread> io_thread;
    int client_id{client_id_counter++};
    mutex instance_mx;
-   State state;
+   State state = State::IDLE;
    int version = 0;
+   int n_reconnects = 0;
+
+   PImpl(WebsocketClientConfig config, boost::asio::io_context *io_context)
+       : config(std::move(config)), io_context(io_context) {
+      client.get_alog().set_name(this->config.name);
+      client.get_elog().set_name(this->config.name);
+      client.set_access_channels(websocketpp::log::alevel::all);
+      // client.clear_access_channels(websocketpp::log::alevel::frame_payload);
+      client.set_error_channels(websocketpp::log::elevel::all);
+      if (io_context == nullptr) {
+         client.init_asio();
+      } else {
+         client.init_asio(io_context);
+      }
+      client.set_message_handler(
+          [this](auto hdl, auto msg) { on_message(hdl, msg); });
+      client.set_open_handler([this](auto hdl) { on_open(hdl); });
+      client.set_ping_handler([](auto /*hdl*/, auto /*msg*/) {
+         LOG(Http, Info, "Received ping");
+         return true;
+      });
+      client.set_pong_timeout(0);
+      client.set_pong_handler([](auto /*hdl*/, auto /*msg*/) {
+         LOG(Http, Info, "Received pong");
+         return true;
+      });
+      client.set_tls_init_handler([&](auto hdl) { return on_tls_init(hdl); });
+      client.set_close_handler([=, this](auto hdl) {
+         on_close(hdl);
+         if (config.reconnect_on_close) {
+            sleep_for(nanoseconds{reconnect_interval(n_reconnects).nanos()});
+            connect();
+         }
+      });
+      client.set_fail_handler([this](auto hdl) {
+         const auto ec = client.get_con_from_hdl(hdl)->get_ec();
+         ERROR_FN("fail_handler. {}; err: {}", snapshot(), ec.message());
+         on_fail(hdl);
+      });
+   }
 
    ~PImpl() {
       if (io_thread) {
@@ -119,7 +214,7 @@ struct WebsocketClient::PImpl {
    }
 
    constexpr WebsocketClientSnapshot snapshot() const {
-      return {client_id, state, version};
+      return {client_id, state, n_reconnects, version};
    }
 
    void close(websocketpp::close::status::value code, std::string reason) {
@@ -163,8 +258,7 @@ struct WebsocketClient::PImpl {
    /**
     * Connects to the websocket server
     */
-   void connect(
-       const map<string, string> &headers, bool run_in_background = true) {
+   void connect() {
       lock_guard<mutex> lock{instance_mx};
       INFO_FN("connect. {}", snapshot());
       switch (state.value) {
@@ -176,9 +270,6 @@ struct WebsocketClient::PImpl {
       }
       set_state(State::OPENING);
       try {
-         init_logging();
-         init_handlers();
-         // Get connection
          websocketpp::lib::error_code ec;
          con = client.get_connection(config.url, ec);
          if (ec) {
@@ -186,46 +277,27 @@ struct WebsocketClient::PImpl {
             return;
          }
          // Add headers
-         for (const auto &[k, v] : headers) {
+         for (const auto &[k, v] : config.headers) {
             con->append_header(k, v);
          }
          // Connect
          client.connect(con);
-         if (run_in_background) {
-            bts_assert_msgf(
-                !io_thread, "Websocket io_thread already instantiated. {}",
-                snapshot());
-            io_thread = std::thread{[this]() { run(); }};
-            if (config.connect_timeout > core::TimeSpan::zero()) {
-               wait_for_connect(config.connect_timeout);
-            }
-         }
       } catch (const std::exception &ex) {
+         std::string str = ex.what();
          ERROR_FN(
              "connect_no_timeout. Failed to connect. {}; err: {}", snapshot(),
-             ex);
+             str);
          set_state(State::IDLE);
          // Don't wait for connect
          return;
       }
    }
 
-   void wait_for_connect(const TimeSpan timeout) {
-      INFO_FN("wait_for_connect. {}; timeout: {}", snapshot(), timeout);
-      const auto start = TimeStamp::now();
-      while (state == State::OPENING) {
-         const auto elapsed = TimeStamp::now() - start;
-         const auto rem = timeout - elapsed;
-         if (rem < TimeSpan::zero()) {
-            throw std::runtime_error{fmt::format(
-                "Timed out waiting for connection after {} secs. {}",
-                elapsed.total_seconds(), snapshot())};
-         }
-         WARN_FN(
-             "wait_for_connect. Still waiting. {}; timeout: {}; rem: {}",
-             snapshot(), timeout, rem);
-         std::this_thread::sleep_for(std::chrono::seconds(1));
-      }
+   void reconnect() {
+      WARN_FN("reconnect. {}", snapshot());
+      sleep_for(nanoseconds{reconnect_interval(n_reconnects).nanos()});
+      ++n_reconnects;
+      connect();
    }
 
    void run() {
@@ -236,11 +308,16 @@ struct WebsocketClient::PImpl {
          ERROR_FN(
              "io_thread. Websocket io threw websocketpp::exception. {}; "
              "err: {}",
-             snapshot(), ex);
+             snapshot(), std::string(ex.what()));
+             bts_assert_msgf(false, "io_thread. Websocket io threw websocketpp::exception. {}; "
+             "err: {}",
+               snapshot(), std::string(ex.what()));
       } catch (const std::exception &ex) {
          ERROR_FN(
              "io_thread. Websocket io threw std::exception. {}; err: {}",
-             snapshot(), ex);
+             snapshot(), std::string(ex.what()));
+         bts_assert_msgf(false, "io_thread. Websocket io threw std::exception. {}; err: {}",
+             snapshot(), std::string(ex.what()));
       } catch (...) {
          bts_assert_msgf(
              false,
@@ -251,10 +328,33 @@ struct WebsocketClient::PImpl {
       INFO("Websocket 'run' thread end. {}", snapshot());
    }
 
+   void run_one() {
+      try {
+         client.run_one();
+      } catch (const websocketpp::exception &ex) {
+         ERROR_FN(
+             "io_thread (run_one). Websocket io threw websocketpp::exception. "
+             "{}; err: {}",
+             snapshot(), ex);
+      } catch (const std::exception &ex) {
+         ERROR_FN(
+             "io_thread (run_one). Websocket io threw std::exception. {}; err: "
+             "{}",
+             snapshot(), ex);
+      } catch (...) {
+         bts_assert_msgf(
+             false,
+             "io_thread (run_one). Websocket io threw unknown exception. Could "
+             "this have been thrown by a callback delegate? {}",
+             snapshot());
+      }
+   }
+
  private:
    static std::atomic<int> client_id_counter;
 
-   auto on_tls_init(ConnectionHdl /*hdl*/) {
+   std::shared_ptr<boost::asio::ssl::context> on_tls_init(
+       ConnectionHdl /*hdl*/) {
       INFO_FN("on_tls_init. state: {}; version: {}", state, version);
       auto result =
           websocketpp::lib::make_shared<SslContext>(SslContext::sslv23_client);
@@ -262,44 +362,13 @@ struct WebsocketClient::PImpl {
       return result;
    }
 
-   void init_handlers() {
-      INFO_FN("init_handlers. state: {}; version: {}", state, version);
-      bts_assert(!io_thread);
-      client.init_asio();
-      client.set_message_handler(
-          [this](auto hdl, auto msg) { on_message(hdl, msg); });
-      client.set_open_handler([this](auto hdl) { on_open(hdl); });
-      client.set_ping_handler([](auto /*hdl*/, auto /*msg*/) {
-         LOG(Http, Info, "Received ping");
-         return true;
-      });
-      client.set_pong_timeout(0);
-      client.set_pong_handler([](auto /*hdl*/, auto /*msg*/) {
-         LOG(Http, Info, "Received pong");
-         return true;
-      });
-      client.set_tls_init_handler(
-          [this](auto hdl) { return on_tls_init(hdl); });
-      client.set_close_handler([this](auto hdl) { on_close(hdl); });
-      client.set_fail_handler([this](auto hdl) {
-         const auto ec = client.get_con_from_hdl(hdl)->get_ec();
-         ERROR_FN("fail_handler. {}; err: {}", snapshot(), ec.message());
-      });
-   }
-
-   void init_logging() {
-      INFO_FN("init_logging. {}", snapshot());
-      client.set_access_channels(websocketpp::log::alevel::all);
-      // client.clear_access_channels(websocketpp::log::alevel::frame_payload);
-      client.set_error_channels(websocketpp::log::elevel::all);
-   }
-
    void on_open(ConnectionHdl hdl) {
       lock_guard<mutex> lock{instance_mx};
       INFO_FN("on_open. {}", snapshot());
       const auto ec = client.get_con_from_hdl(hdl)->get_ec();
       if (ec) {
          ERROR_FN("on_open. {}; err: {}", snapshot(), ec.message());
+         bts_assert(false);
       }
       switch (state.value) {
       case State::OPENING:
@@ -350,11 +419,34 @@ struct WebsocketClient::PImpl {
       if (close_handler) {
          (*close_handler)(ec);
       }
+      if (config.reconnect_on_close) {
+         reconnect();
+      }
+   }
+
+   void on_fail(ConnectionHdl hdl) {
+      lock_guard<mutex> lock{instance_mx};
+      ERROR_FN("on_fail. {}", snapshot());
+      set_state(State::IDLE);
+      const auto con = client.get_con_from_hdl(hdl);
+      const auto ec = con->get_ec();
+      if (ec) {
+         ERROR_FN(
+             "on_fail. {}; local_code: {}; "
+             "local_reason: {}; remote_code: {}; remote_reason: {}; err: {}",
+             snapshot(), con->get_local_close_code(),
+             con->get_local_close_reason(), con->get_remote_close_code(),
+             con->get_remote_close_reason(), ec.message());
+      }
+      if (config.reconnect_on_fail) {
+         reconnect();
+      }
    }
 };
 
-WebsocketClient::WebsocketClient(WebsocketClientConfig config)
-    : me_(std::make_unique<PImpl>(std::move(config))) {}
+WebsocketClient::WebsocketClient(
+    WebsocketClientConfig config, boost::asio::io_context *io_context)
+    : me_(std::make_unique<PImpl>(std::move(config), io_context)) {}
 
 WebsocketClient::~WebsocketClient() {
    me_->close(websocketpp::close::status::going_away, "");
@@ -383,13 +475,16 @@ WebsocketClient::CloseHandler &WebsocketClient::close_handler() {
    return *me_->close_handler;
 }
 
-void WebsocketClient::connect(
-    const std::map<std::string, std::string> &headers, bool run_in_background) {
-   me_->connect(headers, run_in_background);
-}
+void WebsocketClient::connect() { me_->connect(); }
 
 void WebsocketClient::run() { me_->run(); }
 
+void WebsocketClient::run_one() { me_->run_one(); }
+
+boost::asio::io_context *WebsocketClient::io_context() {
+   return me_->io_context;
+}
+
 websocketpp::lib::error_code WebsocketClient::send_text(const string &text) {
    return me_->send_text(text);
 }
@@ -407,4 +502,6 @@ bool WebsocketClient::is_connected() const {
 
 void WebsocketClient::ping(const string &payload) { me_->con->ping(payload); }
 
+const string &WebsocketClient::url() const { return me_->config.url; }
+
 } // namespace bts::websocket

src/lib/io/websocket/websocket_client.h

@@ -3,6 +3,7 @@
 #include "lib/io/websocket/websocket_client_config.h"
 #include "lib/utils/Delegate.h"
 
+#include <boost/asio/io_context.hpp>
 #include <memory>
 #include <string>
 #include <websocketpp/config/core_client.hpp>
@@ -12,28 +13,33 @@ namespace bts::websocket {
 
 class WebsocketClient {
  public:
-   explicit WebsocketClient(WebsocketClientConfig config);
+   WebsocketClient(
+       WebsocketClientConfig config,
+       boost::asio::io_context *io_context = nullptr);
    ~WebsocketClient();
 
    using OpenHandler = bts::core::Delegate<void(const std::error_code &)>;
    using MessageHandler = bts::core::Delegate<void(const std::string &)>;
    using CloseHandler = bts::core::Delegate<void(const std::error_code &)>;
 
-   void connect(
-       const std::map<std::string, std::string> &headers = {},
-       bool run_in_background = true);
+   void connect();
    void disconnect();
    bool is_connected() const;
+   boost::asio::io_context *io_context();
    void run();
+   void run_one();
    CloseHandler &open_handler();
    MessageHandler &message_handler();
    CloseHandler &close_handler();
    websocketpp::lib::error_code send_text(const std::string &text);
    void ping(const std::string &payload = "");
 
+   const std::string &url() const;
+
  private:
    struct PImpl;
    std::unique_ptr<PImpl> me_;
 };
 
 } // namespace bts::websocket
+

src/lib/io/websocket/websocket_client_config.h

@@ -5,9 +5,14 @@
 namespace bts::websocket {
 
 struct WebsocketClientConfig {
-   std::string url;
-   core::TimeSpan connect_timeout;
-   core::TimeSpan pong_timeout;
+   std::string url{};
+   core::TimeSpan connect_timeout = core::TimeSpan::zero();
+   core::TimeSpan pong_timeout = core::TimeSpan::zero();
+   bool reconnect_on_close = false;
+   bool reconnect_on_fail = false;
+   std::map<std::string, std::string> headers{};
+   // name to use in logs
+   std::string name{};
 };
 
 } // namespace bts::websocket

src/lib/io/websocket/websocket_client_logger.h

@@ -0,0 +1,88 @@
+#pragma once
+
+#include "lib/logging/Common.h"
+#include "lib/logging/logging.h"
+
+#include <websocketpp/concurrency/basic.hpp>
+#include <websocketpp/logger/basic.hpp>
+#include <websocketpp/logger/levels.hpp>
+
+namespace websocketpp::log {
+
+struct ElevelAdapter {
+   typedef elevel LevelT;
+
+   constexpr bts::log::Level operator()(level channel) {
+      switch (channel) {
+      case elevel::devel:
+         return bts::Diag;
+      case elevel::library:
+      case elevel::warn:
+         return bts::Warn;
+      case elevel::rerror:
+         return bts::Error;
+      case elevel::fatal:
+         return bts::Fatal;
+      default:
+         return bts::Info;
+      }
+   }
+};
+
+struct AlevelAdapter {
+   typedef alevel LevelT;
+
+   constexpr bts::log::Level operator()(level channel) {
+      switch (channel) {
+      case alevel::control:
+      case alevel::frame_header:
+      case alevel::frame_payload:
+      case alevel::message_header:
+      case alevel::message_payload:
+      case alevel::debug_handshake:
+      case alevel::debug_close:
+      case alevel::devel:
+         return bts::Diag;
+      case alevel::disconnect:
+         return bts::Warn;
+      case alevel::fail:
+         return bts::Error;
+      default:
+         return bts::Info;
+      }
+   }
+};
+
+template <typename LevelAdapter>
+class WebsocketClientLogger
+    : public basic<concurrency::basic, typename LevelAdapter::LevelT> {
+   typedef basic<concurrency::basic, typename LevelAdapter::LevelT> base;
+
+   std::string name;
+   LevelAdapter level_adapter;
+
+ public:
+   WebsocketClientLogger(
+       level c, channel_type_hint::value h = channel_type_hint::access)
+       : base(c, h) {}
+
+   void set_name(std::string name) {
+      concurrency::basic::scoped_lock_type lock{this->m_lock};
+      this->name = std::move(name);
+   }
+
+   void write(level channel, const std::string &msg) {
+      concurrency::basic::scoped_lock_type lock{this->m_lock};
+      if (!this->dynamic_test(channel)) {
+         return;
+      }
+      LOG(Http, level_adapter(channel), "WebsocketClient-{} {}. {}", name,
+          std::string{LevelAdapter::LevelT::channel_name(channel)}, msg);
+   }
+
+   void write(level channel, const char *msg) {
+      write(channel, std::string{msg});
+   }
+};
+
+} // namespace websocketpp::log

src/lib/logging/Sink.cpp

@@ -9,6 +9,7 @@
 #include <boost/date_time/posix_time/posix_time.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/path.hpp>
+#include <cstdint>
 #include <fmt/ostream.h>
 #include <fstream>
 #include <iostream>
@@ -94,7 +95,7 @@ std::ifstream::pos_type filesize(std::string const &filename) {
 Sink file_sink(LoggingConfig const &cfg) {
    static std::ofstream stream;
    static uint64_t bytes_written = 0;
-   const auto max_sz = 120 * 1024 * 1024;
+   const uintmax_t max_sz =  1024 * 1024 * 1024 ;
 
    auto f = formatter(cfg);
    auto path = log_path();
@@ -107,7 +108,7 @@ Sink file_sink(LoggingConfig const &cfg) {
 
    if (boost::filesystem::exists(path)) {
       auto sz = boost::filesystem::file_size(path);
-      if (sz > max_sz && false) {
+      if (sz > max_sz) {
          auto backup = boost::filesystem::path(path.string() + ".old");
          boost::filesystem::rename(path, backup);
       }
@@ -118,8 +119,7 @@ Sink file_sink(LoggingConfig const &cfg) {
    return [=](LogMsg const &m) {
       auto msg = f(m);
       bytes_written += msg.size();
-      if (bytes_written >= max_sz && false) {
-         return;
+      if (bytes_written >= max_sz) {
          bytes_written = 0;
          stream.close();
          auto backup = boost::filesystem::path(path.string() + ".old");

src/lib/md/GeneralMarketDataListener.cpp

@@ -290,7 +290,7 @@ struct GeneralMarketDataListener::PImpl {
    // if this is set to true, then the security key passed on in the
    // on_securities function is used. If this is not set to true, then a new key
    // is generated
-   const bool use_given_sec_key = false;
+   const bool use_given_sec_key = true;
    const std::string cc_name_prefix = "cc_security";
    size_t n_bk_updates = 0;
    bts::persistence::IDGenerator &id_gen =
@@ -682,6 +682,18 @@ static bool get_qty_info(
       qty_tick_unit = 1;
       qty_tick_base = 1000000;
       break;
+   case 10'000'000:
+      qty_tick_unit = 1;
+      qty_tick_base = 10'000'000;
+      break;
+   case 100'000'000:
+      qty_tick_unit = 1;
+      qty_tick_base = 100'000'000;
+      break;
+   case 1'000'000'000:
+      qty_tick_unit = 1;
+      qty_tick_base = 1'000'000'000;
+      break;
    case 4:
       qty_tick_unit = 25;
       qty_tick_base = 100;

src/lib/md/MDConsumer.cpp

@@ -435,7 +435,8 @@ class SyntheticConsumerInst {
    void add_security(const api::Security &sec) {
       Mutex::scoped_lock lock(_workmutex);
       if (_data.find(sec.id()) == _data.end()) {
-         LOG(md, Info, "MDConsumer::add_security. {}", sec);
+         LOG(md, Info, "MDConsumer::add_security. {}",
+             bts::securities::descriptor(sec));
          auto ptr = to_strategy(make_shared<api::Security>(sec));
          vector<IncompleteStrategy> vec;
          vec.push_back(ptr);

src/lib/md/SecurityCache.cpp

@@ -14,6 +14,15 @@
 namespace bts {
 namespace md {
 
+#define ASSERT_FN(test, fn, ...)                                               \
+   bts_assert_msg(test, "SecurityCache::" fn ". " __VA_ARGS__)
+#define LOG_FN(level, fn, ...)                                                 \
+   LOG(md, level, "SecurityCache::" fn ". " __VA_ARGS__)
+#define DIAG_FN(fn, ...) LOG_FN(Diag, fn, __VA_ARGS__)
+#define INFO_FN(fn, ...) LOG_FN(Info, fn, __VA_ARGS__)
+#define WARN_FN(fn, ...) LOG_FN(Warn, fn, __VA_ARGS__)
+#define ERROR_FN(fn, ...) LOG_FN(Error, fn, __VA_ARGS__)
+
 using std::string;
 using std::unordered_map;
 
@@ -99,6 +108,9 @@ struct SecurityCacheIndex {
       // Insert but don't overwrite existing
       auto result =
           cache.insert(std::make_pair(SecurityKey(security->id()), security));
+      venue_type_ids.get_or_add(security->exchange())
+          .get_or_add(bts::securities::security_type(*security))
+          .try_emplace(bts::securities::descriptor(*security), security->id());
       return result.second;
    }
 
@@ -166,8 +178,8 @@ struct SecurityCacheImpl {
 };
 
 void SecurityCacheImpl::enqueue_update_db(const SecurityPtr &security) {
-   LOG(md, Diag, "SecurityCacheImpl::enqueue_update_db security: {}",
-       bts::securities::descriptor(*security));
+   DIAG_FN(
+       "enqueue_update_db", "security: {}", securities::descriptor(*security));
    update_queue.push(security);
 }
 
@@ -180,7 +192,13 @@ void SecurityCacheImpl::flush() {
          work.push_back(temp);
       }
       if (!work.empty()) {
+         INFO_FN("flush", "Saving securities to db. count: {}", work.size());
          if (db::insert_or_replace(work) == Result::error) {
+            WARN_FN(
+                "flush",
+                "Failed to save securities to db. Will retry on next flush. "
+                "count: {}",
+                work.size());
             // If we fail to insert then retry later
             for (auto &s : work)
                update_queue.push(std::move(s));
@@ -270,14 +288,18 @@ std::vector<api::Security const *> SecurityCache::get_securities() {
 }
 
 bool SecurityCache::add_live_security(const SecurityPtr &security) {
-   bts_assert(security);
-   LOG(md, Diag, "SecurityCache::add_live_security security: {}",
-       bts::securities::descriptor(*security));
-   return _me->cache.lock([&](SecurityCacheIndex &cache) {
+   ASSERT_FN(security, "add_live_security", "security is null");
+   const bool is_new_sec = _me->cache.lock([&](SecurityCacheIndex &cache) {
       const bool is_new_sec = cache.add_live_security(security);
       _me->enqueue_update_db(security);
       return is_new_sec;
    });
+   if (is_new_sec) {
+      INFO_FN(
+          "add_live_security", "new security: {}",
+          bts::securities::descriptor(*security));
+   }
+   return is_new_sec;
 }
 
 void SecurityCache::flush_to_disk() { _me->flush(); }

src/lib/md/SecurityIDRecorder.cpp

@@ -7,7 +7,6 @@
 #include "lib/utils/checked_arithmetic.h"
 
 #include <boost/filesystem.hpp>
-#include <boost/pool/object_pool.hpp>
 #include <memory>
 #include <sqlite/sqlite3.h>
 #include <stdexcept>

src/lib/md/Shm/FixedSecurity.cpp

@@ -509,6 +509,91 @@ void convert_security(
    }
 }
 
+void to_json(nlohmann::json &j, const FixedSecurity &f) {
+   j = nlohmann::json{
+       {"key", f.key},
+       {"symbol", f.symbol()},
+       {"type", api::SecurityType_Name(f.type)},
+       {"venue", api::Venue_Name(f.exchange)},
+       {"alt_symbol", f.alt_symbol()},
+       {"cfi_code", f.cfi_code()},
+       {"security_desc", f.security_desc()},
+       {"platform_id", f.platform_id()},
+       {"display_factor", f.display_factor},
+       {"numeric_id", f.numeric_id},
+       {"cboe_class_key", f.cboe_class_key},
+       {"main_frac", (int)f.cme_main_fraction},
+       {"sub_frac", (int)f.cme_sub_fraction},
+       {"price_format", (int)f.cme_price_display_format},
+       {"market_segment", (int)f.cme_market_segment_id},
+
+       {"price_break", f.price_break},
+       {"low_tick", f.low_tick},
+       {"high_tick", f.high_tick},
+       {"cpp", f.cash_per_point},
+
+       {"ice_market_description", f.ice_market_description()},
+       {"ice_cleared_alias", f.ice_cleared_alias()},
+       {"ice_product_name", f.ice_product_name()},
+       {"ice_hub_alias", f.ice_hub_alias()},
+       {"ice_strip_name", f.ice_strip_name()},
+       {"mic_code", f.mic_code()},
+   };
+   // Qty Info
+   {
+      auto qty_infos_j = nlohmann::json::array();
+      float min_value, max_value, tick_unit, tick_base, max_price;
+      bool is_notional;
+      for (int8_t i = 0; f.get_quantity_info(
+               i, min_value, max_value, tick_unit, tick_base, max_price,
+               is_notional);
+           i++) {
+         auto qty_info_j = nlohmann::json::object();
+         qty_info_j["min_value"] = min_value;
+         qty_info_j["max_value"] = max_value;
+         qty_info_j["tick_unit"] = tick_unit;
+         qty_info_j["tick_base"] = tick_base;
+         qty_info_j["max_price"] = max_price;
+         qty_info_j["is_notional"] = is_notional;
+         qty_infos_j.push_back(std::move(qty_info_j));
+      }
+      j["qty_infos"] = std::move(qty_infos_j);
+   }
+
+   switch (f.type) {
+   case api::SECURITYTYPE_SPOT:
+      break;
+   case api::SECURITYTYPE_FUTURE: {
+      auto *fut = static_cast<const FixedFutureSecurity *>(&f);
+      core::TimeStamp exp;
+      core::TimeStamp::from_ymd(
+          fut->expiration_year, fut->expiration_month, fut->expiration_day,
+          &exp);
+      j["expiration"] = exp.to_iso_string();
+      break;
+   }
+   case api::SECURITYTYPE_OPTION: {
+      auto *opt = static_cast<const FixedOptionSecurity *>(&f);
+      core::TimeStamp exp;
+      core::TimeStamp::from_ymd(
+          opt->expiration_year, opt->expiration_month, opt->expiration_day,
+          &exp);
+      j["expiration"] = exp.to_iso_string();
+      j["strike"] = opt->strike;
+      j["callput"] = api::PutOrCall_Name(opt->call_put);
+      break;
+   }
+   case api::SECURITYTYPE_SPREAD: {
+      auto *spd = static_cast<const FixedSpreadSecurityXL *>(&f);
+      std::vector<FixedSpreadLeg> legs{spd->legs, spd->legs + spd->leg_count()};
+      j["legs"] = legs;
+      break;
+   }
+   default:
+      break;
+   }
+}
+
 } // namespace md
 } // namespace bts
 

src/lib/md/Shm/FixedSecurity.h

@@ -273,71 +273,6 @@ inline void to_json(nlohmann::json &j, const FixedSpreadLeg &f) {
    };
 }
 
-inline void to_json(nlohmann::json &j, const FixedSecurity &f) {
-   j = nlohmann::json{
-       {"key", f.key},
-       {"symbol", f.symbol()},
-       {"type", api::SecurityType_Name(f.type)},
-       {"venue", api::Venue_Name(f.exchange)},
-       {"alt_symbol", f.alt_symbol()},
-       {"cfi_code", f.cfi_code()},
-       {"security_desc", f.security_desc()},
-       {"platform_id", f.platform_id()},
-       {"display_factor", f.display_factor},
-       {"numeric_id", f.numeric_id},
-       {"cboe_class_key", f.cboe_class_key},
-       {"main_frac", (int)f.cme_main_fraction},
-       {"sub_frac", (int)f.cme_sub_fraction},
-       {"price_format", (int)f.cme_price_display_format},
-       {"market_segment", (int)f.cme_market_segment_id},
-
-       {"price_break", f.price_break},
-       {"low_tick", f.low_tick},
-       {"high_tick", f.high_tick},
-       {"cpp", f.cash_per_point},
-
-       {"ice_market_description", f.ice_market_description()},
-       {"ice_cleared_alias", f.ice_cleared_alias()},
-       {"ice_product_name", f.ice_product_name()},
-       {"ice_hub_alias", f.ice_hub_alias()},
-       {"ice_strip_name", f.ice_strip_name()},
-       {"mic_code", f.mic_code()},
-   };
-
-   switch (f.type) {
-   case api::SECURITYTYPE_SPOT:
-      break;
-   case api::SECURITYTYPE_FUTURE: {
-      auto *fut = static_cast<const FixedFutureSecurity *>(&f);
-      core::TimeStamp exp;
-      core::TimeStamp::from_ymd(
-          fut->expiration_year, fut->expiration_month, fut->expiration_day,
-          &exp);
-      j["expiration"] = exp.to_iso_string();
-      break;
-   }
-   case api::SECURITYTYPE_OPTION: {
-      auto *opt = static_cast<const FixedOptionSecurity *>(&f);
-      core::TimeStamp exp;
-      core::TimeStamp::from_ymd(
-          opt->expiration_year, opt->expiration_month, opt->expiration_day,
-          &exp);
-      j["expiration"] = exp.to_iso_string();
-      j["strike"] = opt->strike;
-      j["callput"] = api::PutOrCall_Name(opt->call_put);
-      break;
-   }
-   case api::SECURITYTYPE_SPREAD: {
-      auto *spd = static_cast<const FixedSpreadSecurityXL *>(&f);
-      std::vector<FixedSpreadLeg> legs{spd->legs, spd->legs + spd->leg_count()};
-      j["legs"] = legs;
-      break;
-   }
-   default:
-      break;
-   }
-}
-
 inline std::ostream &operator<<(
     std::ostream &out, const FixedSecurity &f) noexcept {
    out << io::JsonDumper::dump(f);

src/lib/mercury/broker/BrokerProxy.cpp

@@ -8,7 +8,10 @@ namespace mercury {
 BrokerProxy::BrokerProxy(ILocalBroker &real_broker)
     : _real_broker(real_broker), _id(real_broker.id()), _cycles(0),
       _outgoing(config::get<config::Mercury>().queue_size()),
-      _incoming(config::get<config::Mercury>().queue_size()), _action_pool() {}
+      _incoming(config::get<config::Mercury>().queue_size()), _action_pool() {
+   _action_pool.set_max_count(4096 * 10);
+   _action_pool.set_log_freq(100);
+}
 
 BrokerProxy::~BrokerProxy() {}
 
@@ -96,5 +99,60 @@ void BrokerProxy::schedule_action(std::function<void()> const &action) {
    send(*m);
 }
 
+void BrokerProxy::do_work(trace::PerfLog &log) {
+   _cycles++;
+   _outgoing.consume_all([this, &log](BrokerMsg *m) {
+      switch (m->msg_type) {
+      case BrokerMsgType::ADD_NEIGHBOUR:
+         _real_broker.add_neighbour(*m->neighbour);
+         break;
+      case BrokerMsgType::REMOVE_NEIGHBOUR:
+         _real_broker.remove_neighbour(m->broker_id);
+         break;
+      case BrokerMsgType::ADVERTISE:
+         _real_broker.advertise(m->broker_id, m->stream_id, m->topic);
+         break;
+      case BrokerMsgType::UNADVERTISE:
+         _real_broker.unadvertise(m->broker_id, m->stream_id);
+         break;
+      case BrokerMsgType::SUBSCRIBE:
+         _real_broker.subscribe(m->broker_id, m->sub_id, m->stream_id);
+         break;
+      case BrokerMsgType::UNSUBSCRIBE:
+         _real_broker.unsubscribe(m->broker_id, m->sub_id, m->stream_id);
+         break;
+      case BrokerMsgType::NOTIFY: {
+         LOG(Latency, Diag, "Notify broker {}. Cycles: {}", _id.value(),
+             _cycles);
+         auto scoped_do_work = trace::log_scope(log, {}, "BrokerProxy::notify");
+         _real_broker.notify(
+             m->broker_id, m->stream_id, m->sub_id, m->notification);
+      } break;
+      case BrokerMsgType::LOCAL_SUBSCRIBE:
+         _real_broker.local_subscribe(
+             m->local_sub_id, m->filter, m->local_handler);
+         break;
+      case BrokerMsgType::LOCAL_UNSUBSCRIBE:
+         _real_broker.local_unsubscribe(m->local_sub_id);
+         break;
+      case BrokerMsgType::EXECUTE_ACTION: {
+         auto start = Timer::now();
+         LOG(Latency, Diag, "Executing lambda on broker {}", _id.value());
+         m->action();
+         auto stop = Timer::now();
+         LOG(Latency, Diag,
+             "Finished executing lambda on broker {}. Took: {}us", _id.value(),
+             us_diff(stop, start));
+         break;
+      }
+      default:
+         bts_assert(false);
+         break;
+      }
+      _incoming.drain();
+      _incoming.push(m);
+   });
+}
+
 } // namespace mercury
 } // namespace bts

src/lib/mercury/broker/BrokerProxy.h

@@ -5,8 +5,8 @@
 #include "lib/mercury/topics/Topic.h"
 #include "lib/tracing/PerfLog.h"
 #include "lib/utils/GrowableSPSCQueue.h"
+#include "lib/utils/object_pool.h"
 
-#include <boost/pool/object_pool.hpp>
 #include <memory>
 
 namespace bts {
@@ -80,61 +80,7 @@ class BrokerProxy : public ILocalBroker {
       send(m);
    }
 
-   void do_work(trace::PerfLog &log) {
-      _cycles++;
-      _outgoing.consume_all([this, &log](BrokerMsg *m) {
-         switch (m->msg_type) {
-         case BrokerMsgType::ADD_NEIGHBOUR:
-            _real_broker.add_neighbour(*m->neighbour);
-            break;
-         case BrokerMsgType::REMOVE_NEIGHBOUR:
-            _real_broker.remove_neighbour(m->broker_id);
-            break;
-         case BrokerMsgType::ADVERTISE:
-            _real_broker.advertise(m->broker_id, m->stream_id, m->topic);
-            break;
-         case BrokerMsgType::UNADVERTISE:
-            _real_broker.unadvertise(m->broker_id, m->stream_id);
-            break;
-         case BrokerMsgType::SUBSCRIBE:
-            _real_broker.subscribe(m->broker_id, m->sub_id, m->stream_id);
-            break;
-         case BrokerMsgType::UNSUBSCRIBE:
-            _real_broker.unsubscribe(m->broker_id, m->sub_id, m->stream_id);
-            break;
-         case BrokerMsgType::NOTIFY: {
-            LOG(Latency, Diag, "Notify broker {}. Cycles: {}", _id.value(),
-                _cycles);
-            auto scoped_do_work =
-                trace::log_scope(log, {}, "BrokerProxy::notify");
-            _real_broker.notify(
-                m->broker_id, m->stream_id, m->sub_id, m->notification);
-         } break;
-         case BrokerMsgType::LOCAL_SUBSCRIBE:
-            _real_broker.local_subscribe(
-                m->local_sub_id, m->filter, m->local_handler);
-            break;
-         case BrokerMsgType::LOCAL_UNSUBSCRIBE:
-            _real_broker.local_unsubscribe(m->local_sub_id);
-            break;
-         case BrokerMsgType::EXECUTE_ACTION: {
-            auto start = Timer::now();
-            LOG(Latency, Diag, "Executing lambda on broker {}", _id.value());
-            m->action();
-            auto stop = Timer::now();
-            LOG(Latency, Diag,
-                "Finished executing lambda on broker {}. Took: {}us",
-                _id.value(), us_diff(stop, start));
-            break;
-         }
-         default:
-            bts_assert(false);
-            break;
-         }
-         _incoming.drain();
-         _incoming.push(m);
-      });
-   }
+   void do_work(trace::PerfLog &log);
 
  private:
    ILocalBroker &_real_broker;
@@ -142,7 +88,7 @@ class BrokerProxy : public ILocalBroker {
    uint64_t _cycles;
    utils::GrowableSPSCQueue<BrokerMsg *> _outgoing;
    utils::GrowableSPSCQueue<BrokerMsg *> _incoming;
-   boost::object_pool<BrokerMsg> _action_pool;
+   bts::utils::object_pool<BrokerMsg> _action_pool;
 
    void send(BrokerMsg &m) {
       _outgoing.push(&m);

src/lib/orderman/CreateOrderMsg.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "lib/api/services/OrderService.h"
 #include "lib/core/FMT_IMPL.h"
 #include "lib/domain_model/Security.h"
 #include "lib/domain_model/common.h"
@@ -24,6 +25,25 @@ struct CreateOrderMsg {
    api::orderman::RoutingRecord const *routing_record;
 };
 
+/**
+ * A CreateOrderMsg wrapper with a custom fmt::formatter that doesn't access the
+ * raw security pointer. This is mainly used for logging purposes (because the
+ * security pointer may be invalid by the time the fmt::formatter is invoked).
+ */
+struct CreateOrderMsg_JsonData {
+   CreateOrderMsg_JsonData() = default;
+   CreateOrderMsg_JsonData(CreateOrderMsg source) : source(std::move(source)) {
+      if (this->source.security != nullptr) {
+         sec_name = securities::descriptor(*this->source.security);
+      }
+   }
+
+   CreateOrderMsg source;
+   std::string sec_name;
+};
+
+void to_json(nlohmann::json &j, const CreateOrderMsg &msg);
+
 inline std::ostream &operator<<(std::ostream &out, CreateOrderMsg const &msg) {
    out << "key: " << msg.key.value() << " price: " << msg.price.value()
        << " qty: " << msg.quantity.full_value() << " security: "
@@ -43,6 +63,29 @@ inline std::ostream &operator<<(std::ostream &out, CreateOrderMsg const &msg) {
    return out;
 }
 
+inline void to_json(
+    nlohmann::json &j, const CreateOrderMsg_JsonData &msg_data) {
+   const auto &msg = msg_data.source;
+   j["key"] = msg.key.value();
+   j["price"] = msg.price.value();
+   j["quantity"] = msg.quantity.value();
+   if (msg.display_quantity.value() != 0) {
+      j["display_quantity"] = msg.quantity.value();
+   }
+   if (!msg_data.sec_name.empty()) {
+      j["security"] = msg_data.sec_name;
+   }
+   j["side"] = msg.side;
+   if (msg.stop_px.value() != 0) {
+      j["stop_px"] = msg.stop_px.value();
+   }
+   j["time_in_force"] = msg.time_in_force;
+   j["manual_order_indicator"] = msg.manual_order_indicator;
+   if (msg.routing_record != nullptr) {
+      j["routing_record"] = *msg.routing_record;
+   }
+}
+
 } // namespace bts::orderman
 
 FMT_IMPL_OSTREAM(bts::orderman::CreateOrderMsg);

src/lib/orderman/OrderServiceServer.cpp

@@ -8,6 +8,7 @@
 #include "lib/config/config.h"
 #include "lib/config/protos/jetfix.pb.h"
 #include "lib/config/protos/orderman_cfg.pb.h"
+#include "lib/core/FMT.h"
 #include "lib/domain_model/Security.pb.h"
 #include "lib/domain_model/common.h"
 #include "lib/logging/logging.h"
@@ -29,11 +30,11 @@
 #include "lib/orderman/actors/order_manager/order_gateway_definition.h"
 #include "lib/orderman/events/ExchangePositionEvent.h"
 #include "lib/orderman/events/OrderOpResultEvent.h"
-#include "lib/orderman/gateways/execution_gateway_provider.h"
 #include "lib/orderman/gateways/i_execution_gateway_provider.h"
 #include "lib/orderman/order_factory/DynamicOrderRegistry.h"
 #include "lib/orderman/orderman_utils/ExchangeOpenOrders.h"
 #include "lib/orderman/orders/Order.h"
+#include "lib/os/os.h"
 #include "lib/os/spark_data_directory.h"
 #include "lib/plugin/SparkServices.h"
 #include "lib/plugin/plugin.h"
@@ -41,7 +42,6 @@
 #include "lib/utils/bts_chrono.h"
 #include "lib/utils/thread_manager.h"
 #include "lib/utils/type_name.h"
-#include "lib_binance_execution/binance_execution/execution_gateway_provider.h"
 
 #include <bits/stdint-uintn.h>
 #include <boost/algorithm/string/predicate.hpp>
@@ -73,16 +73,58 @@ struct ActorThread {
    void start(trace::PerfLog &log) {
       if (cfg.poll())
          thread = threading::make_thread(cfg.thread_name(), [this, &log]() {
-            for (;;)
+            const core::TimeSpan log_interval =
+                core::TimeSpan::from_seconds(30);
+            core::TimeStamp last_log_time{0};
+            core::TimeStamp now;
+            core::TimeSpan elapsed;
+            std::uint32_t last_count = 0;
+            for (;;) {
+               now = core::TimeStamp::now();
+               elapsed = now - last_log_time;
+               if (elapsed > log_interval) {
+                  LOG(OrderMan, Info,
+                      "OrderServiceServer ActorThread work speed is {:.2f} "
+                      "cycles/sec",
+                      static_cast<float>(last_count) / elapsed.total_seconds());
+                  last_log_time = now;
+                  last_count = 0;
+               }
                executor.do_work(log);
+               last_count++;
+            }
          });
       else {
          thread = threading::make_thread(cfg.thread_name(), [this, &log]() {
+            // for logging purposes
+            const std::string t_name = bts::os::get_thread_name();
+            const std::string actors_desc = FMT_JOIN(cfg.hosted_actor(), ", ");
+            const core::TimeSpan log_interval =
+                core::TimeSpan::from_seconds(30);
+
             const auto inner_iter = 1000;
+            bts::core::TimeStamp last_log_time{0};
+            core::TimeStamp now;
+            core::TimeSpan elapsed;
+            std::uint32_t last_count = 0;
             while (!done) {
+               now = core::TimeStamp::now();
+               elapsed = now - last_log_time;
+               auto log_elapsed = now - last_log_time;
+               if (log_elapsed > log_interval) {
+                  LOG(OrderMan, Info,
+                      "OrderServiceServer ActorThread {} work speed is {:.2f} "
+                      "cycles/sec ({})",
+                      t_name,
+                      static_cast<float>(last_count) / elapsed.total_seconds(),
+                      actors_desc);
+                  last_log_time = now;
+                  last_count = 0;
+               }
                auto start = Timer::now();
                for (auto i = 0; i < inner_iter; i++)
                   executor.do_work(log);
+               last_count += inner_iter;
                auto stop = Timer::now();
                loop_metric->observe(ns_diff(stop, start) / inner_iter);
                std::this_thread::sleep_for(10ms);

src/lib/orderman/actors/FillStore.cpp

@@ -24,6 +24,12 @@
 namespace bts {
 namespace orderman {
 
+#define LOG_FN(level, fn, ...)                                                 \
+   LOG(OrderMan, level, "FillStore::" fn ". " __VA_ARGS__)
+#define INFO_FN(fn, ...) LOG_FN(Info, fn, __VA_ARGS__)
+#define WARN_FN(fn, ...) LOG_FN(Warn, fn, __VA_ARGS__)
+#define ERROR_FN(fn, ...) LOG_FN(Error, fn, __VA_ARGS__)
+
 auto const fills_ns = "fills";
 
 struct FillStore::UnresolvedFills {
@@ -160,7 +166,7 @@ void FillStore::on_execution_report(
              ev.data->to_string());
          auto &er = *ev.data;
          auto fk = fmt::format(
-             "{}:{}", er.order_key.value(), to_str(er.exchange_fill_id));
+             "{}:{}", er.order_key.value(), to_strn(er.exchange_fill_id));
          auto is_known = me->known_fills.count(fk);
          if (er.exec_type == api::orderman::EXEC_TYPE_PARTIAL_FILL ||
              er.exec_type == api::orderman::EXEC_TYPE_FILL) {
@@ -176,7 +182,7 @@ void FillStore::on_execution_report(
                   fill.set_fill_kind(api::orderman::Fill::FILL_KIND_STANDARD);
                   fill.set_security_key(er.security->id());
                }
-               fill.set_account(to_str(er.account_name));
+               fill.set_account(to_strn(er.account_name));
                fill.set_last_price(er.last_price.value());
                fill.set_avg_price(er.avg_price.value());
                fill.set_side(er.side);
@@ -207,7 +213,15 @@ void FillStore::on_execution_report(
                auto it = me->known_orders.find(er.order_key);
                if (it != me->known_orders.end()) {
                   auto &order_def = it->second;
-                  fill.set_account(order_def.account());
+                  if (fill.account() != order_def.account()) {
+                     WARN_FN(
+                         "on_execution_report",
+                         "ExecutionReport account '{}' does not match "
+                         "OrderDefinition account '{}' (order_key: {}). "
+                         "Setting fill account to the latter",
+                         fill.account(), order_def.account(), er.order_key);
+                     fill.set_account(order_def.account());
+                  }
                   fill.set_internal_account_id(order_def.account_id());
                } else
                   me->fills_with_unresolved_accounts[er.order_key]
@@ -215,6 +229,7 @@ void FillStore::on_execution_report(
 
                auto fv = utils::pb::to_string(fill);
                me->store->insert(fills_ns, fk, fv);
+               LOG(OrderMan, Info, "Publishing fill. {}", fv);
                publish(me->fill_stream, fill, *er.security);
             } else
                LOG(OrderMan, Info,

src/lib/orderman/actors/OrderStore.cpp

@@ -155,7 +155,7 @@ void OrderStore::store_and_publish_if_complete(
       auto &rec_info = me->known_orders[k];
       auto &rec = rec_info.rec;
       if (rec.has_def() && rec.has_state()) {
-         LOG(OrderMan, Info,
+         LOG(OrderMan, Diag,
              "OrderStore::store_and_publish_if_complete rec: {}",
              utils::pb::to_string(rec));
          publish(me->order_stream, rec);

src/lib/orderman/actors/ServiceBridge.cpp

@@ -37,6 +37,12 @@
 namespace bts {
 namespace orderman {
 
+#define _LOG_FN(level, fn, ...)                                                \
+   LOG(OrderMan, level, "ServiceBridge::" fn ". " __VA_ARGS__)
+#define INFO_FN(fn, ...) _LOG_FN(Info, fn, __VA_ARGS__)
+#define WARN_FN(fn, ...) _LOG_FN(Warn, fn, __VA_ARGS__)
+#define ERROR_FN(fn, ...) _LOG_FN(Error, fn, __VA_ARGS__)
+
 struct ServiceBridge::PImpl {
    PImpl(
        OnReady const &on_ready,
@@ -173,56 +179,72 @@ void ServiceBridge::init() {
 } // namespace orderman
 
 void ServiceBridge::on(mercury::Update<StoredFillEvent> const &update) {
-   if (update.events) {
-      std::vector<api::orderman::Fill> fills;
-      fills.reserve(update.events->size());
-      for (auto ev : *(update.events))
-         fills.push_back(ev.fill);
-      me->on_fills(fills);
+   if (!update.events) {
+      WARN_FN("on StoredFillEvent", "no event data");
+      return;
    }
+   INFO_FN("on StoredFillEvent", "n_events: {}", update.events->size());
+   std::vector<api::orderman::Fill> fills;
+   fills.reserve(update.events->size());
+   for (auto ev : *(update.events))
+      fills.push_back(ev.fill);
+   me->on_fills(fills);
 }
 
 void ServiceBridge::on(mercury::Update<StoredOrderEvent> const &update) {
-   if (update.events) {
-      bool was_ready = is_ready();
-      me->have_orders = true;
-
-      std::vector<api::orderman::OrderRecord> orders;
-      orders.reserve(update.events->size());
-      for (auto ev : *(update.events)) {
-         me->known_orders.insert(OrderKey(ev.order_record.key()));
-         orders.push_back(ev.order_record);
-      }
-      me->on_orders(orders);
-      maybe_signal_ready(was_ready);
+   if (!update.events) {
+      WARN_FN("on StoredOrderEvent", "no event data");
+      return;
    }
+   INFO_FN("on StoredOrderEvent", "n_events: {}", update.events->size());
+   bool was_ready = is_ready();
+   me->have_orders = true;
+
+   std::vector<api::orderman::OrderRecord> orders;
+   orders.reserve(update.events->size());
+   for (auto ev : *(update.events)) {
+      me->known_orders.insert(OrderKey(ev.order_record.key()));
+      orders.push_back(ev.order_record);
+   }
+   me->on_orders(orders);
+   maybe_signal_ready(was_ready);
 }
 
 void ServiceBridge::on(mercury::Update<OrderManagerReadyEvent> const &update) {
-   if (update.events) {
-      bool was_ready = is_ready();
-      me->order_manager_ready = true;
-      maybe_signal_ready(was_ready);
+   if (!update.events) {
+      WARN_FN("on OrderManagerReadyEvent", "no event data");
+      return;
    }
+   INFO_FN("on OrderManagerReadyEvent", "n_events: {}", update.events->size());
+   bool was_ready = is_ready();
+   me->order_manager_ready = true;
+   maybe_signal_ready(was_ready);
 }
 
 void ServiceBridge::on(mercury::Update<NewAdminDataEvent> const &update) {
-   if (update.events)
-      for (auto ev : *(update.events))
-         me->order_router.update(*ev.admin_data);
+   if (!update.events) {
+      WARN_FN("on NewAdminDataEvent", "no event data");
+      return;
+   }
+   INFO_FN("on NewAdminDataEvent", "n_events: {}", update.events->size());
+   for (auto ev : *(update.events)) {
+      me->order_router.update(*ev.admin_data);
+   }
 }
 
 void ServiceBridge::on(mercury::Update<OrderOpResultEvent> const &update) {
-   if (update.events)
-      for (auto ev : *(update.events))
-         me->on_order_op_result(ev);
+   if (!update.events) {
+      WARN_FN("on OrderOpResultEvent", "no event data");
+      return;
+   }
+   INFO_FN("on OrderOpResultEvent", "n_events: {}", update.events->size());
+   for (auto ev : *(update.events)) {
+      me->on_order_op_result(ev);
+   }
 }
 
 void ServiceBridge::create_order(uint32_t rid, api::orderman::Create const &c) {
-
-   LOG(OrderMan, Info, "Received new order from client: {}",
-       utils::pb::to_string(c));
-
+   INFO_FN("create_order", "rid: {}; req: {}", rid, utils::pb::to_string(c));
    me->service_call_marshaller.schedule_action([this, c, rid]() {
       if (is_ready()) {
          auto &order =
@@ -239,9 +261,7 @@ void ServiceBridge::create_order(uint32_t rid, api::orderman::Create const &c) {
 }
 
 void ServiceBridge::update_order(uint32_t rid, api::orderman::Update const &c) {
-   LOG(OrderMan, Info, "ServiceBridge::update_order update: {}",
-       utils::pb::to_string(c));
-
+   INFO_FN("update_order", "rid: {}; req: {}", rid, utils::pb::to_string(c));
    me->service_call_marshaller.schedule_action([this, c, rid]() {
       auto ok = OrderKey(c.order_key());
       if (is_ready()) {
@@ -259,9 +279,7 @@ void ServiceBridge::update_order(uint32_t rid, api::orderman::Update const &c) {
 }
 
 void ServiceBridge::cancel_order(uint32_t rid, api::orderman::Cancel const &c) {
-   LOG(OrderMan, Info, "Received order cancel from client: {}",
-       utils::pb::to_string(c));
-
+   INFO_FN("cancel_order", "rid: {}; req: {}", rid, utils::pb::to_string(c));
    me->service_call_marshaller.schedule_action([this, c, rid]() {
       auto ok = OrderKey(c.order_key());
       if (is_ready()) {
@@ -279,9 +297,7 @@ void ServiceBridge::cancel_order(uint32_t rid, api::orderman::Cancel const &c) {
 }
 
 void ServiceBridge::detach_order(uint32_t rid, api::orderman::Detach const &c) {
-   LOG(OrderMan, Info, "Received order detach from client: {}",
-       utils::pb::to_string(c));
-
+   INFO_FN("detach_order", "rid: {}; req: {}", rid, utils::pb::to_string(c));
    me->service_call_marshaller.schedule_action([this, c, rid]() {
       auto ok = OrderKey(c.order_key());
       if (is_ready()) {
@@ -300,15 +316,13 @@ void ServiceBridge::detach_order(uint32_t rid, api::orderman::Detach const &c) {
 
 void ServiceBridge::manually_bust_all_fills(
     const api::orderman::ManualBustParameters &c) {
-   LOG(OrderMan, Info, "ServiceBridge::manually_bust_all_fills: {}",
-       utils::pb::to_string(c));
+   INFO_FN("manually_bust_all_fills", "req: {}", utils::pb::to_string(c));
    me->service_call_marshaller.schedule_action(
        [this, c]() { publish(me->manually_bust_fills_pub, c.account()); });
 }
 
 void ServiceBridge::send_rfq(api::orderman::RFQRequest const &rfq) {
-   LOG(OrderMan, Info, "ServiceBridge::send_rfq rfq: {}",
-       utils::pb::to_string(rfq));
+   INFO_FN("send_rfq", "req: {}", utils::pb::to_string(rfq));
    me->service_call_marshaller.schedule_action(
        [this, rfq]() { publish(me->rfq_pub, rfq); });
 }
@@ -398,6 +412,11 @@ std::optional<FilledPositionEvent> make_pos_event(
 
 void ServiceBridge::update_position(
     api::position::Position_dataUpdate const &pos) {
+   INFO_FN(
+       "update_position",
+       "n_added: {}; n_changed: {}; n_removed: {}; n_total: {}",
+       pos.added().size(), pos.changed().size(), pos.removed().size(),
+       pos.all().size());
    std::vector<FilledPositionEvent> evs;
    for (auto const &pu : pos.added()) {
       auto ev = make_pos_event(*pu, me->sec_db);
@@ -430,6 +449,9 @@ bool ServiceBridge::is_ready() const {
 }
 
 void ServiceBridge::maybe_signal_ready(bool old_ready) {
+   INFO_FN(
+       "maybe_signal_ready", "old_ready: {}; is_ready: {}", old_ready,
+       is_ready());
    if (!old_ready && is_ready())
       me->on_ready();
 }

src/lib/orderman/actors/order_manager/OrderManager.cpp

@@ -40,6 +40,13 @@
 namespace bts {
 namespace orderman {
 
+#define _LOG_FN(level, fn, ...)                                                \
+   LOG(OrderMan, level, "OrderManager::" fn ". " __VA_ARGS__)
+#define DIAG_FN(fn, ...) _LOG_FN(Diag, fn, __VA_ARGS__)
+#define INFO_FN(fn, ...) _LOG_FN(Info, fn, __VA_ARGS__)
+#define WARN_FN(fn, ...) _LOG_FN(Warn, fn, __VA_ARGS__)
+#define ERROR_FN(fn, ...) _LOG_FN(Error, fn, __VA_ARGS__)
+
 using namespace mercury;
 
 enum class OrderManager::State {
@@ -80,6 +87,23 @@ struct OrderManager::PImpl {
    mercury::Stream<OrderManagerReadyEvent> pub_ready;
    std::thread background_thread;
    std::atomic<bool> shutdown = false;
+
+   constexpr std::string state_name() {
+      switch (state) {
+      case State::WAIT_ADMIN:
+         return "WAIT_ADMIN";
+      case State::WAIT_SEC:
+         return "WAIT_SEC";
+      case State::WAIT_FILLED_POSITION:
+         return "WAIT_FILLED_POSITION";
+      case State::WAIT_ORDERS:
+         return "WAIT_ORDERS";
+      case State::READY:
+         return "READY";
+      default:
+         return "UNKNOWN";
+      }
+   }
 };
 
 OrderManager::OrderManager(
@@ -108,6 +132,7 @@ OrderManager::~OrderManager() {
 }
 
 void OrderManager::init() {
+   INFO_FN("init", "state: {}", me->state_name());
    subscribe<NewAdminDataEvent>(
        [this](auto const &admin_update) { this->on_admin_data(admin_update); });
    subscribe<AccountPnLUpdateEvent>(
@@ -119,300 +144,365 @@ void OrderManager::init() {
 }
 
 void OrderManager::on_admin_data(const mercury::Update<NewAdminDataEvent> &m) {
-   if (m.events) {
-      if (me->state == State::WAIT_ADMIN) {
-         LOG(OrderMan, Info,
-             "Order manager received admin data. Subscribing to security list "
-             "now ");
-         for (auto &osd : me->services_data.sessions)
-            osd->gw->on_admin_data(m);
+   if (!m.events) {
+      WARN_FN("on_admin_data", "no event data. state: {}", me->state_name());
+      return;
+   }
+   INFO_FN("on_admin_data", "state: {}", me->state_name());
+   if (me->state == State::WAIT_ADMIN) {
+      for (auto &osd : me->services_data.sessions)
+         osd->gw->on_admin_data(m);
 
-         me->state = State::WAIT_SEC;
-         subscribe<NewSecuritiesEvent>([this](auto const &sec_update) {
-            this->on_securities(sec_update);
-         });
-      }
+      me->state = State::WAIT_SEC;
+      INFO_FN("on_admin_data", "state: {}", me->state_name());
+      subscribe<NewSecuritiesEvent>(
+          [this](auto const &sec_update) { this->on_securities(sec_update); });
+   }
 
-      for (auto ev : *m.events) {
-         me->services_data.order_router.update(*ev.admin_data);
-         me->services_data.limiter.update(*ev.admin_data);
-      }
+   for (auto ev : *m.events) {
+      me->services_data.order_router.update(*ev.admin_data);
+      me->services_data.limiter.update(*ev.admin_data);
    }
 }
 
 void OrderManager::on_pnl_update(
     const mercury::Update<AccountPnLUpdateEvent> &u) {
-   if (u.events)
-      for (auto ev : *u.events)
-         me->services_data.limiter.set_day_pnl(ev.account_id, ev.day_pnl);
+   if (!u.events) {
+      WARN_FN("on_pnl_update", "no event data. state: {}", me->state_name());
+      return;
+   }
+   // This log is too noisy. It fires regularly on an interval (sometimes when
+   // the pnl hasn't actually changed).
+   // INFO_FN(
+   //     "on_pnl_update", "state: {}; n_events: {}", me->state_name(),
+   //     u.events->size());
+   for (auto ev : *u.events) {
+      me->services_data.limiter.set_day_pnl(ev.account_id, ev.day_pnl);
+   }
 }
 
 void OrderManager::on_excess_liquidity_update(
     const mercury::Update<ExcessLiquidityUpdateEvent> &u) {
-   if (u.events)
-      for (auto ev : *u.events)
-         me->services_data.limiter.set_excess_liquidity(
-             ev.account_id, ev.excess_liquidity);
+   if (!u.events) {
+      WARN_FN(
+          "on_excess_liquidity_update", "no event data. state: {}",
+          me->state_name());
+      return;
+   }
+   INFO_FN(
+       "on_excess_liquidity_update", "state: {}; n_events: {}",
+       me->state_name(), u.events->size());
+   for (auto ev : *u.events) {
+      me->services_data.limiter.set_excess_liquidity(
+          ev.account_id, ev.excess_liquidity);
+   }
 }
 
 void OrderManager::on_rfq_send(const mercury::Update<RFQEvent> &rfq) {
-   if (rfq.events) {
-      std::vector<OrderMsg> msgs = {};
-      for (auto ev : *rfq.events) {
-         RFQMsg rfq_msg = RFQMsg(ev);
-         msgs.emplace_back(rfq_msg);
-      }
-      trace::CorID cid = {};
-      LOG(OrderMan, Info, "OrderManager::on_rfq_send n_events: {}",
-          rfq.events->size());
-      for (auto &osd : me->services_data.sessions)
-         osd->gw->send_msgs(msgs, cid);
+   if (!rfq.events) {
+      WARN_FN("on_rfq_send", "no event data. state: {}", me->state_name());
+      return;
+   }
+   INFO_FN(
+       "on_rfq_send", "state: {}; n_events: {}", me->state_name(),
+       rfq.events->size());
+   std::vector<OrderMsg> msgs = {};
+   for (auto ev : *rfq.events) {
+      RFQMsg rfq_msg = RFQMsg(ev);
+      msgs.emplace_back(rfq_msg);
+   }
+   trace::CorID cid = {};
+   for (auto &osd : me->services_data.sessions) {
+      osd->gw->send_msgs(msgs, cid);
    }
 }
 
 void OrderManager::on_securities(
     const mercury::Update<NewSecuritiesEvent> &update) {
-
+   if (!update.events) {
+      WARN_FN("on_securities", "no event data. state: {}", me->state_name());
+      return;
+   }
+   INFO_FN(
+       "on_securities", "state: {}; n_events: {}", me->state_name(),
+       update.events->size());
    // forward to execution gateways
    for (auto &osd : me->services_data.sessions)
       osd->gw->on_securities(update);
 
-   if (update.events) {
+   // update our security db and accumulate for the sake of the limiter
+   std::vector<api::Security const *> secs;
+   for (auto const &ev : *(update.events)) {
+      auto sec_ptr = ev.security;
+      secs.push_back(sec_ptr);
+   }
+   me->services_data.limiter.add_securities(secs);
 
-      // update our security db and accumulate for the sake of the limiter
-      std::vector<api::Security const *> secs;
-      for (auto const &ev : *(update.events)) {
-         auto sec_ptr = ev.security;
-         secs.push_back(sec_ptr);
-      }
-      me->services_data.limiter.add_securities(secs);
-
-      if (me->state == State::WAIT_SEC) {
-         LOG(OrderMan, Info,
-             "Order manager received initial set of {} "
-             "securities. Waiting for the filled position now",
-             update.events->size());
-         me->state = State::WAIT_FILLED_POSITION;
-         me->filled_position_sub = subscribe<FilledPositionEvent>(
-             [this](auto const &pos) { this->on_filled_position(pos); });
-      }
+   if (me->state == State::WAIT_SEC) {
+      me->state = State::WAIT_FILLED_POSITION;
+      INFO_FN(
+          "on_securities", "received initial set of {} securities. state: {}",
+          update.events->size(), me->state_name());
+      me->filled_position_sub = subscribe<FilledPositionEvent>(
+          [this](auto const &pos) { this->on_filled_position(pos); });
    }
 }
 
 void OrderManager::on_filled_position(
     const mercury::Update<FilledPositionEvent> &update) {
-   if (update.events) {
-      if (me->state == State::WAIT_FILLED_POSITION) {
-         LOG(OrderMan, Info,
-             "Order manager received initial set of {} "
-             "stored filled position records. Waiting for stored orders now",
-             update.events->size());
-         me->state = State::WAIT_ORDERS;
-         me->stored_order_sub = subscribe<StoredOrderEvent>(
-             [this](auto const &orders) { this->on_order_recovery(orders); });
-      }
+   if (!update.events) {
+      WARN_FN(
+          "on_filled_position", "no event data. state: {}", me->state_name());
+      return;
+   }
+   INFO_FN(
+       "on_filled_position", "state: {}; n_events: {}", me->state_name(),
+       update.events->size());
+   if (me->state == State::WAIT_FILLED_POSITION) {
+      me->state = State::WAIT_ORDERS;
+      INFO_FN("on_filled_position", "new_state: {}", me->state_name());
+      me->stored_order_sub = subscribe<StoredOrderEvent>(
+          [this](auto const &orders) { this->on_order_recovery(orders); });
+   }
 
-      for (auto const &ev : *(update.events)) {
-         me->services_data.position.update(ev);
-         me->services_data.limiter.set_filled_position(
-             *ev.security, ev.account_id, ev.long_quantity, ev.short_quantity);
-      }
+   for (auto const &ev : *(update.events)) {
+      me->services_data.position.update(ev);
+      me->services_data.limiter.set_filled_position(
+          *ev.security, ev.account_id, ev.long_quantity, ev.short_quantity);
    }
 }
 
 void OrderManager::on_order_recovery(
     const mercury::Update<StoredOrderEvent> &update) {
-   if (update.events)
-      if (me->state == State::WAIT_ORDERS) {
+   if (!update.events) {
+      WARN_FN(
+          "on_order_recovery", "no event data. state: {}", me->state_name());
+      return;
+   }
+   if (me->state != State::WAIT_ORDERS) {
+      WARN_FN(
+          "on_order_recovery",
+          "Expected state WAIT_ORDERS. state: {}; n_events: {}",
+          me->state_name(), update.events->size());
+      return;
+   }
+   // Got the first batch of persisted orders which has to match the orders
+   // stored on disk (since we are the only ones producing new orders)
+   INFO_FN(
+       "on_order_recovery", "state: {}; n_events: {}", me->state_name(),
+       update.events->size());
+   // Don't need to monitor persisted orders anymore, all the new orders we care
+   // about will be generated by us.
+   unsubscribe(me->stored_order_sub);
 
-         // Got the first batch of persisted orders which has to match
-         // the orders stored on disk (since we are the only ones
-         // producing new orders)
-         LOG(OrderMan, Info,
-             "Order manager received initial set of {} persisted orders.",
-             update.events->size());
+   init_order_services();
 
-         // Don't need to monitor persisted orders anymore, all the
-         // new orders we care about will be generated by us.
-         unsubscribe(me->stored_order_sub);
+   // recover orders from disk
+   for (auto const &ev : *(update.events)) {
+      auto &order = me->services_data.order_factory.recover(ev.order_record);
+      auto const &def = order.def();
+      publish_order_def_update(OrderKey(ev.key()), def);
+      order.init_order_services(me->services_data);
 
-         init_order_services();
+      auto r = me->orders.emplace(order.key(), &order);
+      bts_assert(r.second);
+      auto const &ostate = ev.order_record.state();
+      order.on_activate(
+          Price(ostate.price()), Quantity(ostate.quantity()), *order._services);
+   }
 
-         // recover orders from disk
-         for (auto const &ev : *(update.events)) {
-            auto &order =
-                me->services_data.order_factory.recover(ev.order_record);
-            auto const &def = order.def();
-            publish_order_def_update(OrderKey(ev.key()), def);
-            order.init_order_services(me->services_data);
-
-            auto r = me->orders.emplace(order.key(), &order);
-            bts_assert(r.second);
-            auto const &ostate = ev.order_record.state();
-            order.on_activate(
-                Price(ostate.price()), Quantity(ostate.quantity()),
-                *order._services);
-         }
-
-         // signal that we are ready
-         LOG(OrderMan, Info, "Order manager is ready");
-         publish(me->pub_ready);
-         me->state = State::READY;
-      }
+   // signal that we are ready
+   INFO_FN("on_order_recovery", "OrderManager is ready");
+   publish(me->pub_ready);
+   me->state = State::READY;
 }
 
 void OrderManager::on_execution_report(
     const mercury::Update<NewExecutionReportEvent> &m) {
-   if (m.events)
-      for (auto const &report : *m.events)
-         if (report.data->leg_security == nullptr) {
-            auto k = report.data->order_key;
-            auto it = me->orders.find(k);
-            if (it == me->orders.end()) {
-               LOG(OrderMan, Error,
-                   "Received an execution report for an unknown order with key "
-                   "{}. Order state will be reconstructed from this report but "
-                   "some "
-                   "information will be lost, including CustomerOrFirm,CTI "
-                   "code "
-                   "and "
-                   "parent order id",
-                   k.value());
-               auto order =
-                   &me->services_data.order_factory.recreate_from_exe_report(
-                       *report.data);
-               auto const &def = order->def();
-               publish_order_def_update(OrderKey(k), def);
-               order->init_order_services(me->services_data);
-               auto r = me->orders.emplace(order->key(), order);
-               bts_assert(r.second);
-               order->on_activate(
-                   report.data->price, report.data->quantity,
-                   *order->_services);
-            }
-         }
+   if (!m.events) {
+      WARN_FN("on_execution_report", "no event data");
+      return;
+   }
+   INFO_FN(
+       "on_execution_report", "state: {}; n_events", me->state_name(),
+       m.events->size());
+   for (auto const &report : *m.events) {
+      if (report.data->leg_security != nullptr) {
+         continue;
+      }
+      auto k = report.data->order_key;
+      auto it = me->orders.find(k);
+      if (it == me->orders.end()) {
+         ERROR_FN(
+             "on_execution_report",
+             "Received an execution report for an unknown order with key {}. "
+             "Order state USED TO be reconstructed from this report but some "
+             "information will be lost, including CustomerOrFirm,CTI code and "
+             "parent order id but now these are IGNORED",
+             k.value());
+         continue;
+         // auto order = &me->services_data.order_factory.recreate_from_exe_report(
+         //     *report.data);
+         // auto const &def = order->def();
+         // publish_order_def_update(OrderKey(k), def);
+         // order->init_order_services(me->services_data);
+         // auto r = me->orders.emplace(order->key(), order);
+         // bts_assert(r.second);
+         // order->on_activate(
+         //     report.data->price, report.data->quantity, *order->_services);
+      }
+   }
 }
 
 void OrderManager::on_activate_order(
     const mercury::Update<ActivateOrderEvent> &m) {
-   if (m.events)
-      for (auto const &ev : *m.events) {
-         LOG(OrderMan, Info,
-             "OrderManager::on_activate_order req_id: {} p: {} q: {}",
-             ev.req_id, ev.price, ev.quantity);
-         auto order = ev.order;
-         auto const &def = order->def();
-         publish_order_def_update(order->key(), def);
-         order->init_order_services(me->services_data);
-         auto r = me->orders.emplace(order->key(), order);
-         bts_assert(r.second);
+   if (!m.events) {
+      WARN_FN("on_activate_order", "no event data");
+      return;
+   }
+   INFO_FN(
+       "on_activate_order", "state: {}; n_events: {}", me->state_name(),
+       m.events->size());
+   for (auto const &ev : *m.events) {
+      LOG(OrderMan, Info,
+          "OrderManager::on_activate_order req_id: {} p: {} q: {}", ev.req_id,
+          ev.price, ev.quantity);
+      auto order = ev.order;
+      auto const &def = order->def();
+      publish_order_def_update(order->key(), def);
+      order->init_order_services(me->services_data);
+      auto r = me->orders.emplace(order->key(), order);
+      bts_assert(r.second);
 
-         auto rid = ev.req_id;
-         auto k = order->key();
-         auto on_response = [this, rid, k](auto const &order_op_result) {
-            publish(
-                me->services_data.report_order_op_result_pub, k, rid,
-                order_op_result);
-         };
-         order->_services->activate(ev.price, ev.quantity, on_response);
-      }
+      auto rid = ev.req_id;
+      auto k = order->key();
+      auto on_response = [this, rid, k](auto const &order_op_result) {
+         publish(
+             me->services_data.report_order_op_result_pub, k, rid,
+             order_op_result);
+      };
+      order->_services->activate(ev.price, ev.quantity, on_response);
+   }
 }
 
 void OrderManager::on_update_order(const mercury::Update<UpdateOrderEvent> &m) {
-   if (m.events)
-      for (auto const &ev : *m.events) {
-         auto k = ev.key();
-         auto it = me->orders.find(k);
-         auto rid = ev.req_id();
+   if (!m.events) {
+      WARN_FN("on_update_order", "no event data");
+      return;
+   }
+   INFO_FN(
+       "on_update_order", "state: {}; n_events: {}", me->state_name(),
+       m.events->size());
+   for (auto const &ev : *m.events) {
+      auto k = ev.key();
+      auto it = me->orders.find(k);
+      auto rid = ev.req_id();
 
-         auto on_order_op_result = [this, rid, k](auto const &order_op_result) {
-            publish(
-                me->services_data.report_order_op_result_pub, k, rid,
-                order_op_result);
-         };
+      auto on_order_op_result = [this, rid, k](auto const &order_op_result) {
+         publish(
+             me->services_data.report_order_op_result_pub, k, rid,
+             order_op_result);
+      };
 
-         if (it != me->orders.end()) {
-            if (ev.is_full_update())
-               it->second->_services->update(
-                   ev.full_update(), on_order_op_result);
-            else
-               it->second->_services->update(
-                   Price(ev.price()), Quantity(ev.quantity()),
-                   on_order_op_result);
+      if (it != me->orders.end()) {
+         if (ev.is_full_update()) {
+            it->second->_services->update(ev.full_update(), on_order_op_result);
          } else {
-            on_order_op_result(OrderOpResult{
-                api::orderman::ORDER_ERROR_UNKNOWN_ORDER, "Unknown order"});
-            LOG(OrderMan, Error,
-                "Received an update request for an unknown order {}. "
-                "This should not happen and likely indicates a serious bug. ",
-                ev.key().value());
+            it->second->_services->update(
+                Price(ev.price()), Quantity(ev.quantity()), on_order_op_result);
          }
+      } else {
+         on_order_op_result(OrderOpResult{
+             api::orderman::ORDER_ERROR_UNKNOWN_ORDER, "Unknown order"});
+         ERROR_FN(
+             "on_update_order",
+             "order not found (id={}). This likely indicates a serious bug.",
+             ev.key().value());
       }
+   }
 }
 
 void OrderManager::on_cancel_order(const mercury::Update<CancelOrderEvent> &m) {
-   if (m.events)
-      for (auto const &ev : *m.events) {
-         auto k = ev.key();
-         auto it = me->orders.find(k);
-         auto rid = ev.req_id;
+   if (!m.events) {
+      WARN_FN("on_cancel_order", "no event data");
+      return;
+   }
+   INFO_FN(
+       "on_cancel_order", "state: {}; n_events: {}", me->state_name(),
+       m.events->size());
+   for (auto const &ev : *m.events) {
+      auto k = ev.key();
+      auto it = me->orders.find(k);
+      auto rid = ev.req_id;
 
-         auto on_order_op_result = [this, rid, k](auto const &order_op_result) {
-            publish(
-                me->services_data.report_order_op_result_pub, k, rid,
-                order_op_result);
-         };
+      auto on_order_op_result = [this, rid, k](auto const &order_op_result) {
+         publish(
+             me->services_data.report_order_op_result_pub, k, rid,
+             order_op_result);
+      };
 
-         if (it != me->orders.end())
-            it->second->_services->cancel(on_order_op_result);
-         else {
-            on_order_op_result(OrderOpResult{
-                api::orderman::ORDER_ERROR_UNKNOWN_ORDER, "Unknown order"});
-            LOG(OrderMan, Error,
-                "Received a cancel request for an unknown order {}. "
-                "This should not happen and likely indicates a serious bug. ",
-                ev.key().value());
-         }
+      if (it != me->orders.end()) {
+         it->second->_services->cancel(on_order_op_result);
+      } else {
+         on_order_op_result(OrderOpResult{
+             api::orderman::ORDER_ERROR_UNKNOWN_ORDER, "Unknown order"});
+         ERROR_FN(
+             "on_cancel_order",
+             "order not found (id={}). This likely indicates a serious bug.",
+             ev.key().value());
       }
+   }
 }
 
 void OrderManager::on_detach_order(const mercury::Update<DetachOrderEvent> &m) {
-   if (m.events)
-      for (auto const &ev : *m.events) {
-         auto k = ev.key();
-         auto it = me->orders.find(k);
-         auto rid = ev.req_id;
+   if (!m.events) {
+      WARN_FN("on_detach_order", "no event data");
+      return;
+   }
+   INFO_FN(
+       "on_detach_order", "state: {}; n_events: {}", me->state_name(),
+       m.events->size());
+   for (auto const &ev : *m.events) {
+      auto k = ev.key();
+      auto it = me->orders.find(k);
+      auto rid = ev.req_id;
 
-         auto on_order_op_result = [this, rid, k](auto const &order_op_result) {
-            publish(
-                me->services_data.report_order_op_result_pub, k, rid,
-                order_op_result);
-         };
+      auto on_order_op_result = [this, rid, k](auto const &order_op_result) {
+         publish(
+             me->services_data.report_order_op_result_pub, k, rid,
+             order_op_result);
+      };
 
-         if (it != me->orders.end())
-            it->second->_services->detach(on_order_op_result);
-         else {
-            on_order_op_result(OrderOpResult{
-                api::orderman::ORDER_ERROR_UNKNOWN_ORDER, "Unknown Order"});
+      if (it != me->orders.end()) {
+         it->second->_services->detach(on_order_op_result);
+      } else {
+         on_order_op_result(OrderOpResult{
+             api::orderman::ORDER_ERROR_UNKNOWN_ORDER, "Unknown Order"});
 
-            LOG(OrderMan, Error,
-                "Received a detach request for an unknown order {}. "
-                "This should not happen and likely indicates a serious bug. ",
-                ev.key().value());
-         }
+         ERROR_FN(
+             "on_detach_order",
+             "order not found (id={}). This likely indicates a serious bug.",
+             ev.key().value());
       }
+   }
 }
 
 void OrderManager::on_gateway_state(
     const mercury::Update<GatewayStatusChangeEvent> &m) {
-   if (m.events)
-      for (auto const &ev : *m.events) {
-         LOG(OrderMan, Info,
-             " OrderManager::on_gateway_state gw: {} connected: {}", ev.gw_name,
-             ev.is_connected);
-         auto &sr = me->services_data.get_session(ev.gw_name.value());
-         sr.gw_connected = ev.is_connected;
-      }
+   if (!m.events) {
+      WARN_FN("on_gateway_state", "no event data");
+      return;
+   }
+   INFO_FN(
+       "on_gateway_state", "state: {}; n_events: {}", me->state_name(),
+       m.events->size());
+   for (auto const &ev : *m.events) {
+      LOG(OrderMan, Info,
+          " OrderManager::on_gateway_state gw: {} connected: {}", ev.gw_name,
+          ev.is_connected);
+      auto &sr = me->services_data.get_session(ev.gw_name.value());
+      sr.gw_connected = ev.is_connected;
+   }
 }
 
 void OrderManager::add_child_order(OrderServices &parent, Order &child) {
@@ -470,9 +560,12 @@ void OrderManager::init_order_services() {
                   return s->session;
                }),
        ", "));
-   LOG(OrderMan, Info, "init_order_services. sessions: {}", session_names);
+   INFO_FN(
+       "init_order_services", "state: {}; sessions: {}", me->state_name(),
+       session_names);
 
    for (auto &s : me->services_data.sessions) {
+      INFO_FN("init_order_services", "Initializing session {}", s->session);
       auto gw_name = GatewayName(s->session);
 
       // Now we are ready to subscribe to the execution gateway
@@ -535,9 +628,11 @@ void OrderManager::init_order_services() {
       // reject evets
       auto s_ptr = s.get();
       s->exe_report_pub = make_stream<NewExecutionReportEvent>(gw_name);
-      s->gw->on_exe_report() = [s_ptr, this](auto const &er, auto const &t) {
-         publish(s_ptr->exe_report_pub, &er, t);
-      };
+      s->gw->on_exe_report() =
+          [s_ptr,
+           this](std::shared_ptr<const ExecutionReport> er, auto const &t) {
+             publish(s_ptr->exe_report_pub, std::move(er), t);
+          };
       s->gw_status_change_pub = make_stream<GatewayStatusChangeEvent>(gw_name);
       s->gw->on_gw_status_change() = [s_ptr, this, gw_name](bool b) {
          publish(s_ptr->gw_status_change_pub, gw_name, b);
@@ -563,9 +658,10 @@ void OrderManager::init_order_services() {
       s->gw->on_cancel_reject() = [s_ptr, this](auto const &cr) {
          publish(s_ptr->cancel_reject_pub, cr);
       };
+      INFO_FN("init_order_services", "Starting session {}", s->session);
       // handlers are ready so we can start processing fix events
       s->gw->start();
-
+      INFO_FN("init_order_services", "Started session {}", s->session);
       reload_fix_buffer(*s);
    }
 
@@ -676,9 +772,10 @@ void OrderManager::after_event() {
          if (queued_ops.size()) {
             auto &corid = osd->current_trace;
             auto now = utils::TimeStamp::now();
-            LOG(Latency, Info,
-                "entry_type:after_event channel:{} sn:{} app_arrival:{} now:{} "
-                "delta:{}",
+            INFO_FN(
+                "after_event",
+                "channel: {} sn: {} app_arrival: {} now: {} "
+                "delta: {}",
                 std::string{corid.channel}, corid.sn,
                 corid.app_arrival.to_uint64(), now.to_uint64(),
                 (now - corid.app_arrival).nanoseconds());
@@ -694,9 +791,9 @@ void OrderManager::reload_fix_buffer(SessionServicesData &osd) {
       osd.buffered_fix_msgs = osd.free_buffers.back();
       osd.free_buffers.pop_back();
    } else {
-      LOG(OrderMan, Warn,
-          "Ran out of fix buffers in {}.  Will allocate directly. "
-          "This will have an impact on performance",
+      WARN_FN(
+          "reload_fix_buffer",
+          "Allocating new fix buffers in {}. This will impact performance",
           osd.cfg.name());
       osd.buffered_fix_msgs = std::make_shared<jetfix::OutFIX>(osd.cfg);
    }

src/lib/orderman/actors/order_manager/OrderServicesData.cpp

@@ -1,10 +1,6 @@
 #include "OrderServicesData.h"
-#include "lib/config/config.h"
-#include "lib/config/protos/jetfix.pb.h"
 #include "lib/md/SecurityOracle.h"
 #include "lib/orderman/actors/order_manager/order_gateway_definition.h"
-#include "lib/orderman/gateways/ExecutionGatewayFactory.h"
-#include "lib/os/spark_data_directory.h"
 #include "lib/tracing/PerfLog.h"
 
 namespace bts {
@@ -26,8 +22,6 @@ OrderServicesData::OrderServicesData(
    for (auto &g : gateways) {
       sessions.push_back(std::make_unique<SessionServicesData>(std::move(g)));
    }
-   // spark::SparkDataDirectory data_dir;
-   // auto binance_j = data_dir.load_config_json("binance_execution");
 }
 
 } // namespace orderman

src/lib/orderman/actors/order_manager/OrderServicesData.h

@@ -34,7 +34,6 @@
 #include "lib/orderman/events/TickerUpdateEvent.h"
 #include "lib/orderman/events/TradeSignalEvent.h"
 #include "lib/orderman/events/UpdateOrderEvent.h"
-#include "lib/orderman/gateways/ExecutionGatewayFactory.h"
 #include "lib/orderman/gateways/IExecutionGateway.h"
 #include "lib/orderman/limiter/Limiter.h"
 #include "lib/orderman/order_factory/OrderFactory.h"

src/lib/orderman/actors/order_manager/session_services_data.h

@@ -10,7 +10,6 @@
 #include "lib/orderman/events/ExchangePositionEvent.h"
 #include "lib/orderman/events/GatewayStatusChangeEvent.h"
 #include "lib/orderman/events/NewExecutionReportEvent.h"
-#include "lib/orderman/gateways/ExecutionGatewayFactory.h"
 #include "lib/orderman/gateways/IExecutionGateway.h"
 #include "lib/utils/PBUtils.h"
 

src/lib/orderman/ccapi/CCAPIDeribitOrderMux.cpp

@@ -20,6 +20,7 @@
 #include "lib/utils/bts_assert.h"
 #include <bits/stdint-uintn.h>
 #include <boost/lockfree/spsc_queue.hpp>
+#include <ccapi_cpp/bts_session_options.h>
 #include <ccapi_cpp/ccapi_element.h>
 #include <ccapi_cpp/ccapi_event.h>
 #include <ccapi_cpp/ccapi_macro.h>
@@ -255,7 +256,8 @@ struct CCAPIDeribitOrderMux::PImpl {
           {CCAPI_DERIBIT_CLIENT_ID, d_conn_config.client_id},
           {CCAPI_DERIBIT_CLIENT_SECRET, d_conn_config.client_secret},
       });
-      session = new ccapi::Session({}, session_configs, &event_handler);
+      session = new ccapi::Session(
+          {}, {ccapi::BtsVenues::DERIBIT}, session_configs, &event_handler);
    }
 };
 
@@ -953,4 +955,4 @@ void CCAPIDeribitOrderMux::_send_buffered_orders(uint32_t sec_id) {
        "n_buffered_orders: {}",
        me->buffered_creates.size());
 }
-} // namespace bts::orderman
+} // namespace bts::orderman

src/lib/orderman/events/NewExecutionReportEvent.h

@@ -20,12 +20,13 @@ class NewExecutionReportEvent : public mercury::Event<NewExecutionReportEvent> {
  public:
    using Tag = GatewayName;
    using SynopsisPolicy = mercury::CacheAll<NewExecutionReportEvent>;
-   ExecutionReport const *data;
+   std::shared_ptr<const ExecutionReport> data;
    trace::CorID trace;
    // used by ContentSubscription
    OrderKey key() const { return data->order_key; }
-   NewExecutionReportEvent(const ExecutionReport *dt, trace::CorID const &trace)
-       : data(dt), trace(trace){};
+   NewExecutionReportEvent(
+       std::shared_ptr<const ExecutionReport> dt, trace::CorID const &trace)
+       : data(std::move(dt)), trace(trace){};
 };
 
 } // namespace orderman

src/lib/orderman/gateways/CCAPIExecutionGateway.cpp

@@ -40,7 +40,7 @@
 #include <bits/stdint-uintn.h>
 //#include <boost/container/small_vector.hpp>
 #include "lib/orderman/ccapi/CCAPIConnectionConfig.h"
-#include <boost/pool/object_pool.hpp>
+
 #include <cmath>
 #include <cstdint>
 #include <optional>
@@ -138,7 +138,6 @@ struct CCAPIExecutionGateway::PImpl {
    OnExchangeOpenOrdersState on_exchange_open_orders;
    // orders are sent out and callbacks received via IEMOrderMuxer
    size_t msg_seq_num = 0;
-   boost::object_pool<ExecutionReport> exe_report_pool;
    std::unordered_map<OrderKey, CreateOrderMsg> order_map = {};
    // this is updated as we get exchange positions
    ExchangePosition exchange_position;
@@ -339,8 +338,7 @@ void CCAPIExecutionGateway::_on_order_cb(const CCAPIOrderState &s) {
    const auto &cr_msg = it->second;
 
    // cant create one on the stack
-   ExecutionReport *er_p = me->exe_report_pool.construct();
-   ExecutionReport &er = *er_p;
+   ExecutionReport er;
    // length of these have to be <= 12
    to_array(cr_msg.routing_record->account(), &er.account_name.value());
    to_array(cr_msg.routing_record->operator_id(), &er.operator_id.value());
@@ -390,7 +388,7 @@ void CCAPIExecutionGateway::_on_order_cb(const CCAPIOrderState &s) {
    LOG(OrderMan, Info,
        "CCAPIExecutionGateway::_process_cb_and_fire_er firing er: {}",
        er.to_string());
-   me->on_exe_report(er, {});
+   me->on_exe_report(std::make_shared<ExecutionReport>(std::move(er)), {});
    // _fire_leg_er(em_cb, er);
 }
 

src/lib/orderman/gateways/ExecutionGatewayFactory.cpp

@@ -1,61 +0,0 @@
-#include "ExecutionGatewayFactory.h"
-#include "lib/domain_model/Security.pb.h"
-#include "lib/md/SecurityOracle.h"
-#include "lib/orderman/gateways/CCAPIExecutionGateway.h"
-// #include "lib/orderman/gateways/EMExecutionGateway.h"
-#include "lib/orderman/gateways/FakeExecutionGateway.h"
-#include "lib/orderman/gateways/GeneralExecutionGateway.h"
-#include "lib/orderman/gateways/GLBXExecutionGateway.h"
-#include "lib/orderman/gateways/GLBXiLink3ExecutionGateway.h"
-#include "lib/orderman/gateways/ICEExecutionGateway.h"
-#include "lib/orderman/gateways/IExecutionGenericAdapter.h"
-#include "lib/orderman/gateways/OkxExecutionAdapter.h"
-#include "lib/orderman/gateways/OnixFIXAdapter.h"
-#include "lib/orderman/gateways/sim/SimExecutionGatewayFactory.h"
-#include "lib/orderman/gateways/sim/SimExecutionGatewayUniqueWrapper.h"
-#include "lib/tracing/PerfLog.h"
-#include "lib/utils/PBUtils.h"
-#include "lib/utils/ValueWrapper.h"
-#include <memory>
-#include <variant>
-
-namespace bts::orderman {
-
-std::unique_ptr<IExecutionGateway> ExecutionGatewayFactory::create_gateway(
-    const config::JetFIX::Connection &conn,
-    trace::PerfLog &pl,
-    md::SecurityOracle o) {
-   LOG(OrderMan, Info, "ExecutionGatewayFactory::create_gateway conn: {}",
-       utils::pb::to_string(conn));
-   if (SimExecutionGatewayFactory::is_sim_execution_on(conn.venue())) {
-      return std::make_unique<SimExecutionGatewayUniqueWrapper>(conn.venue());
-   }
-   LOG(OrderMan, Info, "ExecutionGatewayFactory::create_gateway NOT SIM conn: {}",
-       utils::pb::to_string(conn));
-   switch(conn.venue()){
-      case bts::api::VENUE_ICE:
-      case bts::api::VENUE_SIMICE:
-         return std::make_unique<ICEExecutionGateway>(conn, pl, o);
-      case bts::api::VENUE_GLBX:{
-         if (conn.use_ilink3())
-            return std::make_unique<ilink3::GLBXiLink3ExecutionGateway>( conn, pl, o);
-         return std::make_unique<GLBXExecutionGateway>(conn, pl, o);
-      }
-      case bts::api::VENUE_DERIBIT:
-         return std::make_unique<CCAPIExecutionGateway>(conn.venue(), o);
-      case bts::api::VENUE_CBOE:
-         {
-             std::unique_ptr<IExecutionGenericAdapter> adapter = std::make_unique<OnixFIXAdapter>(o);
-            return std::make_unique<GeneralExecutionGateway>(conn.venue(),std::move(adapter),o);
-         }
-      case bts::api::VENUE_OKX:
-         {
-             std::unique_ptr<IExecutionGenericAdapter> adapter = std::make_unique<OkxExecutionAdapter>(o);
-            return std::make_unique<GeneralExecutionGateway>(conn.venue(),std::move(adapter),o);
-         }
-      default:
-         return std::make_unique<FakeExecutionGateway>(o);
-   }
-}
-
-} // namespace bts::orderman

src/lib/orderman/gateways/ExecutionGatewayFactory.h

@@ -1,25 +0,0 @@
-#pragma once
-#include "lib/config/protos/jetfix.pb.h"
-#include <memory>
-
-namespace bts::trace {
-class PerfLog;
-}
-
-namespace bts::md {
-class SecurityOracle;
-}
-
-namespace bts::orderman {
-
-class IExecutionGateway;
-
-class ExecutionGatewayFactory {
- public:
-   static std::unique_ptr<IExecutionGateway> create_gateway(
-       const config::JetFIX::Connection &,
-       trace::PerfLog &,
-       md::SecurityOracle);
-};
-
-} // namespace bts::orderman

src/lib/orderman/gateways/FakeExecutionGateway.cpp

@@ -25,7 +25,6 @@
 #include "server/sparkd/md/platforms/emumba/IEMOrderMuxer.h"
 
 #include <atomic>
-#include <boost/pool/object_pool.hpp>
 #include <cstdint>
 #include <set>
 #include <string>
@@ -54,7 +53,7 @@ struct FakeExecutionGateway::PImpl {
    md::SecurityOracle sec_oracle;
    std::vector<OrderMsg> buffered_msgs = {};
    unsigned long exchange_oid = 0;
-   std::vector<ExecutionReport> exe_reports = {};
+   std::vector<std::shared_ptr<ExecutionReport>> exe_reports = {};
    IEMOrderMuxer *_order_mux = nullptr;
 };
 
@@ -84,7 +83,9 @@ void FakeExecutionGateway::dispatch_incoming_fix_messages() {
              "FakeExecutionGateway::dispatch_incoming_fix_message key: {} sec: "
              "{} p: {} q: {}",
              cr_msg.key, cr_msg.security->id(), cr_msg.price, cr_msg.quantity);
-         ExecutionReport er = {};
+         me->exe_reports.push_back(std::make_shared<ExecutionReport>());
+         std::shared_ptr<ExecutionReport> &er_p = me->exe_reports.back();
+         ExecutionReport &er = *er_p;
          std::string account_name = "ACCA1";
          std::array<char, 12> an_array = {};
          size_t i = 0;
@@ -115,19 +116,18 @@ void FakeExecutionGateway::dispatch_incoming_fix_messages() {
          er.msg = "Message Placed";
          er.exchange = api::VENUE_SIM;
          trace::CorID cid = {};
-         me->exe_reports.push_back(er);
-         const auto &er_pheap = me->exe_reports.back();
          LOG(OrderMan, Info,
              "FakeExecutionGateway::dispatch_incoming_fix_messages er: {}",
-             er_pheap.to_string());
-         me->on_exe_report(er_pheap, cid);
+             er.to_string());
+         me->on_exe_report(er_p, cid);
       } else if (std::holds_alternative<CancelOrderMsg>(m)) {
          CancelOrderMsg cxl_m = std::get<CancelOrderMsg>(m);
-         for (auto &er : me->exe_reports) {
+         for (std::shared_ptr<ExecutionReport> &er_p : me->exe_reports) {
+            auto &er = *er_p;
             if (er.order_key == cxl_m.key) {
                er.order_status = api::orderman::NATIVE_ORDER_STATUS_CANCELED;
                er.msg = "Cancelled";
-               me->on_exe_report(er, {});
+               me->on_exe_report(er_p, {});
                LOG(OrderMan, Info,
                    "FakeExecutionGateway::dispatch_incoming_fix_messages er: "
                    "{}",
@@ -138,13 +138,14 @@ void FakeExecutionGateway::dispatch_incoming_fix_messages() {
       } else if (std::holds_alternative<CancelReplaceOrderMsg>(m)) {
          CancelReplaceOrderMsg cxl_r_msg = std::get<CancelReplaceOrderMsg>(m);
          CreateOrderMsg cr_msg = cxl_r_msg.create_fields;
-         for (auto &er : me->exe_reports) {
+         for (std::shared_ptr<ExecutionReport> &er_p : me->exe_reports) {
+            auto &er = *er_p;
             if (er.order_key == cr_msg.key) {
                er.order_status = api::orderman::NATIVE_ORDER_STATUS_REPLACED;
                er.msg = "Replaced";
                er.price = cr_msg.price;
                er.quantity = cr_msg.quantity;
-               me->on_exe_report(er, {});
+               me->on_exe_report(er_p, {});
                return;
             }
          }
@@ -196,4 +197,4 @@ OnExchangeOpenOrdersState &FakeExecutionGateway::on_exchange_open_orders() {
 }
 
 } // namespace orderman
-} // namespace bts
+} // namespace bts

src/lib/orderman/gateways/GLBXExecutionGateway.cpp

@@ -13,8 +13,6 @@
 #include "lib/utils/TimeStamp.h"
 #include "lib/utils/bts_chrono.h"
 
-#include <boost/pool/object_pool.hpp>
-
 namespace bts {
 namespace orderman {
 
@@ -31,12 +29,11 @@ struct GLBXExecutionGateway::PImpl {
        config::JetFIX::Connection const &cfg,
        trace::PerfLog &l,
        md::SecurityOracle o)
-       : gw(cfg, l), exe_report_pool(), state(State::WAITING_FOR_SECURITIES),
+       : gw(cfg, l), state(State::WAITING_FOR_SECURITIES),
          securities(), securities_by_descriptor(), gw_name(cfg.name()),
          cfg(cfg), perf_log(l), sec_oracle(o) {}
 
    JFGw gw;
-   boost::object_pool<ExecutionReport> exe_report_pool;
    State state;
    std::unordered_map<SecurityKey::Value, api::Security const *> securities;
    std::unordered_map<std::string, api::Security const *>
@@ -209,8 +206,8 @@ void GLBXExecutionGateway::on_exe_report(
       if (msg.has<fix::Text>())
          text_msg = msg.get<fix::Text>();
 
-      // allocate from our pool
-      auto data = me->exe_report_pool.construct();
+      ExecutionReport er;
+      ExecutionReport *data = &er;
 
       data->exec_type = api::orderman::ExecType(msg.get<fix::ExecType>());
       data->msg_seq_num = msg.get<fix::MsgSeqNum>();
@@ -290,7 +287,8 @@ void GLBXExecutionGateway::on_exe_report(
       data->cfg = &me->cfg;
       data->msg = text_msg;
       data->exchange = api::Venue::VENUE_GLBX;
-      me->on_exe_report(*data, msg.trace());
+      me->on_exe_report(
+          std::make_shared<ExecutionReport>(std::move(er)), msg.trace());
    }
 }
 

src/lib/orderman/gateways/GLBXiLink3ExecutionGateway.cpp

@@ -12,6 +12,7 @@
 #include "lib/orderman/ilink3/IL3Reactor.h"
 #include "lib/orderman/ilink3/IL3SecDB.h"
 #include "lib/orderman/ilink3/SessionStore.h"
+#include "lib/orderman/orderman_utils/ExecutionReport.h"
 #include "lib/orderman/orderman_utils/fix_order_codec.h"
 #include "lib/utils/PBUtils.h"
 #include <cstddef>
@@ -149,8 +150,11 @@ void GLBXiLink3ExecutionGateway::connect(jetfix::SessionName const &sn) {
    if (!me->has_replayed) {
       me->has_replayed = true;
       me->replay_messages = me->session_store.value().replay();
-      for (auto const &er : me->replay_messages)
-         me->on_exe_report(er, {});
+      for (ExecutionReport &er : me->replay_messages) {
+         me->on_exe_report(
+             std::make_shared<ExecutionReport>(std::move(er)), {});
+      }
+      me->replay_messages.clear();
    }
 
    c.raw_message_logger =
@@ -169,7 +173,7 @@ void GLBXiLink3ExecutionGateway::connect(jetfix::SessionName const &sn) {
 
    me->gw->subscribe_execution_reports([this](auto er) {
       bts_assert(me->state == State::READY || me->state == State::CONNECTING);
-      me->on_exe_report(*er, {});
+      me->on_exe_report(std::move(er), {});
    });
 
    me->gw->subscribe_session(

src/lib/orderman/gateways/GeneralExecutionGateway.cpp

@@ -64,7 +64,9 @@ void GeneralExecutionGateway::send_msgs(
           msg);
    }
 }
-void GeneralExecutionGateway::dispatch_incoming_fix_messages() {}
+void GeneralExecutionGateway::dispatch_incoming_fix_messages() {
+   me->_adapter->dispatch_incoming_fix_messages();
+}
 void GeneralExecutionGateway::on_securities(
     const mercury::Update<NewSecuritiesEvent> &) {}
 void GeneralExecutionGateway::on_admin_data(

src/lib/orderman/gateways/ICEExecutionGateway.cpp

@@ -18,7 +18,6 @@
 #include "lib/utils/helper_macros.h"
 
 #include <atomic>
-#include <boost/pool/object_pool.hpp>
 #include <set>
 
 namespace bts {
@@ -39,13 +38,12 @@ struct ICEExecutionGateway::PImpl {
        const config::JetFIX::Connection &cfg,
        trace::PerfLog &l,
        md::SecurityOracle o)
-       : gw(cfg, l), exe_report_pool(), state(State::WAITING_FOR_SECURITIES),
-         securities(), securities_by_exchange_id(), gw_name(cfg.name()),
-         cfg(cfg), perf_log(l), admin_data(), nusers(-1), connection_time(),
+       : gw(cfg, l), state(State::WAITING_FOR_SECURITIES), securities(),
+         securities_by_exchange_id(), gw_name(cfg.name()), cfg(cfg),
+         perf_log(l), admin_data(), nusers(-1), connection_time(),
          exec_id_to_memo(), sec_oracle(o) {}
 
    JFGw gw;
-   boost::object_pool<ExecutionReport> exe_report_pool;
    State state;
    std::unordered_map<SecurityKey::Value, api::Security const *> securities;
    std::unordered_map<int64_t, api::Security const *> securities_by_exchange_id;
@@ -373,7 +371,8 @@ void ICEExecutionGateway::on_exe_report(
                      std::to_string(msg.get<fix::DeleteReason>());
 
       // allocate from our pool
-      auto data = me->exe_report_pool.construct();
+      ExecutionReport er;
+      ExecutionReport *data = &er;
 
       if (msg.has<fix::Memo>())
          data->memo = msg.get<fix::Memo>();
@@ -467,7 +466,8 @@ void ICEExecutionGateway::on_exe_report(
       data->cfg = &me->cfg;
       data->msg = text_msg;
       data->exchange = api::Venue::VENUE_ICE;
-      me->on_exe_report(*data, msg.trace());
+      me->on_exe_report(
+          std::make_shared<ExecutionReport>(std::move(er)), msg.trace());
    }
 }
 

src/lib/orderman/gateways/IExecutionGateway.h

@@ -28,8 +28,8 @@ class ExecutionReport;
 class CancelReject;
 
 using OnGwStatusChange = std::function<void(bool)>;
-using OnExeReport =
-    std::function<void(ExecutionReport const &, trace::CorID const &)>;
+using OnExeReport = std::function<void(
+    std::shared_ptr<const ExecutionReport>, trace::CorID const &)>;
 using OnCancelReject = std::function<void(CancelReject const &)>;
 using OnExchangePositionState = std::function<void(ExchangePosition const &)>;
 using OnExchangeOpenOrdersState =

src/lib/orderman/gateways/IExecutionGenericAdapter.h

@@ -20,9 +20,10 @@ class NewAdminDataEvent;
 class ExecutionReport;
 class CancelReject;
 
-using OnExecutionReport = std::function<void(ExecutionReport const &, const trace::CorID &)>;
+using OnExecutionReport = std::function<void(
+    std::shared_ptr<const ExecutionReport>, const trace::CorID &)>;
 
-// fix execution gateway for a single FIX session
+//  execution gateway for a single session
 class IExecutionGenericAdapter {
  public:
    IExecutionGenericAdapter() = default;
@@ -34,6 +35,7 @@ class IExecutionGenericAdapter {
    virtual void launch_new_order(const CreateOrderMsg &) =0;
    virtual void modify_order(const CancelReplaceOrderMsg &) = 0;
    virtual void cancel_order(const CancelOrderMsg &) =0;
+   virtual void dispatch_incoming_fix_messages() = 0;
    // as new securities are added
    // virtual void update_securities(
    //     const mercury::Update<NewSecuritiesEvent> &) = 0;

src/lib/orderman/gateways/OkxExecutionAdapter.cpp

@@ -1,95 +0,0 @@
-#include "OkxExecutionAdapter.h"
-#include "OnixS/FIXEngine/FIX/Engine.h"
-#include "lib/config/protos/marex_fix_cfg.pb.h"
-#include "lib/domain_model/Security.pb.h"
-#include "lib/logging/logging.h"
-#include "lib/orderman/CancelOrderMsg.h"
-#include "lib/orderman/CancelReplaceOrderMsg.h"
-#include "lib/orderman/CreateOrderMsg.h"
-#include "lib/orderman/gateways/OnixFIXListener.h"
-#include "lib/orderman/gateways/TrafixOrderStateHandler.h"
-#include "lib/orderman/orderman_utils/orderman_common.h"
-#include "lib/utils/bts_assert.h"
-#include "lib/utils/helper_macros.h"
-#include <OnixS/FIXEngine/FIX/Exception.h>
-#include <OnixS/FIXEngine/FIX/FIX42.h>
-#include <OnixS/FIXEngine/FIX/Message.h>
-#include <OnixS/FIXEngine/FIX/Numeric.h>
-#include <OnixS/FIXEngine/FIX/ProtocolVersion.h>
-#include <cassert>
-#include <chrono>
-#include <exception>
-#include <memory>
-#include <quickfix/FixFields.h>
-#include <ratio>
-#include <string>
-#include <unistd.h>
-
-namespace bts::orderman {
-
-OkxExecutionAdapter::OkxExecutionAdapter(md::SecurityOracle){}//listener() {}
-OkxExecutionAdapter::~OkxExecutionAdapter() {}
-inline void handleOkException(
-    const std::string &context, const std::string &error) {
-   bts_assert_msgf(false, "{} Exception: {}", context, error);
-}
-void OkxExecutionAdapter::start() {
-    int attempt = 0;
-
-    while (true) {
-        try {
-            LOG(OrderMan, Warn, "OkxExecutionAdapter::start Attempting login (Attempt: {})", attempt + 1);
-            LOG(OrderMan, Warn, "OkxExecutionAdapter::start Login successful");
-            break; // Exit loop on success
-        } catch (const std::exception &ex) {
-            const std::string err = ex.what();
-            LOG(OrderMan, Error, "OkxExecutionAdapter::start Login failed: {}. Retrying in {} seconds.", err, 10);
-
-            // if (maxRetries > 0 && ++attempt >= maxRetries) {
-            //     LOG(OrderMan, Error, "OkxExecutionAdapter::start Exceeded max retries ({}). Aborting.", maxRetries);
-            //     handleOkException("OkxExecutionAdapter::start", err);
-            //     break; // Exit if retries are exhausted
-            // }
-            // std::this_thread::sleep_for(std::chrono::seconds(std::max(60,retryInterval)));
-        }
-    }
-}
-
-void OkxExecutionAdapter::modify_order(
-    const CancelReplaceOrderMsg &msg) { //confirm will have everything
-   try {
-      LOG(OrderMan, Warn, "OkxExecutionAdapter::modify_order modifying order{}",
-          msg);
-   } catch (const std::exception &ex) {
-     const std::string err = ex.what();
-      handleOkException("Modify order", err);
-   }
-}
-
-void OkxExecutionAdapter::launch_new_order(const CreateOrderMsg &msg) {
-   LOG(OrderMan, Warn, "launch order {}", msg);
-   try {
-      LOG(OrderMan, Warn, "OkxExecutionAdapter::launch_new_order creating order");
-   }
-   // Shut down all reactors if they exist.
-   catch (const std::exception &ex) {
-    const std::string err = ex.what();
-      handleOkException("Launch order", err);
-   }
-}
-void OkxExecutionAdapter::register_callback([[maybe_unused]]const OnExecutionReport &cb) {
-   LOG(OrderMan, Warn, "OkxExecutionAdapter::register_callback callback registered");
-   // listener.setCallback(cb);
-}
-void OkxExecutionAdapter::cancel_order(const CancelOrderMsg &msg) {
-   LOG(OrderMan, Warn, "cancel order {}", msg);
-   try {
-      LOG(OrderMan, Warn, "order cancel sent");
-   }
-   // Shut down all reactors if they exist.
-   catch (const std::exception &ex) {
-    const std::string err = ex.what();
-      handleOkException("Cacnel order", err);
-   }
-}
-} // namespace bts::orderman

src/lib/orderman/gateways/OkxExecutionAdapter.h

@@ -1,19 +0,0 @@
-#pragma once
-
-#include "IExecutionGenericAdapter.h"
-#include "lib/md/SecurityOracle.h"
-namespace bts::orderman {
-
-class OkxExecutionAdapter : public IExecutionGenericAdapter {
- public:
-   OkxExecutionAdapter(md::SecurityOracle sec_oracle);
-   ~OkxExecutionAdapter() override;
-   void register_callback(const OnExecutionReport &report) override;
-   void launch_new_order(const CreateOrderMsg &msg) override;
-   void cancel_order(const CancelOrderMsg &msg) override;
-   void modify_order(const CancelReplaceOrderMsg &) override;
-   void start() override;
- private:
-};
-
-} // namespace bts::orderman

src/lib/orderman/gateways/OkxExecutionGateway.cpp

@@ -0,0 +1,755 @@
+#include "OkxExecutionGateway.h"
+#include "lib/api/services/OrderService.pb.h"
+#include "lib/config/protos/okx_exe_cfg.pb.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/domain_model/common.h"
+#include "lib/io/http/http_client.h"
+#include "lib/io/http/http_response.h"
+#include "lib/io/websocket/websocket_client.h"
+#include "lib/io/websocket/websocket_client_config.h"
+#include "lib/logging/logging.h"
+#include "lib/md/SecurityOracle.h"
+#include "lib/orderman/CancelOrderMsg.h"
+#include "lib/orderman/CancelReplaceOrderMsg.h"
+#include "lib/orderman/CreateOrderMsg.h"
+#include "lib/orderman/gateways/IExecutionGateway.h"
+#include "lib/orderman/orderman_utils/ExecutionReport.h"
+#include "lib/orderman/orderman_utils/OkxUtils.h"
+#include "lib/orderman/orderman_utils/orderman_common.h"
+#include "lib/utils/Uuid.h"
+#include "lib/utils/bts_assert.h"
+#include "lib/utils/thread_manager.h"
+#include <OnixS/FIXEngine/FIX/Exception.h>
+#include <OnixS/FIXEngine/FIX/FIX42.h>
+#include <OnixS/FIXEngine/FIX/Message.h>
+#include <OnixS/FIXEngine/FIX/Numeric.h>
+#include <OnixS/FIXEngine/FIX/ProtocolVersion.h>
+#include <atomic>
+#include <cassert>
+#include <chrono>
+#include <cstddef>
+#include <ctime>
+#include <exception>
+#include <fmt/core.h>
+#include <map>
+#include <memory>
+#include <openssl/buffer.h>
+#include <openssl/evp.h>
+#include <openssl/hmac.h>
+#include <quickfix/FixFields.h>
+#include <quickfix/Utility.h>
+#include <string>
+#include <unistd.h>
+#include <unordered_map>
+// Example namespace alias for nlohmann::json
+using json = nlohmann::json;
+
+namespace bts::orderman {
+
+OkxExecutionGateway::OkxExecutionGateway(md::SecurityOracle o)
+    : me_(std::make_unique<PImpl>(o)) {
+   LOG(OrderMan, Info, "OkxExecutionAdapter::constructor ");
+} //listener() {}
+OkxExecutionGateway::~OkxExecutionGateway() {}
+
+enum class OkxExecutionState { OFF, STARTED, LOGGED_IN, SUBSCIRBED };
+void OkxExecutionGateway::send_msgs(
+    std::vector<OrderMsg> const &msgs, trace::CorID) {
+   LOG(OrderMan,Info,"ExecutionGateway::send_msgs" );
+   for (const auto &msg : msgs) {
+      std::visit(
+          [this](auto &&order) {
+             using T = std::decay_t<decltype(order)>;
+             if constexpr (std::is_same_v<T, CreateOrderMsg>) {
+                LOG(OrderMan,Warn,"ExecutionGateway::CreateOrderMsg" );
+                launch_new_order(order);
+             
+             } else if constexpr (std::is_same_v<T, CancelOrderMsg>) {
+                LOG(OrderMan,Warn,"ExecutionGateway::CancelOrderMsg" );
+                cancel_order(order);
+             } else if constexpr (std::is_same_v<T, CancelReplaceOrderMsg>) {
+                LOG(OrderMan,Warn,"ExecutionGateway::CancelReplaceOrderMsg" );
+                modify_order(order);
+             } else if constexpr (std::is_same_v<T, RFQMsg>) {
+                 // Handle RFQMsg if necessary
+                 // adapter->handle_rfq(order); // Example placeholder
+             }
+          },
+          msg);
+   }
+}
+
+static websocket::WebsocketClientConfig get_ws_client_config(
+    const std::string &url,
+    core::TimeSpan connect_timeout,
+    core::TimeSpan pong_timeout) {
+   websocket::WebsocketClientConfig rv{url, connect_timeout, pong_timeout};
+   rv.name = "OkxExecutionGateway";
+   return rv;
+}
+
+struct OkxExecutionGateway::PImpl {
+   const std::string api_key = config::get<config::OkxExeCfg>().api_key();
+   const std::string secret_key = config::get<config::OkxExeCfg>().secret_key();
+   const std::string passphrase = config::get<config::OkxExeCfg>().passphrase();
+   const std::string url =
+       config::get<config::OkxExeCfg>().ws_url() + "/ws/v5/private";
+   const bool simulated_trading = config::get<config::OkxExeCfg>().simulated_trading();
+   const std::string tdmode = config::get<config::OkxExeCfg>().tdmode();
+   const core::TimeSpan connect_timeout = core::TimeSpan::from_seconds(0);
+   const core::TimeSpan pong_timeout = core::TimeSpan::from_seconds(
+       config::get<config::OkxExeCfg>().ws_ping_interval_seconds());
+   websocket::WebsocketClient ws_client;
+   tbb::concurrent_queue<std::shared_ptr<ExecutionReport>> buffered_exec_reps;
+   std::optional<std::thread> ws_thread;
+   OnExeReport cb;
+   OnGwStatusChange on_gw_status_change;
+   OnCancelReject on_cancel_reject;
+   OnExchangePositionState on_exchange_position_state;
+   OnExchangeOpenOrdersState on_exchange_open_orders;
+   md::SecurityOracle o;
+   std::atomic<OkxExecutionState> state = OkxExecutionState::OFF;
+   PImpl(md::SecurityOracle o)
+       : ws_client(websocket::WebsocketClient(
+             get_ws_client_config(url, connect_timeout, pong_timeout))),
+         cb(dummy), o(o) {
+      ws_client.message_handler().bind<&PImpl::on_message>(*this);
+      ws_client.close_handler().bind<&PImpl::on_close>(*this);
+   };
+
+   ~PImpl() {
+      state = OkxExecutionState::OFF;
+      if (ws_run_thread) {
+         ws_run_thread->join();
+      }
+   }
+
+   static void dummy(std::shared_ptr<const ExecutionReport>, const trace::CorID &) {
+      LOG(OrderMan, Error, "OnixFixListener::Dummy fired");
+   }
+   void setCallback(const OnExeReport &callback) {
+      LOG(OrderMan, Info,
+          "OkxExecutionAdapter::setCallback callback registered");
+      cb = callback;
+   }
+
+   std::optional<std::thread> ws_run_thread;
+   std::optional<std::thread> ws_ping_thread;
+
+   void get_exe_report(
+       const nlohmann::basic_json<> &order, ExecutionReport &report) {
+      report.exchange = api::VENUE_OKX;
+      std::string account_name = config::get<config::OkxExeCfg>().account();
+      std::string ops_id = config::get<config::OkxExeCfg>().ops_id();
+
+      std::array<char, 12> account_array = {};
+      std::copy_n(
+          account_name.begin(),
+          std::min(account_name.size(), account_array.size()),
+          account_array.begin());
+      report.account_name = AccountName(account_array);
+
+      to_array(ops_id, &report.operator_id.value());
+
+      report.order_key =
+          OrderKey(std::stoi(order.at("clOrdId").get<std::string>()));
+
+      auto insttype =
+          OkxUtils::getInstType(order.at("instType").get<std::string>());
+      auto venue = api::VENUE_OKX;
+      auto inst_id = order.at("instId").get<std::string>();
+
+      // report.msg_seq_num = /* Generate or map message sequence */;
+      bts_assert(o.get_securities(venue, insttype, inst_id).size() == 1);
+
+      report.security = o.get_securities(venue, insttype, inst_id)[0];
+
+      report.side = (order.at("side").get<std::string>() == "buy")
+                        ? api::Side::SIDE_BUY
+                        : api::Side::SIDE_SELL;
+      report.order_type =
+          OkxUtils::getOrdType(order.at("ordType").get<std::string>());
+      OkxUtils::populateExecTypeAndOrderStatus(
+          report, order.at("state").get<std::string>());
+
+      report.exchange_order_id = ExchangeOrderID(
+          OkxUtils::stringToFixedArray(order.at("ordId").get<std::string>()));
+
+      report.price = Price(std::stod(order.at("px").get<std::string>()));
+
+      report.quantity =
+          Quantity::create_new(std::stod(order.at("sz").get<std::string>()));
+      report.filled_quantity = Quantity::create_new(
+          std::stod(order.at("accFillSz").get<std::string>()));
+      report.last_price =
+          Price(std::stod(order.at("lastPx").get<std::string>()));
+      report.avg_price = Price(std::stod(order.at("avgPx").get<std::string>()));
+      report.last_shares = Quantity::create_new(
+        std::stod(order.at("fillSz").get<std::string>()));
+
+      //there are algos, like spreader
+      // that are super sensitive to messages and cancel themselves. To avoid
+      // this we only pass on the error messages when the order has terminated.
+      // The assumption being that if the order has not terminated it is OK to
+      // ignore the message.
+      // Alternatively we can change the spreader to not do that. Right not this
+      // seems to be the better choice
+      if (!(order.at("code").get<std::string>() == "0")) {
+        report.msg = order.at("msg").get<std::string>();
+     }
+
+
+     //report can be new order, cancel order, order filled, replace order, and partial fill or errror
+     //new order, cancelled and order filled dont need special handling, error gets handled above 
+     //for other three we will look for fields that are only specific to the respectives types
+    //   if (!(order.at("cancelSource").get<std::string>() == "")) {
+
+    //   } 
+      if (!(order.at("amendSource").get<std::string>() == "")) {
+        if((order.at("amendResult").get<std::string>() == "0"))
+            report.exec_type = api::orderman::ExecType::EXEC_TYPE_REPLACED;
+        else{
+            LOG(OrderMan, Warn, "order:{} replace rejected", report.order_key);
+            report.msg = order.at("msg").get<std::string>();
+        }
+      } else if (!(order.at("fillSz").get<std::string>() == "0")) {
+        utils::Uuid guid;
+        std::string fill_id = guid.to_text_string();
+        to_array(fill_id, &report.exchange_fill_id.value());
+      }
+      
+      // report.transact_time = ExchangeTimeStampVar(std::stoll(order.at("uTime").get<std::string>()));
+      ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now());
+      report.transact_time = now;
+      std::string er = report.to_string();
+      LOG(OrderMan, Info, "Created ExecutionReport: {}", er);
+   }
+   void on_close(const std::error_code &ec) {
+      bts_assert_msgf(
+          false, "Websocket stream client disconnected. {}", ec.message());
+   }
+    // fills execution report form basic_json and writes it to
+    // buffered_exec_reps, which is picked up from the other thread
+    // and passed over to the order infrastructure
+    void _push_execution_report(const nlohmann::basic_json<> &order) {
+        ExecutionReport er;
+        get_exe_report(order, er);
+        buffered_exec_reps.push(std::make_shared<ExecutionReport>(std::move(er)));
+    }
+   void on_message(const std::string &msg_str) {
+      LOG(OrderMan, Info, "okxlistener::on_message. {}", msg_str);
+      nlohmann::json msg;
+      try {
+         msg = nlohmann::json::parse(msg_str);
+      } catch (const std::exception &ex) {
+         std::string s = ex.what();
+         LOG(OrderMan, Error, "Failed to parse json. {}\n{}", s, msg_str);
+         return;
+      }
+      const auto event_type_it = msg.find("event");
+      if (event_type_it != msg.end()) {
+         const auto event_type = event_type_it->get<std::string>();
+         if (event_type == "login") {
+            bts_assert(state == OkxExecutionState::STARTED);
+            const auto code_it = msg.find("code");
+            const auto code = code_it->get<std::string>();
+            if (code == "0") {
+               LOG(OrderMan, Info, "Succefully logged in {}", msg_str);
+               state = OkxExecutionState::LOGGED_IN;
+            }
+
+            return;
+         }
+         if (event_type == "subscribe") {
+            bts_assert(state == OkxExecutionState::LOGGED_IN);
+            state = OkxExecutionState::SUBSCIRBED;
+            return;
+         }
+         if (event_type == "channel-conn-count") { //ignore for now
+            return;
+         }
+      }
+      // Handle order updates
+
+      const auto channel_it = msg.find("arg");
+      if (channel_it == msg.end() ||
+          channel_it->at("channel").get<std::string>() != "orders") {
+         LOG(OrderMan, Error, "Unexpected response id:\n{}", msg_str);
+         return;
+      }
+
+      const auto &data = msg.at("data");
+
+      for (const nlohmann::basic_json<> &order : data) {
+        _push_execution_report(order);
+      }
+   }
+   void start() {
+      bts_assert(!ws_thread);
+      state = OkxExecutionState::STARTED;
+      ws_thread.emplace([this]() {
+         bts::threading::configure_current_thread("okx_execution");
+         start_impl();
+      });
+   }
+
+   void start_impl() {
+      size_t attempt = 0;
+      while (state != OkxExecutionState::OFF) {
+         try {
+            LOG(OrderMan, Warn,
+                "OkxExecutionAdapter::start Attempting login (Attempt: {})",
+                attempt + 1);
+            if (ws_run_thread) {
+               // Join this thread to block until the websocket disconnects (then reconnect)
+               if (ws_run_thread->joinable()) {
+                  ws_run_thread->join();
+               }
+               ws_run_thread.reset();
+            }
+            // Connect
+            ws_client.connect();
+            ws_run_thread = threading::make_thread(
+                "okx_execution_ws", [&]() { ws_client.run(); });
+            // Block until the websocket connects (or times out)
+            {
+               const auto start_time = bts::core::TimeStamp::now();
+               for (auto elapsed = core::TimeSpan::zero();
+                    !ws_client.is_connected() &&
+                    elapsed < core::TimeSpan::from_seconds(20);
+                    elapsed = core::TimeStamp::now() - start_time) {
+                  std::this_thread::sleep_for(std::chrono::milliseconds(500));
+               }
+               if (!ws_client.is_connected()) {
+                  LOG(OrderMan, Warn,
+                      "OkxExecutionAdapter::start. Timed out waiting for "
+                      "websocket to connect");
+                  continue;
+               }
+            }
+            if (ws_ping_thread.has_value()) {
+               if (ws_ping_thread->joinable()) {
+                  ws_ping_thread
+                      ->join(); // Ensure the previous thread exits cleanly
+               }
+               ws_ping_thread
+                   .reset(); // Reset the optional before creating a new thread
+            }
+
+            ws_ping_thread.emplace([&]() {
+               LOG(OrderMan, Warn,
+                   "OkxExecutionAdapter::ws_ping_thread thread starting. url: "
+                   "{}",
+                   url);
+               auto last_ping_time = core::TimeStamp::now();
+               auto ping_interval = core::TimeSpan::from_seconds(
+                   config::get<config::OkxExeCfg>().ws_ping_interval_seconds());
+               std::this_thread::sleep_for(
+                   std::chrono::milliseconds(ping_interval.millis()));
+
+               while (ws_client.is_connected()) {
+                  const auto now = bts::core::TimeStamp::now();
+                  if ((now - last_ping_time) > ping_interval) {
+                     LOG(OrderMan, Warn,
+                         "OkxExecutionAdapter::ws_ping_thread Pinging "
+                         "websocket. url: {}",
+                         url);
+                     ws_client.ping();
+                     last_ping_time = now;
+                  }
+                  std::this_thread::sleep_for(
+                      std::chrono::seconds(ping_interval.seconds() / 10));
+               }
+               LOG(OrderMan, Warn,
+                   "OkxExecutionAdapter::ws_ping_thread thread exiting. url: "
+                   "{}",
+                   url);
+            });
+
+            auto [timestamp, sign] = OkxUtils::getLoginSignTime(secret_key);
+            static constexpr const char *AUTH_REQ_TEMPLATE =
+                R"({{"op": "login", "args": [{{ "apiKey": "{}", "passphrase": "{}", "timestamp": "{}", "sign": "{}" }}] }})";
+
+            const std::string auth_req = fmt::format(
+                AUTH_REQ_TEMPLATE, api_key, passphrase, timestamp, sign);
+            LOG(OrderMan, Warn,
+                "OkxExecutionAdapter::start Attempting login (req: {})",
+                auth_req);
+            const auto auth_res = ws_client.send_text(auth_req);
+            LOG(OrderMan, Warn,
+                "OkxExecutionAdapter::start Attempting login {}, {}, {}",
+                std::to_string(auth_res.value()),
+                std::string(auth_res.category().name()), auth_res.message());
+            if (auth_res.value()) {
+               LOG(md, Info, "okx authentication failed. {}",
+                   auth_res.message());
+               attempt++;
+               throw std::runtime_error(
+                   "okx authentication failed: " + auth_res.message());
+            }
+            while (state == OkxExecutionState::STARTED) {
+               LOG(OrderMan, Warn,
+                   "OkxExecutionAdapter::start waiting to login");
+               std::this_thread::sleep_for(1s);
+            }
+            LOG(OrderMan, Warn,
+                "OkxExecutionAdapter::start Login successful {}",
+                auth_res.message());
+            std::string subscribe_req = R"({
+                  "op": "subscribe",
+                  "args": [{"channel": "orders", "instType": "ANY"}]
+               })";
+            LOG(OrderMan, Warn,
+                "OkxExecutionAdapter::start AGttempting sub (req: {})",
+                subscribe_req);
+            const auto auth_res2 = ws_client.send_text(subscribe_req);
+            LOG(OrderMan, Warn, "OkxExecutionAdapter::subscribing to orders{}",
+                auth_res2.message());
+            while (state != OkxExecutionState::SUBSCIRBED) {
+               LOG(OrderMan, Warn,
+                   "OkxExecutionAdapter::start waiting to subscribe");
+
+               std::this_thread::sleep_for(1s);
+            }
+            LOG(OrderMan, Warn,
+                "OkxExecutionAdapter::subscribing to orders successful{}",
+                auth_res2.message());
+            on_gw_status_change(true);
+            break; // Exit loop on success
+         } catch (const std::exception &ex) {
+            const std::string err = ex.what();
+            LOG(OrderMan, Error,
+                "OkxExecutionAdapter::start Login failed: {}. Retrying in {} "
+                "seconds.",
+                err,
+                config::get<config::OkxExeCfg>()
+                    .ws_reconnect_interval_seconds());
+
+            size_t maxRetries =
+                config::get<config::OkxExeCfg>().ws_reconnect_max_count();
+            if (maxRetries > 0 && ++attempt >= maxRetries) {
+               LOG(OrderMan, Error,
+                   "OkxExecutionAdapter::start Exceeded max retries ({}). "
+                   "Aborting.",
+                   maxRetries);
+               OkxUtils::handleOkException("OkxExecutionAdapter::start", err);
+               break; // Exit if retries are exhausted
+            }
+            std::this_thread::sleep_for(
+                std::chrono::seconds(config::get<config::OkxExeCfg>()
+                                         .ws_reconnect_interval_seconds()));
+         }
+      }
+   }
+};
+void OkxExecutionGateway::start() { me_->start(); }
+OnGwStatusChange &OkxExecutionGateway::on_gw_status_change() {
+    return me_->on_gw_status_change;
+ }
+void OkxExecutionGateway::modify_order(
+    const CancelReplaceOrderMsg &msg) { //confirm will have everything
+   try {
+      LOG(OrderMan, Warn, "OkxExecutionAdapter::modify_order modifying order{}",
+          msg);
+      http::HttpResponse res;
+      // static constexpr const char *body_template = R"({"instId": "BTC-USDT", "tdMode": "cash", "side": "buy", "ordType": "limit", "px": "1000", "sz": "0.01"})";
+      static constexpr const char *body_template =
+          R"({{"instId": "{}",  "clordId": "{}", "newPx": "{}", "newSz": "{}"}})";
+      const std::string REQUEST_PATH = "/api/v5/trade/amend-order";
+      const std::string body = fmt::format(
+          body_template, msg.create_fields.security->okx().security_desc(),
+          msg.create_fields.key, msg.create_fields.price,
+          msg.create_fields.quantity.full_value());
+      auto headers = OkxUtils::createHeaders(
+          me_->api_key, me_->passphrase, me_->secret_key, "POST", REQUEST_PATH,
+          me_->simulated_trading,body);
+      // Generate and log the curl command
+      std::string curlCommand = OkxUtils::generateCurlCommand(
+          config::get<config::OkxExeCfg>().rest_url() + REQUEST_PATH, headers,
+          body);
+      LOG(OrderMan, Warn, "Curl Command: \n{}", curlCommand);
+      http::HttpClient::post(
+          config::get<config::OkxExeCfg>().rest_url() + REQUEST_PATH, body,
+          headers, &res);
+      const auto &[suc, top_msg] = OkxUtils::orderAccepted(res.body);
+
+      if (suc) {
+         LOG(md, Info, "Http get succeeded {}", res.body);
+      } else {
+        LOG(md, Info, "Http get failed code: {}, errors: {}, body: {}", res.code, res.errors, res.body);
+      }
+   } catch (const std::exception &ex) {
+      const std::string err = ex.what();
+      OkxUtils::handleOkException("Amend order", err);
+   }
+}
+void OkxExecutionGateway::dispatch_incoming_fix_messages() {
+    if (me_->buffered_exec_reps.empty()) {
+        return;
+     }
+     std::shared_ptr<ExecutionReport> er;
+     while (me_->buffered_exec_reps.try_pop(er)) {
+        trace::CorID cid = {};
+        LOG(OrderMan, Info,
+            "OkxExecutionGateway::dispatch_incoming_fix_messages er: {} ",
+            er->to_string());
+        me_->cb(std::move(er), cid);
+     }
+}
+OnExeReport &OkxExecutionGateway::on_exe_report() { return me_->cb; }
+OnCancelReject &OkxExecutionGateway::on_cancel_reject() {
+   static OnCancelReject default_cancel_reject;
+   return default_cancel_reject;
+}
+OnExchangePositionState &OkxExecutionGateway::on_exchange_position_state() {
+   static OnExchangePositionState default_exchange_position_state;
+   return default_exchange_position_state;
+}
+OnExchangeOpenOrdersState &OkxExecutionGateway::on_exchange_open_orders() {
+   static OnExchangeOpenOrdersState default_exchange_open_orders;
+   return default_exchange_open_orders;
+}
+void OkxExecutionGateway::launch_new_order(const CreateOrderMsg &msg) {
+   LOG(OrderMan, Warn, "launch order {}", msg);
+   try {
+      LOG(OrderMan, Warn,
+          "OkxExecutionAdapter::launch_new_order creating order");
+      static constexpr const char *body_template =
+          R"({{"instId": "{}", "tdMode": "{}", "side": "{}", "ordType": "{}", "px": "{}", "sz": "{}", "clordId": "{}"}})";
+      const std::string REQUEST_PATH = "/api/v5/trade/order";
+      const std::string side = msg.side == api::Side::SIDE_BUY ? "buy" : "sell";
+      const std::string quantity = std::to_string(msg.quantity.full_value());
+      const std::string price = std::to_string(msg.price.value());
+      std::string suffix = "ORDER_TYPE_";
+      auto ord_full_name = NativeOrderType_Name(msg.order_type);
+      auto pos = ord_full_name.find(suffix);
+      const std::string ord_type =
+          FIX::string_toLower(ord_full_name.substr(pos + suffix.size()));
+      const std::string body = fmt::format(
+          body_template, msg.security->okx().security_desc(),me_->tdmode, side, ord_type,
+          price, quantity, msg.key);
+      http::HttpResponse res;
+      auto headers = OkxUtils::createHeaders(
+          me_->api_key, me_->passphrase, me_->secret_key, "POST", REQUEST_PATH,
+          me_->simulated_trading,body);
+      // Generate and log the curl command
+      std::string curlCommand = OkxUtils::generateCurlCommand(
+          config::get<config::OkxExeCfg>().rest_url() + REQUEST_PATH, headers,
+          body);
+      LOG(OrderMan, Warn, "Curl Command: \n{}", curlCommand);
+      ExecutionReport er;
+      http::HttpClient::post(
+          config::get<config::OkxExeCfg>().rest_url() + REQUEST_PATH, body,
+          headers, &res);
+      const auto &[suc, top_msg] = OkxUtils::orderAccepted(res.body);
+
+      OkxUtils::populateCreateOrderFields(er, msg);
+      er.price = msg.price;
+      er.quantity = msg.quantity;
+      er.filled_quantity = 0;
+      er.last_shares = 0;
+      er.avg_price = Price(0);
+      ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now());
+      er.transact_time = now;
+
+      er.order_key = msg.key;
+      const auto &[id, ts, ex_msg] = OkxUtils::extractOrderInfo(res.body);
+      er.exchange_order_id = ExchangeOrderID{OkxUtils::stringToFixedArray(id)};
+
+      if (suc) {
+         LOG(md, Info, "Http get succeeded {}", res.body);
+         er.exec_type = api::orderman::ExecType::EXEC_TYPE_PENDING_NEW;
+         er.order_status = api::orderman::NATIVE_ORDER_STATUS_PENDING_NEW;
+      } else {
+         LOG(md, Info, "Http get failed code: {}, errors: {}, body: {}", res.code, res.errors, res.body);
+         er.exec_type = api::orderman::ExecType::EXEC_TYPE_REJECTED;
+         er.order_status = api::orderman::NATIVE_ORDER_STATUS_REJECTED;
+         er.msg = ex_msg + " " + top_msg;
+    }
+    me_->buffered_exec_reps.push(
+        std::make_shared<ExecutionReport>(std::move(er)));
+   }
+   // Shut down all reactors if they exist.
+   catch (const std::exception &ex) {
+      const std::string err = ex.what();
+      OkxUtils::handleOkException("Launch order", err);
+   }
+}
+void OkxExecutionGateway::register_callback(
+    [[maybe_unused]] const OnExeReport &cb) {
+   LOG(OrderMan, Warn,
+       "OkxExecutionAdapter::register_callback callback registered");
+   me_->setCallback(cb);
+}
+
+void OkxExecutionGateway::cancel_order(const CancelOrderMsg &msg) {
+   LOG(OrderMan, Warn, "cancel order {}", msg);
+   try {
+      http::HttpResponse res;
+      // static constexpr const char *body_template = R"({"instId": "BTC-USDT", "tdMode": "cash", "side": "buy", "ordType": "limit", "px": "1000", "sz": "0.01"})";
+      static constexpr const char *body_template =
+          R"({{"instId": "{}",  "clordId": "{}"}})";
+      const std::string REQUEST_PATH = "/api/v5/trade/cancel-order";
+      const std::string body = fmt::format(
+          body_template, msg.security->okx().security_desc(), msg.key);
+      auto headers = OkxUtils::createHeaders(
+          me_->api_key, me_->passphrase, me_->secret_key, "POST", REQUEST_PATH,
+          me_->simulated_trading,body);
+      // Generate and log the curl command
+      std::string curlCommand = OkxUtils::generateCurlCommand(
+          config::get<config::OkxExeCfg>().rest_url() + REQUEST_PATH, headers,
+          body);
+      LOG(OrderMan, Warn, "Curl Command: \n{}", curlCommand);
+      http::HttpClient::post(
+          config::get<config::OkxExeCfg>().rest_url() + REQUEST_PATH, body,
+          headers, &res);
+      const auto &[suc, top_msg] = OkxUtils::orderAccepted(res.body);
+      //   ExecutionReport *er_p = me_->exe_report_pool.construct();
+      //   ExecutionReport &er = *er_p;
+      //   er.security = msg.security;
+      //   er.side = msg.side;
+      //   er.order_key = msg.key;
+      //   to_array(msg.routing_record->account(), &er.account_name.value());
+      //   to_array(msg.routing_record->operator_id(), &er.operator_id.value());
+      //   er.exchange = msg.security->exchange();
+      //   er.order_key = msg.key;
+      //   const auto &[id, ts, ex_msg] = OkxUtils::extractOrderInfo(res.body);
+      if (suc) {
+         LOG(md, Info, "Http get succeeded {}", res.body);
+         //  er.exec_type = api::orderman::ExecType::EXEC_TYPE_PENDING_CANCEL;
+         //  er.order_status = api::orderman::NATIVE_ORDER_STATUS_PENDING_CANCEL;
+      } else {
+        LOG(md, Info, "Http get failed code: {}, errors: {}, body: {}", res.code, res.errors, res.body);
+         //  er.exec_type = api::orderman::ExecType::EXEC_TYPE_REJECTED;
+         //  er.order_status = api::orderman::NATIVE_ORDER_STATUS_REJECTED;
+         //  er.msg = ex_msg + " " + top_msg;
+      }
+      //   me_->cb(er, {});
+   }
+   // Shut down all reactors if they exist.
+   catch (const std::exception &ex) {
+      const std::string err = ex.what();
+      OkxUtils::handleOkException("Cacnel order", err);
+   }
+}
+} // namespace bts::orderman::okxexe
+
+
+// ExecutionReport OkxExecutionAdapter::fire_dummy_modify_order_report(CancelReplaceOrderMsg msg) {
+//     ExecutionReport *er_p = me_->create_exe_ptr();
+//     ExecutionReport &er = *er_p;
+//     std::string account_name = config::get<config::OkxExeCfg>().account();
+//     std::string ops_id = config::get<config::OkxExeCfg>().ops_id();
+
+//     std::array<char, 12> account_array = {};
+//     std::copy_n(
+//         account_name.begin(),
+//         std::min(account_name.size(), account_array.size()),
+//         account_array.begin());
+//     er.account_name = AccountName(account_array);
+
+//     to_array(ops_id, &er.operator_id.value());
+//     er.order_key = msg.create_fields.key;
+//     er.msg_seq_num = 0;
+//     er.security = msg.create_fields.security;
+//     er.leg_security = nullptr;
+//     er.side = msg.create_fields.side;
+//     er.time_in_force = msg.create_fields.time_in_force;
+//     er.order_type = msg.create_fields.order_type;
+//     er.exec_type = api::orderman::EXEC_TYPE_REPLACED;
+//     er.order_status = api::orderman::NATIVE_ORDER_STATUS_REPLACED;
+//     er.manual_order_indicator = api::orderman::MANUAL_ORDER_INDICATOR_NO;
+//     auto s = std::to_string(msg.create_fields.key.value());
+//     std::array<char, 17> idAsArray{};
+//     std::copy_n(s.begin(), std::min(s.size()-1, idAsArray.size() - 1), idAsArray.begin());
+//     er.exchange_order_id = ExchangeOrderID{idAsArray};
+//     er.price = msg.create_fields.price;
+//     er.quantity = msg.create_fields.quantity;
+//     er.filled_quantity = Quantity(0);
+//     er.last_shares = Quantity(0);
+//     er.msg = "";
+//     er.exchange = msg.create_fields.security->exchange();
+//     // Set transaction time
+//     ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now());
+//     er.transact_time = now;
+//     me_->cb(er, {});
+//     return er;
+// }
+// ExecutionReport OkxExecutionAdapter::fire_dummy_cancel_order_report(CancelOrderMsg msg) {
+//     ExecutionReport *er_p = me_->create_exe_ptr();
+//     ExecutionReport &er = *er_p;
+//     std::string account_name = config::get<config::OkxExeCfg>().account();
+//     std::string ops_id = config::get<config::OkxExeCfg>().ops_id();
+
+//     std::array<char, 12> account_array = {};
+//     std::copy_n(
+//         account_name.begin(),
+//         std::min(account_name.size(), account_array.size()),
+//         account_array.begin());
+//     er.account_name = AccountName(account_array);
+
+//     to_array(ops_id, &er.operator_id.value());
+//     er.order_key = msg.key;
+//     er.msg_seq_num = 0;
+//     er.security = msg.security;
+//     er.leg_security = nullptr;
+//     er.side = msg.side;
+//     er.time_in_force = api::orderman::TIME_IN_FORCE_GTC;
+//     er.order_type = api::orderman::NATIVE_ORDER_TYPE_LIMIT;
+//     er.exec_type = api::orderman::EXEC_TYPE_TRADE_CANCEL;
+//     er.order_status = api::orderman::NATIVE_ORDER_STATUS_CANCELED;
+//     er.manual_order_indicator = api::orderman::MANUAL_ORDER_INDICATOR_NO;
+//     auto s = std::to_string(msg.key.value());
+//     std::array<char, 17> idAsArray{};
+//     std::copy_n(s.begin(), std::min(s.size()-1, idAsArray.size() - 1), idAsArray.begin());
+//     er.exchange_order_id = ExchangeOrderID{idAsArray};
+//     er.filled_quantity = Quantity(0);
+//     er.last_shares = Quantity(0);
+//     er.msg = "";
+//     er.exchange = msg.security->exchange();
+//     // Set transaction time
+//     ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now());
+//     er.transact_time = now;
+//     me_->cb(er, {});
+//     return er;
+// }
+// ExecutionReport OkxExecutionAdapter::fire_dummy_new_order_report(CreateOrderMsg msg) {
+//     ExecutionReport *er_p = me_->create_exe_ptr();
+//     ExecutionReport &er = *er_p;
+//     std::string account_name = config::get<config::OkxExeCfg>().account();
+//     std::string ops_id = config::get<config::OkxExeCfg>().ops_id();
+
+//     std::array<char, 12> account_array = {};
+//     std::copy_n(
+//         account_name.begin(),
+//         std::min(account_name.size(), account_array.size()),
+//         account_array.begin());
+//     er.account_name = AccountName(account_array);
+
+//     to_array(ops_id, &er.operator_id.value());
+//     er.order_key = msg.key;
+//     er.msg_seq_num = 0;
+//     er.security = msg.security;
+//     er.leg_security = nullptr;
+//     er.side = msg.side;
+//     er.time_in_force = api::orderman::TIME_IN_FORCE_GTC;
+//     er.order_type = msg.order_type;
+//     er.exec_type = api::orderman::EXEC_TYPE_NEW;
+//     er.order_status = api::orderman::NATIVE_ORDER_STATUS_NEW;
+//     er.manual_order_indicator = api::orderman::MANUAL_ORDER_INDICATOR_NO;
+//     auto s = std::to_string(msg.key.value());
+//     std::array<char, 17> idAsArray{};
+//     std::copy_n(s.begin(), std::min(s.size()-1, idAsArray.size() - 1), idAsArray.begin());
+//     er.exchange_order_id = ExchangeOrderID{idAsArray};
+//     er.price = msg.price;
+//     er.quantity = msg.quantity;
+//     er.filled_quantity = Quantity(0);
+//     er.last_shares = Quantity(0);
+//     er.msg = "";
+//     er.exchange = msg.security->exchange();
+//     // Set transaction time
+//     ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now());
+//     er.transact_time = now;
+//     me_->cb(er, {});
+//     return er;
+// }

src/lib/orderman/gateways/OkxExecutionGateway.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "lib/logging/logging.h"
+#include "lib/md/SecurityOracle.h"
+#include "lib/orderman/gateways/IExecutionGateway.h"
+namespace bts::orderman {
+
+class OkxExecutionGateway : public IExecutionGateway {
+ public:
+   OkxExecutionGateway(md::SecurityOracle sec_oracle);
+   ~OkxExecutionGateway() override;
+   void register_callback(const OnExeReport &report);
+   void start() override;
+   void dispatch_incoming_fix_messages() override;
+   void on_securities(const mercury::Update<NewSecuritiesEvent> &) override {
+    LOG(md, Info,"Not implemented");
+   };
+   void on_admin_data(const mercury::Update<NewAdminDataEvent> &) override{
+    LOG(md, Info,"Not implemented");
+   };
+   void send_fix(const jetfix::OutFIXPtr &, trace::CorID) override{
+    LOG(md, Info,"Not implemented");
+   }
+   void send_msgs(std::vector<OrderMsg> const &, trace::CorID) override;
+   OnExeReport &on_exe_report() override;
+   OnGwStatusChange &on_gw_status_change() override;
+   OnCancelReject &on_cancel_reject() override;
+   OnExchangePositionState &on_exchange_position_state() override;
+   OnExchangeOpenOrdersState &on_exchange_open_orders() override;
+
+ private:
+   struct PImpl;
+   std::unique_ptr<PImpl> me_;
+   void launch_new_order(const CreateOrderMsg &msg);
+   void cancel_order(const CancelOrderMsg &msg);
+   void modify_order(const CancelReplaceOrderMsg &);
+};
+
+} // namespace bts::orderman

src/lib/orderman/gateways/OnixFIXAdapter.cpp

@@ -238,6 +238,9 @@ void populate_common_order_fields(Message &order, const CreateOrderMsg &msg) {
           .set(OnixS::FIX::FIX42::Tags::Currency, msg.security->currency());
    }
 }
+void OnixFIXAdapter::dispatch_incoming_fix_messages() {
+    listener.dispatch_incoming_fix_messages();
+}
 void OnixFIXAdapter::modify_order(
     const CancelReplaceOrderMsg &msg) { //confirm will have everything
    try {

src/lib/orderman/gateways/OnixFIXAdapter.h

@@ -12,6 +12,7 @@ class OnixFIXAdapter : public IExecutionGenericAdapter {
  public:
    OnixFIXAdapter(md::SecurityOracle sec_oracle);
    ~OnixFIXAdapter() override;
+   void dispatch_incoming_fix_messages() override;
    void register_callback(const OnExecutionReport &report) override;
    void launch_new_order(const CreateOrderMsg &msg) override;
    void cancel_order(const CancelOrderMsg &msg) override;

src/lib/orderman/gateways/OnixFIXListener.cpp

@@ -15,9 +15,8 @@
 #include <string>
 
 namespace bts::orderman {
-ExecutionReport OnixFixListener::fire_dummy_report(CreateOrderMsg msg) {
-    ExecutionReport *er_p = exe_report_pool.construct();
-    ExecutionReport &er = *er_p;
+void OnixFixListener::fire_dummy_report(CreateOrderMsg msg) {
+    ExecutionReport er;
     std::string account_name = "BTSIM";
     std::array<char, 12> an_array = {};
     size_t i = 0;
@@ -51,13 +50,17 @@ ExecutionReport OnixFixListener::fire_dummy_report(CreateOrderMsg msg) {
     // Set transaction time
     ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now());
     er.transact_time = now;
-    cb(er,{});
-    return er;
+    if(er.security==nullptr){
+        LOG(OrderMan, Error,"DID not have sec"); 
+        return;
+    }
+    LOG(OrderMan, Info,"msg {}",er.msg); 
+    buffered_exec_reps.push(std::make_shared<ExecutionReport>(std::move(er)));
+    return;
 }
 
 namespace {
-    ExecutionReport createDefaultExecutionReport() {
-        ExecutionReport er;
+    void createDefaultExecutionReport(ExecutionReport &er) {
         er.exchange = api::VENUE_CBOE;  
         std::string account_name = "BTSIM";
         std::string ops_id = "op_id_bts";
@@ -67,7 +70,6 @@ namespace {
         er.account_name = AccountName(account_array);
 
         to_array(ops_id, &er.operator_id.value());
-        return er;
     }
 
     void populateExecTypeAndOrderStatus(ExecutionReport &er, int exec_type_value) {
@@ -120,9 +122,7 @@ namespace {
         return idAsArray;
     }
 }
-ExecutionReport OnixFixListener::get_exe_report(Message msg) {
-    ExecutionReport *er_p = exe_report_pool.construct();
-    ExecutionReport &er = *er_p;
+void OnixFixListener::get_exe_report(Message msg, ExecutionReport &er) {
 
     auto msg_str = msg.toString();
     LOG(OrderMan,Info, "OnixFixListener::get_exe_report {}", msg_str);
@@ -135,7 +135,7 @@ ExecutionReport OnixFixListener::get_exe_report(Message msg) {
         populateCreateOrderFields(er, m.value());
     }else{ //handle this later
         LOG(OrderMan,Error, "OnixFixListener::get_exe_report {} could not find create order message", msg_str);
-        er = createDefaultExecutionReport();
+        createDefaultExecutionReport(er);
     }  
     // Populate fields from the message
 
@@ -186,8 +186,6 @@ ExecutionReport OnixFixListener::get_exe_report(Message msg) {
     // Set transaction time
     ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now()); 
     er.transact_time = now;
-
-    return er;
 }
 
 void OnixFixListener::onInboundApplicationMsg(Message &msg, Session * /*sn*/) {
@@ -210,13 +208,16 @@ void OnixFixListener::onInboundApplicationMsg(Message &msg, Session * /*sn*/) {
             case '8':
                 {
                     //exection report
-                    auto er = get_exe_report(msg);
+                    ExecutionReport *er_p = exe_report_pool.construct();
+                    ExecutionReport &er = *er_p;
+                    get_exe_report(msg,er);
                     if(er.security==nullptr){
-                    LOG(OrderMan, Error,"DID not have sec"); 
-                    return;
+                        LOG(OrderMan, Error,"DID not have sec"); 
+                        exe_report_pool.destroy(&er);
+                        return;
                     }
-                    LOG(OrderMan, Error,"msg {}",er.msg); 
-                    cb(er,{});
+                    LOG(OrderMan, Info,"msg {}",er.msg); 
+                    buffered_exec_reps.push(std::make_shared<ExecutionReport>(std::move(er)));
                 }
         }
 
@@ -226,7 +227,20 @@ void OnixFixListener::onInboundApplicationMsg(Message &msg, Session * /*sn*/) {
 
     // cb(*er, {});
 }
-
+void OnixFixListener::dispatch_incoming_fix_messages() {
+    if (buffered_exec_reps.empty()) {
+        return;
+     }
+     std::shared_ptr<const ExecutionReport> er;
+     while (buffered_exec_reps.try_pop(er)) {
+        trace::CorID cid = {};
+        LOG(OrderMan, Info,
+            "OnixFixListener::dispatch_incoming_fix_messages er: {} ",
+            er->to_string());
+        cb(std::move(er), cid);
+     }
+    
+ }
 
 void OnixFixListener::onInboundSessionMsg(Message & msg, Session * /*sn*/)
 {
@@ -261,4 +275,4 @@ void OnixFixListener::onWarning(WarningReason::Enum /*reason*/, const std::strin
 }
 
 
-}   // namespace bts::orderman
+}   // namespace bts::orderman

src/lib/orderman/gateways/OnixFIXListener.h

@@ -26,6 +26,7 @@ class OnixFixListener : public OnixS::FIX::ISessionListener {
       LOG(OrderMan, Info, "OnixFixListener::setCallback callback registered");
       cb = callback;
    }
+   void dispatch_incoming_fix_messages();
    /// Is called when the session-level message is received from the counterparty.
    void onInboundSessionMsg(Message &msg, Session *sn) ONIXS_FIXENGINE_FINAL;
    void onStateChange(
@@ -33,9 +34,12 @@ class OnixFixListener : public OnixS::FIX::ISessionListener {
        SessionState::Enum prevState,
        Session *sn) ONIXS_FIXENGINE_FINAL;
    OnExeReport cb;
-   ExecutionReport fire_dummy_report(CreateOrderMsg msg);
-   ExecutionReport get_exe_report(Message msg);
-   static void dummy(const ExecutionReport &, const trace::CorID &) {
+   tbb::concurrent_queue<std::shared_ptr<const ExecutionReport>>
+       buffered_exec_reps;
+   void fire_dummy_report(CreateOrderMsg msg);
+   void get_exe_report(Message msg, ExecutionReport &er);
+   static void dummy(
+       std::shared_ptr<const ExecutionReport>, const trace::CorID &) {
       LOG(OrderMan, Error, "OnixFixListener::Dummy fired");
    }
    void onError(
@@ -49,4 +53,4 @@ class OnixFixListener : public OnixS::FIX::ISessionListener {
    boost::object_pool<ExecutionReport> exe_report_pool;
 };
 
-} // namespace bts::orderman
+} // namespace bts::orderman

src/lib/orderman/gateways/execution_gateway_provider.cpp

@@ -1,31 +0,0 @@
-#include "execution_gateway_provider.h"
-#include "lib/orderman/actors/order_manager/order_gateway_definition.h"
-#include "lib/orderman/gateways/ExecutionGatewayFactory.h"
-
-namespace bts::orderman {
-
-struct ExecutionGatewayProvider::PImpl {
-   PImpl(trace::PerfLog *l, md::SecurityOracle o) : l(l), o(std::move(o)) {}
-
-   trace::PerfLog *l;
-   md::SecurityOracle o;
-};
-
-ExecutionGatewayProvider::ExecutionGatewayProvider(
-    trace::PerfLog *l, md::SecurityOracle o)
-    : me_(std::make_unique<PImpl>(l, std::move(o))) {}
-
-ExecutionGatewayProvider::~ExecutionGatewayProvider() = default;
-
-std::vector<OrderGatewayDefinition> ExecutionGatewayProvider::create() {
-   auto conns = config::get<config::JetFIX>().connection();
-   std::vector<OrderGatewayDefinition> gateways;
-   for (auto c : conns) {
-      auto &gw = gateways.emplace_back();
-      gw.connection = std::move(c);
-      gw.gateway = ExecutionGatewayFactory::create_gateway(gw.connection, *me_->l, me_->o);
-   }
-   return gateways;
-}
-
-} // namespace bts::orderman

src/lib/orderman/gateways/sim/SimExecutionGateway.cpp

@@ -33,7 +33,6 @@
 #include "lib/utils/thread_manager.h"
 
 #include <bits/stdint-uintn.h>
-#include <boost/pool/object_pool.hpp>
 #include <cstdint>
 #include <optional>
 #include <queue>
@@ -280,10 +279,9 @@ struct SimExecutionGateway::PImpl {
    // send_msgs is sent on the order_man thread, its dequeued in the market data
    // thread, and execution reports are queued to buffered_exec_reps
    tbb::concurrent_queue<OrderMsg> buffered_order_ops;
-   tbb::concurrent_queue<ExecutionReport *> buffered_exec_reps;
+   tbb::concurrent_queue<std::shared_ptr<ExecutionReport>> buffered_exec_reps;
    std::unordered_map<bts::SecurityKey, SEGSecurityData> books = {};
    tbb::concurrent_hash_map<uint32_t, api::Security const *> sec_map = {};
-   boost::object_pool<ExecutionReport> exe_report_pool;
    size_t msg_seq_num = 0;
    std::set<SecurityKey> touched_secs;
    api::Security const *find_sec(uint32_t key) const {
@@ -321,13 +319,13 @@ void SimExecutionGateway::dispatch_incoming_fix_messages() {
    if (me->buffered_exec_reps.empty()) {
       return;
    }
-   ExecutionReport *er;
+   std::shared_ptr<ExecutionReport> er;
    while (me->buffered_exec_reps.try_pop(er)) {
       trace::CorID cid = {};
       LOG(OrderMan, Info,
           "ESimExecutionGateway::dispatch_incoming_fix_messages er: {} ",
           er->to_string());
-      me->on_exe_report(*er, cid);
+      me->on_exe_report(std::move(er), cid);
    }
 }
 
@@ -651,10 +649,9 @@ SEGSecurityData *SimExecutionGateway::_find_add_security_entry(
 // buffered_exec_reps, which is picked up from the other thread
 // and passed over to the order infrastructure
 void SimExecutionGateway::_push_execution_report(const SEGOrderState &seg_os) {
-   ExecutionReport *er_p = me->exe_report_pool.construct();
-   ExecutionReport &er = *er_p;
+   ExecutionReport er;
    fill_execution_report(seg_os, er);
-   me->buffered_exec_reps.push(er_p);
+   me->buffered_exec_reps.push(std::make_shared<ExecutionReport>(std::move(er)));
 }
 
 void SimExecutionGateway::_write_book(SecurityKey key) {

src/lib/orderman/ilink3/IL3GW.cpp

@@ -45,7 +45,7 @@ IL3GW::IL3GW(
     const Config &conf,
     config::JetFIX::Connection const *cfg)
     : _config(conf), _x_state(State::INIT), _buffer(buffer),
-      _on_state_change([](auto) {}), _on_exe_report([](auto *) {}),
+      _on_state_change([](auto) {}), _on_exe_report([](auto) {}),
       _on_session_change([](auto) {}), _connection(con), _session(session),
       _msg_count(0), _time(), _pool(), _exe_parser(cfg, db), _retransmit() {
    LOG(OrderMan, Info,

src/lib/orderman/ilink3/IL3GW.h

@@ -10,6 +10,9 @@
 #include "lib/orderman/ilink3/RawMsg.h"
 #include "lib/orderman/gateways/IExecutionGateway.h"
 #include "lib/orderman/orderman_utils/CancelReject.h"
+#include "lib/utils/auto_init_shared_ptr.h"
+#include "lib/utils/object_pool.h"
+
 #include <cstdint>
 #include <functional>
 #include <queue>
@@ -46,7 +49,8 @@ struct Config {
 using State = IL3State;
 
 typedef std::function<void(State)> OnStateChange;
-typedef std::function<void(const ExecutionReport *)> OnExecutionReport;
+typedef std::function<void(std::shared_ptr<const ExecutionReport>)>
+    OnExecutionReport;
 typedef std::function<void(const Session &s)> OnSessionChange;
 typedef std::function<void(CancelReject &)> OnCancelReject;
 
@@ -101,7 +105,7 @@ class IL3GW : IL3ConnectionHandler {
 
    // A pool of pre-allocated ExecutionReport objcets, used to avoid
    // allocation when processing an execution report.
-   boost::object_pool<ExecutionReport> _pool;
+   bts::utils::object_pool<utils::auto_init_shared_ptr<ExecutionReport>> _pool;
 
    // A helper used to turn MDP3 messages into normalized execution
    // report objects
@@ -242,10 +246,10 @@ class IL3GW : IL3ConnectionHandler {
    }
 
    template <class T> void handle_exe_report(const T &msg) {
-      auto *report = _pool.construct();
-      bool result = _exe_parser.parse(msg, report);
+      std::shared_ptr<ExecutionReport> report = _pool.construct()->value();
+      bool result = _exe_parser.parse(msg, report.get());
       if (result) {
-         _on_exe_report(report);
+         _on_exe_report(std::move(report));
       } else {
          LOG(iLink3, Error, "Failed to parse error report.");
       }

src/lib/orderman/ilink3/test/ILinkGWTests.cpp

@@ -377,7 +377,7 @@ DISABLED_TEST_CASE("OutrightVanillaOrder", "[ilink3][execution]") {
 
    size_t exe_reports = 0;
    gw.subscribe_execution_reports(
-       [&](const ExecutionReport *) { exe_reports += 1; });
+       [&](std::shared_ptr<const ExecutionReport>) { exe_reports += 1; });
 
    gw.send_order(order, {});
    gw.flush();

src/lib/orderman/orderman_utils/ExecutionReport.cpp

@@ -12,10 +12,10 @@ using namespace bts::api::orderman;
 
 std::string ExecutionReport::to_string() const {
    std::ostringstream rv;
-   rv << "account_name: " << to_trim_str(account_name)
-      << " operator_id: " << to_trim_str(operator_id)
+   rv << "account_name: " << to_trim_strn(account_name)
+      << " operator_id: " << to_trim_strn(operator_id)
       << " order_key: " << order_key.value() << " msg_seq_num: " << msg_seq_num
-      << " security: " << security->id()
+      << " security: " << (security == nullptr ? 0 : security->id())
       << " leg_security: " << (leg_security == nullptr ? 0 : leg_security->id())
       << " side: " << bts::api::Side_Name(side) << " time_in_force: "
       << bts::api::orderman::TimeInForce_Name(time_in_force)
@@ -32,4 +32,4 @@ std::string ExecutionReport::to_string() const {
 }
 
 } // namespace orderman
-} // namespace bts
+} // namespace bts

src/lib/orderman/orderman_utils/OkxUtils.h

@@ -0,0 +1,270 @@
+#include "lib/api/services/OrderService.pb.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/logging/logging.h"
+#include "lib/md/SecurityOracle.h"
+#include "lib/orderman/CreateOrderMsg.h"
+#include "lib/orderman/gateways/IExecutionGateway.h"
+#include "lib/orderman/orderman_utils/ExecutionReport.h"
+#include "lib/utils/bts_assert.h"
+#include <OnixS/FIXEngine/FIX/Exception.h>
+#include <OnixS/FIXEngine/FIX/FIX42.h>
+#include <OnixS/FIXEngine/FIX/Message.h>
+#include <OnixS/FIXEngine/FIX/Numeric.h>
+#include <OnixS/FIXEngine/FIX/ProtocolVersion.h>
+#include <boost/pool/object_pool.hpp>
+#include <cassert>
+#include <cstddef>
+#include <ctime>
+#include <exception>
+#include <fmt/core.h>
+#include <map>
+#include <nlohmann/json_fwd.hpp>
+#include <openssl/buffer.h>
+#include <openssl/evp.h>
+#include <openssl/hmac.h>
+#include <quickfix/FixFields.h>
+#include <quickfix/Utility.h>
+#include <regex>
+#include <sstream>
+#include <string>
+#include <tuple>
+#include <unistd.h>
+#include <unordered_map>
+namespace bts::orderman {
+using json = nlohmann::json;
+class OkxUtils {
+ public:
+   // Function template to get a value from a JSON object by key
+   template <typename T>
+   static inline T getJsonValue(
+       const json &jsonObject,
+       const std::string &key,
+       const T &defaultValue = T()) {
+      if (jsonObject.contains(key) && jsonObject[key].is_primitive()) {
+         return jsonObject.value(key, defaultValue);
+      }
+      return defaultValue;
+   }
+
+   // Function to parse JSON safely
+   static inline std::optional<json> parseJsonSafely(
+       const std::string &jsonString) {
+      try {
+         return json::parse(jsonString);
+      } catch (const std::exception &e) {
+         std::string s = e.what();
+         LOG(OrderMan, bts::Error, "Failed to parse JSON: {}", s);
+         return std::nullopt;
+      }
+   }
+
+   struct OrderResponse {
+      bool accepted;
+      std::string message;
+   };
+
+   static inline OrderResponse orderAccepted(const std::string &jsonString) {
+      auto body = parseJsonSafely(jsonString);
+      if (!body)
+         return {false, "NO Body"};
+      auto &jsonResponse = *body;
+
+      return {
+          jsonResponse.value("code", "1") == "0",
+          jsonResponse.value("msg", "NO code or message")};
+   }
+
+   // Function to extract order information from JSON
+   static inline std::tuple<std::string, std::string, std::string> extractOrderInfo(
+       const std::string &jsonString) {
+      auto body = parseJsonSafely(jsonString);
+      if (!body.has_value()) {
+         return {"", "", "json parsing error no body"};
+      }
+      auto jsonResponse = body.value();
+      if (jsonResponse.contains("data") && jsonResponse["data"].is_array()) {
+         const auto &dataArray = jsonResponse["data"];
+         if (!dataArray.empty() && dataArray[0].is_object()) {
+            const auto &firstItem = dataArray[0];
+
+            // Extract values using the templated function
+            std::string ordId =
+                OkxUtils::getJsonValue(firstItem, "ordId", std::string());
+            std::string timestamp =
+                OkxUtils::getJsonValue(firstItem, "ts", std::string());
+            std::string sMsg =
+                OkxUtils::getJsonValue(firstItem, "sMsg", std::string());
+
+            return {ordId, timestamp, sMsg};
+         }
+      }
+      return {"", "", "json parsing error"};
+   }
+   // Function to encode a byte array to Base64
+   static std::string base64Encode(const unsigned char *buffer, size_t length) {
+      BIO *bmem = BIO_new(BIO_s_mem());
+      BIO *b64 = BIO_new(BIO_f_base64());
+      BIO_set_flags(b64, BIO_FLAGS_BASE64_NO_NL); // No newlines
+      BIO_push(b64, bmem);
+      BIO_write(b64, buffer, static_cast<int>(length));
+      BIO_flush(b64);
+
+      BUF_MEM *bptr;
+      BIO_get_mem_ptr(b64, &bptr);
+      std::string result(bptr->data, bptr->length);
+
+      BIO_free_all(b64);
+      return result;
+   }
+
+   static inline std::vector<unsigned char> generateSignature(
+       const std::string &prehash, const std::string &secretKey) {
+      unsigned int len = EVP_MAX_MD_SIZE;
+      std::vector<unsigned char> result(len);
+      HMAC(
+          EVP_sha256(), secretKey.c_str(), static_cast<int>(secretKey.length()),
+          reinterpret_cast<const unsigned char *>(prehash.c_str()),
+          prehash.length(), result.data(), &len);
+      result.resize(len);
+      return result;
+   }
+   static inline std::string getSign(
+       std::string message, const std::string &secret_key = "your_secret_key") {
+      auto signature = generateSignature(message, secret_key);
+      std::string sign = base64Encode(signature.data(), signature.size());
+
+      return sign;
+   }
+   static std::tuple<long, std::string> getLoginSignTime(
+       const std::string &secret_key = "your_secret_key",
+       const std::string &method = "GET",
+       const std::string &path = "/users/self/verify") {
+      auto seconds = bts::core::TimeStamp::now().seconds();
+      std::string message = std::to_string(seconds) + method + path;
+      return {seconds, getSign(message, secret_key)};
+   }
+   static std::string getUTCTimestamp(bts::core::TimeStamp t) {
+      auto rawTimestamp =
+          t.to_iso_string() +
+          "Z"; //gives 20241213T175033.840123Z should be  2024-12-13T17:50:33.840Z
+               // Use regex to parse and reformat
+      std::regex re(
+          R"((\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})\.(\d{3})\d*Z)");
+      std::string formattedTimestamp =
+          std::regex_replace(rawTimestamp, re, "$1-$2-$3T$4:$5:$6.$7Z");
+
+      return formattedTimestamp;
+   }
+   static std::array<char, 17> stringToFixedArray(const std::string &s) {
+      std::array<char, 17> idAsArray{};
+      std::copy_n(
+          s.begin(), std::min(s.size(), idAsArray.size() - 1),
+          idAsArray.begin());
+      return idAsArray;
+   }
+   static std::unordered_map<std::string, std::string> createHeaders(
+       const std::string &apiKey,
+       const std::string &passphrase,
+       const std::string &secretKey,
+       const std::string &method,
+       const std::string &requestPath,
+       const bool simulated_trading,
+       const std::string &body = "") {
+      auto ts = bts::core::TimeStamp::now();
+      std::string timestamp = getUTCTimestamp(ts);
+      std::string prehash = timestamp + method + requestPath + body;
+      auto signature = getSign(prehash, secretKey);
+      LOG(OrderMan, bts::Info, "prehash '{}' , key '{}'", prehash, signature);
+      // Prepare headers
+      std::unordered_map<std::string, std::string> headers = {
+          {"accept", "application/json"},
+          {"Content-Type", "application/json"},
+          {"OK-ACCESS-KEY", apiKey},
+          {"OK-ACCESS-SIGN", signature},
+          {"OK-ACCESS-TIMESTAMP", timestamp},
+          {"OK-ACCESS-PASSPHRASE", passphrase}};
+      if (simulated_trading){
+         headers.insert({"x-simulated-trading", "1"});
+      }
+
+      return headers;
+   }
+   static std::string generateCurlCommand(
+       const std::string &url,
+       const std::unordered_map<std::string, std::string> &headers,
+       const std::string &body) {
+      std::ostringstream oss;
+
+      // Start with the curl command
+      std::string curlCommand = "curl -X POST " + url + " \\\n";
+
+      // Add headers to the curl command
+      for (const auto &header : headers) {
+         curlCommand +=
+             "  -H \"" + header.first + ": " + header.second + "\" \\\n";
+      }
+
+      // Add bodyl
+      curlCommand += "  -d '" + body + "'";
+
+      return curlCommand;
+   }
+   static inline api::SecurityType getInstType(const std::string &type) {
+      static const std::unordered_map<std::string, api::SecurityType> type_map =
+          {{"SPOT", api::SECURITYTYPE_SPOT},
+           {"OPTION", api::SECURITYTYPE_OPTION},
+           {"FUTURES", api::SECURITYTYPE_FUTURE},
+           {"SWAP", api::SECURITYTYPE_FUTURE}};
+
+      auto it = type_map.find(type);
+      return it != type_map.end() ? it->second : api::SECURITYTYPE_UNSET;
+   }
+   static inline api::orderman::NativeOrderType getOrdType(const std::string &type) {
+      static const std::unordered_map<std::string, api::orderman::NativeOrderType> type_map =
+          {{"limit", api::orderman::NATIVE_ORDER_TYPE_LIMIT},
+           {"market", api::orderman::NATIVE_ORDER_TYPE_MARKET}};
+
+      auto it = type_map.find(type);
+      return it != type_map.end() ? it->second : api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_UNDEFINED;
+   }
+   static void populateCreateOrderFields(
+       ExecutionReport &er, const CreateOrderMsg &create_order_msg) {
+      er.security = create_order_msg.security;
+      er.order_type = create_order_msg.order_type;
+      er.time_in_force = create_order_msg.time_in_force;
+      er.manual_order_indicator = create_order_msg.manual_order_indicator;
+      er.side = create_order_msg.side;
+      er.order_key = create_order_msg.key;
+      to_array(
+          create_order_msg.routing_record->account(), &er.account_name.value());
+      to_array(
+          create_order_msg.routing_record->operator_id(),
+          &er.operator_id.value());
+      er.exchange = create_order_msg.security->exchange();
+   }
+   static void populateExecTypeAndOrderStatus(
+       ExecutionReport &er, const std::string &state) {
+      if (state == "canceled") {
+         er.exec_type = api::orderman::ExecType::EXEC_TYPE_CANCELED;
+         er.order_status = api::orderman::NATIVE_ORDER_STATUS_CANCELED;
+      } else if (state == "live") {
+         er.exec_type = api::orderman::ExecType::EXEC_TYPE_NEW;
+         er.order_status = api::orderman::NATIVE_ORDER_STATUS_NEW;
+      } else if (state == "partially_filled") {
+         er.exec_type = api::orderman::ExecType::EXEC_TYPE_PARTIAL_FILL;
+         er.order_status = api::orderman::NATIVE_ORDER_STATUS_PARTIALLY_FILLED;
+      } else if (state == "filled") {
+         er.exec_type = api::orderman::ExecType::EXEC_TYPE_FILL;
+         er.order_status = api::orderman::NATIVE_ORDER_STATUS_FILLED;
+      } else {
+         er.exec_type = api::orderman::ExecType::EXEC_TYPE_UNDEFINED;
+         er.order_status = api::orderman::NATIVE_ORDER_STATUS_UNDEFINED;
+      }
+   }
+   static inline void handleOkException(
+       const std::string &context, const std::string &error) {
+      bts_assert_msgf(false, "{} Exception: {}", context, error);
+   }
+};
+
+} // namespace bts::orderman::okxexe

src/lib/orderman/orderman_utils/orderman_common.h

@@ -48,6 +48,15 @@ void to_str(std::array<char, N> const &in, std::string *out) {
    result = std::string(&in[0], N);
 }
 
+template <size_t N>
+void to_strn(std::array<char, N> const &in, std::string *out) {
+   std::size_t n = 0;
+   while (n < N && in[n] != '\0') {
+      n++;
+   }
+   *out = std::string(&in[0], n);
+}
+
 template <class T, class S>
 T to(S const &str) {
    T result;
@@ -67,6 +76,13 @@ inline std::string to_str(T const &t) {
    return result;
 }
 
+template <class T>
+inline std::string to_strn(T const &t) {
+   std::string result;
+   to_strn(t.value(), &result);
+   return result;
+}
+
 template <size_t N>
 std::pair<const char *, size_t> find_trim_str(std::array<char, N> const &in) {
    auto i = N;
@@ -79,12 +95,36 @@ std::pair<const char *, size_t> find_trim_str(std::array<char, N> const &in) {
    return std::make_pair(&in[0], i);
 }
 
+template <size_t N>
+std::pair<const char *, size_t> find_trim_strn(std::array<char, N> const &in) {
+   size_t i = 0;
+   while (i < N && in[i] != '\0') {
+      i++;
+   }
+   while (i > 0 && (in[i - 1] == ' ' || in[i - 1] == 0)) {
+      i--;
+   }
+   if (i == 0) {
+      return {nullptr, 0};
+   }
+   return {&in[0], i};
+}
+
 template <class T>
 std::string to_trim_str(T const &t) {
    auto ts = find_trim_str(t.value());
    return std::string(ts.first, ts.second);
 }
 
+template <class T>
+std::string to_trim_strn(T const &t) {
+   auto ts = find_trim_strn(t.value());
+   if (ts.second == 0) {
+      return {};
+   }
+   return std::string{ts.first, ts.second};
+}
+
 inline bts::api::orderman::OrderStatus get_external_status(
     bts::api::orderman::NativeOrderStatus status) {
    using namespace bts::api::orderman;
@@ -166,7 +206,7 @@ inline void set_exchange_order_id(
        overloaded{
            [out](ExchangeOrderID const &str) {
               out->set_exchange_order_id_u64(0);
-              *(out->mutable_exchange_order_id()) = to_trim_str(str);
+              *(out->mutable_exchange_order_id()) = to_trim_strn(str);
            },
            [out](ExchangeOrderID64 oid) {
               out->set_exchange_order_id_u64(oid.value());
@@ -292,6 +332,29 @@ inline void set_update_quantity(
 } // namespace bts
 
 FMT_IMPL(bts::orderman::AccountID, id, "{}", *id);
+FMT_IMPL(bts::orderman::ExchangeOrderID, id, "{}", bts::orderman::to_strn(id));
 FMT_IMPL(bts::orderman::ExchangeOrderID64, id, "{}", *id);
+
+template <>
+struct fmt::formatter<bts::orderman::ExchangeOrderIDVar> {
+   constexpr auto parse(fmt::format_parse_context &ctx) -> decltype(ctx.end()) {
+      return ctx.end();
+   }
+   template <typename FormatContext>
+   auto format(const bts::orderman::ExchangeOrderIDVar &id, FormatContext &ctx)
+       const -> decltype(ctx.out()) {
+      if (std::holds_alternative<bts::orderman::ExchangeOrderID64>(id)) {
+         return fmt::format_to(
+             ctx.out(), "{}",
+             std::get<bts::orderman::ExchangeOrderID64>(id).value());
+      }
+      if (std::holds_alternative<bts::orderman::ExchangeOrderID>(id)) {
+         return fmt::format_to(
+             ctx.out(), "{}", std::get<bts::orderman::ExchangeOrderID>(id));
+      }
+      return ctx.out();
+   }
+};
+
 FMT_IMPL(bts::orderman::GatewayName, n, "{}", *n);
 FMT_IMPL(bts::orderman::OrderKey, k, "{}", *k);

src/lib/orderman/orders/spreader_orders/SpreaderOrder.cpp

@@ -3,6 +3,7 @@
 #include "lib/config/protos/spreader.pb.h"
 #include "lib/logging/logging.h"
 #include "lib/orderman/events/SettlementUpdateEvent.h"
+#include "lib/orderman/orderman_utils/orderman_common.h"
 #include "lib/orderman/orders/spreader_orders/spreader/LegPropertyData.h"
 #include "lib/orderman/orders/spreader_orders/spreader/NLegsHedgeRecompData.h"
 #include "lib/orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.h"
@@ -67,6 +68,8 @@ OrderOpResult SpreaderOrder::on_activate(
       return OrderOpResult{api::orderman::ORDER_ERROR_OP_NOT_SUPPORTED};
    quantity =
        _bookkeeping_data().fqty_helper().in_transform_spreader_qty(quantity);
+   LOG(OrderMan, Info, "SpreaderOrder::on_activate price: {} quantity: {}",
+       price, quantity);
    SpreaderOrderServices spreader_order_services(&services);
    _on_activate(price, quantity, spreader_order_services);
    // the reason why this is immediatly reported (vs in the timer
@@ -81,8 +84,9 @@ OrderOpResult SpreaderOrder::on_update(Price price, Quantity quantity) {
       return OrderOpResult{api::orderman::ORDER_ERROR_OP_NOT_SUPPORTED};
    quantity =
        _bookkeeping_data().fqty_helper().in_transform_spreader_qty(quantity);
-   LOG(OrderMan, Info, "SpreaderOrder.on_update key:{} price:{} quantity:{}",
-       key().value(), price.value(), quantity.value());
+
+   LOG(OrderMan, Info, "SpreaderOrder::on_update price: {} quantity: {}", price,
+       quantity);
    _state_worker->on_update(price, quantity);
    _report_state();
    return OrderOpResult{};
@@ -328,8 +332,11 @@ void SpreaderOrder::_on_activate(
     Price price, Quantity quantity, SpreaderOrderServices &order_services) {
    bts_assert(_inited());
    bts_assert(!_activated());
-   quantity =
-       _bookkeeping_data().fqty_helper().in_transform_spreader_qty(quantity);
+   LOG(OrderMan, Info, "SpreaderOrder::_on_activate price: {} quantity: {}",
+       price, quantity);
+   // quantity has already been fractional_normalized
+   // quantity =
+   //     _bookkeeping_data().fqty_helper().in_transform_spreader_qty(quantity);
    LOG(OrderMan, Info, "SpreaderOrder.on_activate key:{} price:{} quantity:{}",
        key().value(), price.value(), quantity.value());
    _order_services = new SpreaderOrderServices(order_services);
@@ -491,15 +498,18 @@ void SpreaderOrder::_comp_and_report_state() {
          reported_state.set_status(status);
          reported_state.set_price(bookkeeping_data.input_price().value());
          reported_state.set_quantity(bookkeeping_data.quantity().value());
-         reported_state.set_filled_quantity(filled_qty.value());
+         bts::orderman::set_quantity(
+             reported_state, StateQuantityType::FilledQuantity,
+             filled_qty.full_value());
+         //reported_state.set_filled_quantity(filled_qty.value());
          reported_state.set_avg_price(avg_filled_price.value());
          reported_state.set_last_price(avg_filled_price.value());
       }
       // if order is cons'ed from disc, with inited_new==false,
       // we just publish what was given to us
       LOG(OrderMan, Info,
-          "SpreaderOrder::_report_state key:{} status:{} "
-          "price:{} qty:{} filled:{} avg_price:{}",
+          "SpreaderOrder::_report_state key: {} status: {} "
+          "price: {} qty: {} filled: {} avg_price: {}",
           key(), status, reported_state.price(), reported_state.quantity(),
           reported_state.filled_quantity(), reported_state.avg_price());
       auto rerported_state_adj = reported_state;

src/lib/orderman/orders/spreader_orders/SpreaderOrder.h

@@ -285,6 +285,9 @@ class SpreaderOrder : public Order, public ISpreaderManager {
    }
    // just sets the price and qty in the bookkeeping data
    void _set_price_quantity(Price input_price, Quantity quantity) {
+      LOG(OrderMan, Info,
+          "SpreaderOrder::_set_price_quantity input_price: {} quantity: {}",
+          input_price, quantity);
       Price effective_price = _compute_effective_price(input_price);
       _bookkeeping_data().update_order(input_price, effective_price, quantity);
    }

src/lib/orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp

@@ -1,4 +1,6 @@
 #include "SpreaderBookkeepingData.h"
+#include "lib/domain_model/common.h"
+#include "lib/logging/logging.h"
 #include "lib/orderman/orders/spreader_orders/spreader/LegPropertyData.h"
 #include "lib/orderman/orders/spreader_orders/spreader/SpreaderFractionalQtyHelper.h"
 #include "lib/os/os.h"
@@ -25,7 +27,9 @@ SpreaderBookkeepingData::SpreaderBookkeepingData(
                                                         .spreader()
                                                         .execution_strategy()
                                                         .hedge_pricing()),
-      _fqty_helper(spreader_order_key, legs) {
+      _fqty_helper(
+          spreader_order_key, order_def.custom_def().spreader(), legs) {
+   //_order_def.custom_def().spreader()
    LOG(OrderMan, Info,
        "SpreaderBookkeepingData.cons started "
        "key: {} n_legs: {}",
@@ -97,8 +101,12 @@ void SpreaderBookkeepingData::total_fills(
          avg_price += leg_fill_price.value() *
                       leg_data.leg_property_data().price_multiplier();
    }
+   LOG(OrderMan, Info,
+       "SpreaderBookkeepingData::total_fills avg_price: {} fills: {}",
+       avg_price, fills);
+
    avg_price = fills == 0 ? 0.0 : avg_price; // should it be nan?
-   out_quantity = Quantity(fills);
+   out_quantity = Quantity::create_new(fills);
    out_price = Price(avg_price);
 }
 } // namespace orderman

src/lib/orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.h

@@ -1,4 +1,5 @@
 #pragma once
+#include "lib/logging/logging.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"
 #include <boost/container/flat_map.hpp>
 #include <memory>
@@ -76,7 +77,11 @@ class SpreaderBookkeepingData {
    }
    Price input_price() const { return _input_price; }
    Price effective_price() const { return _effective_price; }
-   Quantity quantity() const { return _quantity; }
+   Quantity quantity() const {
+      LOG(OrderMan, Info, "SpreaderBookkeepingData::quantity _quantity: {}",
+          _quantity);
+      return _quantity;
+   }
    NLegsHedgeRecompData &hedge_recomp_data() { return _hedge_recomp_data; }
    // gets leg data from leg's sec key
    SpreaderLegData &get_leg_data(SecurityKey security_key) {
@@ -146,6 +151,10 @@ class SpreaderBookkeepingData {
    }
    void update_order(
        Price input_price, Price effective_price, Quantity quantity) {
+      LOG(OrderMan, Info,
+          "SpreaderBookkeepingData::update_order input_price: {} "
+          "effective_price: {} quantity: {}",
+          input_price, effective_price, quantity.full_value());
       _input_price = input_price;
       _effective_price = effective_price;
       _quantity = quantity;
@@ -157,9 +166,6 @@ class SpreaderBookkeepingData {
    }
    size_t n_quoting_legs() const { return _n_quoting_legs; }
    SpreaderFractionalQtyHelper &fqty_helper() { return _fqty_helper; }
-   // SpreaderFractionalQtyHelper const *spreader_qty_helper() {
-   //    return &_fqty_helper;
-   // }
 };
 } // namespace orderman
 } // namespace bts

src/lib/orderman/orders/spreader_orders/spreader/SpreaderFractionalQtyHelper.cpp

@@ -8,6 +8,8 @@
 #include "lib/md/PriceSize.h"
 #include "lib/orderman/events/BookUpdateEvent.h"
 #include "lib/orderman/events/OrderStateUpdateEvent.h"
+//#include "lib/orderman/orders/spreader_orders/SpreaderOrder.h"
+#include "lib/orderman/orderman_utils/orderman_common.h"
 #include "lib/orderman/orders/spreader_orders/spreader/LegPropertyData.h"
 #include "lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h"
 #include "lib/os/os.h"
@@ -36,15 +38,6 @@ bool has_crypto_leg(const std::vector<LegPropertyData> &legs) {
    return false;
 }
 
-bool has_notional_leg(const std::vector<LegPropertyData> &legs) {
-   return has_crypto_leg(legs);
-   // TODO
-   // for (const auto &l : legs) {
-   //    return l.is_notional();
-   // }
-   // return false;
-}
-
 struct QtyTickInfo {
  public:
    const double tick_base;
@@ -71,22 +64,20 @@ struct SpreaderFractionalQtyHelper ::PImpl {
    Book op_book = {};
    OrderStateUpdateEvent op_ord = {};
    const bool passthrough = false;
-   //const double in_qty_mult;
+   // we can keep the mult per leg, based on their tick info, but its easier
+   // (and correct) if we do it
+   const double in_qty_mult = 100000;
    const bool is_notional;
    const QtyTickInfo def_qty_tick_info = QtyTickInfo(10000.0, 1.0);
    std::map<bts::SecurityKey, QtyTickInfo> qty_info_map = {};
 
-   PImpl(const std::vector<LegPropertyData> &lpd_vec)
-       : passthrough(has_crypto_leg(lpd_vec) ? false : true),
-         // in_qty_mult(
-         //     has_crypto_leg(lpd_vec)
-         //         ? static_cast<double>(
-         //               config::get<bts::config::Spreader>().qty_multiplier())
-         //         : 1.0),
-         is_notional(has_notional_leg(lpd_vec)) {
-      // LOG(OrderMan, Info,
-      //     "SpreaderFractionalQtyHelper::cons, in_qty_mult: {} is_notional: {}",
-      //     in_qty_mult, is_notional);
+   PImpl(
+       const api::orderman::SpreaderOrderDefinition &spreader_order_def,
+       const std::vector<LegPropertyData> &lpd_vec)
+       : passthrough(!has_crypto_leg(lpd_vec)),
+         is_notional(
+             spreader_order_def.security().synthetic().price_calc() ==
+             api::SyntheticContract_PriceCalc_PRICECALC_NOTIONAL_LEG) {
       auto i = 0;
       for (const auto &l : lpd_vec) {
          const auto &qiv = l.security().contract_spec().quantity_infos();
@@ -104,50 +95,73 @@ struct SpreaderFractionalQtyHelper ::PImpl {
    }
 };
 
+double SpreaderFractionalQtyHelper::_get_mult(size_t) const {
+   return me->in_qty_mult;
+}
+
+Quantity SpreaderFractionalQtyHelper::in_transform_spreader_qty(
+    Quantity qty) const {
+   // nothing needs to be done with spreader qty
+   if (me->passthrough)
+      return qty;
+   double qm = _get_mult(0);
+   double rv = qty.full_value() * qm;
+   auto rv_qty = Quantity::create_new(rv);
+   LOG(OrderMan, Info,
+       "SpreaderFractionalQtyHelper::in_transform_spreader_qty {} => {}", qty,
+       rv_qty);
+   return rv_qty;
+}
+
 std::vector<LegPropertyData> SpreaderFractionalQtyHelper::get_transformed_legs(
     const std::vector<LegPropertyData> &legs) const {
-   std::vector<LegPropertyData> rv = {};
-   for (const auto &l : legs) {
-      bts::api::ContractLeg leg_adj = l.leg_contract();
-      const auto &q_tinfo = get_qty_tick_info(
-          me->qty_info_map, bts::SecurityKey(l.leg_security_key()),
-          me->def_qty_tick_info);
-      double qm = q_tinfo.tick_mult;
-      leg_adj.set_quantity(l.size_multiplier() * qm);
-      double qty_tick_size = q_tinfo.tick_sz;
-      LegPropertyData lpd_adj(
-          l.leg_index(), leg_adj, l.leg_params(), l.trading_side(),
-          bts::api::SyntheticContract::PriceCalc::
-              SyntheticContract_PriceCalc_PRICECALC_STANDARD,
-          qty_tick_size);
-      rv.push_back(lpd_adj);
-   }
-   return rv;
+   // std::vector<LegPropertyData> rv = {};
+   // for (const auto &l : legs) {
+   //    bts::api::ContractLeg leg_adj = l.leg_contract();
+   //    const auto &q_tinfo = get_qty_tick_info(
+   //        me->qty_info_map, bts::SecurityKey(l.leg_security_key()),
+   //    double qm = q_tinfo.tick_mult;
+   //    leg_adj.set_quantity(l.size_multiplier() * qm);
+   //    double qty_tick_size = q_tinfo.tick_sz;
+   //    LegPropertyData lpd_adj(
+   //        l.leg_index(), leg_adj, l.leg_params(), l.trading_side(),
+   //        bts::api::SyntheticContract::PriceCalc::
+   //            SyntheticContract_PriceCalc_PRICECALC_STANDARD,
+   //        qty_tick_size);
+   //    rv.push_back(lpd_adj);
+   // }
+   // return rv;
+   return legs;
 }
 
 SpreaderFractionalQtyHelper::SpreaderFractionalQtyHelper(
-    OrderKey /*spreader_ord_key*/, const std::vector<LegPropertyData> &lpd_vec)
-    : me(new PImpl(lpd_vec)) {}
+    OrderKey spreader_ord_key,
+    const api::orderman::SpreaderOrderDefinition &spreader_order_def,
+    const std::vector<LegPropertyData> &lpd_vec)
+    : me(new PImpl(spreader_order_def, lpd_vec)) {
+   LOG(OrderMan, Info,
+       "SpreaderFractionalQtyHelper::cons spreader_ord_key: {} passthrough: {} "
+       "is_notional: {}",
+       spreader_ord_key, me->passthrough, me->is_notional);
+}
 
 const Book &SpreaderFractionalQtyHelper::in_transform_book(
     const Book &bk) const {
    if (me->passthrough) {
-      LOG(OrderMan, Info,
+      LOG(OrderMan, Diag,
           "SpreaderFractionalQtyHelper::in_transform_book passthrough");
       return bk;
    }
    if (me->is_notional) {
       SpreaderUtils::compute_notional_book(bk, me->op_book);
-      LOG(OrderMan, Info,
+      LOG(OrderMan, Diag,
           "SpreaderFractionalQtyHelper::in_transform_book sec: {} bk: {} "
           "notional: {}",
           bk.security_key().value(), bk, me->op_book);
    } else {
       me->op_book = bk;
    }
-   const auto &q_tinfo = get_qty_tick_info(
-       me->qty_info_map, bk.security_key(), me->def_qty_tick_info);
-   double qm = q_tinfo.tick_mult;
+   double qm = _get_mult(0);
    for (auto &ps : me->op_book.bid()) {
       ps = bts::md::PriceSize(ps.int_price(), ps.size() * qm);
    }
@@ -155,7 +169,7 @@ const Book &SpreaderFractionalQtyHelper::in_transform_book(
       ps = bts::md::PriceSize(ps.int_price(), ps.size() * qm);
    }
    // mult by leg_mult
-   LOG(OrderMan, Info,
+   LOG(OrderMan, Diag,
        "SpreaderFractionalQtyHelper::in_transform_book bk: {} output: {}", bk,
        me->op_book);
    return me->op_book;
@@ -177,18 +191,18 @@ SpreaderFractionalQtyHelper::in_transform_order_event(
       log_ev(sec_key, ev, ev);
       return ev;
    }
-   me->op_ord = ev;
    // if notional, then order qtys needs to be converted to notional, since spreader sees books
    // in notionals and fire orders in notionals
-   me->op_ord.quantity =
-       SpreaderUtils::to_notional(Price(ev.price), ev.quantity);
-   me->op_ord.filled_quantity =
-       SpreaderUtils::to_notional(Price(ev.avg_price), ev.filled_quantity);
-   me->op_ord.filled_at_attach_time_quantity = os::nan();
+   me->op_ord = ev;
+   if (me->is_notional) {
+      me->op_ord.quantity =
+          SpreaderUtils::to_notional(Price(ev.price), ev.quantity);
+      me->op_ord.filled_quantity =
+          SpreaderUtils::to_notional(Price(ev.avg_price), ev.filled_quantity);
+      me->op_ord.filled_at_attach_time_quantity = os::nan();
+   }
    // factor mult
-   const auto &q_tinfo =
-       get_qty_tick_info(me->qty_info_map, sec_key, me->def_qty_tick_info);
-   double qmult = q_tinfo.tick_mult;
+   double qmult = _get_mult(sec_key.value());
    me->op_ord.quantity *= qmult;
    me->op_ord.filled_quantity *= qmult;
    me->op_ord.filled_at_attach_time_quantity *= qmult;
@@ -196,14 +210,6 @@ SpreaderFractionalQtyHelper::in_transform_order_event(
    return me->op_ord;
 }
 
-Quantity SpreaderFractionalQtyHelper::in_transform_spreader_qty(
-    Quantity qty) const {
-   // nothing needs to be done with spreader qty
-   if (me->passthrough)
-      return qty;
-   return qty;
-}
-
 Quantity SpreaderFractionalQtyHelper::out_transform_leg_order_qty(
     SecurityKey skey, Price price, Quantity qty) const {
    auto qty_init = qty;
@@ -217,13 +223,10 @@ Quantity SpreaderFractionalQtyHelper::out_transform_leg_order_qty(
              "notional -> physical {} -> {}",
              qty_init, qty);
       }
-      const auto &q_tinfo =
-          get_qty_tick_info(me->qty_info_map, skey, me->def_qty_tick_info);
-      double tick_sz = q_tinfo.tick_sz;
-      double qm = q_tinfo.tick_mult;
+      double qm = _get_mult(skey.value());
       double qty_out = qty.full_value() / qm;
       // rouning it to tick size
-      qty_out = std::floor(qty_out / tick_sz) * tick_sz;
+      //qty_out = std::floor(qty_out / tick_sz) * tick_sz;
       return Quantity::create_new(qty_out);
    };
    auto rv_qty = tr();
@@ -236,12 +239,23 @@ Quantity SpreaderFractionalQtyHelper::out_transform_leg_order_qty(
 }
 
 void SpreaderFractionalQtyHelper::out_transform_order_state(
-    bts::api::orderman::OrderState &) const {
+    bts::api::orderman::OrderState &s) const {
+   auto s_txt = utils::pb::to_string(s);
+   LOG(OrderMan, Info,
+       "SpreaderFractionalQtyHelper::out_transform_order_state state: "
+       "{} filled_qty: {} avg_price: {}",
+       s_txt, s.filled_quantity(), s.avg_price());
    // todo:
    // this needs to divide the filled qty of the spreader
-   // if (me->passthrough)
-   //    return ord_st;
-   // return ord_st;
+   if (me->passthrough)
+      return;
+   double qm = _get_mult(0);
+   set_quantity(
+       s, StateQuantityType::OrderQuantity,
+       Quantity::create_new(static_cast<double>(s.quantity()) / qm));
+   set_quantity(
+       s, StateQuantityType::FilledQuantity,
+       static_cast<double>(s.filled_quantity()) / qm);
 }
 
 } // namespace bts::orderman

src/lib/orderman/orders/spreader_orders/spreader/SpreaderFractionalQtyHelper.h

@@ -3,7 +3,9 @@
 //#include "SpreaderLegData.h"
 //#include "lib/domain_model/common.h"
 //#include "lib/orderman/orderman_utils/orderman_common.h"
+#include "lib/api/services/OrderService.pb.h"
 #include "lib/domain_model/SecurityKey.h"
+//#include "lib/orderman/orders/spreader_orders/SpreaderOrder.h"
 #include "lib/orderman/orders/spreader_orders/spreader/LegPropertyData.h"
 
 namespace bts::api::orderman {
@@ -13,29 +15,52 @@ class OrderState;
 namespace bts::orderman {
 class Book;
 class OrderStateUpdateEvent;
-// forinstruments with fractional qtys this sits on the outside layer and does conversions,
-//both for market data and orders
-// this resdies inside SpeaderBookkeepingData
+// for instruments with fractional qtys this sits on the outside layer and does conversions,
+// both for market data and orders
+// it resides inside SpeaderBookkeepingData
+// these are the trasnfo
+// spreader_qty in *mult
+// books coming in *mult
+// order out /mult
+// order cb in * mult
+// report out /mult
 class SpreaderFractionalQtyHelper {
  public:
    SpreaderFractionalQtyHelper(
-       OrderKey spreader_ord_key, const std::vector<LegPropertyData> &legs);
-   // the leg qtys need to be translated
+       OrderKey spreader_ord_key,
+       const api::orderman::SpreaderOrderDefinition &spreader_order_def,
+       const std::vector<LegPropertyData> &legs);
+   double _get_mult(size_t leg_idx) const;
+   ////////////////////////////////////
+   // 1. qty (in)
+   ////////////////////////////////////
+   Quantity in_transform_spreader_qty(Quantity qty) const;
+   ////////////////////////////////////
+   // 2. leg sizes (in)
+   ////////////////////////////////////
    std::vector<LegPropertyData> get_transformed_legs(
        const std::vector<LegPropertyData> &legs) const;
-   // in_ and out_ signify market data / orders coming in or out.
+   ////////////////////////////////////
+   // 3. books (in)
+   ////////////////////////////////////
    const Book &in_transform_book(const Book &book) const;
    // this is called when the order feedback fomr the outside in to the spreader
+   ////////////////////////////////////
+   // 4. orders event (in)
+   ////////////////////////////////////
    const OrderStateUpdateEvent &in_transform_order_event(
        SecurityKey sec_key, const OrderStateUpdateEvent &) const;
-   Quantity in_transform_spreader_qty(Quantity qty) const;
    // TODO add price
+   ////////////////////////////////////
+   // 5. orders (out)
+   ////////////////////////////////////
    Quantity out_transform_leg_order_qty(
        SecurityKey key, Price price, Quantity qty) const;
    // the state reported ot he rest of the system also has to be converted
+   ////////////////////////////////////
+   // 6. order report (out)
+   ////////////////////////////////////
    void out_transform_order_state(bts::api::orderman::OrderState &) const;
-   // struct QtyTickInfo;
-   // const QtyTickInfo &_get_tickinfo(SecurityKey skey);
 
  private:
    struct PImpl;

src/lib/orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp

@@ -1,4 +1,5 @@
 #include "SpreaderStartStateWorker.h"
+#include "lib/logging/logging.h"
 #include "lib/orderman/OrderBuilder.h"
 #include "lib/orderman/orders/PreActivationOrderServices.h"
 #include "lib/orderman/orders/spreader_orders/SpreaderOrder.h"
@@ -47,6 +48,8 @@ void SpreaderStartStateWorker::on_init_new(
       std::string msg = "Could not resolve securities";
       _spreader->_transform_to_END(&msg);
    }
+   LOG(OrderMan, Info, "SpreaderStartStateWorker::on_init_new qty: {}",
+       _spreader->_bookkeeping_data().quantity().full_value());
 }
 void SpreaderStartStateWorker::on_init_recovered(
     const bts::api::orderman::OrderRecord &order_record,

src/lib/orderman/tests/CCApiExecutionTests.cpp

@@ -8,6 +8,7 @@
 #include "lib/utils/bts_chrono.h"
 #include "lib/utils/helper_macros.h"
 #include <boost/move/detail/placement_new.hpp>
+#include <ccapi_cpp/bts_session_options.h>
 #include <ccapi_cpp/ccapi_message.h>
 #include <ccapi_cpp/ccapi_util_private.h>
 #include <chrono>
@@ -65,7 +66,9 @@ class Bootstrapper {
       ccapi::SessionOptions sessionOptions;
       ccapi::SessionConfigs sessionConfigs;
       MyEventHandler eventHandler;
-      ccapi::Session session(sessionOptions, sessionConfigs, &eventHandler);
+      ccapi::Session session(
+          sessionOptions, {ccapi::BtsVenues::BINANCE_USDS_FUTURES},
+          sessionConfigs, &eventHandler);
       ccapi::Request request(
           ccapi::Request::Operation::CREATE_ORDER,
           CCAPI_EXCHANGE_NAME_BINANCE_USDS_FUTURES, "BTCUSD");
@@ -85,7 +88,9 @@ class Bootstrapper {
            "_FY2PcXYNc8VdlfkZO5c-pkFJQFhn8P9z4_p8YA2fS0"},
       });
       MyEventHandler eventHandler;
-      ccapi::Session session(sessionOptions, sessionConfigs, &eventHandler);
+      ccapi::Session session(
+          sessionOptions, {ccapi::BtsVenues::DERIBIT}, sessionConfigs,
+          &eventHandler);
       ccapi::Request request(
           ccapi::Request::Operation::CREATE_ORDER, CCAPI_EXCHANGE_NAME_DERIBIT,
           "BTC-29SEP23");
@@ -135,4 +140,4 @@ TEST_CASE("Simple", "[ccapi]") {
    }
 }
 
-} // namespace bts::orderman
+} // namespace bts::orderman

src/lib/os/spark_data_directory.cpp

@@ -26,7 +26,7 @@ boost::filesystem::path SparkDataDirectory::config_json_dir() const {
    if (p.has_parent_path()) {
       p = p.parent_path();
    }
-   return p / "conf_json/";
+   return p / "conf/";
 }
 
 boost::filesystem::path SparkDataDirectory::config_json_file(