Merge branch 'development.cboe-prod' of bitbucket.org:bluetradesys/spark into development.cboe-prod

It was black in a couple places before, making it hard to read.

.devcontainer/Dockerfile

@@ -90,6 +90,12 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev
 
+# Docker in Docker
+RUN apt-get update && apt-get install -y \
+    docker.io \
+    docker-compose \
+    docker-compose-v2
+
 # Set up default executables (update-alternatives)
 RUN \
    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \

.devcontainer/devcontainer.json

@@ -1,15 +1,10 @@
 {
   "dockerFile": "Dockerfile",
-  // Creates local dir to mount to HOME in docker 
-  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
   "hostRequirements": {
-    "memory": "4gb",
-    "storage": "6gb"
+    "memory": "8gb",
+    "storage": "10gb"
   },
-  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
   "containerUser": "bts",
-  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
-  "workspaceFolder": "/home/bts/spark",
   "updateRemoteUserUID": true,
   "shutdownAction": "none",
   // Note: If running multiple containers on the same host, each must use a
@@ -17,34 +12,20 @@
   // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
   "runArgs": [
     "--shm-size",
-    "2gb",
-    "-p",
-    "8000:8000",
-    "-p",
-    "8001:8001",
-    "-p",
-    "8002:8002",
-    "-p",
-    "8003:8003",
-    "-p",
-    "12345:12345"
+    "8gb"
   ],
   "mounts": [
-    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
   ],
   "customizations": {
     "vscode": {
-      "settings": {
-        "clangd.arguments": [
-          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
-        ]
-      },
       "extensions": [
         "eamodio.gitlens",
         "llvm-vs-code-extensions.vscode-clangd",
         "ms-vscode.cmake-tools",
-        "ms-vscode.cpptools"
+        "ms-vscode.cpptools",
+        "xaver.clang-format"
       ]
     }
   }
-}
+}

.env

@@ -0,0 +1,47 @@
+# This file defines the global default configuration for docker deployments.
+# It is tracked in source control. For temporary modifications, please create
+# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
+# If a variable is defined in both files, the value in .env.local will be used.
+#
+# About the deployment procedure:
+#   We use Docker Compose to build images and launch containers for various services
+#   that make up our software suite:
+#     - sparkd
+#     - TODO: pubsub
+#     - TODO: md-publisher
+#     - TODO: webapp (DSQA on the web)
+#     - TODO: api-server (websocket based)
+#   From inside a devcontainer, you can launch all services in separate containers
+#   on your machine by running:
+#
+#     tools/deploy.sh
+#
+#   This script builds release versions for each service, packaged (generally) in
+#   self-contained directories. Next, it runs a few "docker compose" commands to
+#   build docker images loaded with the self-contained binaries from the previous
+#   step, and start containers with those images to run each service. To view the
+#   commands that deploy.sh runs internally (without actually running them), run:
+#
+#     tools/deploy.sh --dry-run
+#
+#   For additional information/help, run:
+#
+#     tools/deploy.sh --help
+#
+
+### Sparkd ###
+# The internal image for sparkd to use. For more info, view:
+#   src/server/sparkd/Dockerfile
+SPARKD__IMAGE=sparkd
+# The config dir for this service to use
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
+# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
+# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
+# SPARKD__DATA_DIR=/home/bts/sparkdata/data
+SPARKD__SOURCE_DIR=install/sparkd-release-gcc
+SPARKD__CONFIG_PORT=8000
+SPARKD__QUICKBUS_PORT=8001
+SPARKD__SIM_FIX_PORT=8002
+SPARKD__SHM_SIZE=4gb
+
+# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitignore

@@ -59,3 +59,4 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
+.env.local

.vscode/launch.json

@@ -2,9 +2,10 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": ".NET Core Attach",
+      "name": "Attach Spark",
       "type": "coreclr",
-      "request": "attach"
+      "request": "attach",
+      "processId": "${command:pickProcess}"
     },
     {
       "name": "DataStreamPublisher",
@@ -326,4 +327,4 @@
       "description": "Test to run",
     },
   ]
-}
+}

compose.yaml

@@ -0,0 +1,63 @@
+services:
+  sparkd:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+    build:
+      dockerfile: src/server/sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+
+  sparkd-tail:
+    image: bluetradingsystems/linux_dotnet7
+    volumes:
+      - type: volume
+        read_only: true
+        source: sparkd_data
+        target: /home/bts/sparkdata/data/logging.sparkd
+        volume:
+          subpath: logging.sparkd
+    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
+
+  sparkd-ping-test:
+    depends_on:
+      - sparkd
+    profiles:
+      - ping-test
+    build:
+      context: .
+      dockerfile_inline: |
+        FROM bluetradingsystems/linux_dotnet7
+        USER "root"
+        RUN apt-get update && apt-get -y install iputils-ping
+    command: [ "ping", "sparkd" ]
+
+volumes:
+  sparkd_data:
+    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"

docker/images/bluetradingsystems/linux_dotnet7/Dockerfile

@@ -0,0 +1,38 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y pkg-config
+RUN apt-get update && apt-get install -y locales
+RUN apt-get update && apt-get install -y ca-certificates
+RUN apt-get update && apt-get install -y libssh2-1-dev
+RUN apt-get update && apt-get install -y software-properties-common
+RUN apt-get update && apt-get install -y sudo
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+  && apt-get update \
+  && apt-get install -y g++-$gcc_major
+
+RUN apt-get update \
+  && apt-get install -y dotnet7
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+  groupadd -g "$docker_uid" "$docker_user" \
+  && useradd \
+  -m \
+  -u "$docker_uid" \
+  -g "$docker_uid" \
+  -G sudo \
+  --password "$(openssl passwd -1 "$docker_pass")" \
+  "$docker_user"
+
+USER bts
+WORKDIR /home/bts

global.json

@@ -1,6 +0,0 @@
-{
-  "sdk": {
-    "version": "6.0.200",
-    "rollForward": "latestMajor"
-  }
-}

requirements.txt

@@ -1,10 +1,13 @@
 astroid==2.7.3
 autopep8==1.5.7
+boto3==1.37.9
+botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
 colorama==0.4.4
 idna==3.4
 isort==5.9.3
+jmespath==1.0.1
 lazy-object-proxy==1.6.0
 mccabe==0.6.1
 platformdirs==2.3.0
@@ -14,7 +17,11 @@ pycodestyle==2.7.0
 pycparser==2.20
 pylint==2.10.2
 PyMeta3==0.5.1
+python-dotenv==1.0.1
+python-dateutil==2.9.0.post0
 requests==2.31.0
+s3transfer==0.11.4
+six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3

src/client/spark/AW2/Bus/AnalyticsParametersManager.cs

@@ -45,7 +45,8 @@ namespace mbts.spark {
             Func<AnalyticsExpirationParameters, bool> is_unexpired = (p) => {
                var st = p.driver_security.security_tuple.to_api_sec_tuple();
                var exp = p.security.expiration;
-               return SecurityExpirationHelper.is_unexpired(st, exp);
+               return exp.Date >= DateTime.Now.Date;
+               //return SecurityExpirationHelper.is_unexpired(st, exp);
             };
             exp_params = exp_params.Where(t => is_unexpired(t));
             // listens to the commit tag and saves it

src/client/spark/AW2/Bus/SingleVolSurfaceInterpGreeksPublisher.cs

@@ -12,8 +12,8 @@ using bts.api;
 using mbts.ui.converters;
 
 namespace mbts.spark {
-   // This class gathers all the inputs needed to calculate InterpGreeks, pass it to 
-   // VolSurfacePrecomputedTheosProducer, and publishes the InterpGreeks to the bus
+   // This class gathers all the inputs needed to calculate InterpGreeks for a single vol surface,
+   // passes it to VolSurfacePrecomputedTheosProducer, and publishes the InterpGreeks to the bus
    // (called SingleVolSurfaceClientPrecomputedTheosPublisher in Edge)
    // It listens to
    // a. vol surface inputs

src/client/spark/AW2/Bus/VolSurfaceBlackboardManager.cs

@@ -20,6 +20,7 @@ using Spark.Core.Protobuf;
 using mbts.ui;
 using bts.config.toggles;
 using System.Diagnostics.CodeAnalysis;
+using BCG;
 // using mbts.bb;
 
 namespace mbts.spark
@@ -92,11 +93,11 @@ namespace mbts.spark
       // in tags
       private readonly ITag<SystemContractData> _in_scdata_tag =
          SparkTags.SystemContractsTag;
+      private readonly ITag<NMSheetSymbol> _archive_ivs_tag = SparkTags.TC2Tags.ArchiveIVsTag;
       private readonly ITag<VolSurfaceSettings> _in_bb_create_tag = SparkTags.VolSurfaceCreateTag;
       private readonly ITag<int> _in_bb_delete_tag = SparkTags.VolSurfaceDeleteTag;
       private readonly ITag<AnalyticsExpirationParametersMap> _in_bb_exp_params_tag =
          SparkTags.ExpirationParametersInputTag;
-      private readonly ITag<ClientArchivedVolRequest> _in_bb_req_vol = SparkTags.RequestArchivedVols;
       // state
       private SystemContractData _contract_data = null;
       private IDictionary<int, SingleVolSurfaceManager> _vs_manager_map =
@@ -106,6 +107,43 @@ namespace mbts.spark
       private bts.api.IBus _bus;
       private List<IDisposable> _subs = new List<IDisposable>();
 
+      private static void add_temp_vsu_entries(IDictionary<string, int> name_id_map,
+                                                 VolSurfaceUsageList ulist) {
+         Action<string, Venue, string> add_entry = (option_name, option_venue, vs_name) => {
+
+            int vs_key;
+            if (name_id_map.TryGetValue(vs_name, out vs_key)) {
+               ulist.UsageList.Add(new VolSurfaceUsageEntry() {
+                  VsId = vs_key,
+                  SymbolName = option_name,
+                  SecTup = new bts.api.SecurityTuple {
+                     Type = SecurityType.Option,
+                     Venue = option_venue,
+                     Symbol = option_name
+                  }
+               });
+            }
+
+         };
+         add_entry("SPX", Venue.Cboe, "CBOE_skews");
+         add_entry("SPXW", Venue.Cboe, "CBOE_skews");
+         add_entry("E1B", Venue.Glbx, "CME_Skews");
+         add_entry("E1C", Venue.Glbx, "CME_Skews");
+         add_entry("E3B", Venue.Glbx, "CME_Skews");
+         add_entry("E3C", Venue.Glbx, "CME_Skews");
+         add_entry("E3D", Venue.Glbx, "CME_Skews");
+         add_entry("E4A", Venue.Glbx, "CME_Skews");
+         add_entry("E4B", Venue.Glbx, "CME_Skews");
+         add_entry("E4C", Venue.Glbx, "CME_Skews");
+         add_entry("E4C", Venue.Glbx, "CME_Skews");
+         add_entry("E4D", Venue.Glbx, "CME_Skews");
+         add_entry("ES", Venue.Glbx, "CME_Skews");
+         add_entry("EW1", Venue.Glbx, "CME_Skews");
+         add_entry("EW2", Venue.Glbx, "CME_Skews");
+         add_entry("EW3", Venue.Glbx, "CME_Skews");
+         add_entry("EW4", Venue.Glbx, "CME_Skews");
+      }
+
       public VolSurfaceBusManager(bts.api.IBus bus,
                                          bts.api.analytics.AnalyticsService analytics_service,
                                          bts.api.orderman.OrderService order_service)
@@ -119,7 +157,7 @@ namespace mbts.spark
          _bb.subscribe(_in_bb_create_tag);
          _bb.subscribe(_in_bb_delete_tag);
          _bb.subscribe(_in_bb_exp_params_tag);
-         _bb.subscribe(_in_bb_req_vol);
+         _bb.subscribe(_archive_ivs_tag);
          _bb.data_callback += _on_bb_update;
          _bb.timer_callback += _on_timer;
          var contracts_throttle = TimeSpan.FromMilliseconds(200);
@@ -127,11 +165,25 @@ namespace mbts.spark
          var vs_sub = _analytics_service.subscribei_volsurfaces()
             .ObserveOn(SynchronizationContext.Current)
             .Subscribe(_on_vss);
+         //var hist_theos_sub_obs = _analytics_service.subscribei_archived_theos_table();
+         //var obs = hist_theos_sub_obs.ObserveOn(SynchronizationContext.Current);//.Subcribe(_on_hist_theos);
+         //obs.Subscribe(tu => {
+         //   Log.Gen.info("Received historical theos");
+         //   _on_hist_theos(tu);
+         //});
+
+         var archived_theos_obs = _analytics_service.subscribei_archived_theos().ObserveOn(SynchronizationContext.Current).Subscribe(tu => {
+            Log.Gen.info("Received historical theos");
+            _on_hist_theos(tu);
+         });
+         
+
          var vs_usage_sub = _analytics_service.subscribei_volsurface_usage()
             .ObserveOn(SynchronizationContext.Current)
             .Subscribe(_on_vs_usage_update);
          _subs.Add(vs_sub);
          _subs.Add(vs_usage_sub);
+         _subs.Add(archived_theos_obs);
          _subs.Add(new AnalyticsParametersManager(bus, _analytics_service));
          _subs.Add(new ResourceAllocator(bus, _bb, contracts_throttle));
          _subs.Add(new SymbolAggregationPublisher<OptionDriverPriceMap>(
@@ -143,7 +195,7 @@ namespace mbts.spark
          _subs.Add(new LiveTheoPublisher(_bb));
          _subs.Add(new InterpTheosServerPublisher(_bb, bus, order_service, contracts_throttle));
          _subs.Add(new VolSurfaceInterpGreeksMultiplexer(_bb));
-         // usage         
+         // list of all volsurface. in this callback, we read the usage config, and detemine the id of the named vol surface         
          var list_sub = _bus.subscribe(SparkTags.VolSurfaceListTag).OnDispatcher().Subscribe(t =>
          {
             VolSurfaceUsageList ulist = new VolSurfaceUsageList();
@@ -154,9 +206,12 @@ namespace mbts.spark
             }
             Log.Ui.info($"VolSurfaceUsageList name_to_ids count: {name_to_ids.Count}");
             var vs_usage = mbts.config.Configuration.get<bts.config.toggles.VolSurfaceUsage>();
-            Log.Ui.info($"VolSurfaceUsageList vs_usage acount: {vs_usage.Items.Count}");
+            Log.Ui.info($"VolSurfaceUsageList vs_usage acount: {vs_usage.Items.Count}");            
             foreach (var u in vs_usage.Items)
             {
+               if (string.Compare(u.OptionName, "SPX", 0) == 0 || string.Compare(u.OptionName, "SPXW", 0) == 0){
+                  continue;
+               }
                if (string.Compare(u.OptionName, "SPX", 0) == 0 || string.Compare(u.OptionName, "SPXW", 0) == 0 && u.OptionVenue == Venue.Unset)
                {
                   u.OptionVenue = Venue.Cboe;
@@ -186,10 +241,11 @@ namespace mbts.spark
                else
                {
                   Log.Gen.error($"Could not find vol surface with name: {vs_name}");
-               }
+               }               
             }
+            add_temp_vsu_entries(name_to_ids, ulist);
             var list_imm = new VolSurfaceUsageListImmutable(ulist);
-            Log.Ui.info(list_imm.log_string());
+            Log.Ui.info("VolSurfaceUsage: " + list_imm.log_string());
             _bb.post(SparkTags.VolSurfaceUsageTag, list_imm);
             _spin_vss(list_imm);
          });
@@ -292,6 +348,8 @@ namespace mbts.spark
 
       void _on_vs_usage_update(Update<VolSurfaceUsageListImmutable, int> update)
       {
+         var state = update.all;
+         Log.Gen.info($"VSBBMan._on_vs_usage_update size: {state.Count()}");
          // TODO
          return;
       }
@@ -310,6 +368,17 @@ namespace mbts.spark
          _init_vss(to_publish.Keys);
       }
 
+      void _on_hist_theos(Update<bts.api.analytics.ArchivedTheoListImmutable, ulong> update) {
+         //var total_state = update.state;
+         Log.Gen.info($"VSBBMan._on_hist_theos update: {update}");
+         foreach (var kv in update.all) {
+            bts.Log.Gen.info($"kv: {kv}");
+            _bb.post(SparkTags.ArchivedTheos, kv);
+            break;
+         }
+         //Log.assert(false);
+      }
+
       private void _init_vss(IEnumerable<int> vs_ids)
       {
          // we consider every vs in the db as "activated" which might prove to be expensive
@@ -362,29 +431,56 @@ namespace mbts.spark
             _bb.post(SparkTags.VolSurfaceNonVolInputTag,
                                           new mbts.analytics.VolSurfaceNonVolInput(exp_param_map.exp_map.ToDictionary()));
          }
-         else if (_in_bb_req_vol.Equals(tag))
-         {
-            ClientArchivedVolRequest client_req = _bb.read_state(_in_bb_req_vol);
-            bts.Log.Ui.info($"VolSurfaceBusManager._on_bb_update ClientArchivedVolRequest: {client_req.ToString()}");
-            ArchivedTheoRetrieval ret = client_req.ret;
-            if (ret == null || ret.SecTup == null)
-            {
-               bts.Log.Ui.warn($"VolSurfaceBusManager malformed ArchivedTheoRetrieval");
-               ArchivedTheoList empty = new ArchivedTheoList();
-               _bb.post(SparkTags.ResponseArchivedVols, Tuple.Create(client_req, empty));
-               return;
+         // this is publishe dby the sheets 
+         else if (_archive_ivs_tag.Equals(tag)){
+            //Log.assert(false);
+            NMSheetSymbol sym = data as NMSheetSymbol;
+            SystemContractData sc_data = _bb.read_state(SparkTags.SystemContractsTag);            
+            var associated_syms = _get_associated_symbols(sym);
+            List<ArchivedSecurityTheo> theo_list = new List<ArchivedSecurityTheo>();
+            foreach (NMSheetSymbol s in associated_syms){
+               ExpStrikeKeyedData<TheoPoint> symbol_theos = _bb.read_state(SparkTags.TC2Tags.LiveTheosTag((s.option_name, s.option_exchange)));
+               foreach (var t in symbol_theos.get_data_linearly()){
+                  Venue v = s.option_exchange;
+                  string symbol = s.option_name;
+                  DateTime exp = t.first;
+                  double strike = t.second;
+                  TheoPoint tp = t.third;
+                  Tuple<uint?, uint?> cp_id = sc_data.get_option_sec_id(v, symbol, exp, strike);                  
+                  if (cp_id != null && cp_id.Item1.HasValue && cp_id.Item2.HasValue){
+                     // call
+                     ArchivedSecurityTheo c_theo = new ArchivedSecurityTheo();
+                     c_theo.Id = cp_id.Item1.Value;
+                     c_theo.Driver = tp.driver;
+                     c_theo.Vol = tp.vol;
+                     c_theo.Theo = tp.call;
+                     c_theo.Delta = tp.cdelta;
+                     theo_list.Add(c_theo);
+                     // put
+                     ArchivedSecurityTheo p_theo = new ArchivedSecurityTheo();
+                     p_theo.Id = cp_id.Item1.Value;
+                     p_theo.Driver = tp.driver;
+                     p_theo.Vol = tp.vol;
+                     p_theo.Theo = tp.put;
+                     p_theo.Delta = tp.pdelta;
+                     theo_list.Add(p_theo);
+                  }
+               }
             }
-            bts.Log.Ui.info($"VolSurfaceBusManager ArchivedTheoRetrieval id: {client_req.id} save_time: {ret.SaveTime} " +
-                           $"sec_tup: {ret.SecTup.Venue}_{ret.SecTup.Type}_{ret.SecTup.Symbol}");
-            var timer = Stopwatch.StartNew();
-            ArchivedTheoList arch_theos = _analytics_service.get_archived_theos(ret);
-            timer.Stop();
-            bts.Log.Ui.info($"VolSurfaceBusManager ArchivedTheoRetrieval time: {timer.Elapsed.TotalMilliseconds}ms " +
-                            $"n_entries: {arch_theos.SecTheo.Count}");
-            _bb.post(SparkTags.ResponseArchivedVols, Tuple.Create(client_req, arch_theos));
+            ArchivedTheoList rv_list = new ArchivedTheoList();
+            // we dont care abouyt the save time any more, and dont want it to te the ky of the table anymore
+            rv_list.SaveTime = 0;
+            rv_list.SecTheo.AddRange(theo_list);
+            bts.Log.Gen.info($"VSBBMan archiving {rv_list.SecTheo.Count} entries");
+            _analytics_service.clean_and_archive_theos(rv_list);
+            // send it to the server
          }
       }
 
+      private List<NMSheetSymbol> _get_associated_symbols(NMSheetSymbol symbol){
+         return new List<NMSheetSymbol>() {new NMSheetSymbol(Venue.Cboe, "SPX"),new NMSheetSymbol(Venue.Cboe, "SPXW")};
+      }
+
       private int _timer_count = 0;
       private void _on_timer()
       {

src/client/spark/AW2/LiveTheoPublisher.cs

@@ -8,11 +8,17 @@ using MathNet.Numerics.Providers.FourierTransform;
 using BCG;
 using System.Windows.Automation;
 using bts.collections;
+using mbts.co2;
 
 namespace mbts.spark {
    // Publishes theos for all options (on the side, also publishes the theos for TC2).
    // Listens to driver and interp theos, on a change, calls _interp_and_publish
    public class LiveTheoPublisher : IDisposable {
+      public enum Impetus { 
+         DRIVER,
+         SKEW,
+      }
+
       private ThreadedBlackboard _bb;
       private readonly ITag<IReadOnlyDictionary<(string, Venue), OptionTheoInterpolationData>>
          _greeks_tag = SparkTags.AllInterpGreeks;
@@ -29,13 +35,14 @@ namespace mbts.spark {
          _bb.subscribe(_drivers_tag);
          _bb.data_callback += _on_data;
          _bb.timer_callback += _on_timer;
-         _interp_and_publish();
+         _interp_and_publish(Impetus.SKEW);
       }
 
       private void _on_data(object tag, object data) {
          if (tag.Equals(_drivers_tag) ||
             tag.Equals(_greeks_tag)) {
-            _interp_and_publish();
+            Impetus imp = tag.Equals(_drivers_tag) ? Impetus.DRIVER : Impetus.SKEW;
+            _interp_and_publish(imp);
          }
       }
 
@@ -102,7 +109,7 @@ namespace mbts.spark {
          return TheoPoint.getNulledObject();
       }
 
-      private void _publish_tc2_theos(IDictionary<SecurityStrike, TheoPoint> theos) {
+      private void _publish_tc2_theos(IDictionary<SecurityStrike, TheoPoint> theos, Impetus imp) {
          var rv = new Dictionary<(string, Venue), ExpStrikeKeyedData<TheoPoint>>();
          foreach (var kv in theos) {
             SecurityStrike k = kv.Key;
@@ -114,13 +121,62 @@ namespace mbts.spark {
             }
             sym_theos.set_data(k, tp);
          }
+         _override_theos(ref rv);
          foreach (var kv in rv) {
             _bb.post(SparkTags.TC2Tags.LiveTheosTag(kv.Key), kv.Value);
+            if (imp == Impetus.SKEW) {
+               _bb.post(SparkTags.TC2Tags.VolSaveEventTheosTag(kv.Key), kv.Value);
+            }
+         }
+      }
+
+      private void _override_theos(ref Dictionary<(string, Venue), ExpStrikeKeyedData<TheoPoint>> theos) {
+         if (_compute_midmarket) {
+            SystemContractData sc_data = _bb.read_state(SparkTags.SystemContractsTag);
+            foreach (var kv in theos) {
+               string sym = kv.Key.Item1;
+               Venue exchange = kv.Key.Item2;
+               foreach (var t in kv.Value.get_data_linearly()) {
+                  DateTime exp = t.first;
+                  double strike = t.second;
+                  TheoPoint tp = t.third;
+                  var call_sec_md = sc_data.get_option(exchange, sym, exp, strike, PutOrCall.Call);
+                  if (call_sec_md == null)
+                     continue;
+                  {
+                     NativePriceSize call_bid, call_ask;
+                     ulong time = 0;
+                     call_sec_md.Value.md.get_bbo(ref time, out call_bid, out call_ask);
+                     double call_mid = (call_bid.price + call_ask.price) / 2;
+                     tp.call = call_mid;
+                  }
+                  {
+                     var put_sec_md = sc_data.get_option(exchange, sym, exp, strike, PutOrCall.Put);
+                     if (put_sec_md == null)
+                        continue;
+                     NativePriceSize put_bid, put_ask;
+                     ulong time = 0;
+                     put_sec_md.Value.md.get_bbo(ref time, out put_bid, out put_ask);
+                     double put_mid = (put_bid.price + put_ask.price) / 2;
+                     tp.put = put_mid;
+                  }
+               }
+            }
+            //replace theos with market bid
+            // get it from the bus and override
+         } else {
+            // do nothing, passes unchanged
+         }
+      }
+
+      private bool _compute_midmarket {
+         get {
+            return true;
          }
       }
 
 
-      private void _interp_and_publish() {
+      private void _interp_and_publish(Impetus imp) {
          var timer = Stopwatch.StartNew();
          var res = new Dictionary<uint, GreeksTuple>();
          var driver_map = _bb.read_state(_drivers_tag);
@@ -162,13 +218,13 @@ namespace mbts.spark {
          }
          // copy and publish
          IReadOnlyDictionary<uint, GreeksTuple> res_ro = res;
-         bts.Log.Gen.info($"LiveTheoPublisher._interp_and_publish posting " +
+         bts.Log.Gen.info($"LiveTheoPublisher._interp_and_publish Impetus: {imp} posting " +
                              $"count: {res_ro.Count}");
          timer.Stop();
          _logger.log(_log_id, _log_id, timer.Elapsed.TotalMilliseconds);
          _bb.post(SparkTags.Greeks, res_ro);
          // oh and those TC2 theos for comptability
-         _publish_tc2_theos(k_theos);
+         _publish_tc2_theos(k_theos, imp);
       }
 
       // If we do any rolls like SP for futures, it will be here

src/client/spark/AW2/Misc/TheoPoint.cs

@@ -281,5 +281,16 @@ namespace mbts.spark {
          else
             return tp.put + (tp.pdelta * driver_diff) + (0.5 * tp.pgamma * driver_diff * driver_diff);
       }
+
+      public static TheoPoint extrapolate(TheoPoint tp, double driver, bool is_call) {
+         TheoPoint rv = tp.Clone() as TheoPoint;
+         double h = driver - tp.driver;
+         if (is_call) {
+            rv.call = tp.call + h * (tp.cdelta);
+         } else {
+            rv.put = tp.put + h * (tp.cdelta);
+         }
+         return rv;
+      }
    }
 }

src/client/spark/Analytics/Core/VolSurfaceConstants.cs

@@ -8,7 +8,7 @@ using System.Threading.Tasks;
 
 namespace mbts.analytics {
    public static class VolSurfaceConstants {
-      public static int vol_time_sample_mins = 1;
+      public static int vol_time_sample_mins = 5;
    }
    // TODO: remove this
    public static class VolSurfaceTemp {

src/client/spark/Analytics/Misc/VolSurfaceTimeCalculatorFactory.cs

@@ -21,13 +21,23 @@ namespace mbts.analytics {
       private class TimeCalculator : IVolSurfaceTimeCalculator {
          // TODO, this should be passed in the exp itself
          private Dictionary<string, TimeSpan> utc_time_of_exp_map = new Dictionary<string, TimeSpan>() {
+            // TODO: get it from config
             {"BTC", TimeSpan.FromHours(8)},
-            {"ETH", TimeSpan.FromHours(8)}
+            {"ETH", TimeSpan.FromHours(8)},
+            {"SPXW", TimeSpan.FromHours(20)},
+            {"SPX", new TimeSpan(13, 30, 0)}
          };
          public void compute_vol_time(VolSurfaceInput vs_input, VolSurfaceExpiration exp,
                                       DateTime voltime, double days_shift,
                                       out double yrs, out double days_per_yr) {
+
             days_per_yr = 365;
+            if (vs_input != null && vs_input.surface_settings != null && vs_input.surface_settings.core_settings != null
+               && vs_input.surface_settings.core_settings.time_settings != null) {
+               if (vs_input.surface_settings.core_settings.time_settings.vol_time_measure.account_for_holidays_and_weekends) {
+                  days_per_yr = vs_input.surface_settings.core_settings.time_settings.vol_time_measure.days_per_year;
+               }
+            }
             TimeSpan time_of_exp = TimeSpan.Zero;
             utc_time_of_exp_map.TryGetValue(exp.symbol, out time_of_exp);
             // assumes exp is the utc date of expiration

src/client/spark/Analytics/Pricing/VolSurfaceInterpGreeksProducer.cs

@@ -9,11 +9,11 @@ using bts.api;
 using mbts.spark;
 
 namespace mbts.analytics {
-   // This class computes the theos/greeks for a number of driver
-   // prices (specified in the vs settings).
+   // This class computes interp theos/greeks (ie for a number of driver
+   // prices (specified in the vs settings)) for a volsurface
    // It has public setters that get
    // 1. contracts
-   // 2. vs inout
+   // 2. vs input
    // 3. exp params(drivers, interst rate etc)
    // 4. driver prices
    // 5. vol time
@@ -41,7 +41,8 @@ namespace mbts.analytics {
       private IDictionary<VolSurfaceExpiration, Tuple<double, double>>
       _computed_driver_range_map = new Dictionary<VolSurfaceExpiration, Tuple<double, double>>();
       private readonly IEnumerable<double> _default_abs_moves =
-         new double[] { 300, -200, -125, -100, -75, -50, -25, 0, 25, 50, 75, 100, 125, 200, 300 };
+         //new double[] { 300, -200, -125, -100, -75, -50, -25, 0, 25, 50, 75, 100, 125, 200, 300 };
+         new double[] { -100, -50, 0, 50, 100 };
       private readonly double _cushion_percentage = 25.0;
 
       public event
@@ -135,8 +136,8 @@ namespace mbts.analytics {
                      false, true, false, false, false, _filter_map);
                timer.stop();
                vol_compute_time = timer.last_interval_time_ms;
-               Log.Gen.info($"VolSurfacePrecomputedTheosProducer._compute_and_fire volmat time " +
-                           $"{timer.last_interval_time_ms}");
+               Log.Gen.info($"VolSurfacePrecomputedTheosProducer._compute_and_fire volmat input_type: {input_type} vs_id: {_vs_id} time " +
+                           $"{timer.last_interval_time_ms} tid: {System.Threading.Thread.CurrentThread.ManagedThreadId}");
                timer.start();
                IDictionary<VolSurfaceExpiration, IEnumerable<Theos>> exp_theo_map =
                  VolSurfaceTheoCalculator.compute_standard_greeks(synced_vs_input.surface_settings,

src/client/spark/Daemons/MDDaemon.cs

@@ -133,17 +133,31 @@ namespace bts {
 
             // set 'unit quantity' for each sec
             foreach (var smd in exch_secs.Values) {
-               var meta = _security_metadata.get(smd.security);
-               if (meta.HasValue) {
-                  exch_secs[smd.id] = new(
-                     smd.security,
-                     smd.md,
-                     meta.Value.HasPriceDisplay ? meta.Value.PriceDisplay : null,
-                     meta.Value.HasQuantityDisplay ? meta.Value.QuantityDisplay : null);
-               }
+               var (priceDisplay, qtyDisplay) = GetDisplaySettings(smd.security);
+               exch_secs[smd.id] = new(smd.security, smd.md, priceDisplay, qtyDisplay);
             }
             _bus.post(SparkTags.ServerSecuritiesTag, exch_secs);
          }
+
+         (TickDisplaySettingsImmutable? Price, TickDisplaySettingsImmutable? Quantity) GetDisplaySettings(SecurityImmutable sec) {
+            var meta = _security_metadata.get(sec);
+            var priceDisplay = meta?.PriceDisplay ?? Security.Defaults.price_display_settingsi;
+            var qtyDisplay = meta?.QuantityDisplay ?? Security.Defaults.quantity_display_settingsi;
+            if (sec.HasContractSpec) {
+               var pd = priceDisplay.CloneToMutable();
+               pd.TickFactors = TickUtils.get_tick(sec, 0).to_api_tick_factors();
+               priceDisplay = new(pd);
+               if (sec.ContractSpec.QuantityInfos.TryGetFirst(out var qtyInfo)) {
+                  var qd = qtyDisplay.CloneToMutable();
+                  qd.ShowDecimalPart = qtyInfo.TickBase.is_real_non_zero() && !(qtyInfo.TickUnit / qtyInfo.TickBase).is_int();
+                  qd.TickFactors ??= new();
+                  qd.TickFactors.TickBase = qtyInfo.TickBase;
+                  qd.TickFactors.TickUnit = qtyInfo.TickUnit;
+                  qtyDisplay = new(qd);
+               }
+            }
+            return (priceDisplay, qtyDisplay);
+         }
       }
 
       public void Dispose() =>

src/client/spark/Daemons/MergedMDDaemon.cs

@@ -111,13 +111,43 @@ namespace bts
          _merge_and_post();
       }
 
+
+
+      private static bool is_active(SecurityMD sec_md) {
+         SecurityImmutable sec = sec_md.security;
+         // temp: to see if spreads work
+         bool fut_or_option = (sec.security_type() == bts.api.SecurityType.Future || sec.security_type() == bts.api.SecurityType.Option);
+         if (!fut_or_option)
+            return true;
+         Func<bool> is_daily_option = () => {
+            bool is_daily = false;
+            if (sec.Exchange == Venue.Cboe) {
+               is_daily = sec.Symbol.ToLower() == "spxw";
+            }
+            if (sec.Exchange == Venue.Glbx) {
+               string symbol = sec.Symbol.ToLower();
+               int unused;
+               is_daily = symbol.Length >= 3 && symbol[0] == 'e' && Int32.TryParse(symbol[1].ToString(), out unused);
+            }
+            return is_daily;
+         };
+         bool is_expired = !SecurityExpirationHelper.is_unexpired(sec);
+         if (is_expired) {
+            return false;
+         }
+         if (sec.security_type() == bts.api.SecurityType.Option) {
+            int days_to_exp = Convert.ToInt32((sec.earliest_expiration() - DateTime.Now).TotalDays);
+            if (is_daily_option()) {
+               return days_to_exp <= 14;
+            } else {
+               return days_to_exp <= 90;
+            }
+         }
+         return true;            
+      }   
+
       private void _merge_and_post()
       {
-         Func<SecurityMD, bool> is_active = (sec_md) =>
-         {
-            var sec = sec_md.security;
-            return SecurityExpirationHelper.is_unexpired(sec);
-         };
          var to_pub = _server_sec_map.Where(kv => is_active(kv.Value)).ToDictionary(kv => kv.Key, kv => kv.Value);
          foreach (var kv in _fake_sec_map)
          {

src/client/spark/Modules/Market/MarketView/MarketViewFilterStates.cs

@@ -12,8 +12,13 @@ namespace bts.shell.market_view {
    class MarketViewFilterStates {
       [ProtoMember(37)]
       public Venue[] venue { get; set; }
+
+      [ProtoMember(38)]
+      public string[] currency { get; set; }
+
       [ProtoMember(1)]
       public string security { get; set; }
+
       [ProtoMember(2)]
       public DoubleMinMaxState bid_price { get; set; }
 
@@ -128,6 +133,7 @@ namespace bts.shell.market_view {
          var pb = new ColumnPredicateBuilder<MarketViewVM.Row>();
          // security
          pb.add_discrete_values(venue, r => r.venue);
+         pb.add_discrete_values(currency, r => r.currency);
          pb.add_regex(security, r => r.security.ToString());
          pb.add_regex(security_key, r => r.security_key.ToString());
 

src/client/spark/Modules/Market/MarketView/MarketViewVM.cs

@@ -98,7 +98,14 @@ namespace mbts.spark {
          _sec_meta_store = sec_meta_store;
          _settings_hub = settings_hub;
          security_filter = new RegexFilterVM<Row>(r => r.security.ToString(), nameof(Row.security));
-         currency_filter =new RegexFilterVM<Row>(r => r.currency, nameof(Row.currency));
+         currency_filter = DiscreteValuesFilterVM.Create<Row, string>(
+            bus.subscribe(SparkTags.SecuritiesTag)
+               .Sample(TimeSpan.FromSeconds(1))
+               .Select(sec_mds => sec_mds.Values.Select(smd => smd.security.Currency).Distinct().ToList()),
+            r => r.currency,
+            nameof(Row.currency),
+            EqualityComparer<string>.Default,
+            Comparer<string>.Default);
          // rows
          _wrapped_rows = _rows.WrapSelector(r => r.security_md, s => {
             var r = new Row(_bus, s, _orders_model, _position_model);
@@ -124,7 +131,7 @@ namespace mbts.spark {
 
          add_col(ColKey.Venue, r => r.venue.ToString(), nameof(Row.venue), venue_filter, FilterType.DiscreteValues, ColWidth.Exchange);
          add_col(ColKey.Security, r => r.security.ToString(), nameof(Row.security), security_filter, FilterType.Regex, ColWidth.Security);
-         add_col(ColKey.Currency, r => r.currency, nameof(Row.currency), currency_filter, FilterType.Regex, ColWidth.Currency);
+         add_col(ColKey.Currency, r => r.currency, nameof(Row.currency), currency_filter, FilterType.DiscreteValues, ColWidth.Currency);
          add_col(ColKey.Volume, r => r.volume_text, nameof(Row.volume), volume_filter, FilterType.ULongMinMax, ColWidth.Volume);
 
          // last
@@ -391,7 +398,7 @@ namespace mbts.spark {
 
       public IDataGridColumnPersistentFilter<Row, Venue[]> venue_filter { get; } = DiscreteValuesFilterVM.Common.exchange<Row>(r => r.venue, nameof(Row.venue));
       public IDataGridColumnPersistentFilter<Row, string> security_filter { get; }
-      public IDataGridColumnPersistentFilter<Row, string> currency_filter { get; }
+      public IDataGridColumnPersistentFilter<Row, string[]> currency_filter { get; }
       public IDataGridColumnPersistentFilter<Row, DoubleMinMaxState> bid_price_filter { get; } = new DoubleMinMax<Row>(r => r.bid_price, nameof(Row.bid_price));
       public IDataGridColumnPersistentFilter<Row, DoubleMinMaxState> bid_qty_filter { get; } = new DoubleMinMax<Row>(r => r.bid_qty_display, nameof(Row.bid_qty_display));
       public IDataGridColumnPersistentFilter<Row, DoubleMinMaxState> dbid_qty_filter { get; } = new DoubleMinMax<Row>(r => r.dbid_qty_display, nameof(Row.dbid_qty_display));
@@ -538,6 +545,7 @@ namespace mbts.spark {
       public MarketViewFilterStates get_filter_states() =>
          new() {
             venue = venue_filter.get_state(),
+            currency = currency_filter.get_state(),
             security = security_filter.get_state(),
             bid_price = bid_price_filter.get_state(),
             bid_merged_size = bid_qty_filter.get_state(),
@@ -584,6 +592,7 @@ namespace mbts.spark {
 
          using var _ = rows_view.DeferRefresh();
          venue_filter.load_state(filters.venue);
+         currency_filter.load_state(filters.currency);
          security_filter.load_state(filters.security);
          bid_price_filter.load_state(filters.bid_price);
          bid_qty_filter.load_state(filters.bid_merged_size);
@@ -620,16 +629,16 @@ namespace mbts.spark {
 
       static (string desc, Color fg) _fills_title(ColKey key) =>
          key switch {
-            ColKey.Position => ("Day Fills", Colors.Black),
-            ColKey.RawDayPosition => ("Raw Day Fills", Colors.Black),
-            _ => ("Fills", Colors.Black),
+            ColKey.Position => ("Day Fills", BTSColors.DayFills.foreground),
+            ColKey.RawDayPosition => ("Raw Day Fills", BTSColors.DayFills.foreground),
+            _ => ("Fills", BTSColors.UI.default_text),
          };
 
       static (string desc, Color fg) _orderbook_title(ColKey key) =>
          key switch {
             ColKey.WorkingBids => ((string desc, Color fg))("Working Buy Orders", BTSColors.MarketData.bid),
             ColKey.WorkingAsks => ((string desc, Color fg))("Working Sell Orders", BTSColors.MarketData.ask),
-            _ => ("Working Orders", Colors.Black),
+            _ => ("Working Orders", BTSColors.UI.default_text),
          };
 
       static GridSettings _hide_cols(GridSettings grid, IEnumerable<OrderBookGridView.ColumnKeys> columns) =>

src/client/spark/Modules/Trading/Fills/FillsVM.cs

@@ -237,6 +237,17 @@ namespace mbts.spark {
             DefaultColumnWidths.Qty,
             get_fg: r => r.side == Side.Buy ? BTSColors.MarketData.bid : BTSColors.MarketData.ask);
 
+         add_col(
+            Columns.NotionalValue,
+            DefaultColumnHeaders.NotionalValue,
+            r => _get_row(r).security.spec_or_default().format(r.fill.notional_value()),
+            new DoubleMinMax<FillTuple>(r => r.fill.notional_value()),
+            DataGridFilterKnownTemplates.DoubleMinMax,
+            fs => fs.notional_value,
+            (fs, cf) => fs.notional_value = cf,
+            DefaultColumnWidths.NotionalValue,
+            get_fg: r => r.side == Side.Buy ? BTSColors.MarketData.bid : BTSColors.MarketData.ask);
+
          add_col(
             Columns.Price,
             DefaultColumnHeaders.Price,

src/client/spark/Modules/Trading/Fills/FillsView.xaml.cs

@@ -21,6 +21,7 @@ namespace mbts.ui.spark {
          Side,
          Price,
          Size,
+         NotionalValue,
          FillKind,
          FillStatus,
          FillKey,

src/client/spark/Modules/Trading/Fills/Settings/FillsFilterStates.cs

@@ -61,6 +61,10 @@ namespace bts.shell.fills {
       [DIYGridColumnID(nameof(Columns.Size))]
       public DoubleMinMaxState size { get; set; }
 
+      [ProtoMember(18)]
+      [DIYGridColumnID(nameof(Columns.NotionalValue))]
+      public DoubleMinMaxState notional_value { get; set; }
+
       [ProtoMember(11)]
       [DIYGridColumnID(nameof(Columns.FillKey))]
       public string fill_key { get; set; }
@@ -113,6 +117,7 @@ namespace bts.shell.fills {
          pb.add_discrete_values(fs.fill_status, ft => ft.fill.FillStatus);
          // numeric
          pb.add_numeric(fs.size, ft => ft.fill.LastShares);
+         pb.add_numeric(fs.notional_value, ft => ft.fill.notional_value());
          pb.add_numeric(fs.price, ft => ft.fill.LastPrice);
 
          if (fs.exp_date.HasValue) {

src/client/spark/Modules/Trading/Ladder/LadderSettings.cs

@@ -408,18 +408,18 @@ namespace mbts.spark {
 
    [ProtoContract]
    class LadderPanelPricePointSettings : IEquatable<LadderPanelPricePointSettings> {
-      public static LadderPanelPricePointSettings historic_default =>
+      public static LadderPanelPricePointSettings Default =>
          new LadderPanelPricePointSettings {
             day_change = new DaySettings {
                // (neutral)
-               font = _default_font,
-               background_color = Colors.Transparent.to_uint(),
+               font = _default_font.With(font_color: Colors.Black.to_uint()),
+               background_color = Color.FromRgb(113, 201, 69).to_uint(),
                // negative
                negative_font_color = Colors.White.to_uint(),
                negative_background_color = Colors.Red.to_uint(),
                // positive
                positive_font_color = Colors.White.to_uint(),
-               positive_background_color = Colors.Green.to_uint()
+               positive_background_color = Colors.Green.to_uint(),
             },
             session_low = _default_item.With(_default_item.font.With(font_color: BTSColors.MarketData.session_low)),
             session_high = _default_item.With(_default_item.font.With(font_color: BTSColors.MarketData.session_high)),
@@ -431,21 +431,9 @@ namespace mbts.spark {
             theo = _default_item,
             delta = _default_item,
             vega = _default_item,
-            volume = _default_item
+            volume = _default_item,
          };
 
-      public static LadderPanelPricePointSettings tt_default {
-         get {
-            var rv = historic_default;
-            rv.day_change.background_color = Color.FromRgb(113, 201, 69).to_uint();
-            rv.day_change.font.font_color = Colors.Black.to_uint();
-            return rv;
-         }
-      }
-
-      public static LadderPanelPricePointSettings Default => LadderSettings.mimic_tt ?
-         tt_default : historic_default;
-
       [ProtoMember(1)]
       public DaySettings day_change { get; set; }
       [ProtoMember(2)]
@@ -607,11 +595,6 @@ namespace mbts.spark {
       uint _ticks_per_row;
       List<FairValueLine> _fair_value_lines;
 
-      static LadderSettings() {
-         var knobs = config.Configuration.get<bts.config.LadderKnobs>();
-         mimic_tt = knobs.MimicTt;
-      }
-
       // Default proto constructor
       // Added this so that when clients upgrade to a version with new settings,
       // the default values are applied correctly; thx to John for this tip
@@ -643,9 +626,7 @@ namespace mbts.spark {
          status_message_visible = true;
       }
 
-      public static bool mimic_tt { get; }
-
-      public static LadderSettings historic_default {
+      public static LadderSettings Default {
          get {
             LadderColumnSettings
                orders_column = LadderColumnSettings.default_orders,
@@ -713,11 +694,6 @@ namespace mbts.spark {
          }
       }
 
-      public static LadderSettings tt_default => historic_default;
-
-      public static LadderSettings Default => mimic_tt ?
-         tt_default : historic_default;
-
       public enum BadgeStyle { Normal, Alternate }
 
       // TODO: rename to PriceDisplayFormat (today is merge monday, since this isn't a functional change, it is not worth the potential merge headaches right now)
@@ -1039,7 +1015,6 @@ namespace mbts.spark {
          a.ladder_order_limits.safe_equals(b.ladder_order_limits) &&
          a._ticks_per_row == b._ticks_per_row &&
          a.panel_price_point_settings.safe_equals(b.panel_price_point_settings) &&
-         a._ticks_per_row == b._ticks_per_row &&
          EqualityComparer.safe_sequence_equals(a.fair_value_lines, b.fair_value_lines);
 
       public bool Equals(LadderSettings other) =>

src/client/spark/Modules/Trading/Ladder/VM/LadderVM.cs

@@ -107,6 +107,7 @@ namespace mbts.spark {
       ImmutableDictionary<AlgoKey, LadderFavoriteAlgoMetadata> _favorite_algo_metadata_map =
          ImmutableDictionary<AlgoKey, LadderFavoriteAlgoMetadata>.Empty;
       RegionalTickFactors? _spec;
+      IReadOnlyDictionary<uint, SecurityMD> _securities = new Dictionary<uint, SecurityMD>(0);
 
       LadderVM(
          IBus bus,
@@ -329,7 +330,8 @@ namespace mbts.spark {
             }),
             WorkLoop.Start($"Ladder '{security}'", _tick),
             ObservableAutoWrapper.dispose_on_removed(orders, true),
-            Disposable.Create(() => _on_disposed_impl?.Invoke(true)));
+            Disposable.Create(() => _on_disposed_impl?.Invoke(true)),
+            bus.subscribe(SparkTags.SecuritiesTag).Subscribe(secs => _securities = secs));
 
          PropertyAutoWrapper<LadderSettings, bool> bool_prop(
             string name, Func<LadderSettings, bool> get_bool) =>
@@ -405,7 +407,9 @@ namespace mbts.spark {
       event Action<bool>? _on_disposed_impl;
 
       public IMDConsumerHandle md => _model.md;
+      public SecurityMD security_md => _model.security_md;
       public IBus bus => _bus;
+      public IReadOnlyDictionary<uint, SecurityMD> securities => _securities;
 
       // we return only entries who have piqs and whose piqs are not uint.Max
       // uint.Max happens when the server is restarted but the client is still up

src/client/spark/Modules/Trading/Ladder/View/LadderGridHost.cs

@@ -17,7 +17,6 @@ using Spark.Core.Disposables;
 using Spark.Core.Math;
 
 using System;
-using System.Collections;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using System.Collections.Specialized;
@@ -38,9 +37,6 @@ namespace mbts.spark {
    partial class LadderGridHost : Control {
       static readonly int _initial_row_count;
       static readonly bool _show_scrollbar;
-      static readonly bool _lighten_market_cells;
-      static readonly Color? _bid_market_cell_background;
-      static readonly Color? _ask_market_cell_background;
 
       readonly VisualHost _visual_host = new();
       readonly LadderOrdersHost _ordershost;
@@ -84,6 +80,7 @@ namespace mbts.spark {
       LadderTactileAdorner _tactile_adorner;
       ulong _prices_updated;
       NativePriceSize[] _bids = new NativePriceSize[NativePriceHelpers.COCONUT_MAX_DEPTH], _asks = new NativePriceSize[NativePriceHelpers.COCONUT_MAX_DEPTH];
+      SecurityMDExtensions.Book _depth = SecurityMDExtensions.Book.Empty;
       bool _price_dirty = false, _piq_dirty = false;
       LadderPriceList _price_list = new() {
          length = _initial_row_count,
@@ -96,6 +93,8 @@ namespace mbts.spark {
       Func<NativePriceSize, string> _fmt_mkt_depth_item_impl = MDDisplayHelper.get_formatter_nps(
          MarketSizeDisplay.get_default(), Security.Defaults.quantity_display_settingsi, Security.Defaults.price_display_settingsi);
       Func<double, double, string> _fmt_mkt_quantity_impl = (v, p) => v.ToString();
+      Func<LadderPriceLevel, string> _fmt_price_level_impl = MDDisplayHelper.get_formatter_lpl(
+         MarketSizeDisplay.get_default(), Security.Defaults.quantity_display_settingsi, Security.Defaults.price_display_settingsi);
       int _old_tooltip_row = int.MaxValue;
       NativePriceSize? _old_last_trade;
       string _direction_arrow = string.Empty;
@@ -107,17 +106,7 @@ namespace mbts.spark {
       static LadderGridHost() {
          var knobs = config.Configuration.get<bts.config.LadderKnobs>();
          _show_scrollbar = knobs.ShowScrollbar;
-         _lighten_market_cells = knobs.LightenMarketCells;
          _initial_row_count = (int)knobs.ScrollableTickWindowSize;
-         // For market cell backgrounds, 0 means null.  Note that the full
-         // range of colors can still be expressed, as zero is merely one of
-         // 2^24 possible representations of 'transparent.'
-         if (knobs.BidMarketCellBackground != 0) {
-            _bid_market_cell_background = BTSColors.Ladder.bid_column_background;
-         }
-         if (knobs.AskMarketCellBackground != 0) {
-            _ask_market_cell_background = BTSColors.Ladder.ask_column_background;
-         }
       }
 
       public LadderGridHost() {
@@ -872,13 +861,13 @@ namespace mbts.spark {
             switch (column.column_type) {
                case ColumnType.Order: _ordershost.set_col_width(column.settings.width); break;
                case ColumnType.Bid:
-                  render_market_col(_bids, _bid_market_cell_background);
+                  render_market_col(_depth.Bids);
                   break;
                case ColumnType.Price:
                   _refresh_prices(column);
                   break;
                case ColumnType.Ask:
-                  render_market_col(_asks, _ask_market_cell_background);
+                  render_market_col(_depth.Asks);
                   break;
                case ColumnType.Volume:
                   _volume_container.mark_dirty();
@@ -891,9 +880,9 @@ namespace mbts.spark {
                if (_mkt_break_lines.high_render_index.HasValue)
                   _render_session_line(_session_high_vis, _mkt_break_lines.high_render_index.Value, _settings.session_high_line.color);
             }
-            void render_market_col(NativePriceSize[] depth, Color? market_cell_background) {
+            void render_market_col(IEnumerable<(double Price, double Volume, double NotionalValue)> depth) {
                var mid = LadderRender.index_of(_price_list, _last_origin_price, _price_comparer);
-               LadderRender.render_sizes(column.visual_tup.visual, column.visual_tup.glyph_context, mid, _price_list, depth, _settings.row_height, _spec, column.settings, _fmt_mkt_depth_item, _lighten_market_cells, market_cell_background);
+               LadderRender.render_sizes(column.visual_tup.visual, column.visual_tup.glyph_context, mid, _price_list, depth, _settings.row_height, _spec, column.settings, _fmt_price_level);
             }
          }
          _apply_column_visual_offset();
@@ -939,6 +928,7 @@ namespace mbts.spark {
             // **use local vm to protect against re-assignment,
             // so the events are unhooked from the same instance to which they were added
             // when the subs are disposed.
+            Disposable.dispose(ref _vm_sub);
             var local_vm = _vm;
             local_vm.on_tick += _tick;
             local_vm.request_recenter += on_request_recenter;
@@ -1110,6 +1100,7 @@ namespace mbts.spark {
             ulong unused = 0;
             bool last_trade_changed = _vm.md.get_last_trade(ref unused, out _last_trade, out DateTime _, out ulong __);
             bool depth_changed = _vm.md.get_depth(ref _prices_updated, ref _bids, ref _asks, NativePriceHelpers.COCONUT_MAX_DEPTH);
+            _depth = _vm.security_md.Depth(_bids, _asks, _vm.securities);
             _mkt_break_lines.refresh();
             if (_price_list is null) {
                Log.assert(_price_dirty);
@@ -1322,8 +1313,8 @@ namespace mbts.spark {
       void _refresh_sizes() {
          Log.assert_d(_price_list != null);
          int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer);
-         LadderRender.render_sizes(_bid_vis_tup.visual, _bid_vis_tup.glyph_context, origin_index, _price_list, _bids, _settings.row_height, _spec, _settings.bid_column, _fmt_mkt_depth_item, _lighten_market_cells, _bid_market_cell_background);
-         LadderRender.render_sizes(_ask_vis_tup.visual, _ask_vis_tup.glyph_context, origin_index, _price_list, _asks, _settings.row_height, _spec, _settings.ask_column, _fmt_mkt_depth_item, _lighten_market_cells, _ask_market_cell_background);
+         LadderRender.render_sizes(_bid_vis_tup.visual, _bid_vis_tup.glyph_context, origin_index, _price_list, _depth.Bids, _settings.row_height, _spec, _settings.bid_column, _fmt_price_level);
+         LadderRender.render_sizes(_ask_vis_tup.visual, _ask_vis_tup.glyph_context, origin_index, _price_list, _depth.Asks, _settings.row_height, _spec, _settings.ask_column, _fmt_price_level);
       }
 
       /// <summary>
@@ -1506,11 +1497,13 @@ namespace mbts.spark {
          static GlyphContext glyph_context(LadderColumnSettings cs) =>
             GlyphContext.create(cs.font_family, cs.font_weight);
 
-         var price_display = _spec.tick_factors(0).to_api_tick_display_settingsi();
+         var price_display = security.regional_ticks().tick_factors(0).to_api_tick_display_settingsi();
          _fmt_mkt_depth_item_impl = MDDisplayHelper.get_formatter_nps(
             _settings.size_display, _qty_display, price_display);
          _fmt_mkt_quantity_impl = MDDisplayHelper.get_value_formatter(
             _settings.size_display, _qty_display, price_display);
+         _fmt_price_level_impl = MDDisplayHelper.get_formatter_lpl(
+            _settings.size_display, _qty_display, price_display);
          _set_columns();
          _apply_column_visual_offset();
          if (prev_row_height != _settings.row_height)
@@ -1520,18 +1513,16 @@ namespace mbts.spark {
          _mkt_break_lines.fair_value_line_views.Clear();
          // Clear and re-add the columns to the viewer to ensure offset/visibility is correct
          _visual_host.Clear();
-         if (LadderSettings.mimic_tt) {
-            // The session_range_vis sits beneath the grid backgrounds (on the
-            // z-axis) to avoid obfuscating the font color (dj wants it to be
-            // pure white).  Historically, it sat atop the price column, with a
-            // low opacity to slightly darken the range of prices inside of the
-            // session range (i.e., the price-band of the day's trades).  In
-            // conjuction with this change, the price column's background
-            // opacity is made slightly less than 1 (in the default settings),
-            // while the font color remains fully opaque, so this layer will
-            // darken the background without affecting the font color.
-            _visual_host.AddVisualChild(_session_range_vis);
-         }
+         // The session_range_vis sits beneath the grid backgrounds (on the
+         // z-axis) to avoid obfuscating the font color (dj wants it to be
+         // pure white).  Historically, it sat atop the price column, with a
+         // low opacity to slightly darken the range of prices inside of the
+         // session range (i.e., the price-band of the day's trades).  In
+         // conjuction with this change, the price column's background
+         // opacity is made slightly less than 1 (in the default settings),
+         // while the font color remains fully opaque, so this layer will
+         // darken the background without affecting the font color.
+         _visual_host.AddVisualChild(_session_range_vis);
          // Renders the grid lines and column backgrounds
          _visual_host.AddVisualChild(_gridvis);
          // Orders column
@@ -1571,8 +1562,6 @@ namespace mbts.spark {
             with_vis => _visual_host.RemoveVisualChild(with_vis.vis));
          _visual_host.AddVisualChild(_session_low_vis);
          _visual_host.AddVisualChild(_session_high_vis);
-         if (!LadderSettings.mimic_tt)
-            _visual_host.AddVisualChild(_session_range_vis);
          _render_grid_background();
          _init_tactile_adorner();
          // this effectively sets the background of the "Last" column //deleted to fix color
@@ -1594,6 +1583,16 @@ namespace mbts.spark {
          return impl(value);
       }
 
+      // This is designed to be a redundant null check for _fmt_mkt_depth_item_impl.
+      string _fmt_price_level(LadderPriceLevel value) {
+         // Assign the impl delegate to a local variable, just in case it is reassigned on another thread.
+         // This shouldn't happen, but better to avoid the potential null ref just in case it does. Expect the unexpected.
+         var impl = _fmt_price_level_impl;
+         impl ??= MDDisplayHelper.get_formatter_lpl(
+            MarketSizeDisplay.get_default(), _qty_display, security.regional_ticks().tick_factors(0).to_api_tick_display_settingsi());
+         return impl(value);
+      }
+
       // This is designed to be a redundant null check for _fmt_mkt_quantity_impl.
       string _fmt_mkt_quantity(double value, double price) {
          // Assign the impl delegate to a local variable, just in case it is reassigned on another thread.

src/client/spark/Modules/Trading/Ladder/View/LadderPriceLevel.cs

@@ -0,0 +1,98 @@
+namespace mbts.spark;
+
+using bts;
+using bts.utils;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+
+readonly struct LadderPriceLevel : IEquatable<LadderPriceLevel> {
+   public LadderPriceLevel(double price = double.NaN, double volume = 0, double notionalValue = double.NaN) {
+      Price = price;
+      Volume = volume;
+      NotionalValue = notionalValue.is_nan() ? volume * price : notionalValue;
+   }
+
+   public static LadderPriceLevel Empty => new();
+
+   public double Price { get; }
+   public double Volume { get; }
+   public double NotionalValue { get; }
+
+   public LadderPriceLevel Add(double price, double volume) {
+      if (!(volume > 0)) {
+         return this;
+      }
+      if (!(Volume > 0)) {
+         return new(price, volume);
+      }
+      Log.assert(Price.is_real());
+      Log.assert(price.is_real());
+      var newVolume = Volume + volume;
+      var newPrice = (Price * Volume + price * volume) / newVolume;
+      return new(newPrice, newVolume);
+   }
+
+   public LadderPriceLevel Add(IEnumerable<(double Price, double Volume)> items) {
+      var (totalPrice, totalVolume) = items.Sum();
+      return totalVolume > 0
+         ? Add(totalPrice / totalVolume, totalVolume)
+         : this;
+   }
+
+   public Builder ToBuilder() =>
+      new() { Cost = Price, Volume = Volume };
+
+   public LadderPriceLevel With(double price = double.NaN, double volume = double.NaN, double notionalValue = double.NaN) =>
+      new(price.if_nan(Price), volume.if_nan(Volume), notionalValue.if_nan(NotionalValue));
+
+   public bool Equals(LadderPriceLevel other) =>
+      MathUtils.epsilon_eq_comparer.Equals(Price, other.Price)
+      && MathUtils.epsilon_eq_comparer.Equals(Volume, other.Volume);
+
+   public override bool Equals([NotNullWhen(true)] object obj) =>
+      obj is LadderPriceLevel other && Equals(other);
+
+   public override int GetHashCode() =>
+      HashCode.Combine(
+         MathUtils.epsilon_eq_comparer.GetHashCode(Price),
+         MathUtils.epsilon_eq_comparer.GetHashCode(Volume));
+
+   public class Builder {
+      double notionalValue = double.NaN;
+
+      public double Cost { get; set; } = double.NaN;
+      public double Volume { get; set; } = 0;
+      public double NotionalValue => notionalValue.is_nan() ? Volume * Cost : notionalValue;
+
+      public void Add(double price, double volume) {
+         if (!(volume > 0)) {
+            return;
+         }
+         Log.assert(price.is_real());
+         Cost += price * volume;
+         Volume += volume;
+      }
+
+      public void Add(double price, double volume, double notionalValue) {
+         if (!(volume > 0)) {
+            return;
+         }
+         Log.assert(price.is_real());
+         Cost += price * volume;
+         Volume += volume;
+         if (this.notionalValue.is_nan()) {
+            this.notionalValue = notionalValue;
+         } else{
+            this.notionalValue += notionalValue;
+         }
+      }
+
+      public LadderPriceLevel ToImmutable() =>
+         Volume > 0
+            ? new(Cost / Volume, Volume, NotionalValue)
+            : Empty;
+   }
+}

src/client/spark/Modules/Trading/Ladder/View/LadderRender.cs

@@ -57,8 +57,7 @@ namespace mbts.spark {
    static class LadderRender {
       // Note: the orders-column has a separate "row_padding" value: Badge.DefaultMargin
       // (see how it is incorporated in measure_row_height() below)
-      public static Thickness row_padding => LadderSettings.mimic_tt ?
-         new Thickness(4, 2, 4, 2) : new Thickness(6);
+      public static Thickness row_padding => new Thickness(4, 2, 4, 2);
 
       const string tick_separator_period = ".";
       const string tick_separator_dash = "-";
@@ -214,70 +213,78 @@ namespace mbts.spark {
          }
       }
 
+      public static void RenderSizes(
+         DrawingVisual dv,
+         GlyphContext glyphContext,
+         int originIndex,
+         IList<double> prices,
+         IEnumerable<LadderPriceLevel> levels,
+         double rowHeight,
+         RegionalTickFactors spec,
+         LadderColumnSettings settings,
+         Func<LadderPriceLevel, string> fmt) {
+         var bg = settings.background.rgb_color().blend((BTSColor)0x2AFFFFFF).to_brush();
+         using (var dc = dv.RenderOpen())
+         using (_draw_with_clip(dc, settings, rowHeight, prices.Count))
+         using (var levelsEn = (levels ?? Enumerable.Empty<LadderPriceLevel>()).GetEnumerator()) {
+            var fg = settings.foreground.rgb_brush();
+            var currIdx = originIndex;
+            var currPrice = prices[currIdx];
+            while (levelsEn.MoveNext()) {
+               var level = levelsEn.Current;
+               var ticks = spec.count_ticks(level.Price);
+               var price = ticks.HasValue ? spec.tickup(0, ticks.Value) : level.Price;
+               // prices are in descending order, so get diff from curr_price - price
+               var tickDiff = spec.tickdiff(currPrice, price).assert_value();
+               if (!MathUtils.try_increment(ref currIdx, tickDiff) || currIdx < 0 || currIdx >= prices.Count) {
+                  continue;
+               }
+               currPrice = price;
+               if (bg is not null) {
+                  var y = currIdx * rowHeight + GRID_LINE_HEIGHT;
+                  var height = rowHeight - GRID_LINE_HEIGHT;
+                  dc.DrawRectangle(bg, null, new Rect(0, y, settings.width, height));
+               }
+               var gr = format_text(fmt(level), currIdx, rowHeight, settings, glyphContext);
+               dc.DrawGlyphRun(fg, gr);
+            }
+         }
+      }
+
+      public static IEnumerable<LadderPriceLevel> GetLevels(IEnumerable<(double Price, double Volume, double NotionalValue)> depth, RegionalTickFactors spec) =>
+          depth.TakeWhile(nps => nps.Volume > 0 && nps.Price.is_real())
+            .AggregateConsecutiveGroups(
+               LadderPriceLevel.Empty.ToBuilder,
+               (bd, rung) => {
+                  bd.Add(rung.Price, rung.Volume, rung.NotionalValue);
+                  return bd;
+               },
+               rung => {
+                  var ticks = spec.count_ticks(rung.Price);
+                  return ticks.HasValue ? spec.tickup(0, ticks.Value) : rung.Price;
+               })
+            .Select(bd => bd.ToImmutable());
+
       public static void render_sizes(
          DrawingVisual dv,
          GlyphContext glyph_context,
          int origin_index,
          IList<double> prices,
-         NativePriceSize[] rungs,
+         IEnumerable<(double Price, double Volume, double NotionalValue)> rungs,
          double row_height,
          RegionalTickFactors spec,
          LadderColumnSettings settings,
-         Func<NativePriceSize, string> fmt,
-         bool lighten_market_cells,
-         Color? market_cell_background) {
-         //var background = new SolidColorBrush(settings.background.rgb_color());
-         Brush market_cell_background_brush;
-         if (market_cell_background.HasValue) {
-            market_cell_background_brush = new SolidColorBrush(market_cell_background.Value);
-         } else if (lighten_market_cells) {
-            market_cell_background_brush = settings.background.rgb_color()
-            .blend((BTSColor)0x2AFFFFFF)
-            .to_brush();
-         } else {
-            market_cell_background_brush = null;
-         }
-         // market_cell_background_brush = settings.background.rgb_color().blend((BTSColor)0xFF000000).to_brush();
-         using (var dc = dv.RenderOpen())
-         using (_draw_with_clip(dc, settings, row_height, prices.Count)) {
-            //dc.DrawRectangle(background, null, new Rect(0, 0, settings.width, prices.Count * row_height));
-            if (rungs.is_null_or_empty()) {
-               return;
-            }
-            var foreground = settings.foreground.rgb_brush();
-            int curr_index = origin_index;
-            double curr_price = prices[curr_index];
-            NativePriceSize curr_size = NativePriceSize.NO_MARKET;
-            var row_tick_groups = rungs
-               .TakeWhile(nps => !nps.is_nan)
-               .AggregateConsecutiveGroups(() => NativePriceSize.NO_MARKET,
-                  (aggr, rung) => rung.with(
-                     direct_size: aggr.direct_size + rung.direct_size,
-                     implied_size: aggr.implied_size + rung.implied_size),
-                  rung => {
-                     var ticks = spec.count_ticks(rung.price);
-                     return ticks.HasValue ? spec.tickup(0, ticks.Value) : rung.price;
-                  });
-            foreach (var rung in row_tick_groups) {
-               //double price = spec.round_to_tick(rung.price);
-               var ticks = spec.count_ticks(rung.price);
-               var price = ticks.HasValue ? spec.tickup(0, ticks.Value) : rung.price;
-               // prices are in descending order, so get diff from curr_price - price
-               var tick_diff = spec.tickdiff(curr_price, price).assert_value();
-               if (!MathUtils.try_increment(ref curr_index, tick_diff)) {
-                  continue;
-               }
-               curr_price = price;
-               if (curr_index >= 0 && curr_index < prices.Count) {
-                  if (market_cell_background_brush != null)
-                     dc.DrawRectangle(market_cell_background_brush, null,
-                        new Rect(0, curr_index * row_height + GRID_LINE_HEIGHT,
-                           settings.width, row_height - GRID_LINE_HEIGHT));
-                  var gr = format_text(fmt(rung), curr_index, row_height, settings, glyph_context);
-                  dc.DrawGlyphRun(foreground, gr);
-               } else break;
-            }
-         }
+         Func<LadderPriceLevel, string> fmt) {
+         RenderSizes(
+            dv,
+            glyph_context,
+            origin_index,
+            prices,
+            GetLevels(rungs, spec),
+            row_height,
+            spec,
+            settings,
+            fmt);
       }
 
       public static void render_volumes(

src/client/spark/Modules/Trading/Ladder/View/LadderView.xaml

@@ -182,7 +182,7 @@
       </Style>
 
       <Style x:Key="CancelAll" TargetType="Button" BasedOn="{StaticResource LadderMenuButton}">
-        <Setter Property="Grid.Row" Value="2" />
+        <Setter Property="Grid.Row" Value="0"/>
         <Setter Property="Content">
           <Setter.Value>
             <TextBlock VerticalAlignment="Center">Cancel All</TextBlock>
@@ -220,10 +220,7 @@
           </Setter.Value>
         </Setter>
       </Style>
-      <Style x:Key="TT_CancelAll" TargetType="Button" BasedOn="{StaticResource CancelAll}">
-        <Setter Property="Grid.Row" Value="0"/>
-      </Style>
-      <Style x:Key="TT_CancelBuys" TargetType="Button" BasedOn="{StaticResource CancelAll}">
+      <Style x:Key="CancelBuys" TargetType="Button" BasedOn="{StaticResource CancelAll}">
         <Setter Property="Grid.Row" Value="1" />
         <Setter Property="Content">
           <Setter.Value>
@@ -231,7 +228,7 @@
           </Setter.Value>
         </Setter>
       </Style>
-      <Style x:Key="TT_CancelSells" TargetType="Button" BasedOn="{StaticResource CancelAll}">
+      <Style x:Key="CancelSells" TargetType="Button" BasedOn="{StaticResource CancelAll}">
         <Setter Property="Grid.Row" Value="2" />
         <Setter Property="Content">
           <Setter.Value>
@@ -239,83 +236,6 @@
           </Setter.Value>
         </Setter>
       </Style>
-      <Style x:Key="CancelBuys" TargetType="Button" BasedOn="{StaticResource LadderMenuButton}">
-        <Setter Property="Grid.Row" Value="0" />
-        <Setter Property="Content">
-          <Setter.Value>
-            <TextBlock VerticalAlignment="Center" Foreground="{Binding bid_foreground}">
-              Cancel Buys
-            </TextBlock>
-          </Setter.Value>
-        </Setter>
-        <Setter Property="Template">
-          <Setter.Value>
-            <ControlTemplate TargetType="Button">
-              <Border
-                x:Name="Border"
-                BorderThickness="{TemplateBinding BorderThickness}"
-                BorderBrush="{u:BTSColor UI.Button.normal_border}"
-                Background="{Binding bid_background}"
-                Padding="{TemplateBinding Padding}"
-                SnapsToDevicePixels="True">
-                <ContentPresenter
-                  HorizontalAlignment="Center"
-                  VerticalAlignment="Center"
-                  RecognizesAccessKey="True"
-                  Name="button_content" />
-              </Border>
-              <ControlTemplate.Triggers>
-                <Trigger Property="IsMouseOver" Value="True">
-                  <Setter TargetName="Border" Property="Background" Value="#99167ed3" />
-                  <Setter
-                    TargetName="Border"
-                    Property="BorderBrush"
-                    Value="{u:BTSColor UI.Button.mouseover_border}" />
-                </Trigger>
-              </ControlTemplate.Triggers>
-            </ControlTemplate>
-          </Setter.Value>
-        </Setter>
-      </Style>
-      <Style x:Key="CancelSells" TargetType="Button" BasedOn="{StaticResource LadderMenuButton}">
-        <Setter Property="Background" Value="{u:BTSColor MarketData.ask}" />
-        <Setter Property="Grid.Row" Value="1" />
-        <Setter Property="Content">
-          <Setter.Value>
-            <TextBlock VerticalAlignment="Center" Foreground="{Binding ask_foreground}">
-              Cancel Sells
-            </TextBlock>
-          </Setter.Value>
-        </Setter>
-        <Setter Property="Template">
-          <Setter.Value>
-            <ControlTemplate TargetType="Button">
-              <Border
-                x:Name="Border"
-                BorderThickness="{TemplateBinding BorderThickness}"
-                BorderBrush="{u:BTSColor UI.Button.normal_border}"
-                Background="{Binding ask_background}"
-                Padding="{TemplateBinding Padding}"
-                SnapsToDevicePixels="True" >
-                <ContentPresenter
-                  HorizontalAlignment="Center"
-                  VerticalAlignment="Center"
-                  RecognizesAccessKey="True"
-                  Name="button_content" />
-              </Border>
-              <ControlTemplate.Triggers>
-                <Trigger Property="IsMouseOver" Value="True">
-                  <Setter TargetName="Border" Property="Background" Value="#99c00000" />
-                  <Setter
-                    TargetName="Border"
-                    Property="BorderBrush"
-                    Value="{u:BTSColor UI.Button.mouseover_border}" />
-                </Trigger>
-              </ControlTemplate.Triggers>
-            </ControlTemplate>
-          </Setter.Value>
-        </Setter>
-      </Style>
       <Style x:Key="SettingsButton" TargetType="Button" BasedOn="{StaticResource LadderMenuButton}">
         <Setter Property="Content">
           <Setter.Value>
@@ -1177,19 +1097,19 @@
               <Button
                 Command="{Binding cancel_cmd}"
                 CommandParameter="Bids"
-                Style="{StaticResource {x:Static sp:LadderView.CancelBuysButtonStyleKey}}"
+                Style="{StaticResource CancelBuys}"
                 Visibility="{Binding
                   show_cancel_buys, Converter={StaticResource BooleanToVisibilityConverter}}" />
               <Button
                 Command="{Binding cancel_cmd}"
                 CommandParameter="Asks"
-                Style="{StaticResource {x:Static sp:LadderView.CancelSellsButtonStyleKey}}"
+                Style="{StaticResource CancelSells}"
                 Visibility="{Binding
                   show_cancel_sells, Converter={StaticResource BooleanToVisibilityConverter}}" />
               <Button
                 Command="{Binding cancel_cmd}"
                 CommandParameter="All"
-                Style="{StaticResource {x:Static sp:LadderView.CancelAllButtonStyleKey}}"
+                Style="{StaticResource CancelAll}"
                 Visibility="{Binding
                   show_cancel_all, Converter={StaticResource BooleanToVisibilityConverter}}" />
             </Grid>

src/client/spark/Modules/Trading/Ladder/View/LadderView.xaml.cs

@@ -27,22 +27,6 @@ namespace mbts.spark {
          MinZoomFactor = 0.5,
          MaxZoomFactor = 1.5;
 
-      static LadderView() {
-         var knobs = config.Configuration.get<bts.config.LadderKnobs>();
-         MimicTT = knobs.MimicTt;
-      }
-
-      public static bool MimicTT { get; }
-
-      public static string CancelBuysButtonStyleKey =>
-         MimicTT ? "TT_CancelBuys" : "CancelBuys";
-
-      public static string CancelSellsButtonStyleKey =>
-         MimicTT ? "TT_CancelSells" : "CancelSells";
-
-      public static string CancelAllButtonStyleKey =>
-         MimicTT ? "TT_CancelAll" : "CancelAll";
-
       public static IMultiValueConverter PositionToolTipConverter =
          ValueConverter.create<AccountTuple?, bool, bool, bool, string>((account, show_day_pos, show_raw_day_pos, show_total_pos) => {
             const string day_pos_text = "day", raw_day_pos_text = "raw day", total_pos_text = "total";

src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs

@@ -1,4 +1,4 @@
-using bts;
+using bts;
 using bts.api;
 using bts.api.core;
 using bts.api.orderman;
@@ -223,6 +223,10 @@ namespace mbts.spark {
             Add(qty_column = new SizeColumn(
                new DoubleMinMax<OrderRecordTupleWithRoot>(
                   ort => ort.order.order.quantity(), nameof(OrderBookRowVM.quantity))));
+            // notional value
+            Add(notional_value_column = new NotionalValueColumn(
+               new DoubleMinMax<OrderRecordTupleWithRoot>(
+                  ort => ort.notional_value, nameof(OrderBookRowVM.notional_value))));
             // edit quantity column
             // not usually part of the main order book, but it's useful to add it
             // rather than debugging the popup window
@@ -232,6 +236,10 @@ namespace mbts.spark {
                new DoubleMinMax<OrderRecordTupleWithRoot>(
                   ort => ort.order.order.filled_quantity(),
                   nameof(OrderBookRowVM.filled_quantity))));
+            // filled notional value
+            Add(filled_notional_value_column = new FilledNotionalValueColumn(
+               new DoubleMinMax<OrderRecordTupleWithRoot>(
+                  ort => ort.filled_notional_value, nameof(OrderBookRowVM.filled_notional_value))));
             // price
             Add(price_column = new SimpleTextColumn<DoubleMinMaxState>(
                Columns.Price,
@@ -363,9 +371,11 @@ namespace mbts.spark {
          public SimpleTextColumn<OrderSubType[]> sub_type_column { get; }
          public SideColumn side_column { get; }
          public SizeColumn qty_column { get; }
+         public NotionalValueColumn notional_value_column { get; }
          public EditableQtyColumn edit_qty_column { get; }
          public SimpleTextColumn<DoubleMinMaxState> price_column { get; }
          public FilledColumn filled_qty_column { get; }
+         public FilledNotionalValueColumn filled_notional_value_column { get; }
          public SimpleTextColumn<DoubleMinMaxState> last_price_column { get; }
          public SimpleTextColumn<DoubleMinMaxState> avg_price_column { get; }
          public SimpleTextColumn<OrderStatus[]> status_column { get; }
@@ -391,8 +401,10 @@ namespace mbts.spark {
                order_sub_types = sub_type_column.filter.get_state(),
                sides = side_column.filter.get_state(),
                qty = qty_column.filter.get_state(),
+               notional_value = notional_value_column.filter.get_state(),
                price = price_column.filter.get_state(),
                qty_filled = filled_qty_column.filter.get_state(),
+               filled_notional_value = filled_notional_value_column.filter.get_state(),
                avg_price = avg_price_column.filter.get_state(),
                last_price = last_price_column.filter.get_state(),
                statuses = status_column.filter.get_state(),
@@ -415,7 +427,9 @@ namespace mbts.spark {
                account_column.filter.load_state(fs.accounts);
                price_column.filter.load_state(fs.price);
                qty_column.filter.load_state(fs.qty);
+               notional_value_column.filter.load_state(fs.notional_value);
                filled_qty_column.filter.load_state(fs.qty_filled);
+               filled_notional_value_column.filter.load_state(fs.filled_notional_value);
                side_column.filter.load_state(fs.sides);
                order_type_column.filter.load_state(fs.order_types);
                root_algo_type_column.filter.load_state(fs.root_algo_types);
@@ -645,6 +659,32 @@ namespace mbts.spark {
                DefaultCellPadding);
       }
 
+      public class NotionalValueColumn : OrderbookColumnBase<DoubleMinMaxState>, IRenderColumn<OrderBookRowVM> {
+         public NotionalValueColumn(IDataGridColumnPersistentFilter<OrderRecordTupleWithRoot, DoubleMinMaxState> filter)
+            : base(
+                 Columns.NotionalValue,
+                 DefaultColumnHeaders.NotionalValue,
+                 r => r.notional_value_text,
+                 DefaultColumnWidths.NotionalValue,
+                 filter,
+                 DataGridFilterKnownTemplates.DoubleMinMax,
+                 _wrap_comparer(r => r.notional_value)) { }
+
+         public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) =>
+            CellRender.draw_text(
+               dc,
+               _get_glyph_context(row),
+               font_size,
+               row.notional_value_text,
+               position,
+               row_height,
+               RenderUtils.get_side_brush(row),
+               background_brush,
+               horizontal_alignment,
+               VerticalAlignment.Center,
+               DefaultCellPadding);
+      }
+
       public class FilledColumn : OrderbookColumnBase<DoubleMinMaxState>, IRenderColumn<OrderBookRowVM> {
          public FilledColumn(IDataGridColumnPersistentFilter<OrderRecordTupleWithRoot, DoubleMinMaxState> filter)
             : base(Columns.Filled, "Filled Qty", r => r.display_filled_quantity.ToString(), DefaultColumnWidths.Qty, filter, DataGridFilterKnownTemplates.DoubleMinMax, _wrap_comparer(r => r.filled_quantity)) { }
@@ -664,6 +704,32 @@ namespace mbts.spark {
                DefaultCellPadding);
       }
 
+      public class FilledNotionalValueColumn : OrderbookColumnBase<DoubleMinMaxState>, IRenderColumn<OrderBookRowVM> {
+         public FilledNotionalValueColumn(IDataGridColumnPersistentFilter<OrderRecordTupleWithRoot, DoubleMinMaxState> filter)
+            : base(
+                 Columns.FilledNotionalValue,
+                 DefaultColumnHeaders.FilledNotionalValue,
+                 r => r.filled_notional_value_text,
+                 DefaultColumnWidths.FilledNotionalValue,
+                 filter,
+                 DataGridFilterKnownTemplates.DoubleMinMax,
+                 _wrap_comparer(r => r.filled_notional_value)) { }
+
+         public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) =>
+            CellRender.draw_text(
+               dc,
+               _get_glyph_context(row),
+               font_size,
+               row.filled_notional_value_text,
+               position,
+               row_height,
+               RenderUtils.get_side_brush(row),
+               background_brush,
+               horizontal_alignment,
+               VerticalAlignment.Center,
+               DefaultCellPadding);
+      }
+
       public class OrderBookOrderTypeColumn : OrderTypeColumn<Columns>, ICVOrderBookColumn {
          readonly BindingBase _foreground_binding;
          ListSortDirection? _cached_sort_direction;

src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs

@@ -1,4 +1,4 @@
-using bts;
+using bts;
 using bts.api;
 using bts.api.orderman;
 using bts.collections;
@@ -108,6 +108,8 @@ namespace mbts.spark {
       public double display_quantity => quantity.safe_divide(_display_qty_divisor);
       public string display_quantity_text =>
          quantity_display_settings.to_tick_display_settings().format(display_quantity);
+      public double notional_value => order_tuple.notional_value;
+      public string notional_value_text => _fmt_price(notional_value);
       public TickDisplaySettingsImmutable quantity_display_settings {
          get => _quantity_display_settings;
          set {
@@ -128,6 +130,8 @@ namespace mbts.spark {
          filled_quantity.safe_divide(_display_qty_divisor);
       public string display_filled_quantity_text =>
          quantity_display_settings.to_tick_display_settings().format(display_filled_quantity);
+      public double filled_notional_value => order_tuple.filled_notional_value;
+      public string filled_notional_value_text => _fmt_price(filled_notional_value);
       public string last_msg => order.last_msg();
       // piq
       public uint piq => order_tuple.order.piq ?? uint.MaxValue;
@@ -355,9 +359,15 @@ namespace mbts.spark {
             if (_prospective_qty_translator_backing != null)
                _prospective_qty_translator_backing.actual_value = quantity;
          }
+         if ((Columns.NotionalValue & dirty) == Columns.NotionalValue) {
+            RaisePropertyChanged(nameof(notional_value), nameof(notional_value_text));
+         }
          if ((Columns.Filled & dirty) != 0) {
             RaisePropertyChanged(nameof(filled_quantity), nameof(display_filled_quantity));
          }
+         if ((Columns.FilledNotionalValue & dirty) == Columns.FilledNotionalValue) {
+            RaisePropertyChanged(nameof(filled_notional_value), nameof(filled_notional_value_text));
+         }
          // badge fill
          if (old_order_tuple.order.meta?.badge_color != order_tuple.order.meta?.badge_color) {
             // invalidate by setting to null (will be re-instantiated in the public getter)
@@ -521,6 +531,12 @@ namespace mbts.spark {
             if (oldval.order.piq != newval.order.piq) {
                dirty |= Columns.Piq;
             }
+            if (oldval.notional_value != newval.notional_value) {
+               dirty |= Columns.NotionalValue;
+            }
+            if (oldval.filled_notional_value != newval.filled_notional_value) {
+               dirty |= Columns.FilledNotionalValue;
+            }
             return dirty;
          }
       }

src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookFilterStates.cs

@@ -1,160 +1,174 @@
-using bts.api;
-using bts.api.orderman;
-using bts.utils;
-
-using mbts;
-using mbts.spark;
-using mbts.spark.filters;
-using mbts.ui;
-using mbts.ui.utility;
-
-using ProtoBuf;
-
-using Spark.Api.Orders;
-
-using System;
-
-namespace bts.shell.orderbook {
-   [ProtoContract]
-   class FilterStates : IEquatable<FilterStates> {
-      [ProtoMember(1)]
-      public SecurityCriteria security_criteria;
-      [ProtoMember(2)]
-      public string order_key_regex;
-      [ProtoMember(3)]
-      public TimeFilterTuple time_created;
-      [ProtoMember(4)]
-      public string[] users;
-      [ProtoMember(5)]
-      public string[] accounts;
-      [ProtoMember(6)]
-      public Venue[] exchanges;
-      [ProtoMember(7)]
-      public TimeInForce[] tifs;
-      //[ProtoMember(8)]
-      //public CustomOrderDefinition.CustomOneofCase[] order_types;
-      [ProtoMember(23)]
-      public AlgoID[] order_types;
-      [ProtoMember(24)]
-      public AlgoID[] root_algo_types;
-      [ProtoMember(9)]
-      public Side[] sides;
-      [ProtoMember(10)]
-      public DoubleMinMaxState qty;
-      [ProtoMember(11)]
-      public DoubleMinMaxState price;
-      [ProtoMember(12)]
-      public DoubleMinMaxState qty_filled;
-      [ProtoMember(13)]
-      public DoubleMinMaxState avg_price;
-      [ProtoMember(14)]
-      public DoubleMinMaxState last_price;
-      [ProtoMember(15)]
-      public OrderStatus[] statuses;
-      [ProtoMember(16)]
-      public string exchange_message_regex;
-      [ProtoMember(17)]
-      public TimeFilterTuple last_change;
-      //[ProtoMember(18)]
-      //public string security_regex;
-      [ProtoMember(19)]
-      public api.orderman.OrderSubType[] order_sub_types;
-      [ProtoMember(20)]
-      public string[] symbols;
-      [ProtoMember(21)]
-      public string parent_key_regex;
-      [ProtoMember(22)]
-      public string attached_target_order_key_regex;
-      [ProtoMember(25)]
-      public UIntMinMaxState piq;
-
-      public bool Equals(FilterStates other) =>
-         other != null &&
-         security_criteria.safe_equals(other.security_criteria) &&
-         order_key_regex == other.order_key_regex &&
-         parent_key_regex == other.parent_key_regex &&
-         attached_target_order_key_regex == other.attached_target_order_key_regex &&
-         time_created.Equals(other.time_created) &&
-         EqualityComparer.safe_sequence_equals(users, other.users) &&
-         EqualityComparer.safe_sequence_equals(accounts, other.accounts) &&
-         EqualityComparer.safe_sequence_equals(exchanges, other.exchanges) &&
-         EqualityComparer.safe_sequence_equals(tifs, other.tifs) &&
-         EqualityComparer.safe_sequence_equals(order_types, other.order_types) &&
-         EqualityComparer.safe_sequence_equals(root_algo_types, other.root_algo_types) &&
-         EqualityComparer.safe_sequence_equals(sides, other.sides) &&
-         qty.safe_equals(other.qty) &&
-         price.safe_equals(other.price) &&
-         qty_filled.safe_equals(other.qty_filled) &&
-         avg_price.safe_equals(other.avg_price) &&
-         last_price.safe_equals(other.last_price) &&
-         EqualityComparer.safe_sequence_equals(statuses, other.statuses) &&
-         exchange_message_regex == other.exchange_message_regex &&
-         last_change.Equals(other.last_change) &&
-         EqualityComparer.safe_sequence_equals(order_sub_types, other.order_sub_types) &&
-         EqualityComparer.safe_sequence_equals(symbols, other.symbols) &&
-         piq.safe_equals(other.piq);
-
-      public override bool Equals(object obj) => Equals(obj as FilterStates);
-
-      public override int GetHashCode() =>
-         EqualityComparer.aggregate_hash(
-            security_criteria?.GetHashCode() ?? 0,
-            order_key_regex?.GetHashCode() ?? 0,
-            parent_key_regex?.GetHashCode() ?? 0,
-            attached_target_order_key_regex?.GetHashCode() ?? 0,
-            time_created.GetHashCode(),
-            EqualityComparer.safe_aggregate_hash(users),
-            EqualityComparer.safe_aggregate_hash(accounts),
-            EqualityComparer.safe_aggregate_hash(exchanges),
-            EqualityComparer.safe_aggregate_hash(tifs),
-            EqualityComparer.safe_aggregate_hash(order_types),
-            EqualityComparer.safe_aggregate_hash(root_algo_types),
-            EqualityComparer.safe_aggregate_hash(sides),
-            qty?.GetHashCode() ?? 0,
-            price?.GetHashCode() ?? 0,
-            qty_filled?.GetHashCode() ?? 0,
-            avg_price?.GetHashCode() ?? 0,
-            last_price?.GetHashCode() ?? 0,
-            EqualityComparer.safe_aggregate_hash(statuses),
-            exchange_message_regex?.GetHashCode() ?? 0,
-            last_change.GetHashCode(),
-            EqualityComparer.safe_aggregate_hash(order_sub_types),
-            EqualityComparer.safe_aggregate_hash(symbols),
-            piq?.GetHashCode() ?? 0);
-   }
-
-   static class FilterStatesExtensions {
-      public static Func<OrderRecordTupleWithRoot, bool> build_predicate(this FilterStates fs) {
-         var pb = new ColumnPredicateBuilder<OrderRecordTupleWithRoot>();
-         // security
-         pb.add_security_criteria(fs.security_criteria, o => o.order.security);
-         // regex
-         pb.add_regex(fs.order_key_regex, o => o.order.order.Key.ToString());
-         pb.add_regex(fs.exchange_message_regex, o => o.order.order.State.LastMsg);
-         pb.add_regex(fs.attached_target_order_key_regex, o => o.order.order.attached_target_order_key().ToString());
-         pb.add_regex(fs.parent_key_regex, o => o.order.order.ParentKey.ToString());
-         // time
-         pb.add_time_filter(fs.time_created, o => o.order.order.created());
-         pb.add_time_filter(fs.last_change, o => o.order.order.last_change());
-         // discrete values
-         pb.add_discrete_values(fs.users, o => o.order.order.user());
-         pb.add_discrete_values(fs.accounts, o => o.order.order.account());
-         pb.add_discrete_values(fs.exchanges, o => o.order.security?.Exchange ?? Venue.Unset);
-         pb.add_discrete_values(fs.tifs, o => o.order.order.time_in_force());
-         pb.add_discrete_values(fs.order_types, o => AlgoID.from_custom_def(o.order.order.Def.CustomDef));
-         pb.add_discrete_values(fs.root_algo_types, o => AlgoID.from_algo_key(o.root.algo?.key ?? default));
-         pb.add_discrete_values(fs.order_sub_types, o => o.order.order.sub_type());
-         pb.add_discrete_values(fs.sides, o => o.order.order.side());
-         pb.add_discrete_values(fs.statuses, o => o.order.order.status());
-         pb.add_discrete_values(fs.symbols, o => o.order.security?.Symbol);
-         // numeric
-         pb.add_numeric(fs.qty, o => o.order.order.quantity());
-         pb.add_price(fs.price, o => o.order.order.ui_price(), o => o.order.security);
-         pb.add_numeric(fs.qty_filled, o => o.order.order.filled_quantity());
-         pb.add_numeric(fs.avg_price, o => o.order.order.avg_price());
-         pb.add_numeric(fs.last_price, o => o.order.order.last_price());
-         pb.add_numeric(fs.piq, o => o.order.piq ?? uint.MaxValue);
-         return pb.to_predicate();
-      }
-   }
-}
+using bts.api;
+using bts.api.orderman;
+using bts.utils;
+
+using mbts;
+using mbts.spark;
+using mbts.spark.filters;
+using mbts.ui;
+using mbts.ui.utility;
+
+using ProtoBuf;
+
+using Spark.Api.Orders;
+
+using System;
+
+namespace bts.shell.orderbook {
+   [ProtoContract]
+   class FilterStates : IEquatable<FilterStates> {
+      [ProtoMember(1)]
+      public SecurityCriteria security_criteria;
+      [ProtoMember(2)]
+      public string order_key_regex;
+      [ProtoMember(3)]
+      public TimeFilterTuple time_created;
+      [ProtoMember(4)]
+      public string[] users;
+      [ProtoMember(5)]
+      public string[] accounts;
+      [ProtoMember(6)]
+      public Venue[] exchanges;
+      [ProtoMember(7)]
+      public TimeInForce[] tifs;
+      //[ProtoMember(8)]
+      //public CustomOrderDefinition.CustomOneofCase[] order_types;
+      [ProtoMember(23)]
+      public AlgoID[] order_types;
+      [ProtoMember(24)]
+      public AlgoID[] root_algo_types;
+      [ProtoMember(9)]
+      public Side[] sides;
+      [ProtoMember(10)]
+      public DoubleMinMaxState qty;
+      [ProtoMember(26)]
+      public DoubleMinMaxState notional_value;
+      [ProtoMember(11)]
+      public DoubleMinMaxState price;
+      [ProtoMember(12)]
+      public DoubleMinMaxState qty_filled;
+      [ProtoMember(27)]
+      public DoubleMinMaxState filled_notional_value;
+      [ProtoMember(13)]
+      public DoubleMinMaxState avg_price;
+      [ProtoMember(14)]
+      public DoubleMinMaxState last_price;
+      [ProtoMember(15)]
+      public OrderStatus[] statuses;
+      [ProtoMember(16)]
+      public string exchange_message_regex;
+      [ProtoMember(17)]
+      public TimeFilterTuple last_change;
+      //[ProtoMember(18)]
+      //public string security_regex;
+      [ProtoMember(19)]
+      public api.orderman.OrderSubType[] order_sub_types;
+      [ProtoMember(20)]
+      public string[] symbols;
+      [ProtoMember(21)]
+      public string parent_key_regex;
+      [ProtoMember(22)]
+      public string attached_target_order_key_regex;
+      [ProtoMember(25)]
+      public UIntMinMaxState piq;
+
+      public bool Equals(FilterStates other) =>
+         other != null &&
+         security_criteria.safe_equals(other.security_criteria) &&
+         order_key_regex == other.order_key_regex &&
+         parent_key_regex == other.parent_key_regex &&
+         attached_target_order_key_regex == other.attached_target_order_key_regex &&
+         time_created.Equals(other.time_created) &&
+         EqualityComparer.safe_sequence_equals(users, other.users) &&
+         EqualityComparer.safe_sequence_equals(accounts, other.accounts) &&
+         EqualityComparer.safe_sequence_equals(exchanges, other.exchanges) &&
+         EqualityComparer.safe_sequence_equals(tifs, other.tifs) &&
+         EqualityComparer.safe_sequence_equals(order_types, other.order_types) &&
+         EqualityComparer.safe_sequence_equals(root_algo_types, other.root_algo_types) &&
+         EqualityComparer.safe_sequence_equals(sides, other.sides) &&
+         qty.safe_equals(other.qty) &&
+         notional_value.safe_equals(other.notional_value) &&
+         price.safe_equals(other.price) &&
+         qty_filled.safe_equals(other.qty_filled) &&
+         filled_notional_value.safe_equals(other.filled_notional_value) &&
+         avg_price.safe_equals(other.avg_price) &&
+         last_price.safe_equals(other.last_price) &&
+         EqualityComparer.safe_sequence_equals(statuses, other.statuses) &&
+         exchange_message_regex == other.exchange_message_regex &&
+         last_change.Equals(other.last_change) &&
+         EqualityComparer.safe_sequence_equals(order_sub_types, other.order_sub_types) &&
+         EqualityComparer.safe_sequence_equals(symbols, other.symbols) &&
+         piq.safe_equals(other.piq);
+
+      public override bool Equals(object obj) => Equals(obj as FilterStates);
+
+      public override int GetHashCode() =>
+         EqualityComparer.aggregate_hash(
+            security_criteria?.GetHashCode() ?? 0,
+            order_key_regex?.GetHashCode() ?? 0,
+            parent_key_regex?.GetHashCode() ?? 0,
+            attached_target_order_key_regex?.GetHashCode() ?? 0,
+            time_created.GetHashCode(),
+            EqualityComparer.safe_aggregate_hash(users),
+            EqualityComparer.safe_aggregate_hash(accounts),
+            EqualityComparer.safe_aggregate_hash(exchanges),
+            EqualityComparer.safe_aggregate_hash(tifs),
+            EqualityComparer.safe_aggregate_hash(order_types),
+            EqualityComparer.safe_aggregate_hash(root_algo_types),
+            EqualityComparer.safe_aggregate_hash(sides),
+            qty?.GetHashCode() ?? 0,
+            notional_value?.GetHashCode() ?? 0,
+            price?.GetHashCode() ?? 0,
+            qty_filled?.GetHashCode() ?? 0,
+            filled_notional_value?.GetHashCode() ?? 0,
+            avg_price?.GetHashCode() ?? 0,
+            last_price?.GetHashCode() ?? 0,
+            EqualityComparer.safe_aggregate_hash(statuses),
+            exchange_message_regex?.GetHashCode() ?? 0,
+            last_change.GetHashCode(),
+            EqualityComparer.safe_aggregate_hash(order_sub_types),
+            EqualityComparer.safe_aggregate_hash(symbols),
+            piq?.GetHashCode() ?? 0);
+   }
+
+   static class FilterStatesExtensions {
+      public static Func<OrderRecordTupleWithRoot, bool> build_predicate(this FilterStates fs) {
+         var pb = new ColumnPredicateBuilder<OrderRecordTupleWithRoot>();
+         // security
+         pb.add_security_criteria(fs.security_criteria, o => o.order.security);
+         // regex
+         pb.add_regex(fs.order_key_regex, o => o.order.order.Key.ToString());
+         pb.add_regex(fs.exchange_message_regex, o => o.order.order.State.LastMsg);
+         pb.add_regex(fs.attached_target_order_key_regex, o => o.order.order.attached_target_order_key().ToString());
+         pb.add_regex(fs.parent_key_regex, o => o.order.order.ParentKey.ToString());
+         // time
+         pb.add_time_filter(fs.time_created, o => o.order.order.created());
+         pb.add_time_filter(fs.last_change, o => o.order.order.last_change());
+         // discrete values
+         pb.add_discrete_values(fs.users, o => o.order.order.user());
+         pb.add_discrete_values(fs.accounts, o => o.order.order.account());
+         pb.add_discrete_values(fs.exchanges, o => o.order.security?.Exchange ?? Venue.Unset);
+         pb.add_discrete_values(fs.tifs, o => o.order.order.time_in_force());
+         pb.add_discrete_values(fs.order_types, o => AlgoID.from_custom_def(o.order.order.Def.CustomDef));
+         pb.add_discrete_values(
+            Array.ConvertAll(fs.root_algo_types, a => (AlgoID?)a),
+            o => o.root.algo.HasValue
+               ? AlgoID.from_algo_key(o.root.algo.Value.key)
+               : null);
+         pb.add_discrete_values(fs.order_sub_types, o => o.order.order.sub_type());
+         pb.add_discrete_values(fs.sides, o => o.order.order.side());
+         pb.add_discrete_values(fs.statuses, o => o.order.order.status());
+         pb.add_discrete_values(fs.symbols, o => o.order.security?.Symbol);
+         // numeric
+         pb.add_numeric(fs.qty, o => o.order.order.quantity());
+         pb.add_numeric(fs.notional_value, o => o.notional_value);
+         pb.add_price(fs.price, o => o.order.order.ui_price(), o => o.order.security);
+         pb.add_numeric(fs.qty_filled, o => o.order.order.filled_quantity());
+         pb.add_numeric(fs.filled_notional_value, o => o.filled_notional_value);
+         pb.add_numeric(fs.avg_price, o => o.order.order.avg_price());
+         pb.add_numeric(fs.last_price, o => o.order.order.last_price());
+         pb.add_numeric(fs.piq, o => o.order.piq ?? uint.MaxValue);
+         return pb.to_predicate();
+      }
+   }
+}

src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookGridSettingsManager.cs

@@ -29,7 +29,7 @@ namespace bts.shell.orderbook {
       static OrderBookGridSettingsManager() {
          // make font-weight immutable for all columns (the font-weight rule is hard-coded: Regular for root-orders, Light for all others)
          // TODO: Remove the immutable font-weight flags when cell animation is ready
-         var col_settings_flags = EnumHelper.get_values<Columns>().ToDictionary(
+         var col_settings_flags = Enum.GetValuesAsUnderlyingType<Columns>().Cast<Columns>().ToDictionary(
             c => c.ToString(), c => GridColumnSettingsFeatureFlags.ImmutableFontWeight);
          // make font-color immutable for Side, Size, and Filled columns (the font-color rule is hard-coded: MarketData.bid for Buy orders, MarketData.ask for Sells)
          // TODO: Remove the immutable font-color flags when cell animation is ready

src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookSettings.cs

@@ -1,270 +1,277 @@
-using bts.api.orderman;
-using bts.shell.fills;
-
-using mbts;
-using mbts.ui;
-using mbts.ui.controls;
-using mbts.ui.settings;
-
-using ProtoBuf;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Reactive.Linq;
-using System.Windows;
-
-using Columns = mbts.spark.OrderBookGridView.ColumnKeys;
-using GridSettings = mbts.ui.settings.GridSettings;
-
-namespace bts.shell.orderbook {
-   using ColHeader = mbts.ui.controls.DIYGrid.DefaultColumnHeaders;
-   using ColWidth = mbts.ui.controls.DIYGrid.DefaultColumnWidths;
-
-   [ProtoContract]
-   class OrderBookStartupData {
-      const bool
-         _default_show_right_panel = true,
-         _default_show_order_params = true;
-      const double
-         _default_right_panel_width = 400,
-         _default_order_params_height = 250,
-         _default_fills_height = 300,
-         _default_ladder_height = 350;
-      [ProtoMember(16)]
-      OrderBookRightPanelSettings _right_panel;
-      // ALREADY USED TAGS
-      // [ProtoMember(5)]
-      // public readonly List<SortingCriteria> sort_criteria;
-
-      public OrderBookRightPanelSettings right_panel => _right_panel ?? (_right_panel = new OrderBookRightPanelSettings());
-      /// TODO: remove properties below, and redirect callers to <see cref="right_panel"/>
-      // right panel (host)
-      public bool show_right_panel { get => right_panel.show; set => right_panel.show = value; }
-      public double right_panel_width { get => right_panel.width; set => right_panel.width = value; }
-      // order params
-      public bool show_order_params { get => right_panel.stack.show_order_params; set => right_panel.stack.show_order_params = value; }
-      public double order_params_height { get; set; }
-      // ladder
-      public bool show_ladder { get; set; }
-      public double ladder_height { get => right_panel.stack.ladder_height; set => right_panel.stack.ladder_height = value; }
-      // fills
-      public bool show_fills { get => right_panel.stack.show_fills; set => right_panel.stack.show_fills = value; }
-      public double fills_height { get => right_panel.stack.fills_height; set => right_panel.stack.fills_height = value; }
-
-      // filters
-      [ProtoMember(10)]
-      public FilterStates filters;
-      // grid
-      [ProtoMember(15)]
-      [Obsolete]
-      public GridSettings grid { get; set; }
-      // orderbook grid
-      [ProtoMember(17)]
-      public OrderBookGridSettings orderbook_grid { get; set; }
-      // fills settings
-      [ProtoMember(26)]
-      public OrderBookFillsGridSettings fills_settings { get; set; }
-      // predicate animations
-      [ProtoMember(20)]
-      [Obsolete]
-      List<OrderRecordTuplePredicateAnimations> _predicate_animations;
-      // use defaults
-      [ProtoMember(25)]
-      public bool use_defaults;
-      // menu display
-      [ProtoMember(27)]
-      public SparkMenuDisplaySettings menu_display { get; set; }
-
-      /// <summary>
-      /// Please use <see cref="Default"/> to get a new instance (the only reason the default constructor still exists is for ProtoBuf deserialization).
-      /// </summary>
-      [Obsolete]
-      public OrderBookStartupData() {
-         show_right_panel = _default_show_right_panel;
-         show_order_params = _default_show_order_params;
-         right_panel_width = _default_right_panel_width;
-         order_params_height = _default_order_params_height;
-         fills_height = _default_fills_height;
-         ladder_height = _default_ladder_height;
-      }
-
-      public OrderBookStartupData(FilterStates filters, OrderBookGridSettings orderbook_grid = default, bool show_right_panel = false, double right_panel_width = 0, double fills_height = 0, bool show_fills = false, bool use_defaults = false) {
-         this.filters = filters;
-         this.use_defaults = use_defaults;
-         this.show_right_panel = show_right_panel;
-         this.right_panel_width = right_panel_width;
-         this.fills_height = fills_height;
-         this.show_fills = show_fills;
-         this.orderbook_grid = orderbook_grid;
-#pragma warning disable CS0612 // Type or member is obsolete
-         _predicate_animations = new List<OrderRecordTuplePredicateAnimations>();
-         if (predicate_animations != null)
-            _predicate_animations.AddRange(predicate_animations);
-#pragma warning restore CS0612 // Type or member is obsolete
-      }
-
-#pragma warning disable CS0612 // Type or member is obsolete
-      [Obsolete]
-      public IReadOnlyList<OrderRecordTuplePredicateAnimations> predicate_animations => _predicate_animations ?? (_predicate_animations = new List<OrderRecordTuplePredicateAnimations>());
-      public OrderBookGridSettings grid_settings_wrapper() {
-         if (use_defaults)
-            return null;
-         if (orderbook_grid == null) {
-            if (grid == null)
-               grid = _default_general_grid;
-            orderbook_grid = new OrderBookGridSettings(grid.Clone(), _predicate_animations);
-         }
-         return orderbook_grid;
-      }
-#pragma warning restore CS0612 // Type or member is obsolete
-
-      public static OrderBookStartupData Default =>
-        new OrderBookStartupData(
-           new FilterStates { statuses = new[] { OrderStatus.Working } },
-           DefaultGrid,
-           _default_show_right_panel,
-           _default_right_panel_width,
-           _default_fills_height,
-           use_defaults: true) {
-           show_order_params = _default_show_order_params,
-           order_params_height = _default_order_params_height,
-           fills_settings = DefaultFillsGrid
-        };
-
-      public static OrderBookGridSettings DefaultGrid => new OrderBookGridSettings(_default_general_grid) {
-         algo_tree_display_mode = AlgoTreeDisplayMode.RootAndLeafNodes
-      };
-
-      public static OrderBookFillsGridSettings DefaultFillsGrid {
-         get {
-            var default_fills = FillsSettings.Default();
-            var rv = new OrderBookFillsGridSettings {
-               sorting_criteria = default_fills.sorting_criteria.ToList(),
-               filters = default_fills.filters,
-               general = FillsSettings.MiniGridSettings
-            };
-            return rv;
-         }
-      }
-      static GridSettings _default_general_grid => GridSettings.Default.with(DefaultColumns.standard, DefaultColumns.sorting_criteria);
-      public static OrderBookGridSettings default_mini_grid => new OrderBookGridSettings(GridSettings.Default.with(DefaultColumns.mini, DefaultColumns.sorting_criteria)) {
-         algo_tree_display_mode = AlgoTreeDisplayMode.RootAndLeafNodes
-      };
-
-      static class DefaultColumns {
-         // standard
-         public static GridColumnSettings security => new GridColumnSettings(Columns.Security.ToString(), DIYGrid.DefaultColumnHeaders.Security, width: DIYGrid.DefaultColumnWidths.Security);
-         public static GridColumnSettings symbol => new GridColumnSettings(Columns.Symbol.ToString(), DIYGrid.DefaultColumnHeaders.Symbol, width: DIYGrid.DefaultColumnWidths.Symbol, horizontal_alignment: HorizontalAlignment.Left);
-         public static GridColumnSettings time_created => new GridColumnSettings(Columns.Time.ToString(), "Time", width: DIYGrid.DefaultColumnWidths.Time, horizontal_alignment: HorizontalAlignment.Left);
-         public static GridColumnSettings last_change => new GridColumnSettings(Columns.LastChange.ToString(), "Last Change", width: DIYGrid.DefaultColumnWidths.Time, horizontal_alignment: HorizontalAlignment.Left);
-         public static GridColumnSettings user => new GridColumnSettings(Columns.User.ToString(), DIYGrid.DefaultColumnHeaders.User, width: DIYGrid.DefaultColumnWidths.User, horizontal_alignment: HorizontalAlignment.Left);
-         public static GridColumnSettings account => new GridColumnSettings(Columns.Account.ToString(), DIYGrid.DefaultColumnHeaders.Account, width: DIYGrid.DefaultColumnWidths.Account, horizontal_alignment: HorizontalAlignment.Left);
-         public static GridColumnSettings exchange => new GridColumnSettings(Columns.Exchange.ToString(), DIYGrid.DefaultColumnHeaders.Exchange, width: DIYGrid.DefaultColumnWidths.Exchange, horizontal_alignment: HorizontalAlignment.Left);
-         public static GridColumnSettings time_in_force => new GridColumnSettings(Columns.TimeInForce.ToString(), DIYGrid.DefaultColumnHeaders.TimeInForce, width: DIYGrid.DefaultColumnWidths.TimeInForce);
-         public static GridColumnSettings order_type => new GridColumnSettings(Columns.OrderType.ToString(), DIYGrid.DefaultColumnHeaders.OrderType, width: DIYGrid.DefaultColumnWidths.OrderType);
-         public static GridColumnSettings root_algo_type => new GridColumnSettings(Columns.RootAlgoType.ToString(), DIYGrid.DefaultColumnHeaders.RootAlgoType, width: DIYGrid.DefaultColumnWidths.OrderType);
-         public static GridColumnSettings sub_type => new GridColumnSettings(Columns.SubType.ToString(), "Sub Type", width: 80, horizontal_alignment: HorizontalAlignment.Left);
-
-         public static GridColumnSettings side => new(
-            Columns.Side.ToString(), ColHeader.Side,
-            width: ColWidth.Side,
-            horizontal_alignment: HorizontalAlignment.Left);
-
-         public static GridColumnSettings quantity => new GridColumnSettings(Columns.Size.ToString(), "Qty", width: DIYGrid.DefaultColumnWidths.Qty, horizontal_alignment: HorizontalAlignment.Right);
-         public static GridColumnSettings price => new GridColumnSettings(Columns.Price.ToString(), DIYGrid.DefaultColumnHeaders.Price, width: DIYGrid.DefaultColumnWidths.Price, horizontal_alignment: HorizontalAlignment.Right);
-
-         public static GridColumnSettings filled_quantity => new(
-            Columns.Filled.ToString(),
-            ColHeader.FilledQuantity,
-            width: ColWidth.Filled,
-            horizontal_alignment: HorizontalAlignment.Right);
-
-         public static GridColumnSettings avg_price => new(
-            Columns.AvgFilled.ToString(),
-            ColHeader.AvgFilled,
-            width: ColWidth.AveragePrice,
-            horizontal_alignment: HorizontalAlignment.Right);
-
-         public static GridColumnSettings last_price => new GridColumnSettings(Columns.LastPrice.ToString(), "Last Filled", width: DIYGrid.DefaultColumnWidths.LastPrice, horizontal_alignment: HorizontalAlignment.Right);
-         public static GridColumnSettings status => new GridColumnSettings(Columns.Status.ToString(), "Status", width: 80, horizontal_alignment: HorizontalAlignment.Left);
-         public static GridColumnSettings last_message => new GridColumnSettings(Columns.LastMessage.ToString(), "Last Message", width: DIYGrid.DefaultColumnWidths.LastMessage, horizontal_alignment: HorizontalAlignment.Left);
-         public static GridColumnSettings security_key => new GridColumnSettings(Columns.SecurityKey.ToString(), DIYGrid.DefaultColumnHeaders.SecurityKey, is_visible: false, width: DIYGrid.DefaultColumnWidths.SecurityKey, horizontal_alignment: HorizontalAlignment.Left);
-         public static GridColumnSettings order_key => new GridColumnSettings(Columns.OrderKey.ToString(), DIYGrid.DefaultColumnHeaders.OrderKey, width: DIYGrid.DefaultColumnWidths.OrderKey, horizontal_alignment: HorizontalAlignment.Left);
-         public static GridColumnSettings parent_key => new GridColumnSettings(Columns.ParentKey.ToString(), "Parent Key", width: DIYGrid.DefaultColumnWidths.ParentKey, horizontal_alignment: HorizontalAlignment.Left);
-         public static GridColumnSettings attached_target_order_key => new GridColumnSettings(Columns.AttachedTargetOrderKey.ToString(), DIYGrid.DefaultColumnHeaders.AttachedTargetOrderKey, width: DIYGrid.DefaultColumnWidths.ParentKey, horizontal_alignment: HorizontalAlignment.Left);
-         public static GridColumnSettings piq => new GridColumnSettings(Columns.Piq.ToString(), "PIQ", width: DIYGrid.DefaultColumnWidths.Qty, horizontal_alignment: HorizontalAlignment.Right);
-         // editable
-         public static GridColumnSettings cancel => new GridColumnSettings(Columns.Cancel.ToString(), DIYGrid.DefaultColumnHeaders.Cancel, width: DIYGrid.DefaultColumnWidths.MiniOrderBook.Cancel);
-         public static GridColumnSettings attach_detach => new GridColumnSettings(Columns.Attach.ToString(), DIYGrid.DefaultColumnHeaders.AttachDetach, width: DIYGrid.DefaultColumnWidths.MiniOrderBook.AttachDetach);
-         public static GridColumnSettings update => new GridColumnSettings(Columns.Update.ToString(), DIYGrid.DefaultColumnHeaders.Update, width: DIYGrid.DefaultColumnWidths.MiniOrderBook.Update);
-         public static GridColumnSettings edit_quantity => new GridColumnSettings(Columns.EditableQty.ToString(), DIYGrid.DefaultColumnHeaders.Qty, width: DIYGrid.DefaultColumnWidths.MiniOrderBook.EditableSize);
-         public static GridColumnSettings edit_price => new GridColumnSettings(Columns.EditablePrice.ToString(), DIYGrid.DefaultColumnHeaders.Price, width: DIYGrid.DefaultColumnWidths.MiniOrderBook.EditablePrice);
-         public static GridColumnSettings commit => new GridColumnSettings(Columns.Commit.ToString(), width: DIYGrid.DefaultColumnWidths.MiniOrderBook.Commit);
-
-         public static IEnumerable<SortingCriteria> sorting_criteria {
-            get {
-               yield return new SortingCriteria(time_created.id, false);
-            }
-         }
-
-         public static IEnumerable<GridColumnSettings> standard {
-            get {
-               yield return security;
-               yield return symbol;
-               yield return time_created;
-               yield return last_change;
-               yield return user;
-               yield return account;
-               yield return exchange;
-               yield return time_in_force;
-               yield return order_type;
-               yield return root_algo_type;
-               yield return sub_type;
-               yield return side;
-               yield return quantity;
-               yield return price;
-               yield return piq;
-               yield return filled_quantity;
-               yield return avg_price;
-               yield return last_price;
-               yield return status;
-               yield return last_message;
-               yield return security_key;
-               yield return order_key;
-               yield return parent_key;
-               yield return attached_target_order_key;
-            }
-         }
-
-         public static IEnumerable<GridColumnSettings> editable {
-            get {
-               yield return cancel;
-               yield return update;
-               yield return edit_quantity;
-               yield return edit_price;
-               yield return commit;
-               yield return attach_detach;
-            }
-         }
-
-         private static HashSet<GridColumnSettings> _mini_mode_columns_to_hide = new HashSet<GridColumnSettings> {
-               symbol, last_change, exchange, sub_type, quantity, price, avg_price, last_price, status, last_message, security_key, order_key, parent_key, attached_target_order_key
-            };
-
-         public static IEnumerable<GridColumnSettings> mini {
-            get {
-               foreach (var col_setting in editable)
-                  yield return col_setting;
-               foreach (var col_setting in standard) {
-                  if (_mini_mode_columns_to_hide.Contains(col_setting)) {
-                     yield return col_setting.with(is_visible: false);
-                  } else {
-                     yield return col_setting;
-                  }
-               }
-            }
-         }
-      }
-   }
-}
+using bts.api.orderman;
+using bts.shell.fills;
+
+using mbts;
+using mbts.ui;
+using mbts.ui.controls;
+using mbts.ui.settings;
+
+using ProtoBuf;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Windows;
+
+using Columns = mbts.spark.OrderBookGridView.ColumnKeys;
+using GridSettings = mbts.ui.settings.GridSettings;
+
+namespace bts.shell.orderbook {
+   using ColHeader = mbts.ui.controls.DIYGrid.DefaultColumnHeaders;
+   using ColWidth = mbts.ui.controls.DIYGrid.DefaultColumnWidths;
+
+   [ProtoContract]
+   class OrderBookStartupData {
+      const bool
+         _default_show_right_panel = true,
+         _default_show_order_params = true;
+      const double
+         _default_right_panel_width = 400,
+         _default_order_params_height = 250,
+         _default_fills_height = 300,
+         _default_ladder_height = 350;
+      [ProtoMember(16)]
+      OrderBookRightPanelSettings _right_panel;
+      // ALREADY USED TAGS
+      // [ProtoMember(5)]
+      // public readonly List<SortingCriteria> sort_criteria;
+
+      public OrderBookRightPanelSettings right_panel => _right_panel ?? (_right_panel = new OrderBookRightPanelSettings());
+      /// TODO: remove properties below, and redirect callers to <see cref="right_panel"/>
+      // right panel (host)
+      public bool show_right_panel { get => right_panel.show; set => right_panel.show = value; }
+      public double right_panel_width { get => right_panel.width; set => right_panel.width = value; }
+      // order params
+      public bool show_order_params { get => right_panel.stack.show_order_params; set => right_panel.stack.show_order_params = value; }
+      public double order_params_height { get; set; }
+      // ladder
+      public bool show_ladder { get; set; }
+      public double ladder_height { get => right_panel.stack.ladder_height; set => right_panel.stack.ladder_height = value; }
+      // fills
+      public bool show_fills { get => right_panel.stack.show_fills; set => right_panel.stack.show_fills = value; }
+      public double fills_height { get => right_panel.stack.fills_height; set => right_panel.stack.fills_height = value; }
+
+      // filters
+      [ProtoMember(10)]
+      public FilterStates filters;
+      // grid
+      [ProtoMember(15)]
+      [Obsolete]
+      public GridSettings grid { get; set; }
+      // orderbook grid
+      [ProtoMember(17)]
+      public OrderBookGridSettings orderbook_grid { get; set; }
+      // fills settings
+      [ProtoMember(26)]
+      public OrderBookFillsGridSettings fills_settings { get; set; }
+      // predicate animations
+      [ProtoMember(20)]
+      [Obsolete]
+      List<OrderRecordTuplePredicateAnimations> _predicate_animations;
+      // use defaults
+      [ProtoMember(25)]
+      public bool use_defaults;
+      // menu display
+      [ProtoMember(27)]
+      public SparkMenuDisplaySettings menu_display { get; set; }
+
+      /// <summary>
+      /// Please use <see cref="Default"/> to get a new instance (the only reason the default constructor still exists is for ProtoBuf deserialization).
+      /// </summary>
+      [Obsolete]
+      public OrderBookStartupData() {
+         show_right_panel = _default_show_right_panel;
+         show_order_params = _default_show_order_params;
+         right_panel_width = _default_right_panel_width;
+         order_params_height = _default_order_params_height;
+         fills_height = _default_fills_height;
+         ladder_height = _default_ladder_height;
+      }
+
+      public OrderBookStartupData(FilterStates filters, OrderBookGridSettings orderbook_grid = default, bool show_right_panel = false, double right_panel_width = 0, double fills_height = 0, bool show_fills = false, bool use_defaults = false) {
+         this.filters = filters;
+         this.use_defaults = use_defaults;
+         this.show_right_panel = show_right_panel;
+         this.right_panel_width = right_panel_width;
+         this.fills_height = fills_height;
+         this.show_fills = show_fills;
+         this.orderbook_grid = orderbook_grid;
+#pragma warning disable CS0612 // Type or member is obsolete
+         _predicate_animations = new List<OrderRecordTuplePredicateAnimations>();
+         if (predicate_animations != null)
+            _predicate_animations.AddRange(predicate_animations);
+#pragma warning restore CS0612 // Type or member is obsolete
+      }
+
+#pragma warning disable CS0612 // Type or member is obsolete
+      [Obsolete]
+      public IReadOnlyList<OrderRecordTuplePredicateAnimations> predicate_animations => _predicate_animations ?? (_predicate_animations = new List<OrderRecordTuplePredicateAnimations>());
+      public OrderBookGridSettings grid_settings_wrapper() {
+         if (use_defaults)
+            return null;
+         if (orderbook_grid == null) {
+            if (grid == null)
+               grid = _default_general_grid;
+            orderbook_grid = new OrderBookGridSettings(grid.Clone(), _predicate_animations);
+         }
+         return orderbook_grid;
+      }
+#pragma warning restore CS0612 // Type or member is obsolete
+
+      public static OrderBookStartupData Default =>
+        new OrderBookStartupData(
+           new FilterStates { statuses = new[] { OrderStatus.Working } },
+           DefaultGrid,
+           _default_show_right_panel,
+           _default_right_panel_width,
+           _default_fills_height,
+           use_defaults: true) {
+           show_order_params = _default_show_order_params,
+           order_params_height = _default_order_params_height,
+           fills_settings = DefaultFillsGrid
+        };
+
+      public static OrderBookGridSettings DefaultGrid => new OrderBookGridSettings(_default_general_grid) {
+         algo_tree_display_mode = AlgoTreeDisplayMode.RootAndLeafNodes
+      };
+
+      public static OrderBookFillsGridSettings DefaultFillsGrid {
+         get {
+            var default_fills = FillsSettings.Default();
+            var rv = new OrderBookFillsGridSettings {
+               sorting_criteria = default_fills.sorting_criteria.ToList(),
+               filters = default_fills.filters,
+               general = FillsSettings.MiniGridSettings
+            };
+            return rv;
+         }
+      }
+      static GridSettings _default_general_grid => GridSettings.Default.with(DefaultColumns.standard, DefaultColumns.sorting_criteria);
+      public static OrderBookGridSettings default_mini_grid => new OrderBookGridSettings(GridSettings.Default.with(DefaultColumns.mini, DefaultColumns.sorting_criteria)) {
+         algo_tree_display_mode = AlgoTreeDisplayMode.RootAndLeafNodes
+      };
+
+      static class DefaultColumns {
+         // standard
+         public static GridColumnSettings security => new GridColumnSettings(Columns.Security.ToString(), DIYGrid.DefaultColumnHeaders.Security, width: DIYGrid.DefaultColumnWidths.Security);
+         public static GridColumnSettings symbol => new GridColumnSettings(Columns.Symbol.ToString(), DIYGrid.DefaultColumnHeaders.Symbol, width: DIYGrid.DefaultColumnWidths.Symbol, horizontal_alignment: HorizontalAlignment.Left);
+         public static GridColumnSettings time_created => new GridColumnSettings(Columns.Time.ToString(), "Time", width: DIYGrid.DefaultColumnWidths.Time, horizontal_alignment: HorizontalAlignment.Left);
+         public static GridColumnSettings last_change => new GridColumnSettings(Columns.LastChange.ToString(), "Last Change", width: DIYGrid.DefaultColumnWidths.Time, horizontal_alignment: HorizontalAlignment.Left);
+         public static GridColumnSettings user => new GridColumnSettings(Columns.User.ToString(), DIYGrid.DefaultColumnHeaders.User, width: DIYGrid.DefaultColumnWidths.User, horizontal_alignment: HorizontalAlignment.Left);
+         public static GridColumnSettings account => new GridColumnSettings(Columns.Account.ToString(), DIYGrid.DefaultColumnHeaders.Account, width: DIYGrid.DefaultColumnWidths.Account, horizontal_alignment: HorizontalAlignment.Left);
+         public static GridColumnSettings exchange => new GridColumnSettings(Columns.Exchange.ToString(), DIYGrid.DefaultColumnHeaders.Exchange, width: DIYGrid.DefaultColumnWidths.Exchange, horizontal_alignment: HorizontalAlignment.Left);
+         public static GridColumnSettings time_in_force => new GridColumnSettings(Columns.TimeInForce.ToString(), DIYGrid.DefaultColumnHeaders.TimeInForce, width: DIYGrid.DefaultColumnWidths.TimeInForce);
+         public static GridColumnSettings order_type => new GridColumnSettings(Columns.OrderType.ToString(), DIYGrid.DefaultColumnHeaders.OrderType, width: DIYGrid.DefaultColumnWidths.OrderType);
+         public static GridColumnSettings root_algo_type => new GridColumnSettings(Columns.RootAlgoType.ToString(), DIYGrid.DefaultColumnHeaders.RootAlgoType, width: DIYGrid.DefaultColumnWidths.OrderType);
+         public static GridColumnSettings sub_type => new GridColumnSettings(Columns.SubType.ToString(), "Sub Type", width: 80, horizontal_alignment: HorizontalAlignment.Left);
+
+         public static GridColumnSettings side => new(
+            Columns.Side.ToString(), ColHeader.Side,
+            width: ColWidth.Side,
+            horizontal_alignment: HorizontalAlignment.Left);
+
+         public static GridColumnSettings quantity => new GridColumnSettings(Columns.Size.ToString(), "Qty", width: DIYGrid.DefaultColumnWidths.Qty, horizontal_alignment: HorizontalAlignment.Right);
+         public static GridColumnSettings notional_value => new GridColumnSettings(Columns.NotionalValue.ToString(), DIYGrid.DefaultColumnHeaders.NotionalValue, width: DIYGrid.DefaultColumnWidths.NotionalValue, horizontal_alignment: HorizontalAlignment.Right);
+         public static GridColumnSettings price => new GridColumnSettings(Columns.Price.ToString(), DIYGrid.DefaultColumnHeaders.Price, width: DIYGrid.DefaultColumnWidths.Price, horizontal_alignment: HorizontalAlignment.Right);
+
+         public static GridColumnSettings filled_quantity => new(
+            Columns.Filled.ToString(),
+            ColHeader.FilledQuantity,
+            width: ColWidth.Filled,
+            horizontal_alignment: HorizontalAlignment.Right);
+
+         public static GridColumnSettings filled_notional_value => new(
+            Columns.FilledNotionalValue.ToString(),
+            ColHeader.FilledNotionalValue,
+            width: ColWidth.FilledNotionalValue,
+            horizontal_alignment: HorizontalAlignment.Right);
+
+         public static GridColumnSettings avg_price => new(
+            Columns.AvgFilled.ToString(),
+            ColHeader.AvgFilled,
+            width: ColWidth.AveragePrice,
+            horizontal_alignment: HorizontalAlignment.Right);
+
+         public static GridColumnSettings last_price => new GridColumnSettings(Columns.LastPrice.ToString(), "Last Filled", width: DIYGrid.DefaultColumnWidths.LastPrice, horizontal_alignment: HorizontalAlignment.Right);
+         public static GridColumnSettings status => new GridColumnSettings(Columns.Status.ToString(), "Status", width: 80, horizontal_alignment: HorizontalAlignment.Left);
+         public static GridColumnSettings last_message => new GridColumnSettings(Columns.LastMessage.ToString(), "Last Message", width: DIYGrid.DefaultColumnWidths.LastMessage, horizontal_alignment: HorizontalAlignment.Left);
+         public static GridColumnSettings security_key => new GridColumnSettings(Columns.SecurityKey.ToString(), DIYGrid.DefaultColumnHeaders.SecurityKey, is_visible: false, width: DIYGrid.DefaultColumnWidths.SecurityKey, horizontal_alignment: HorizontalAlignment.Left);
+         public static GridColumnSettings order_key => new GridColumnSettings(Columns.OrderKey.ToString(), DIYGrid.DefaultColumnHeaders.OrderKey, width: DIYGrid.DefaultColumnWidths.OrderKey, horizontal_alignment: HorizontalAlignment.Left);
+         public static GridColumnSettings parent_key => new GridColumnSettings(Columns.ParentKey.ToString(), "Parent Key", width: DIYGrid.DefaultColumnWidths.ParentKey, horizontal_alignment: HorizontalAlignment.Left);
+         public static GridColumnSettings attached_target_order_key => new GridColumnSettings(Columns.AttachedTargetOrderKey.ToString(), DIYGrid.DefaultColumnHeaders.AttachedTargetOrderKey, width: DIYGrid.DefaultColumnWidths.ParentKey, horizontal_alignment: HorizontalAlignment.Left);
+         public static GridColumnSettings piq => new GridColumnSettings(Columns.Piq.ToString(), "PIQ", width: DIYGrid.DefaultColumnWidths.Qty, horizontal_alignment: HorizontalAlignment.Right);
+         // editable
+         public static GridColumnSettings cancel => new GridColumnSettings(Columns.Cancel.ToString(), DIYGrid.DefaultColumnHeaders.Cancel, width: DIYGrid.DefaultColumnWidths.MiniOrderBook.Cancel);
+         public static GridColumnSettings attach_detach => new GridColumnSettings(Columns.Attach.ToString(), DIYGrid.DefaultColumnHeaders.AttachDetach, width: DIYGrid.DefaultColumnWidths.MiniOrderBook.AttachDetach);
+         public static GridColumnSettings update => new GridColumnSettings(Columns.Update.ToString(), DIYGrid.DefaultColumnHeaders.Update, width: DIYGrid.DefaultColumnWidths.MiniOrderBook.Update);
+         public static GridColumnSettings edit_quantity => new GridColumnSettings(Columns.EditableQty.ToString(), DIYGrid.DefaultColumnHeaders.Qty, width: DIYGrid.DefaultColumnWidths.MiniOrderBook.EditableSize);
+         public static GridColumnSettings edit_price => new GridColumnSettings(Columns.EditablePrice.ToString(), DIYGrid.DefaultColumnHeaders.Price, width: DIYGrid.DefaultColumnWidths.MiniOrderBook.EditablePrice);
+         public static GridColumnSettings commit => new GridColumnSettings(Columns.Commit.ToString(), width: DIYGrid.DefaultColumnWidths.MiniOrderBook.Commit);
+
+         public static IEnumerable<SortingCriteria> sorting_criteria {
+            get {
+               yield return new SortingCriteria(time_created.id, false);
+            }
+         }
+
+         public static IEnumerable<GridColumnSettings> standard {
+            get {
+               yield return security;
+               yield return symbol;
+               yield return time_created;
+               yield return last_change;
+               yield return user;
+               yield return account;
+               yield return exchange;
+               yield return time_in_force;
+               yield return order_type;
+               yield return root_algo_type;
+               yield return sub_type;
+               yield return side;
+               yield return quantity;
+               yield return price;
+               yield return piq;
+               yield return filled_quantity;
+               yield return avg_price;
+               yield return last_price;
+               yield return status;
+               yield return last_message;
+               yield return security_key;
+               yield return order_key;
+               yield return parent_key;
+               yield return attached_target_order_key;
+            }
+         }
+
+         public static IEnumerable<GridColumnSettings> editable {
+            get {
+               yield return cancel;
+               yield return update;
+               yield return edit_quantity;
+               yield return edit_price;
+               yield return commit;
+               yield return attach_detach;
+            }
+         }
+
+         private static HashSet<GridColumnSettings> _mini_mode_columns_to_hide = new HashSet<GridColumnSettings> {
+               symbol, last_change, exchange, sub_type, quantity, price, avg_price, last_price, status, last_message, security_key, order_key, parent_key, attached_target_order_key
+            };
+
+         public static IEnumerable<GridColumnSettings> mini {
+            get {
+               foreach (var col_setting in editable)
+                  yield return col_setting;
+               foreach (var col_setting in standard) {
+                  if (_mini_mode_columns_to_hide.Contains(col_setting)) {
+                     yield return col_setting.with(is_visible: false);
+                  } else {
+                     yield return col_setting;
+                  }
+               }
+            }
+         }
+      }
+   }
+}

src/client/spark/Modules/Trading/OrderBook/View/OrderBookGridView.xaml.cs

@@ -1,107 +1,109 @@
-using bts.api.orderman;
-
-using mbts.ui;
-using mbts.ui.controls;
-using mbts.ui.converters;
-
-using System.Windows;
-using System.Windows.Controls;
-using System.Windows.Data;
-using System.Windows.Media;
-
-namespace mbts.spark {
-   internal partial class OrderBookGridView : DIYGrid {
-      public const string
-         SubTypeFilterTemplateKey = "SubTypeFilterTemplate",
-         ExpandCollapseToggleStyleKey = "ExpandCollapseToggleStyle",
-         RowStyleKey = "RowStyle",
-         RowTemplateKey = "RowTemplateKey",
-         CVRowStyleKey = "CVRowStyle",
-         CVRowTemplateKey = "CVRowTemplateKey";
-
-      // IMPORTANT: these enums are stored as strings in the general-purpose GridColumnSettings proto, and are used to ID THE COLUMNS!
-      // Do NOT RENAME them!  However, if you must, be sure to write an IShellWindowMigrationRule or something to account for backwards compatability.
-      public enum ColumnKeys {
-         Security = 1,
-         Symbol = 1 << 1,
-         Time = 1 << 2,
-         LastChange = 1 << 3,
-         User = 1 << 4,
-         Account = 1 << 5,
-         Exchange = 1 << 6,
-         TimeInForce = 1 << 7,
-         OrderType = 1 << 8,
-         SubType = 1 << 9,
-         Side = 1 << 10,
-         Size = 1 << 11,
-         Price = 1 << 12,
-         Filled = 1 << 13,
-         AvgFilled = 1 << 14,
-         LastPrice = 1 << 15,
-         Status = 1 << 16,
-         LastMessage = 1 << 17,
-         SecurityKey = 1 << 18,
-         OrderKey = 1 << 19,
-         ParentKey = 1 << 20,
-         AttachedTargetOrderKey = 1 << 21,
-         Cancel = 1 << 22,
-         EditableQty = 1 << 23,
-         EditablePrice = 1 << 24,
-         Attach = 1 << 25,
-         Detach = 1 << 26,
-         Update = 1 << 27,
-         Commit = 1 << 28,
-         RootAlgoType = 1 << 29,
-         Piq = 1 << 30,
-         Check = 1 << 31
-      }
-
-      public OrderBookGridView() {
-         InitializeComponent();
-         var row_template_factory = new FrameworkElementFactory(typeof(OrderBookRowView));
-         row_template_factory.SetValue(OrderBookRowView.BackgroundProperty, RowBackgroundHelper.row_background_binding);
-         RowTemplate = new DataTemplate { VisualTree = row_template_factory };
-      }
-
-      static class RowBackgroundHelper {
-         public static readonly BindingBase row_background_binding;
-
-         static RowBackgroundHelper() {
-            var bg_binding = new MultiBinding();
-            bg_binding.Bindings.Add(new Binding(nameof(OrderBookRowVM.status)));
-            bg_binding.Bindings.Add(new Binding(nameof(ListBoxItem.IsSelected)) { RelativeSource = new RelativeSource(RelativeSourceMode.FindAncestor, typeof(ListBoxItem), 1) });
-            double overlay_opacity = config.Configuration.get<bts.config.OrderBookOptions>().StatusColorOpacityWhenSelected;
-            if (overlay_opacity > 0.1 && overlay_opacity < 0.9) {
-               bg_binding.Converter = ValueConverter.create<OrderStatus, bool, Brush>((status, is_selected) => {
-                  var color = _status_color(status);
-                  return color.HasValue ? new SolidColorBrush(color.Value) { Opacity = is_selected ? overlay_opacity : 1.0 } : null;
-               });
-            } else {
-               bg_binding.Converter = ValueConverter.create<OrderStatus, bool, Brush>((status, is_selected) => {
-                  if (is_selected)
-                     return null;
-                  var color = _status_color(status);
-                  return color.HasValue ? new SolidColorBrush(color.Value) : null;
-               });
-            }
-            row_background_binding = bg_binding;
-         }
-
-         static Color? _status_color(OrderStatus status) {
-            switch (status) {
-               case OrderStatus.Canceled: return BTSColors.Orders.canceled_order_background;
-               case OrderStatus.Completed: return BTSColors.Orders.completed_order_background;
-               case OrderStatus.Disconnected:
-               case OrderStatus.RejectedBySpark:
-               case OrderStatus.Error:
-                  return BTSColors.Orders.error_order_background;
-               default: return null;
-            }
-         }
-      }
-   }
-
-   class OrderBookRowView : DIYGrid.HighPerformanceRow<OrderBookRowVM> {
-      public OrderBookRowView() : base() { }
-   }
-}
+using bts.api.orderman;
+
+using mbts.ui;
+using mbts.ui.controls;
+using mbts.ui.converters;
+
+using System.Windows;
+using System.Windows.Controls;
+using System.Windows.Data;
+using System.Windows.Media;
+
+namespace mbts.spark {
+   internal partial class OrderBookGridView : DIYGrid {
+      public const string
+         SubTypeFilterTemplateKey = "SubTypeFilterTemplate",
+         ExpandCollapseToggleStyleKey = "ExpandCollapseToggleStyle",
+         RowStyleKey = "RowStyle",
+         RowTemplateKey = "RowTemplateKey",
+         CVRowStyleKey = "CVRowStyle",
+         CVRowTemplateKey = "CVRowTemplateKey";
+
+      // IMPORTANT: these enums are stored as strings in the general-purpose GridColumnSettings proto, and are used to ID THE COLUMNS!
+      // Do NOT RENAME them!  However, if you must, be sure to write an IShellWindowMigrationRule or something to account for backwards compatability.
+      public enum ColumnKeys : long {
+         Security = 1,
+         Symbol = 1 << 1,
+         Time = 1 << 2,
+         LastChange = 1 << 3,
+         User = 1 << 4,
+         Account = 1 << 5,
+         Exchange = 1 << 6,
+         TimeInForce = 1 << 7,
+         OrderType = 1 << 8,
+         SubType = 1 << 9,
+         Side = 1 << 10,
+         Size = 1 << 11,
+         Price = 1 << 12,
+         Filled = 1 << 13,
+         AvgFilled = 1 << 14,
+         LastPrice = 1 << 15,
+         Status = 1 << 16,
+         LastMessage = 1 << 17,
+         SecurityKey = 1 << 18,
+         OrderKey = 1 << 19,
+         ParentKey = 1 << 20,
+         AttachedTargetOrderKey = 1 << 21,
+         Cancel = 1 << 22,
+         EditableQty = 1 << 23,
+         EditablePrice = 1 << 24,
+         Attach = 1 << 25,
+         Detach = 1 << 26,
+         Update = 1 << 27,
+         Commit = 1 << 28,
+         RootAlgoType = 1 << 29,
+         Piq = 1 << 30,
+         Check = 1 << 31,
+         NotionalValue = 1L << 32,
+         FilledNotionalValue = 1L << 33,
+      }
+
+      public OrderBookGridView() {
+         InitializeComponent();
+         var row_template_factory = new FrameworkElementFactory(typeof(OrderBookRowView));
+         row_template_factory.SetValue(OrderBookRowView.BackgroundProperty, RowBackgroundHelper.row_background_binding);
+         RowTemplate = new DataTemplate { VisualTree = row_template_factory };
+      }
+
+      static class RowBackgroundHelper {
+         public static readonly BindingBase row_background_binding;
+
+         static RowBackgroundHelper() {
+            var bg_binding = new MultiBinding();
+            bg_binding.Bindings.Add(new Binding(nameof(OrderBookRowVM.status)));
+            bg_binding.Bindings.Add(new Binding(nameof(ListBoxItem.IsSelected)) { RelativeSource = new RelativeSource(RelativeSourceMode.FindAncestor, typeof(ListBoxItem), 1) });
+            double overlay_opacity = config.Configuration.get<bts.config.OrderBookOptions>().StatusColorOpacityWhenSelected;
+            if (overlay_opacity > 0.1 && overlay_opacity < 0.9) {
+               bg_binding.Converter = ValueConverter.create<OrderStatus, bool, Brush>((status, is_selected) => {
+                  var color = _status_color(status);
+                  return color.HasValue ? new SolidColorBrush(color.Value) { Opacity = is_selected ? overlay_opacity : 1.0 } : null;
+               });
+            } else {
+               bg_binding.Converter = ValueConverter.create<OrderStatus, bool, Brush>((status, is_selected) => {
+                  if (is_selected)
+                     return null;
+                  var color = _status_color(status);
+                  return color.HasValue ? new SolidColorBrush(color.Value) : null;
+               });
+            }
+            row_background_binding = bg_binding;
+         }
+
+         static Color? _status_color(OrderStatus status) {
+            switch (status) {
+               case OrderStatus.Canceled: return BTSColors.Orders.canceled_order_background;
+               case OrderStatus.Completed: return BTSColors.Orders.completed_order_background;
+               case OrderStatus.Disconnected:
+               case OrderStatus.RejectedBySpark:
+               case OrderStatus.Error:
+                  return BTSColors.Orders.error_order_background;
+               default: return null;
+            }
+         }
+      }
+   }
+
+   class OrderBookRowView : DIYGrid.HighPerformanceRow<OrderBookRowVM> {
+      public OrderBookRowView() : base() { }
+   }
+}

src/client/spark/Modules/Trading/PNL/VM/PNLRowVM.cs

@@ -1,378 +1,380 @@
-using bts.api;
-using bts.utils;
-
-using mbts;
-using mbts.spark;
-using mbts.ui;
-using mbts.ui.shell;
-
-using Spark.Core.Collections;
-using Spark.Core.Data;
-
-using System;
-using System.Collections.Generic;
-using System.Collections.Immutable;
-using System.Linq;
-using System.Reactive.Linq;
-
-using ColumnKeys = bts.shell.pnl.PNLView.Columns;
-using PNLKey = bts.AccountSecurityKey;
-
-namespace bts.shell.pnl {
-   class PNLRowVM : DisposableViewModelBase, IEquatable<PNLRowVM> {
-      public static readonly IComparer<PNLRowVM> key_comparer = Comparer<PNLKey>.Default.wrap<PNLKey, PNLRowVM>(r => r.key);
-      static readonly Func<bool, string> _fmt_has_day_trades = hdt => hdt ? "Yes" : "No";
-      //
-      readonly PNLCellIndexBuilder<PNLRowData> _cell_map = new PNLCellIndexBuilder<PNLRowData>();
-      readonly PNLCellEmbeddedAnimationModel<PNLRowData> _animator;
-      readonly ISecurityMetadataStore _sec_meta_store;
-      bool _refresh_normalized_data = true, _is_selected;
-      PNLNormalizedData _normalized_data;
-      api.domain.TickDisplaySettingsImmutable? _quantity_display_backing;
-
-      public PNLRowVM(
-         PNLRowData data,
-         ISecurityMetadataStore sec_meta_store,
-         ImmutableDictionary<ColumnKeys, PNLCellAppearance> default_appearances,
-         IReadOnlyList<IPNLAnimationTrigger<PNLRowData>> triggers,
-         Func<double, string> fmt_fill_price,
-         Func<double, string> fmt_market_price) {
-         //
-         this.data = data;
-         _sec_meta_store = sec_meta_store;
-         _animator = new PNLCellEmbeddedAnimationModel<PNLRowData>(default_appearances, triggers);
-         _animator.on_row(data);
-         pnl_price = _cell_map.add_price_double(ColumnKeys.PNLPrice, d => d.pnl_price);
-         account_cell = _cell_map.add(ColumnKeys.Account, d => d.account);
-         security_type_cell = _cell_map.add(ColumnKeys.SecurityType, d => d.security?.ContractCase ?? Security.ContractOneofCase.None);
-         exchange_cell = _cell_map.add(ColumnKeys.Exchange, d => d.security?.Exchange ?? Venue.Unset);
-         expiration_cell = _cell_map.add(
-            ColumnKeys.Expiration,
-            d => d.expiration,
-            e => SecurityFormatter.format_expiration(e));
-         symbol_cell = _cell_map.add(ColumnKeys.Symbol, d => d.symbol);
-         security_cell = _cell_map.add(
-            ColumnKeys.Security,
-            d => d.security,
-            null,
-            EqualityComparer<SecurityImmutable?>.Default,
-            Comparer<SecurityImmutable?>.Default);
-         security_key = _cell_map.add(ColumnKeys.SecurityKey, d => d.key.security_id);
-         cpp = _cell_map.add(ColumnKeys.CPP, d => d.effective_cpp);
-         // sod
-         sod_pnl = _cell_map.add_pnl_double(ColumnKeys.SODPNL, d => d.sod_pnl);
-         sod_net_pos = _cell_map.add_net_pos(
-            ColumnKeys.SODNetPos, d => d.display_sod_net_pos, fmt_qty);
-         sod_long_pos = _cell_map.add_long_pos(
-            ColumnKeys.SODLongPos, d => d.display_sod_long_pos, fmt_qty);
-         sod_long_avg_price = _cell_map.add_long_price(ColumnKeys.SODLongAvgPrice, d => d.sod_long_avg_price);
-         sod_short_pos = _cell_map.add_short_pos(
-            ColumnKeys.SODShortPos, d => d.display_sod_short_pos, fmt_qty);
-         sod_short_avg_price = _cell_map.add_short_price(ColumnKeys.SODShortAvgPrice, d => d.sod_short_avg_price);
-         sod_delta = _cell_map.add_pnl_double(ColumnKeys.SODDelta, d => d.sod_delta, fmt_greek);
-         sod_gamma = _cell_map.add_pnl_double(ColumnKeys.SODGamma, d => d.sod_gamma, fmt_greek);
-         sod_vega = _cell_map.add_pnl_double(ColumnKeys.SODVega, d => d.sod_vega, fmt_greek);
-         sod_theta = _cell_map.add_pnl_double(ColumnKeys.SODTheta, d => d.sod_theta, fmt_greek);
-
-         // day
-         day_pnl = _cell_map.add_pnl_double(ColumnKeys.DayPNL, d => d.day_pnl);
-         day_net_pos = _cell_map.add_net_pos(
-            ColumnKeys.DayNetPos, d => d.display_day_net_pos, fmt_qty);
-         day_long_pos = _cell_map.add_long_pos(
-            ColumnKeys.DayLongPos, d => d.display_day_long_pos, fmt_qty);
-         day_long_avg_price = _cell_map.add_long_price(ColumnKeys.DayLongAvgPrice, d => d.day_long_avg_price);
-         day_short_pos = _cell_map.add_short_pos(
-            ColumnKeys.DayShortPos, d => d.display_day_short_pos, fmt_qty);
-         day_short_avg_price = _cell_map.add_short_price(ColumnKeys.DayShortAvgPrice, d => d.day_short_avg_price);
-         day_delta = _cell_map.add_pnl_double(ColumnKeys.DayDelta, d => d.day_delta, fmt_greek);
-         day_gamma = _cell_map.add_pnl_double(ColumnKeys.DayGamma, d => d.day_gamma, fmt_greek);
-         day_vega = _cell_map.add_pnl_double(ColumnKeys.DayVega, d => d.day_vega, fmt_greek);
-         day_theta = _cell_map.add_pnl_double(ColumnKeys.DayTheta, d => d.day_theta, fmt_greek);
-         has_day_trades = _cell_map.add(ColumnKeys.HasDayTrades, d => d.has_day_trades, _fmt_has_day_trades);
-         // working
-         working_net_pos = _cell_map.add_net_pos(
-            ColumnKeys.WorkingNetPos, d => d.display_working_net_pos, fmt_qty);
-         var qty_display = _sec_meta_store.get_qty_display(security);
-         working_long_pos = _cell_map.add_working_long(
-            ColumnKeys.WorkingLongPos,
-            d => d.working.long_data,
-            d => d.bid_price,
-            qty_display);
-         working_long_avg_price = _cell_map.add_long_price(ColumnKeys.WorkingLongAvgPrice, d => d.working_long_avg_price);
-         working_short_pos = _cell_map.add_working_short(
-            ColumnKeys.WorkingShortPos, r => r.working.short_data, r => r.ask_price, qty_display);
-         working_short_avg_price = _cell_map.add_short_price(ColumnKeys.WorkingShortAvgPrice, d => d.working_short_avg_price);
-         // total
-         total_pnl = _cell_map.add_pnl_double(ColumnKeys.TotalPNL, d => d.total_pnl);
-         total_net_pos = _cell_map.add_net_pos(
-            ColumnKeys.TotalNetPos, d => d.display_total_net_pos, fmt_qty);
-         total_long_pos = _cell_map.add_long_pos(
-            ColumnKeys.TotalLongPos, d => d.display_total_long_pos, fmt_qty);
-         total_long_avg_price = _cell_map.add_long_price(ColumnKeys.TotalLongAvgPrice, d => d.total_long_avg_price);
-         total_short_pos = _cell_map.add_short_pos(
-            ColumnKeys.TotalShortPos, d => d.display_total_short_pos, fmt_qty);
-         total_short_avg_price = _cell_map.add_short_price(ColumnKeys.TotalShortAvgPrice, d => d.total_short_avg_price);
-         total_delta = _cell_map.add_pnl_double(
-            ColumnKeys.TotalDelta, d => d.total_delta, fmt_greek);
-         total_gamma = _cell_map.add_pnl_double(
-            ColumnKeys.TotalGamma, d => d.total_gamma, fmt_greek);
-         total_vega = _cell_map.add_pnl_double(ColumnKeys.TotalVega, d => d.total_vega, fmt_greek);
-         total_theta = _cell_map.add_pnl_double(
-            ColumnKeys.TotalTheta, d => d.total_theta, fmt_greek);
-
-         // market
-         bid_price = _cell_map.add_price_double(ColumnKeys.BidPrice, d => d.bid_price);
-         ask_price = _cell_map.add_price_double(ColumnKeys.AskPrice, d => d.ask_price);
-         mid_price = _cell_map.add_price_double(ColumnKeys.MidPrice, d => d.mid_price);
-         last_price = _cell_map.add_price_double(ColumnKeys.LastPrice, d => d.last_price);
-         settlement_price = _cell_map.add_price_double(ColumnKeys.SettlementPrice, d => d.settlement_price);
-
-         _set_price_formatting_delegates(fmt_fill_price, fmt_market_price);
-         _cell_map.on_row(data);
-         _cell_map.set_appearances(_animator.cell_appearances);
-         _dispose = _cell_map.animation_model.OnDispatcher().Subscribe(new_animations => {
-            animations = new_animations.Select(kv => new mbts.ui.spark.CellAnimation(kv.Key.column, kv.Key.type, kv.Value.color.to_brush())).ToList();
-            RaisePropertyChanged(nameof(animations));
-         });
-
-         static string fmt_greek(double v) =>
-            v.is_nan() ? "-" : v.ToString("#,0.00");
-
-         string fmt_qty(double v) =>
-            quantity_display.format(v);
-      }
-
-      // events
-      public event Action<PNLNormalizedData> updated;
-      public event Action<bool> is_selected_changed;
-
-      // data
-      public PNLKey key => data.key;
-      public PNLRowData data { get; private set; }
-      public PNLNormalizedData normalized_data {
-         get {
-            if (_refresh_normalized_data) {
-               _refresh_normalized_data = false;
-               // Only count outrights (spread positions should be normalized into outrights)
-               var sec_type = security?.ContractCase ?? Security.ContractOneofCase.None;
-               var (
-                  sod_long,
-                  sod_short,
-                  sod_delta,
-                  sod_gamma,
-                  sod_vega,
-                  sod_theta,
-                  day_long,
-                  day_short,
-                  day_delta,
-                  day_gamma,
-                  day_vega,
-                  day_theta) = sec_type switch {
-                  Security.ContractOneofCase.Future or Security.ContractOneofCase.Option => (
-                     data.sod.long_pos.round_to_decimal(),
-                     data.sod.short_pos.round_to_decimal(),
-                     data.sod.delta.round_to_decimal(),
-                     data.sod.gamma.round_to_decimal(),
-                     data.sod.vega.round_to_decimal(),
-                     data.sod.theta.round_to_decimal(),
-                     data.day.long_pos.round_to_decimal(),
-                     data.day.short_pos.round_to_decimal(),
-                     data.day.delta.round_to_decimal(),
-                     data.day.gamma.round_to_decimal(),
-                     data.day.vega.round_to_decimal(),
-                     data.day.theta.round_to_decimal()),
-                  _ => (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
-               };
-
-               var working_long = data.working_long_pos.round_to_decimal();
-               var working_short = data.working_short_pos.round_to_decimal();
-
-               var nd = new PNLNormalizedData() {
-                  // sod
-                  sod_pnl = sod_pnl.value.round_to_decimal(),
-                  sod_long_pos = sod_long,
-                  sod_short_pos = sod_short,
-                  sod_delta = sod_delta,
-                  sod_gamma = sod_gamma,
-                  sod_vega = sod_vega,
-                  sod_theta = sod_theta,
-                  display_sod_long_pos = sod_long,
-                  display_sod_short_pos = sod_short,
-
-                  // day
-                  day_pnl = day_pnl.value.round_to_decimal(),
-                  day_long_pos = day_long,
-                  day_short_pos = day_short,
-                  day_delta = day_delta,
-                  day_gamma = day_gamma,
-                  day_vega = day_vega,
-                  day_theta = day_theta,
-                  display_day_long_pos = day_long,
-                  display_day_short_pos = day_short,
-
-                  // working
-                  working_long_pos = working_long,
-                  working_short_pos = working_short,
-                  display_working_long_pos = working_long,
-                  display_working_short_pos = working_short
-               };
-               if (_normalized_data != nd) {
-                  _normalized_data = nd;
-                  updated?.Invoke(nd);
-               }
-            }
-            return _normalized_data;
-         }
-      }
-      // security
-      public string account_name => data.account_name;
-      public Venue exchange => security?.Exchange ?? Venue.Unset;
-      public string symbol => security.symbol_or_default();
-      public DateTime expiration => data.expiration;
-      public Security.ContractOneofCase security_type => security?.ContractCase ?? Security.ContractOneofCase.None;
-      public SecurityImmutable? security => data.security;
-      // cells
-      public PNLCellVM<PNLRowData, AccountTuple> account_cell { get; }
-      public PNLCellVM<PNLRowData, Venue> exchange_cell { get; }
-      public PNLCellVM<PNLRowData, DateTime> expiration_cell { get; }
-      public PNLCellVM<PNLRowData, string> symbol_cell { get; }
-      public PNLCellVM<PNLRowData, Security.ContractOneofCase> security_type_cell { get; }
-      public PNLCellVM<PNLRowData, SecurityImmutable?> security_cell { get; }
-      public PNLCellVM<PNLRowData, uint> security_key { get; }
-      public PNLCellVM<PNLRowData, double> cpp { get; }
-      // sod
-      public PNLCellVM<PNLRowData, double> sod_pnl { get; }
-      public PNLCellVM<PNLRowData, double> sod_net_pos { get; }
-      public PNLCellVM<PNLRowData, double> sod_long_pos { get; }
-      public PNLCellVM<PNLRowData, double> sod_long_avg_price { get; }
-      public PNLCellVM<PNLRowData, double> sod_short_pos { get; }
-      public PNLCellVM<PNLRowData, double> sod_short_avg_price { get; }
-      public PNLCellVM<PNLRowData, double> sod_delta { get; }
-      public PNLCellVM<PNLRowData, double> sod_gamma { get; }
-      public PNLCellVM<PNLRowData, double> sod_vega { get; }
-      public PNLCellVM<PNLRowData, double> sod_theta { get; }
-
-      // day
-      public PNLCellVM<PNLRowData, double> day_pnl { get; }
-      public PNLCellVM<PNLRowData, double> day_net_pos { get; }
-      public PNLCellVM<PNLRowData, double> day_long_pos { get; }
-      public PNLCellVM<PNLRowData, double> day_long_avg_price { get; }
-      public PNLCellVM<PNLRowData, double> day_short_pos { get; }
-      public PNLCellVM<PNLRowData, double> day_short_avg_price { get; }
-      public PNLCellVM<PNLRowData, double> day_delta { get; }
-      public PNLCellVM<PNLRowData, double> day_gamma { get; }
-      public PNLCellVM<PNLRowData, double> day_vega { get; }
-      public PNLCellVM<PNLRowData, double> day_theta { get; }
-      public PNLCellVM<PNLRowData, bool> has_day_trades { get; }
-      // working
-      public PNLCellVM<PNLRowData, double> working_net_pos { get; }
-      public PNLCellVM<PNLRowData, WorkingOrderSideData> working_long_pos { get; }
-      public PNLCellVM<PNLRowData, double> working_long_avg_price { get; }
-      public PNLCellVM<PNLRowData, WorkingOrderSideData> working_short_pos { get; }
-      public PNLCellVM<PNLRowData, double> working_short_avg_price { get; }
-      // total
-      public PNLCellVM<PNLRowData, double> total_pnl { get; }
-      public PNLCellVM<PNLRowData, double> total_net_pos { get; }
-      public PNLCellVM<PNLRowData, double> total_long_pos { get; }
-      public PNLCellVM<PNLRowData, double> total_long_avg_price { get; }
-      public PNLCellVM<PNLRowData, double> total_short_pos { get; }
-      public PNLCellVM<PNLRowData, double> total_short_avg_price { get; }
-      public PNLCellVM<PNLRowData, double> total_delta { get; }
-      public PNLCellVM<PNLRowData, double> total_gamma { get; }
-      public PNLCellVM<PNLRowData, double> total_vega { get; }
-      public PNLCellVM<PNLRowData, double> total_theta { get; }
-
-      // (market)
-      public PNLCellVM<PNLRowData, double> bid_price { get; }
-      public PNLCellVM<PNLRowData, double> mid_price { get; }
-      public PNLCellVM<PNLRowData, double> ask_price { get; }
-      public PNLCellVM<PNLRowData, double> last_price { get; }
-      public PNLCellVM<PNLRowData, double> settlement_price { get; }
-      public PNLCellVM<PNLRowData, double> pnl_price { get; }
-
-      // display
-      public api.domain.TickDisplaySettingsImmutable quantity_display {
-         get {
-            if (_quantity_display_backing.HasValue) {
-               return _quantity_display_backing.Value;
-            }
-            if (security.HasValue) {
-               _quantity_display_backing = _sec_meta_store.get_qty_display(security.Value);
-            }
-            return _quantity_display_backing ??= Security.Defaults.quantity_display_settingsi;
-         }
-      }
-
-      public IList<mbts.ui.spark.CellAnimation> animations { get; private set; }
-
-      public bool is_selected {
-         get => _is_selected;
-         set {
-            if (Set(ref _is_selected, value))
-               is_selected_changed?.Invoke(value);
-         }
-      }
-
-      IEnumerable<PNLCellVM<PNLRowData, double>> _market_price_cells {
-         get {
-            yield return pnl_price;
-            yield return last_price;
-            yield return bid_price;
-            yield return mid_price;
-            yield return ask_price;
-            yield return settlement_price;
-         }
-      }
-
-      // includes sod, day, total, and working avg price columns
-      IEnumerable<PNLCellVM<PNLRowData, double>> _fill_price_cells {
-         get {
-            // sod
-            yield return sod_long_avg_price;
-            yield return sod_short_avg_price;
-            // day
-            yield return day_long_avg_price;
-            yield return day_short_avg_price;
-            // total
-            yield return total_long_avg_price;
-            yield return total_short_avg_price;
-            // working
-            yield return working_long_avg_price;
-            yield return working_short_avg_price;
-         }
-      }
-
-      public void update(PNLRowData data) {
-         _quantity_display_backing = null;
-         _refresh_normalized_data = true;
-         this.data = data;
-         _cell_map.on_row(data);
-         _animator.on_row(data);
-         _cell_map.set_appearances(_animator.cell_appearances);
-      }
-
-      public void on_settings(ImmutableDictionary<ColumnKeys, PNLCellAppearance> defaults, IReadOnlyList<IPNLAnimationTrigger<PNLRowData>> triggers, Func<double, string> fmt_fill_price, Func<double, string> fmt_market_price) {
-         _animator.on_settings(defaults, triggers);
-         _cell_map.set_appearances(_animator.cell_appearances);
-         _set_price_formatting_delegates(fmt_fill_price, fmt_market_price);
-      }
-
-      public bool Equals(PNLRowVM other) =>
-         other != null && key == other.key;
-
-      public override int GetHashCode() =>
-         key.GetHashCode();
-
-      public override string ToString() =>
-         $"account: {account_name} exchange: {exchange} symbol: {symbol} security: {security} buy_quantity: {day_long_pos} sell_quantity: {day_short_pos} pnl: {total_pnl}";
-
-      void _set_price_formatting_delegates(Func<double, string> fmt_fill_price, Func<double, string> fmt_market_price) {
-         foreach (var cell in _fill_price_cells) {
-            cell.format = fmt_fill_price;
-         }
-         foreach (var cell in _market_price_cells) {
-            cell.format = fmt_market_price;
-         }
-      }
-   }
-}
+using bts.api;
+using bts.utils;
+
+using mbts;
+using mbts.spark;
+using mbts.ui;
+using mbts.ui.shell;
+
+using Spark.Core.Collections;
+using Spark.Core.Data;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Reactive.Linq;
+
+using ColumnKeys = bts.shell.pnl.PNLView.Columns;
+using PNLKey = bts.AccountSecurityKey;
+
+namespace bts.shell.pnl {
+   class PNLRowVM : DisposableViewModelBase, IEquatable<PNLRowVM> {
+      public static readonly IComparer<PNLRowVM> key_comparer = Comparer<PNLKey>.Default.wrap<PNLKey, PNLRowVM>(r => r.key);
+      static readonly Func<bool, string> _fmt_has_day_trades = hdt => hdt ? "Yes" : "No";
+      //
+      readonly PNLCellIndexBuilder<PNLRowData> _cell_map = new PNLCellIndexBuilder<PNLRowData>();
+      readonly PNLCellEmbeddedAnimationModel<PNLRowData> _animator;
+      readonly ISecurityMetadataStore _sec_meta_store;
+      bool _refresh_normalized_data = true, _is_selected;
+      PNLNormalizedData _normalized_data;
+      api.domain.TickDisplaySettingsImmutable? _quantity_display_backing;
+
+      public PNLRowVM(
+         PNLRowData data,
+         ISecurityMetadataStore sec_meta_store,
+         ImmutableDictionary<ColumnKeys, PNLCellAppearance> default_appearances,
+         IReadOnlyList<IPNLAnimationTrigger<PNLRowData>> triggers,
+         Func<double, string> fmt_fill_price,
+         Func<double, string> fmt_market_price) {
+         //
+         this.data = data;
+         _sec_meta_store = sec_meta_store;
+         _animator = new PNLCellEmbeddedAnimationModel<PNLRowData>(default_appearances, triggers);
+         _animator.on_row(data);
+         pnl_price = _cell_map.add_price_double(ColumnKeys.PNLPrice, d => d.pnl_price);
+         account_cell = _cell_map.add(ColumnKeys.Account, d => d.account);
+         security_type_cell = _cell_map.add(ColumnKeys.SecurityType, d => d.security?.ContractCase ?? Security.ContractOneofCase.None);
+         exchange_cell = _cell_map.add(ColumnKeys.Exchange, d => d.security?.Exchange ?? Venue.Unset);
+         expiration_cell = _cell_map.add(
+            ColumnKeys.Expiration,
+            d => d.expiration,
+            e => e == DateTime.MinValue
+               ? "N/A"
+               : SecurityFormatter.format_expiration(e));
+         symbol_cell = _cell_map.add(ColumnKeys.Symbol, d => d.symbol);
+         security_cell = _cell_map.add(
+            ColumnKeys.Security,
+            d => d.security,
+            null,
+            EqualityComparer<SecurityImmutable?>.Default,
+            Comparer<SecurityImmutable?>.Default);
+         security_key = _cell_map.add(ColumnKeys.SecurityKey, d => d.key.security_id);
+         cpp = _cell_map.add(ColumnKeys.CPP, d => d.effective_cpp);
+         // sod
+         sod_pnl = _cell_map.add_pnl_double(ColumnKeys.SODPNL, d => d.sod_pnl);
+         sod_net_pos = _cell_map.add_net_pos(
+            ColumnKeys.SODNetPos, d => d.display_sod_net_pos, fmt_qty);
+         sod_long_pos = _cell_map.add_long_pos(
+            ColumnKeys.SODLongPos, d => d.display_sod_long_pos, fmt_qty);
+         sod_long_avg_price = _cell_map.add_long_price(ColumnKeys.SODLongAvgPrice, d => d.sod_long_avg_price);
+         sod_short_pos = _cell_map.add_short_pos(
+            ColumnKeys.SODShortPos, d => d.display_sod_short_pos, fmt_qty);
+         sod_short_avg_price = _cell_map.add_short_price(ColumnKeys.SODShortAvgPrice, d => d.sod_short_avg_price);
+         sod_delta = _cell_map.add_pnl_double(ColumnKeys.SODDelta, d => d.sod_delta, fmt_greek);
+         sod_gamma = _cell_map.add_pnl_double(ColumnKeys.SODGamma, d => d.sod_gamma, fmt_greek);
+         sod_vega = _cell_map.add_pnl_double(ColumnKeys.SODVega, d => d.sod_vega, fmt_greek);
+         sod_theta = _cell_map.add_pnl_double(ColumnKeys.SODTheta, d => d.sod_theta, fmt_greek);
+
+         // day
+         day_pnl = _cell_map.add_pnl_double(ColumnKeys.DayPNL, d => d.day_pnl);
+         day_net_pos = _cell_map.add_net_pos(
+            ColumnKeys.DayNetPos, d => d.display_day_net_pos, fmt_qty);
+         day_long_pos = _cell_map.add_long_pos(
+            ColumnKeys.DayLongPos, d => d.display_day_long_pos, fmt_qty);
+         day_long_avg_price = _cell_map.add_long_price(ColumnKeys.DayLongAvgPrice, d => d.day_long_avg_price);
+         day_short_pos = _cell_map.add_short_pos(
+            ColumnKeys.DayShortPos, d => d.display_day_short_pos, fmt_qty);
+         day_short_avg_price = _cell_map.add_short_price(ColumnKeys.DayShortAvgPrice, d => d.day_short_avg_price);
+         day_delta = _cell_map.add_pnl_double(ColumnKeys.DayDelta, d => d.day_delta, fmt_greek);
+         day_gamma = _cell_map.add_pnl_double(ColumnKeys.DayGamma, d => d.day_gamma, fmt_greek);
+         day_vega = _cell_map.add_pnl_double(ColumnKeys.DayVega, d => d.day_vega, fmt_greek);
+         day_theta = _cell_map.add_pnl_double(ColumnKeys.DayTheta, d => d.day_theta, fmt_greek);
+         has_day_trades = _cell_map.add(ColumnKeys.HasDayTrades, d => d.has_day_trades, _fmt_has_day_trades);
+         // working
+         working_net_pos = _cell_map.add_net_pos(
+            ColumnKeys.WorkingNetPos, d => d.display_working_net_pos, fmt_qty);
+         var qty_display = _sec_meta_store.get_qty_display(security);
+         working_long_pos = _cell_map.add_working_long(
+            ColumnKeys.WorkingLongPos,
+            d => d.working.long_data,
+            d => d.bid_price,
+            qty_display);
+         working_long_avg_price = _cell_map.add_long_price(ColumnKeys.WorkingLongAvgPrice, d => d.working_long_avg_price);
+         working_short_pos = _cell_map.add_working_short(
+            ColumnKeys.WorkingShortPos, r => r.working.short_data, r => r.ask_price, qty_display);
+         working_short_avg_price = _cell_map.add_short_price(ColumnKeys.WorkingShortAvgPrice, d => d.working_short_avg_price);
+         // total
+         total_pnl = _cell_map.add_pnl_double(ColumnKeys.TotalPNL, d => d.total_pnl);
+         total_net_pos = _cell_map.add_net_pos(
+            ColumnKeys.TotalNetPos, d => d.display_total_net_pos, fmt_qty);
+         total_long_pos = _cell_map.add_long_pos(
+            ColumnKeys.TotalLongPos, d => d.display_total_long_pos, fmt_qty);
+         total_long_avg_price = _cell_map.add_long_price(ColumnKeys.TotalLongAvgPrice, d => d.total_long_avg_price);
+         total_short_pos = _cell_map.add_short_pos(
+            ColumnKeys.TotalShortPos, d => d.display_total_short_pos, fmt_qty);
+         total_short_avg_price = _cell_map.add_short_price(ColumnKeys.TotalShortAvgPrice, d => d.total_short_avg_price);
+         total_delta = _cell_map.add_pnl_double(
+            ColumnKeys.TotalDelta, d => d.total_delta, fmt_greek);
+         total_gamma = _cell_map.add_pnl_double(
+            ColumnKeys.TotalGamma, d => d.total_gamma, fmt_greek);
+         total_vega = _cell_map.add_pnl_double(ColumnKeys.TotalVega, d => d.total_vega, fmt_greek);
+         total_theta = _cell_map.add_pnl_double(
+            ColumnKeys.TotalTheta, d => d.total_theta, fmt_greek);
+
+         // market
+         bid_price = _cell_map.add_price_double(ColumnKeys.BidPrice, d => d.bid_price);
+         ask_price = _cell_map.add_price_double(ColumnKeys.AskPrice, d => d.ask_price);
+         mid_price = _cell_map.add_price_double(ColumnKeys.MidPrice, d => d.mid_price);
+         last_price = _cell_map.add_price_double(ColumnKeys.LastPrice, d => d.last_price);
+         settlement_price = _cell_map.add_price_double(ColumnKeys.SettlementPrice, d => d.settlement_price);
+
+         _set_price_formatting_delegates(fmt_fill_price, fmt_market_price);
+         _cell_map.on_row(data);
+         _cell_map.set_appearances(_animator.cell_appearances);
+         _dispose = _cell_map.animation_model.OnDispatcher().Subscribe(new_animations => {
+            animations = new_animations.Select(kv => new mbts.ui.spark.CellAnimation(kv.Key.column, kv.Key.type, kv.Value.color.to_brush())).ToList();
+            RaisePropertyChanged(nameof(animations));
+         });
+
+         static string fmt_greek(double v) =>
+            v.is_nan() ? "-" : v.ToString("#,0.00");
+
+         string fmt_qty(double v) =>
+            quantity_display.format(v);
+      }
+
+      // events
+      public event Action<PNLNormalizedData> updated;
+      public event Action<bool> is_selected_changed;
+
+      // data
+      public PNLKey key => data.key;
+      public PNLRowData data { get; private set; }
+      public PNLNormalizedData normalized_data {
+         get {
+            if (_refresh_normalized_data) {
+               _refresh_normalized_data = false;
+               // Only count outrights (spread positions should be normalized into outrights)
+               var sec_type = security?.ContractCase ?? Security.ContractOneofCase.None;
+               var (
+                  sod_long,
+                  sod_short,
+                  sod_delta,
+                  sod_gamma,
+                  sod_vega,
+                  sod_theta,
+                  day_long,
+                  day_short,
+                  day_delta,
+                  day_gamma,
+                  day_vega,
+                  day_theta) = sec_type switch {
+                  Security.ContractOneofCase.Future or Security.ContractOneofCase.Option => (
+                     data.sod.long_pos.round_to_decimal(),
+                     data.sod.short_pos.round_to_decimal(),
+                     data.sod.delta.round_to_decimal(),
+                     data.sod.gamma.round_to_decimal(),
+                     data.sod.vega.round_to_decimal(),
+                     data.sod.theta.round_to_decimal(),
+                     data.day.long_pos.round_to_decimal(),
+                     data.day.short_pos.round_to_decimal(),
+                     data.day.delta.round_to_decimal(),
+                     data.day.gamma.round_to_decimal(),
+                     data.day.vega.round_to_decimal(),
+                     data.day.theta.round_to_decimal()),
+                  _ => (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
+               };
+
+               var working_long = data.working_long_pos.round_to_decimal();
+               var working_short = data.working_short_pos.round_to_decimal();
+
+               var nd = new PNLNormalizedData() {
+                  // sod
+                  sod_pnl = sod_pnl.value.round_to_decimal(),
+                  sod_long_pos = sod_long,
+                  sod_short_pos = sod_short,
+                  sod_delta = sod_delta,
+                  sod_gamma = sod_gamma,
+                  sod_vega = sod_vega,
+                  sod_theta = sod_theta,
+                  display_sod_long_pos = sod_long,
+                  display_sod_short_pos = sod_short,
+
+                  // day
+                  day_pnl = day_pnl.value.round_to_decimal(),
+                  day_long_pos = day_long,
+                  day_short_pos = day_short,
+                  day_delta = day_delta,
+                  day_gamma = day_gamma,
+                  day_vega = day_vega,
+                  day_theta = day_theta,
+                  display_day_long_pos = day_long,
+                  display_day_short_pos = day_short,
+
+                  // working
+                  working_long_pos = working_long,
+                  working_short_pos = working_short,
+                  display_working_long_pos = working_long,
+                  display_working_short_pos = working_short
+               };
+               if (_normalized_data != nd) {
+                  _normalized_data = nd;
+                  updated?.Invoke(nd);
+               }
+            }
+            return _normalized_data;
+         }
+      }
+      // security
+      public string account_name => data.account_name;
+      public Venue exchange => security?.Exchange ?? Venue.Unset;
+      public string symbol => security.symbol_or_default();
+      public DateTime expiration => data.expiration;
+      public Security.ContractOneofCase security_type => security?.ContractCase ?? Security.ContractOneofCase.None;
+      public SecurityImmutable? security => data.security;
+      // cells
+      public PNLCellVM<PNLRowData, AccountTuple> account_cell { get; }
+      public PNLCellVM<PNLRowData, Venue> exchange_cell { get; }
+      public PNLCellVM<PNLRowData, DateTime> expiration_cell { get; }
+      public PNLCellVM<PNLRowData, string> symbol_cell { get; }
+      public PNLCellVM<PNLRowData, Security.ContractOneofCase> security_type_cell { get; }
+      public PNLCellVM<PNLRowData, SecurityImmutable?> security_cell { get; }
+      public PNLCellVM<PNLRowData, uint> security_key { get; }
+      public PNLCellVM<PNLRowData, double> cpp { get; }
+      // sod
+      public PNLCellVM<PNLRowData, double> sod_pnl { get; }
+      public PNLCellVM<PNLRowData, double> sod_net_pos { get; }
+      public PNLCellVM<PNLRowData, double> sod_long_pos { get; }
+      public PNLCellVM<PNLRowData, double> sod_long_avg_price { get; }
+      public PNLCellVM<PNLRowData, double> sod_short_pos { get; }
+      public PNLCellVM<PNLRowData, double> sod_short_avg_price { get; }
+      public PNLCellVM<PNLRowData, double> sod_delta { get; }
+      public PNLCellVM<PNLRowData, double> sod_gamma { get; }
+      public PNLCellVM<PNLRowData, double> sod_vega { get; }
+      public PNLCellVM<PNLRowData, double> sod_theta { get; }
+
+      // day
+      public PNLCellVM<PNLRowData, double> day_pnl { get; }
+      public PNLCellVM<PNLRowData, double> day_net_pos { get; }
+      public PNLCellVM<PNLRowData, double> day_long_pos { get; }
+      public PNLCellVM<PNLRowData, double> day_long_avg_price { get; }
+      public PNLCellVM<PNLRowData, double> day_short_pos { get; }
+      public PNLCellVM<PNLRowData, double> day_short_avg_price { get; }
+      public PNLCellVM<PNLRowData, double> day_delta { get; }
+      public PNLCellVM<PNLRowData, double> day_gamma { get; }
+      public PNLCellVM<PNLRowData, double> day_vega { get; }
+      public PNLCellVM<PNLRowData, double> day_theta { get; }
+      public PNLCellVM<PNLRowData, bool> has_day_trades { get; }
+      // working
+      public PNLCellVM<PNLRowData, double> working_net_pos { get; }
+      public PNLCellVM<PNLRowData, WorkingOrderSideData> working_long_pos { get; }
+      public PNLCellVM<PNLRowData, double> working_long_avg_price { get; }
+      public PNLCellVM<PNLRowData, WorkingOrderSideData> working_short_pos { get; }
+      public PNLCellVM<PNLRowData, double> working_short_avg_price { get; }
+      // total
+      public PNLCellVM<PNLRowData, double> total_pnl { get; }
+      public PNLCellVM<PNLRowData, double> total_net_pos { get; }
+      public PNLCellVM<PNLRowData, double> total_long_pos { get; }
+      public PNLCellVM<PNLRowData, double> total_long_avg_price { get; }
+      public PNLCellVM<PNLRowData, double> total_short_pos { get; }
+      public PNLCellVM<PNLRowData, double> total_short_avg_price { get; }
+      public PNLCellVM<PNLRowData, double> total_delta { get; }
+      public PNLCellVM<PNLRowData, double> total_gamma { get; }
+      public PNLCellVM<PNLRowData, double> total_vega { get; }
+      public PNLCellVM<PNLRowData, double> total_theta { get; }
+
+      // (market)
+      public PNLCellVM<PNLRowData, double> bid_price { get; }
+      public PNLCellVM<PNLRowData, double> mid_price { get; }
+      public PNLCellVM<PNLRowData, double> ask_price { get; }
+      public PNLCellVM<PNLRowData, double> last_price { get; }
+      public PNLCellVM<PNLRowData, double> settlement_price { get; }
+      public PNLCellVM<PNLRowData, double> pnl_price { get; }
+
+      // display
+      public api.domain.TickDisplaySettingsImmutable quantity_display {
+         get {
+            if (_quantity_display_backing.HasValue) {
+               return _quantity_display_backing.Value;
+            }
+            if (security.HasValue) {
+               _quantity_display_backing = _sec_meta_store.get_qty_display(security.Value);
+            }
+            return _quantity_display_backing ??= Security.Defaults.quantity_display_settingsi;
+         }
+      }
+
+      public IList<mbts.ui.spark.CellAnimation> animations { get; private set; }
+
+      public bool is_selected {
+         get => _is_selected;
+         set {
+            if (Set(ref _is_selected, value))
+               is_selected_changed?.Invoke(value);
+         }
+      }
+
+      IEnumerable<PNLCellVM<PNLRowData, double>> _market_price_cells {
+         get {
+            yield return pnl_price;
+            yield return last_price;
+            yield return bid_price;
+            yield return mid_price;
+            yield return ask_price;
+            yield return settlement_price;
+         }
+      }
+
+      // includes sod, day, total, and working avg price columns
+      IEnumerable<PNLCellVM<PNLRowData, double>> _fill_price_cells {
+         get {
+            // sod
+            yield return sod_long_avg_price;
+            yield return sod_short_avg_price;
+            // day
+            yield return day_long_avg_price;
+            yield return day_short_avg_price;
+            // total
+            yield return total_long_avg_price;
+            yield return total_short_avg_price;
+            // working
+            yield return working_long_avg_price;
+            yield return working_short_avg_price;
+         }
+      }
+
+      public void update(PNLRowData data) {
+         _quantity_display_backing = null;
+         _refresh_normalized_data = true;
+         this.data = data;
+         _cell_map.on_row(data);
+         _animator.on_row(data);
+         _cell_map.set_appearances(_animator.cell_appearances);
+      }
+
+      public void on_settings(ImmutableDictionary<ColumnKeys, PNLCellAppearance> defaults, IReadOnlyList<IPNLAnimationTrigger<PNLRowData>> triggers, Func<double, string> fmt_fill_price, Func<double, string> fmt_market_price) {
+         _animator.on_settings(defaults, triggers);
+         _cell_map.set_appearances(_animator.cell_appearances);
+         _set_price_formatting_delegates(fmt_fill_price, fmt_market_price);
+      }
+
+      public bool Equals(PNLRowVM other) =>
+         other != null && key == other.key;
+
+      public override int GetHashCode() =>
+         key.GetHashCode();
+
+      public override string ToString() =>
+         $"account: {account_name} exchange: {exchange} symbol: {symbol} security: {security} buy_quantity: {day_long_pos} sell_quantity: {day_short_pos} pnl: {total_pnl}";
+
+      void _set_price_formatting_delegates(Func<double, string> fmt_fill_price, Func<double, string> fmt_market_price) {
+         foreach (var cell in _fill_price_cells) {
+            cell.format = fmt_fill_price;
+         }
+         foreach (var cell in _market_price_cells) {
+            cell.format = fmt_market_price;
+         }
+      }
+   }
+}

src/client/spark/Modules/Trading/PNL/VM/PNLVM.cs

@@ -774,6 +774,17 @@ namespace bts.shell.pnl {
             case ColumnKeys.Account: filter.accounts = new[] { g.Name.ToString() }; break;
             case ColumnKeys.Symbol: filter.symbols = new[] { g.Name.ToString() }; break;
             case ColumnKeys.Exchange: filter.exchanges = new[] { (Venue)g.Name }; break;
+            case ColumnKeys.Expiration: {
+                  if (g.Name is DateTime exp) {
+                     if (filter.security_criteria is null) {
+                        filter.security_criteria = SecurityCriteria.from_expirations(exp);
+                     } else {
+                        filter.security_criteria.clauses = ArrayHelper.add(
+                           filter.security_criteria.clauses, SecurityCriteriaClause.from_expirations(exp));
+                     }
+                  }
+                  break;
+               }
             case ColumnKeys.Security:
                if (g.Name is SecurityImmutable sec)
                   filter.security_criteria = SecurityCriteria.from_securities(sec);

src/client/spark/NMSheets/BlackboardStructure/NMSheetSingleOptionData.cs

@@ -237,7 +237,7 @@ namespace BCG {
          PNLRollOverDataUnit u;
          if (_roll_over_data != null && (u = _roll_over_data.getValue(exp, strike, true)) != null) {
             cp_row[(int)NMSheetColumnName.vol_change] = tp.vol - u.vol;
-            cp_row[(int)NMSheetColumnName.straddle_change] = ((tp.vol - u.vol) * tp.cvega * 2.0 * 100.0) / tp.driver;
+            cp_row[(int)NMSheetColumnName.straddle_change] = (tp.vol - u.vol) * tp.cvega * 2.0 * 100.0;
          } else {
             cp_row[(int)NMSheetColumnName.vol_change] = Double.NaN;
             cp_row[(int)NMSheetColumnName.straddle_change] = Double.NaN;

src/client/spark/NMSheets/DataProviders/NMSheetArchivedVolProvider.cs

@@ -14,103 +14,23 @@ using mbts.analytics;
 using bts.api.analytics;
 
 namespace mbts.spark {
-   public class ClientArchivedVolRequest {
-      public readonly int id;
-      public readonly ArchivedTheoRetrieval ret;
-      public ClientArchivedVolRequest(Int32 id, ArchivedTheoRetrieval ret) {
-         this.id = id;
-         this.ret = ret;
-      }
-      private static int _id = 0;
-      public static int get_new_id() {
-         int rv = 0;
-         lock (typeof(ClientArchivedVolRequest)) {
-            rv = ++_id;
-         }
-         return rv;
-      }
-      public override string ToString() {
-         return $"id: {id} sym: {ret.SecTup.Symbol} vol_time: {ret.SaveTime}";
-      }
-   }
    // This is created by the NMSheetSingleOptionMainDataSource - it downloads the
    // archived theos and provides access to it
    public class NMSheetArchivedVolProvider : IDisposable {
       private readonly ThreadedBlackboard _bb;
-      private readonly NMSheetSymbol _sym;
-      private readonly bts.api.SecurityTuple _sec_tup;
       private Action _cb = () => { };
       private readonly ITag<SystemContractData> _sc_tag = SparkTags.SystemContractsTag;
-      private readonly ITag<NMSheetSettings> _settings_tag;
-      private readonly ITag<Tuple<ClientArchivedVolRequest, bts.api.analytics.ArchivedTheoList>>
-         _response_tag = SparkTags.ResponseArchivedVols;
-      private bool _waiting_for_reponse = false;
-      private ClientArchivedVolRequest _last_req;
-      bts.api.analytics.ArchivedTheoList _cached_data = null;
+      private Nullable<ArchivedTheoListImmutable> _cached_data;
 
       public NMSheetArchivedVolProvider(string req_id, ThreadedBlackboard bb,
                                        NMSheetSymbol symbol,
                                        Action cb) {
          bts.Log.Ui.info($"NMSheetArchivedVolProvider req_id: {req_id} symbol: {symbol}");
          _bb = bb;
-         _sym = symbol;
-         _sec_tup = new bts.api.SecurityTuple() {
-            Venue = _sym.option_exchange,
-            Type = SecurityType.Option,
-            Symbol = _sym.option_name
-         };
-         _last_req = new ClientArchivedVolRequest(
-            0,
-            new ArchivedTheoRetrieval() {
-               SaveTime = 0,
-               SecTup = _sec_tup
-            });
          _cb = cb;
-         _settings_tag = SparkTags.TC2Tags.NMSheetSettingsTag(req_id);
          _bb.subscribe(_sc_tag);
-         _bb.subscribe(_settings_tag);
-         _bb.subscribe(_response_tag);
-         _bb.data_callback += _on_data;
-         _check_and_send_request(true);
-      }
-
-      private static UInt64 _get_vol_time(NMSheetSettings settings) {
-         NMSheetSettingLayout.VolDay vol_day = settings.setting_layout.vol_day;
-         TimeSpan vol_time = settings.setting_layout.vol_time;
-         DateTime date = DateTime.Today.Date;
-         switch (vol_day) {
-            case NMSheetSettingLayout.VolDay.DayBeforeYesterday:
-               date = date.Subtract(TimeSpan.FromDays(2));
-               break;
-            case NMSheetSettingLayout.VolDay.Yesterday:
-               date = date.Subtract(TimeSpan.FromDays(1));
-               break;
-            case NMSheetSettingLayout.VolDay.Today:
-               break;
-         }
-         DateTime dt = date.Add(vol_time);
-         var rv = VolSurfaceUtilityFunctions.get_unix_epoch_msecs(dt);
-         bts.Log.Ui.info($"NMSheetArchivedVolProvider._get_vol_time vol_day: {vol_day} " +
-            $"vol_time: {vol_time} dt: {dt} epoch_md: {rv}");
-         return rv;
-      }
-
-      private void _check_and_send_request(bool force = false) {
-         NMSheetSettings settings = _bb.read_state(_settings_tag);
-         if (settings != null && settings.setting_layout != null) {
-            UInt64 vol_time = _get_vol_time(settings);
-            if (force || _last_req.ret.SaveTime != vol_time) {
-               _last_req = new ClientArchivedVolRequest(
-                     ClientArchivedVolRequest.get_new_id(),
-                     new ArchivedTheoRetrieval() {
-                        SaveTime = vol_time,
-                        SecTup = _sec_tup
-                     });
-               _waiting_for_reponse = true;
-               bts.Log.Gen.info($"NMSheetArchivedVolProvider._check_and_send_request req: {_last_req}");
-               _bb.post(SparkTags.RequestArchivedVols, _last_req);
-            }
-         }
+         _bb.subscribe(SparkTags.ArchivedTheos);
+         _bb.data_callback += _on_data;         
       }
 
       private void _compute_and_post() {
@@ -118,34 +38,22 @@ namespace mbts.spark {
       }
 
       private void _on_data(object tag, object data) {
-         if (_settings_tag.Equals(tag)) {
-            _check_and_send_request();
-         } else if (_response_tag.Equals(tag)) {
-            if (_waiting_for_reponse) {
-               Tuple<ClientArchivedVolRequest, bts.api.analytics.ArchivedTheoList> resp =
-                  _bb.read_state(_response_tag);
-               if (resp != null) {
-                  ClientArchivedVolRequest responsed_req = resp.Item1;
-                  if (responsed_req.id == _last_req.id) {
-                     // got we were looking for
-                     _cached_data = resp.Item2;
-                     _waiting_for_reponse = false;
-                     bts.Log.Gen.info($"NMSheetArchivedVolProvider._on_data recvd data, req: {_last_req}");
-                     _compute_and_post();
-                  }
-               }
-            }
-         } else if (_sc_tag.Equals(tag)) {
+         // just listen to archihved theos and fire callback
+         if (tag.Equals(SparkTags.ArchivedTheos)) {
+            _cached_data = new Nullable<ArchivedTheoListImmutable>((ArchivedTheoListImmutable)data);
+            _compute_and_post();
+          }
+         else if (_sc_tag.Equals(tag)) {
             _compute_and_post();
          }
       }
 
       public OptionContractKeyedData<PNLRollOverDataUnit> get_all_data() {
          SystemContractData sc_data = _bb.read_state(_sc_tag);
-         if (_cached_data != null && sc_data != null) {
+         if (sc_data != null && _cached_data != null && _cached_data.HasValue) {
             OptionContractKeyedData<PNLRollOverDataUnit> rv =
                new OptionContractKeyedData<PNLRollOverDataUnit>();
-            foreach (var v in _cached_data.SecTheo) {
+            foreach (var v in _cached_data.Value.SecTheo) {
                var sec_md = sc_data.get_sec_md(v.Id);
                try {
                   if (sec_md != null) {

src/client/spark/NMSheets/INMSheetGridFacingData.cs

@@ -65,9 +65,9 @@ namespace BCG {
       public static NMSheetTheoBBOCross get_cross_state(double bid_price, double ask_price, double theo, double delta, bool is_call, double driver_tick_size, bool is_proximity_alert_enabled) {
          double abs_delta = Math.Abs(delta);
          abs_delta = abs_delta == 0 ? Double.Epsilon : abs_delta;
-         if (!Double.IsNaN(theo) && !Double.IsNaN(bid_price) && theo < bid_price)
+         if (!Double.IsNaN(theo) && !Double.IsNaN(bid_price) && (bid_price - theo) > _basic_crosss_eps)
             return new NMSheetTheoBBOCross(CrossState.bid_cross, 0);
-         else if (!Double.IsNaN(theo) && !Double.IsNaN(ask_price) && theo > ask_price)
+         else if (!Double.IsNaN(theo) && !Double.IsNaN(ask_price) && (theo - ask_price) > _basic_crosss_eps)
             return new NMSheetTheoBBOCross(CrossState.ask_cross, 0);
          else if (is_proximity_alert_enabled) {
             /// bool is passed to bypass doing the below
@@ -87,6 +87,7 @@ namespace BCG {
          return NMSheetTheoBBOCross.get_nulled_obj();
       }
 
+      private static double _basic_crosss_eps = 0.01;
       private static double _n_max_ticks = 100.0;
       private static NMSheetTheoBBOCross _nulled_obj = new NMSheetTheoBBOCross(CrossState.none, 0);
       public static NMSheetTheoBBOCross get_nulled_obj() { return _nulled_obj; }

src/client/spark/NMSheets/UIComponents/MainSheet/NMSheetControl.Designer.cs

@@ -1,5 +1,7 @@
-namespace BCG {
-   partial class NMSheetControl {
+namespace BCG
+{
+   partial class NMSheetControl
+   {
       /// <summary> 
       /// Required designer variable.
       /// </summary>
@@ -9,9 +11,11 @@
       /// Clean up any resources being used.
       /// </summary>
       /// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
-      protected override void Dispose(bool disposing) {
+      protected override void Dispose(bool disposing)
+      {
          _logical_dispose(disposing);
-         if (disposing && (components != null)) {
+         if (disposing && (components != null))
+         {
             components.Dispose();
 
          }
@@ -24,7 +28,8 @@
       /// Required method for Designer support - do not modify 
       /// the contents of this method with the code editor.
       /// </summary>
-      private void InitializeComponent() {
+      private void InitializeComponent()
+      {
          this.components = new System.ComponentModel.Container();
          this._menu_panel = new System.Windows.Forms.Panel();
          this._button_minimize_spreads = new System.Windows.Forms.Button();
@@ -67,6 +72,7 @@
          this._toolstripmenuitem_task_manage_templates = new System.Windows.Forms.ToolStripMenuItem();
          this._toolstripseperator_task_eye = new System.Windows.Forms.ToolStripSeparator();
          this._toolstrip_menuitem_save_data_to_file = new System.Windows.Forms.ToolStripMenuItem();
+         this._toolstrip_menuitem_save_ivs = new System.Windows.Forms.ToolStripMenuItem();
          this._toolstripseperator_export_data = new System.Windows.Forms.ToolStripSeparator();
          this._toolstripmenuitem_card = new System.Windows.Forms.ToolStripMenuItem();
          this._toolstripseperator_expirations = new System.Windows.Forms.ToolStripSeparator();
@@ -293,7 +299,8 @@
             this._toolstripmenuitem_atm,
             this._toolseperator_atm,
          };
-         if (_api_tasking_enabled) {
+         if (_api_tasking_enabled)
+         {
             right_click_menu.AddRange(new System.Collections.Generic.List<System.Windows.Forms.ToolStripItem>() {
                this._toolstripmenuitem_task_eye,
                // no quoter
@@ -307,7 +314,8 @@
          //       this._toolstripmenuitem_task_override_size,
          //    });
          // }
-         if (_api_tasking_enabled) {
+         if (_api_tasking_enabled)
+         {
             right_click_menu.AddRange(new System.Collections.Generic.List<System.Windows.Forms.ToolStripItem>() {
                this._toolstripmenuitem_task_eye_switch_all_off,
                // We can do this from the algo studio now
@@ -317,6 +325,7 @@
          right_click_menu.AddRange(new System.Collections.Generic.List<System.Windows.Forms.ToolStripItem>() {
             // this._toolstripseperator_task_eye,
             this._toolstrip_menuitem_save_data_to_file,
+            this._toolstrip_menuitem_save_ivs,
             this._toolstripseperator_export_data,
             this._toolstripmenuitem_card,
             this._toolstripseperator_expirations,
@@ -324,7 +333,8 @@
             this._toolstripmenuitem_minimize_spreads,
             this._toolstripseperator_filter,
          });
-         if (_api_tasking_enabled) {
+         if (_api_tasking_enabled)
+         {
             right_click_menu.AddRange(new System.Collections.Generic.List<System.Windows.Forms.ToolStripItem>() {
                this._toolstripmenuitem_filter,
                this._toolseperator_quotesets
@@ -497,6 +507,13 @@
          this._toolstripseperator_task_eye.Name = "_toolstripseperator_task_eye";
          this._toolstripseperator_task_eye.Size = new System.Drawing.Size(213, 6);
          // 
+         // _toolstrip_menuitem_save_ivs
+         // 
+         this._toolstrip_menuitem_save_ivs.Name = "_toolstrip_menuitem_save_ivs";
+         this._toolstrip_menuitem_save_ivs.Size = new System.Drawing.Size(216, 22);
+         this._toolstrip_menuitem_save_ivs.Text = "Save IVs";
+         //
+         // 
          // _toolstrip_menuitem_save_data_to_file
          // 
          this._toolstrip_menuitem_save_data_to_file.Name = "_toolstrip_menuitem_save_data_to_file";
@@ -581,7 +598,8 @@
             this._toolstripmenuitem_settings_volchange,
             this._toolstripmenuitem_settings_straddle
          };
-         if (_api_tasking_enabled) {
+         if (_api_tasking_enabled)
+         {
             sheet_settings_items.AddRange(new System.Collections.Generic.List<System.Windows.Forms.ToolStripItem>() {
                this._toolseperator_task_ee,
                this._toolstripmenuitem_settings_show_task_ee_data,
@@ -900,6 +918,7 @@
       private System.Windows.Forms.ToolStripSeparator _toolstripseparator_task_size;
       private System.Windows.Forms.ToolStripMenuItem _toolstripmenuitem_task_undo_size_override;
       private System.Windows.Forms.ToolStripMenuItem _toolstrip_menuitem_save_data_to_file;
+      private System.Windows.Forms.ToolStripMenuItem _toolstrip_menuitem_save_ivs;
       private System.Windows.Forms.ToolStripSeparator _toolstripseperator_export_data;
       private System.Windows.Forms.Button _button_theo_acceptance_confirmation;
       private System.Windows.Forms.ToolStripMenuItem _toolstripmenuitem_filter_strikes_with_pos_edge;

src/client/spark/NMSheets/UIComponents/MainSheet/NMSheetControl.cs

@@ -20,6 +20,8 @@ using System.Collections;
 using System.Reactive.Linq;
 using System.Threading;
 using bts.api.orderman;
+using bts;
+using mbts.analytics;
 
 namespace BCG {
    public partial class NMSheetControl : UserControl {
@@ -252,7 +254,7 @@ namespace BCG {
          /// windows timer
          _windows_timer = new System.Windows.Forms.Timer(components);
          _windows_timer.Enabled = true;
-         _windows_timer.Interval = 20;
+         _windows_timer.Interval = 100;
          _windows_timer.Tick += _windows_timer_tick;
          _toolstripmenuitem_card.Visible = false;
          /// status bar
@@ -317,6 +319,9 @@ namespace BCG {
             _set_new_settings(_settings);
             _theo_grid.theo_grid.scroll_to_point(Point.Empty);
          };
+         _toolstrip_menuitem_save_ivs.Click += (s, e) => {
+            _threaded_bb.post(SparkTags.TC2Tags.ArchiveIVsTag, _nmsheet_symbol_list[0]);
+         };
          _toolstripmenuitem_filter_strikes_in_tasks.Click += (s, e) => {
             _settings.setting_filter.filter_ee_task_strikes = !_settings.setting_filter.filter_ee_task_strikes;
             _set_new_settings(_settings);
@@ -612,7 +617,7 @@ namespace BCG {
                cell.col == NMSheetColumnName.put_ask_price || cell.col == NMSheetColumnName.put_ask_size) {
                _launch_order_entry_window(cell, _is_control_down() ? OrderEntryWindow.Ladder : OrderEntryWindow.Ticket);
             } else if (cell.col == NMSheetColumnName.call_theo || cell.col == NMSheetColumnName.put_theo ||
-                       cell.col == NMSheetColumnName.straddle || cell.col == NMSheetColumnName.strike) {
+                         cell.col == NMSheetColumnName.straddle || cell.col == NMSheetColumnName.strike) {
                if (cell.call_selected)
                   _add_spread_leg(cell.expiration, cell.strike, true, false, leg_size, state);
                if (cell.put_selected)
@@ -717,9 +722,9 @@ namespace BCG {
 
       private void _windows_timer_tick(object sender, EventArgs e) {
          _publish_activation_state();
-         //_threaded_bb.sec_thread_dowork();
+         // this computes the data and then paints it
          _check_and_propagate_data();
-         _poll_theo_acceptance_button();
+         //_poll_theo_acceptance_button();
          _refresh_statusbar_data(false);
       }
 
@@ -780,21 +785,31 @@ namespace BCG {
       #region Coalescer NMSheetData interaction
 
       private DateTime _last_data_draw_time = DateTime.MinValue;
-      private const double _min_draw_period_ms = 333.3;
+      private const double _min_draw_period_ms = 500;
       private NMSheetGeometricLayout _last_geo_lo_info = null;
+      private VolSurfaceTimer _vs_timer = new VolSurfaceTimer();
+      // this is where we assemble the data (in _coalescer.get_data())
+      // and then painted in _respond_to_data
       private void _check_and_propagate_data() {
          if (is_visible) {
             if ((DateTime.Now - _last_data_draw_time).Milliseconds >= Math.Max(_settings.setting_grid_internal.grid_paint_throttle_ms, _min_draw_period_ms)) {
+               _vs_timer.start();
                NMSheetData new_sheet_data = _coalescer.get_data();
+               _vs_timer.stop();
+               Log.Gen.info($"NMSheetControl._check_and_propagate_data get_data time: {_vs_timer.last_interval_time_ms} msecs symbols: {_title_prefix} tid: {System.Threading.Thread.CurrentThread.ManagedThreadId}");
                if (new_sheet_data != null) {
                   _last_data_draw_time = DateTime.Now;
                   _sheet_data = new_sheet_data;
+                  _vs_timer.start();
                   _respond_to_data();
+                  _vs_timer.stop();
+                  Log.Gen.info($"NMSheetControl._check_and_propagate_data _respond_to_data time: {_vs_timer.last_interval_time_ms} msecs symbols: {_title_prefix} tid: {System.Threading.Thread.CurrentThread.ManagedThreadId}");
                }
             }
          }
       }
 
+      // this is the painting
       private void _respond_to_data() {
          if (_sheet_data != null) {
             _set_driver_control_display();

src/client/spark/Orders/OrderTicket.xaml

@@ -1,161 +1,161 @@
-<Window
-  x:Class="mbts.ui.spark.OrderTicket"
-  xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
-  xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
-  xmlns:msys="clr-namespace:bts.api;assembly=spark_api"
-  xmlns:u="clr-namespace:mbts.ui"
-  xmlns:cv="clr-namespace:mbts.ui.converters"
-  xmlns:usp="clr-namespace:mbts.ui.spark"
-  xmlns:sys="clr-namespace:System;assembly=mscorlib"
-  u:WindowStyle.SizeToContent="WidthAndHeight"
-  u:WindowStyle.TitleContent="{Binding bbo, Mode=OneWay}"
-  Style="{StaticResource BTSWindowTitleTemplate}"
->
-  <Window.Resources>
-    <ResourceDictionary>
-      <ResourceDictionary.MergedDictionaries>
-        <ResourceDictionary Source="/view/Resources/Converters.xaml"/>
-      </ResourceDictionary.MergedDictionaries>
-      <sys:Double x:Key="HorizontalSpacing">8.0</sys:Double>
-      <sys:Double x:Key="VerticalSpacing">4.0</sys:Double>
-      <sys:Double x:Key="ButtonHeight">22.0</sys:Double>
-      <Style x:Key="SellButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
-         <Setter Property="Background" Value="{u:BTSColor MarketData.ask}" />
-         <Setter Property="Foreground" Value="White" />
-         <Setter Property="MinHeight" Value="40" />
-         <Setter Property="FontWeight" Value="Bold" />
-         <Setter Property="HorizontalAlignment" Value="Stretch" />
-         <Setter Property="VerticalAlignment" Value="Stretch" />
-         <Setter Property="Content">
-            <Setter.Value>
-              <TextBlock VerticalAlignment="Center" Text="Sell" />
-            </Setter.Value>
-         </Setter>
-         <Setter Property="Template">
-            <Setter.Value>
-               <ControlTemplate TargetType="Button">
-                  <Border 
-                        x:Name="Border"
-                        CornerRadius="0" 
-                        BorderThickness="1"
-                        BorderBrush="{u:BTSColor UI.Button.normal_border}"
-                        Background="{u:BTSColor MarketData.ask}"
-                        Padding="2"
-                        Height="Auto"
-                        SnapsToDevicePixels="True">
-                        <ContentPresenter 
-                           HorizontalAlignment="Center"
-                           VerticalAlignment="Center"
-                           RecognizesAccessKey="True"
-                           Name="button_content" 
-                        />
-                     </Border>
-                  <ControlTemplate.Triggers>
-                        <Trigger Property="IsMouseOver" Value="True">
-                           <Setter TargetName="Border" Property="Background" Value="#99c00000" />
-                           <Setter TargetName="Border" Property="BorderBrush" Value="{u:BTSColor UI.Button.mouseover_border}" />
-                        </Trigger>
-                     </ControlTemplate.Triggers>
-               </ControlTemplate>
-            </Setter.Value>
-         </Setter>
-      </Style>
-      <Style x:Key="BuyButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
-         <Setter Property="Background" Value="{u:BTSColor MarketData.bid}" />
-         <Setter Property="Foreground" Value="White" />
-         <Setter Property="MinHeight" Value="40" />
-         <Setter Property="FontWeight" Value="Bold" />
-         <Setter Property="HorizontalAlignment" Value="Stretch" />
-         <Setter Property="VerticalAlignment" Value="Stretch" />
-         <Setter Property="Content">
-            <Setter.Value>
-              <TextBlock VerticalAlignment="Center" Text="Buy" />
-            </Setter.Value>
-         </Setter>
-         <Setter Property="Template">
-            <Setter.Value>
-               <ControlTemplate TargetType="Button">
-                  <Border 
-                     x:Name="Border"
-                     CornerRadius="0" 
-                     BorderThickness="1"
-                     BorderBrush="{u:BTSColor UI.Button.normal_border}"
-                     Background="{u:BTSColor MarketData.bid}"
-                     Padding="2"
-                     Height="Auto"
-                     SnapsToDevicePixels="True">
-                     <ContentPresenter 
-                        HorizontalAlignment="Center"
-                        VerticalAlignment="Center"
-                        RecognizesAccessKey="True"
-                        Name="button_content" 
-                     />
-                  </Border>
-                  <ControlTemplate.Triggers>
-                     <Trigger Property="IsMouseOver" Value="True">
-                        <Setter TargetName="Border" Property="Background" Value="#99167ed3" />
-                        <Setter TargetName="Border" Property="BorderBrush" Value="{u:BTSColor UI.Button.mouseover_border}" />
-                     </Trigger>
-                  </ControlTemplate.Triggers>
-               </ControlTemplate>
-            </Setter.Value>
-         </Setter>
-      </Style>
-      </ResourceDictionary>
-  </Window.Resources>
-  <Window.InputBindings>
-    <KeyBinding Key="Esc" Command="{x:Static SystemCommands.CloseWindowCommand}"/>
-  </Window.InputBindings>
-  <Grid Margin="10">
-    <Grid.RowDefinitions>
-      <RowDefinition Height="Auto"/>
-      <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}"/>
-      <RowDefinition Height="Auto"/>
-    </Grid.RowDefinitions>
-    <Grid.ColumnDefinitions>
-      <ColumnDefinition MinWidth="120"/>
-      <ColumnDefinition Width="{Binding Source={StaticResource HorizontalSpacing}}"/>
-      <ColumnDefinition MinWidth="120"/>
-    </Grid.ColumnDefinitions>
-    <usp:OrderModificationControl Grid.ColumnSpan="3" />
-      <Button Grid.Row="2"                 Style="{StaticResource BuyButton}"  Command="{Binding send_order}" CommandParameter="{u:Enum Buy, Type=msys:Side}" />
-      <Button Grid.Row="2" Grid.Column="2" Style="{StaticResource SellButton}" Command="{Binding send_order}" CommandParameter="{u:Enum Sell, Type=msys:Side}" />
-   </Grid>
-  <u:WindowStyle.TitleContentTemplate>
-    <DataTemplate>
-      <TextBlock Style="{StaticResource TextBlock.Dark}">
-        <TextBlock.Inlines>
-          <Run FontWeight="Bold" Text="{Binding security, Mode=OneWay}" />
-          <Run Foreground="{u:BTSColor MarketData.bid}" FontSize="10">
-            <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}" StringFormat="({0})">
-              <Binding Path="bid_size" Mode="OneWay"/>
-              <Binding Path="DataContext.qty_display" RelativeSource="{RelativeSource AncestorType=usp:OrderTicket}"/>
-            </MultiBinding>
-          </Run>
-          <Run Foreground="{u:BTSColor MarketData.bid}" FontWeight="Bold">
-            <Run.Text>
-              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
-                <Binding Path="bid"/>
-                <Binding Path="security.ContractSpec"/>
-              </MultiBinding>
-            </Run.Text>
-          </Run>
-          <Run Foreground="{u:BTSColor MarketData.ask}" FontWeight="Bold">
-            <Run.Text>
-              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
-                <Binding Path="ask"/>
-                <Binding Path="security.ContractSpec"/>
-              </MultiBinding>
-            </Run.Text>
-          </Run>
-          <Run Foreground="{u:BTSColor MarketData.ask}" FontSize="10">
-            <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}" StringFormat="({0})">
-              <Binding Path="ask_size" Mode="OneWay"/>
-              <Binding Path="DataContext.qty_display" RelativeSource="{RelativeSource AncestorType=usp:OrderTicket}"/>
-            </MultiBinding>
-          </Run>
-        </TextBlock.Inlines>
-      </TextBlock>
-    </DataTemplate>
-  </u:WindowStyle.TitleContentTemplate>
-</Window>
+<Window
+  x:Class="mbts.ui.spark.OrderTicket"
+  xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+  xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+  xmlns:msys="clr-namespace:bts.api;assembly=spark_api"
+  xmlns:u="clr-namespace:mbts.ui"
+  xmlns:cv="clr-namespace:mbts.ui.converters"
+  xmlns:usp="clr-namespace:mbts.ui.spark"
+  xmlns:sys="clr-namespace:System;assembly=mscorlib"
+  u:WindowStyle.SizeToContent="WidthAndHeight"
+  u:WindowStyle.TitleContent="{Binding bbo, Mode=OneWay}"
+  Style="{StaticResource BTSWindowTitleTemplate}"
+>
+  <Window.Resources>
+    <ResourceDictionary>
+      <ResourceDictionary.MergedDictionaries>
+        <ResourceDictionary Source="/view/Resources/Converters.xaml"/>
+      </ResourceDictionary.MergedDictionaries>
+      <sys:Double x:Key="HorizontalSpacing">8.0</sys:Double>
+      <sys:Double x:Key="VerticalSpacing">4.0</sys:Double>
+      <sys:Double x:Key="ButtonHeight">22.0</sys:Double>
+      <Style x:Key="SellButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
+         <Setter Property="Background" Value="{u:BTSColor MarketData.ask_dark}" />
+         <Setter Property="Foreground" Value="White" />
+         <Setter Property="MinHeight" Value="40" />
+         <Setter Property="FontWeight" Value="Bold" />
+         <Setter Property="HorizontalAlignment" Value="Stretch" />
+         <Setter Property="VerticalAlignment" Value="Stretch" />
+         <Setter Property="Content">
+            <Setter.Value>
+              <TextBlock VerticalAlignment="Center" Text="Sell" />
+            </Setter.Value>
+         </Setter>
+         <Setter Property="Template">
+            <Setter.Value>
+               <ControlTemplate TargetType="Button">
+                  <Border 
+                        x:Name="Border"
+                        CornerRadius="0" 
+                        BorderThickness="1"
+                        BorderBrush="{u:BTSColor UI.Button.normal_border}"
+                        Background="{u:BTSColor MarketData.ask}"
+                        Padding="2"
+                        Height="Auto"
+                        SnapsToDevicePixels="True">
+                        <ContentPresenter 
+                           HorizontalAlignment="Center"
+                           VerticalAlignment="Center"
+                           RecognizesAccessKey="True"
+                           Name="button_content" 
+                        />
+                     </Border>
+                  <ControlTemplate.Triggers>
+                        <Trigger Property="IsMouseOver" Value="True">
+                           <Setter TargetName="Border" Property="Background" Value="#99c00000" />
+                           <Setter TargetName="Border" Property="BorderBrush" Value="{u:BTSColor UI.Button.mouseover_border}" />
+                        </Trigger>
+                     </ControlTemplate.Triggers>
+               </ControlTemplate>
+            </Setter.Value>
+         </Setter>
+      </Style>
+      <Style x:Key="BuyButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
+         <Setter Property="Background" Value="{u:BTSColor MarketData.bid_dark}" />
+         <Setter Property="Foreground" Value="White" />
+         <Setter Property="MinHeight" Value="40" />
+         <Setter Property="FontWeight" Value="Bold" />
+         <Setter Property="HorizontalAlignment" Value="Stretch" />
+         <Setter Property="VerticalAlignment" Value="Stretch" />
+         <Setter Property="Content">
+            <Setter.Value>
+              <TextBlock VerticalAlignment="Center" Text="Buy" />
+            </Setter.Value>
+         </Setter>
+         <Setter Property="Template">
+            <Setter.Value>
+               <ControlTemplate TargetType="Button">
+                  <Border 
+                     x:Name="Border"
+                     CornerRadius="0" 
+                     BorderThickness="1"
+                     BorderBrush="{u:BTSColor UI.Button.normal_border}"
+                     Background="{u:BTSColor MarketData.bid}"
+                     Padding="2"
+                     Height="Auto"
+                     SnapsToDevicePixels="True">
+                     <ContentPresenter 
+                        HorizontalAlignment="Center"
+                        VerticalAlignment="Center"
+                        RecognizesAccessKey="True"
+                        Name="button_content" 
+                     />
+                  </Border>
+                  <ControlTemplate.Triggers>
+                     <Trigger Property="IsMouseOver" Value="True">
+                        <Setter TargetName="Border" Property="Background" Value="#99167ed3" />
+                        <Setter TargetName="Border" Property="BorderBrush" Value="{u:BTSColor UI.Button.mouseover_border}" />
+                     </Trigger>
+                  </ControlTemplate.Triggers>
+               </ControlTemplate>
+            </Setter.Value>
+         </Setter>
+      </Style>
+      </ResourceDictionary>
+  </Window.Resources>
+  <Window.InputBindings>
+    <KeyBinding Key="Esc" Command="{x:Static SystemCommands.CloseWindowCommand}"/>
+  </Window.InputBindings>
+  <Grid Margin="10">
+    <Grid.RowDefinitions>
+      <RowDefinition Height="Auto"/>
+      <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}"/>
+      <RowDefinition Height="Auto"/>
+    </Grid.RowDefinitions>
+    <Grid.ColumnDefinitions>
+      <ColumnDefinition MinWidth="120"/>
+      <ColumnDefinition Width="{Binding Source={StaticResource HorizontalSpacing}}"/>
+      <ColumnDefinition MinWidth="120"/>
+    </Grid.ColumnDefinitions>
+    <usp:OrderModificationControl Grid.ColumnSpan="3" />
+      <Button Grid.Row="2"                 Style="{StaticResource BuyButton}"  Command="{Binding send_order}" CommandParameter="{u:Enum Buy, Type=msys:Side}" />
+      <Button Grid.Row="2" Grid.Column="2" Style="{StaticResource SellButton}" Command="{Binding send_order}" CommandParameter="{u:Enum Sell, Type=msys:Side}" />
+   </Grid>
+  <u:WindowStyle.TitleContentTemplate>
+    <DataTemplate>
+      <TextBlock Style="{StaticResource TextBlock.Dark}">
+        <TextBlock.Inlines>
+          <Run FontWeight="Bold" Text="{Binding security, Mode=OneWay}" />
+          <Run Foreground="{u:BTSColor MarketData.bid}" FontSize="10">
+            <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}" StringFormat="({0})">
+              <Binding Path="bid_size" Mode="OneWay"/>
+              <Binding Path="DataContext.qty_display" RelativeSource="{RelativeSource AncestorType=usp:OrderTicket}"/>
+            </MultiBinding>
+          </Run>
+          <Run Foreground="{u:BTSColor MarketData.bid}" FontWeight="Bold">
+            <Run.Text>
+              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
+                <Binding Path="bid"/>
+                <Binding Path="security.ContractSpec"/>
+              </MultiBinding>
+            </Run.Text>
+          </Run>
+          <Run Foreground="{u:BTSColor MarketData.ask}" FontWeight="Bold">
+            <Run.Text>
+              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
+                <Binding Path="ask"/>
+                <Binding Path="security.ContractSpec"/>
+              </MultiBinding>
+            </Run.Text>
+          </Run>
+          <Run Foreground="{u:BTSColor MarketData.ask}" FontSize="10">
+            <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}" StringFormat="({0})">
+              <Binding Path="ask_size" Mode="OneWay"/>
+              <Binding Path="DataContext.qty_display" RelativeSource="{RelativeSource AncestorType=usp:OrderTicket}"/>
+            </MultiBinding>
+          </Run>
+        </TextBlock.Inlines>
+      </TextBlock>
+    </DataTemplate>
+  </u:WindowStyle.TitleContentTemplate>
+</Window>

src/client/spark/Risk/PNL/PNLReportCalculator.cs

@@ -64,8 +64,7 @@ namespace mbts.analytics {
                if (tl.tl_row.size > 0) {
                   start_long_pos += tl.tl_row.size;
                   start_long_total_cost += cost;
-               }
-               else {
+               } else {
                   start_short_pos += Math.Abs(tl.tl_row.size);
                   start_short_total_cost += cost;
                }
@@ -78,8 +77,7 @@ namespace mbts.analytics {
                if (tl.tl_row.size > 0) {
                   day_long_pos += tl.tl_row.size;
                   day_long_total_cost += cost;
-               }
-               else {
+               } else {
                   day_short_pos += Math.Abs(tl.tl_row.size);
                   day_short_total_cost += cost;
                }
@@ -165,6 +163,13 @@ namespace mbts.analytics {
                   is_day_trade = _is_day_trade(tl)
                };
                pnl_trades.Add(pnl_tl);
+            } else if (tl.security_type == SecurityType.Spot) {
+               PNLTradeRow pnl_tl = new PNLTradeRow() {
+                  tl_row = tl,
+                  md = tl_md,
+                  is_day_trade = _is_day_trade(tl)
+               };
+               pnl_trades.Add(pnl_tl);
             }
          }
          return pnl_trades;

src/client/spark/Risk/PNL/PNLTradeLogRow.cs

@@ -4,6 +4,9 @@ using bts.api;
 
 using BCG;
 using mbts.co2;
+using System.Xml.Serialization;
+using ZstdSharp.Unsafe;
+using bts.proto;
 
 namespace mbts.analytics {
    // Pairs ITradeLogRow along with the data (theo, md) that is needed to compute
@@ -18,20 +21,88 @@ namespace mbts.analytics {
 
       public SecurityImmutable security { get { return md.security; } }
 
-      private double _option_pnl(double trade_price, double now_price) {
-         double coin_terms = (now_price - trade_price) * tl_row.size;
-         double usd_terms = coin_terms * driver;
-         return usd_terms;
+      // private double _option_pnl(double trade_price, double now_price) {
+      //    double coin_terms = (now_price - trade_price) * tl_row.size;
+      //    double usd_terms = coin_terms * driver;
+      //    return usd_terms;
+      // }
+
+      private bool _is_notional {
+         get {
+            return false;
+         }
       }
 
-      private double _future_pnl(double trade_price, double now_price) {
-         // the qty is in USD, so this is the initial cash flow
-         double init_asset_value = tl_row.size;
-         double asset_size = init_asset_value / tl_row.price;
-         double now_asset_value = asset_size * now_price;
-         return now_asset_value - init_asset_value;
+      private bool _is_inverse_option_contract {
+         get {
+            return false;
+         }
       }
 
+      private double _cpp {
+         get {
+            return 1.0;
+         }
+      }
+
+      private bool _is_reverse_option_contract {
+         get {
+            // check for Deribit options
+            return false;
+         }
+      }
+
+      private double _option_pnl {
+         get {
+            if (_is_notional) {
+               var now_price = theo;
+               double coin_terms = (now_price - tl_row.price) * tl_row.size;
+               double usd_terms = coin_terms * driver;
+               return usd_terms;
+            } else {
+               return _cpp * tl_row.size * (theo - tl_row.price);
+            }
+         }
+      }
+
+      private double _spot_pnl {
+         get {
+            // both are computed the same way
+            return _future_pnl;
+         }
+      }
+
+      private double _future_pnl {
+         get {
+            if (_is_notional) {
+               // TODO:
+               // let it fall through right now
+               // the qty is in USD, so this is the initial cash flow
+               var now_price = theo;
+               double init_asset_value = tl_row.size;
+               double asset_size = init_asset_value / tl_row.price;
+               double now_asset_value = asset_size * now_price;
+               return now_asset_value - init_asset_value;
+            } else {
+               // TODO: is position pnl, compute it wrt settlement price?
+               return tl_row.size * (theo - tl_row.price);
+            }
+         }
+      }
+
+      // private double _future_pnl(double trade_price, double now_price) {
+      //    if (_is_notional) {
+      //       // the qty is in USD, so this is the initial cash flow
+      //       double init_asset_value = tl_row.size;
+      //       double asset_size = init_asset_value / tl_row.price;
+      //       double now_asset_value = asset_size * now_price;
+      //       return now_asset_value - init_asset_value;
+      //    } else {
+      //       return tl_row.size * (theo - tl_row.price);
+      //    }
+      // }
+
+
       public double driver {
          get {
             if (tl_row.security_type == SecurityType.Option) {
@@ -44,35 +115,46 @@ namespace mbts.analytics {
       }
       public double theo {
          get {
-            if (tl_row.security_type == SecurityType.Option) {
-               return tl_row.call_put == PutOrCall.Call ?
-                  opt_theo.fwd_normalized_call : opt_theo.fwd_normalized_put;
-            } else if (tl_row.security_type == SecurityType.Future) {
-               return fut_theo.theo;
+            switch (tl_row.security_type) {
+               // TODO:
+               case SecurityType.Option:
+                  if (_is_inverse_option_contract) {
+                     return tl_row.call_put == PutOrCall.Call ?
+                        opt_theo.fwd_normalized_call : opt_theo.fwd_normalized_put;
+                  } else {
+                     return tl_row.call_put == PutOrCall.Call ?
+                        opt_theo.call : opt_theo.put;
+                  }
+               // future and options, just get mid market
+               case SecurityType.Future:
+               //return fut_theo.theo;
+               case SecurityType.Spot:
+                  var bbo = top_of_book;
+                  return (bbo.Item1 + bbo.Item2) / 2;
+               default:
+                  return Double.NaN;
             }
-            return Double.NaN;
          }
       }
       public double pnl {
          get {
             double rv = Double.NaN;
-            if (is_day_trade) {
-               if (tl_row.security_type == SecurityType.Future) {
-                  rv = _future_pnl(tl_row.price, theo);
-               } else if (tl_row.security_type == SecurityType.Option) {
-                  rv = _option_pnl(tl_row.price, theo);
-               }
-               return rv;
-            }
-            // use settle price ?
-            if (tl_row.security_type == SecurityType.Future) {
-               rv = _future_pnl(tl_row.price, theo);
-            } else if (tl_row.security_type == SecurityType.Option) {
-               rv = _option_pnl(tl_row.price, theo);
+            switch (tl_row.security_type) {
+               case SecurityType.Option:
+                  rv = _option_pnl;
+                  break;
+               case SecurityType.Future:
+                  rv = _future_pnl;
+                  break;
+               case SecurityType.Spot:
+                  // = tl_row.size * (theo - tl_row.price);
+                  rv = _spot_pnl;
+                  break;
             }
             return rv;
          }
       }
+
       public double settle_price {
          get {
             return md.md == null ? Double.NaN : md.md.settlement_price;

src/client/spark/Security/SecurityExpirationHelper.cs

@@ -36,6 +36,9 @@ namespace mbts {
       }
 
       public static bool is_unexpired(bts.api.SecurityTuple st, DateTime exp_date) {
+         if (st.Venue == Venue.Cboe) {
+            return exp_date.Date >= DateTime.Now.Date;
+         }
          TimeSpan exp_time = get_exp_time(st);
          DateTime full_exp_utc = exp_date.ToUniversalTime().Date.Add(exp_time);
          DateTime now_utc = DateTime.UtcNow;
@@ -43,6 +46,12 @@ namespace mbts {
       }
 
       public static bool is_unexpired(bts.api.SecurityImmutable sec) {
+         if (sec.security_tuple().Venue == Venue.Cboe || sec.security_tuple().Venue == Venue.Glbx) {
+            var exp = sec.to_sec_exp().expiration;
+            bool is_unexp = exp.Date >= DateTime.Now.Date;
+//            Log.Gen.info($"SecurityExpirationMapper.is_unexpired exp: {exp} is_unexp: {is_unexp}");
+            return is_unexp;
+         }
          var sec_type = sec.security_type();
          switch (sec_type) {
             case bts.api.SecurityType.Future:

src/client/spark/SpreadBox/Calculators/StatelessSpreadCalculator.cs

@@ -58,9 +58,10 @@ namespace BCG {
             }
             // TODO: put it as a setting
             bts.Log.assert(theos.Length == drivers.Length);
-            for (int i = 0; i < theos.Length; i++) {
-               theos[i] /= drivers[i];
-            }
+            // only do this when its deribit crypto option, TODO
+            // for (int i = 0; i < theos.Length; i++) {
+            //    theos[i] /= drivers[i];
+            // }
             return new SpreadValue(spread_spec, theos, deltas, gammas, vegas, adj_vegas, oevs, thetas, slope_vegas, drivers, qs_midpoint_theos, pos, additional_delta);
          }
       }
@@ -142,9 +143,21 @@ namespace BCG {
          return tp ?? mbts.spark.TheoPoint.getNulledObject();
       }
 
-      private static mbts.spark.TheoPoint _compute_single_option_leg_at_tied_price(DateTime option_expiration, double strike, bool is_call, OptionTheoInterpolationData sparse_theo_map, double driver_price) {
-         mbts.spark.TheoPoint tp = sparse_theo_map == null ? mbts.spark.TheoPoint.getNulledObject() : sparse_theo_map.interpolate_theos(option_expiration, strike, driver_price);
-         return tp ?? mbts.spark.TheoPoint.getNulledObject();
+      private static mbts.spark.TheoPoint _compute_single_option_leg_at_tied_price(DateTime option_expiration, double strike, bool is_call, OptionTheoInterpolationData sparse_theo_map,
+                                                                                    ExpStrikeKeyedData<mbts.spark.TheoPoint> live_symbol_theo_map, double driver_price) {
+         // mbts.spark.TheoPoint tp = sparse_theo_map == null ? mbts.spark.TheoPoint.getNulledObject() : sparse_theo_map.interpolate_theos(option_expiration, strike, driver_price);
+         // return tp ?? mbts.spark.TheoPoint.getNulledObject();
+         // TOD: config here
+         return _compute_single_option_leg_at_tied_price_from_live(option_expiration, strike, is_call, live_symbol_theo_map, driver_price);
+      }
+
+      private static mbts.spark.TheoPoint _compute_single_option_leg_at_tied_price_from_live(DateTime option_expiration, double strike, bool is_call,
+                                                                                             ExpStrikeKeyedData<mbts.spark.TheoPoint> live_symbol_theo_map, double driver_price) {
+         var tp = live_symbol_theo_map.get_data(option_expiration, strike);
+         if (tp == null)
+            return mbts.spark.TheoPoint.getNulledObject();
+         return TheoPoint.extrapolate(tp, driver_price, is_call);
+
       }
 
       /// this is the only relatively tricky thing here      
@@ -174,7 +187,7 @@ namespace BCG {
                   if (leg_price_spec.primaryFrozen)   /// tied
                   {
                      effective_driver = leg_price_spec.primary;
-                     return _compute_single_option_leg_at_tied_price(spread_option_leg.contract.expiration, spread_option_leg.contract.strike, spread_option_leg.contract.isCall, symbol_sparse_theo_map, leg_price_spec.primary);
+                     return _compute_single_option_leg_at_tied_price(spread_option_leg.contract.expiration, spread_option_leg.contract.strike, spread_option_leg.contract.isCall, symbol_sparse_theo_map, live_symbol_theo_map, leg_price_spec.primary);
                   } else {
                      mbts.spark.TheoPoint tp = _compute_single_option_leg_at_live_price(spread_option_leg.contract.expiration, spread_option_leg.contract.strike, spread_option_leg.contract.isCall, live_symbol_theo_map);
                      effective_driver = tp.driver;
@@ -188,7 +201,7 @@ namespace BCG {
                         future_theo_map.TryGetValue(driver_price_info.first, out symbol_future_theo_map);
                      double driver_price = StatelessFutureTheoCalculator.transform_price(symbol_future_theo_map, leg_price_spec.driverExpiration, leg_price_spec.primary, driver_price_info.second);
                      effective_driver = leg_price_spec.primary;
-                     return _compute_single_option_leg_at_tied_price(spread_option_leg.contract.expiration, spread_option_leg.contract.strike, spread_option_leg.contract.isCall, symbol_sparse_theo_map, driver_price);
+                     return _compute_single_option_leg_at_tied_price(spread_option_leg.contract.expiration, spread_option_leg.contract.strike, spread_option_leg.contract.isCall, symbol_sparse_theo_map, live_symbol_theo_map, driver_price);
                   } else {
                      bts.Log.Ui.error(String.Format("This spread is live but the driver expiration {0} is different from the system driver expiration {1} for {2}",
                                                       spread_option_leg.legPriceSpec.driverExpiration, driver_price_info.second, spread_option_leg.contract.product));

src/client/spark/core/Bus/SparkTags.cs

@@ -353,11 +353,8 @@ namespace mbts.spark {
 
       public static ITag<Nothing> ShowGlobalAlertsTag { get; } = _mk_tag<Nothing>();
 
-      public static ITag<ClientArchivedVolRequest> RequestArchivedVols { get; } = _mk_tag<ClientArchivedVolRequest>();
-
-      public static ITag<Tuple<ClientArchivedVolRequest, bts.api.analytics.ArchivedTheoList>>
-      ResponseArchivedVols { get; } =
-         _mk_tag<Tuple<ClientArchivedVolRequest, bts.api.analytics.ArchivedTheoList>>();
+      public static ITag<ArchivedTheoListImmutable>
+      ArchivedTheos { get; } = _mk_tag<ArchivedTheoListImmutable>();
 
       static SparkTag<T> _mk_tag<T>([CallerMemberName] string name = null) =>
          new(name ?? throw new ArgumentNullException(nameof(name)));
@@ -373,6 +370,7 @@ namespace mbts.spark {
       public class Nothing { Nothing() { } }
 
       public class TC2Tags {
+         public static ITag<NMSheetSymbol> ArchiveIVsTag {get;} = _mk_tag<NMSheetSymbol>();
          public static ITag<NMSheetAllTaskingData> TaskData { get; } =
             _mk_tag<NMSheetAllTaskingData>();
          // Data request for a sheet 
@@ -386,6 +384,11 @@ namespace mbts.spark {
          public static ITag<ExpStrikeKeyedData<TheoPoint>> LiveTheosTag((string, Venue) symbol) =>
             _mk_tag<ExpStrikeKeyedData<TheoPoint>, (string, Venue)>(symbol);
 
+         // these are the theos at time of the vol save (we use it to extrapolate the theos prices for the spreads)
+         public static ITag<ExpStrikeKeyedData<TheoPoint>> VolSaveEventTheosTag((string, Venue) symbol) =>
+            _mk_tag<ExpStrikeKeyedData<TheoPoint>, (string, Venue)>(symbol);
+
+
          // Aggregated version of the above
          public static ITag<IReadOnlyDictionary<(string, Venue), ExpStrikeKeyedData<TheoPoint>>>
          AllLiveTheosTag { get; } =

src/client/spark/core/FontsAndColors/BTSColors.cs

@@ -216,8 +216,10 @@ namespace mbts.ui {
          public static BTSColor ask_text => BTSCoreColors.red_dark;
          public static BTSColor bid_text => BTSCoreColors.blue;
          public static BTSColor bid => 0xff76b7fa;
+         public static BTSColor bid_dark => 0xff26239a;
          public static BTSColor bid_alt => BTSCoreColors.blue_light.with(0x70);
          public static BTSColor ask => 0xffff6666;
+         public static BTSColor ask_dark => 0xff7a2331;
          public static BTSColor ask_alt => 0x70ffb4b4;
          public static BTSColor last => BTSCoreColors.black;
          public static BTSColor bid_ask_alt_blend => BTSCoreColors.purple_light;
@@ -1038,6 +1040,7 @@ namespace mbts.ui {
       }
       public static class DayFills {
          public static BTSColor background => UI.common_background;
+         public static BTSColor foreground => UI.default_foreground;
       }
       public static class ComboReport {
          public static BTSColor foreground => UI.default_text;

src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs

@@ -1,4 +1,4 @@
-namespace mbts.ui.controls {
+namespace mbts.ui.controls {
    using bts;
    using bts.utils;
 
@@ -26,7 +26,7 @@
          public const double
             Account = 62,
             CheckMark = 42,
-            Currency = 54,
+            Currency = 68,
             Exchange = 66,
             FillKey = 68,
             OrderKey = 78,
@@ -43,7 +43,9 @@
             Side = 47,
             Qty = 48,
             MergedQty = 68,
+            NotionalValue = 68,
             Filled = 56,
+            FilledNotionalValue = 81,
             Symbol = 60,
             Time = 112,
             TimeOfDay = 84,
@@ -122,6 +124,7 @@
             SecurityType = "Security Type",
             Side = "B/S",
             Qty = "Qty",
+            NotionalValue = "Notional",
             MergedQty = "Qty",
             DirectQty = "Qty (D)",
             ImpliedQty = "Qty (I)",
@@ -144,6 +147,7 @@
             FillKind = "Fill Kind",
             FilledPrice = "Fill Price",
             FilledQuantity = "Fill Qty",
+            FilledNotionalValue = "Fill Notional",
             FillStatus = "Fill Status",
             Name = "Name",
             Description = "Description",

src/client/spark/view/Controls/Order/OrderTicketResources.xaml

@@ -19,7 +19,7 @@
   <sys:Double x:Key="ButtonHeight">22.0</sys:Double>
 
   <Style x:Key="SellButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
-    <Setter Property="Background" Value="{u:BTSColor MarketData.ask}"/>
+    <Setter Property="Background" Value="{u:BTSBrush MarketData.ask_dark}"/>
     <Setter Property="BorderBrush" Value="{u:BTSColor UI.Button.normal_border}"/>
     <Setter Property="Foreground" Value="{u:BTSColor UI.default_text}" />
     <Setter Property="MinHeight" Value="40"/>
@@ -32,7 +32,7 @@
   </Style>
 
   <Style x:Key="BuyButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
-    <Setter Property="Background" Value="{u:BTSColor MarketData.bid}"/>
+    <Setter Property="Background" Value="{u:BTSBrush MarketData.bid_dark}"/>
     <Setter Property="BorderBrush" Value="{u:BTSColor UI.Button.normal_border}"/>
     <Setter Property="Padding" Value="2"/>
     <Setter Property="Foreground" Value="{u:BTSColor UI.default_text}" />

src/client/spark/view/Controls/TickUpDown/PriceEntryCommitControl.xaml

@@ -8,7 +8,8 @@
              xmlns:cv="clr-namespace:mbts.ui.converters"
              xmlns:c="clr-namespace:mbts.ui.controls"
              xmlns:usp="clr-namespace:mbts.ui.spark"
-             d:DesignHeight="100" d:DesignWidth="800" x:Name="_me_price">
+             d:DesignHeight="100" d:DesignWidth="800" x:Name="_me_price"
+             Foreground="{u:BTSColor UI.input_text}">
   <UserControl.Resources>
     <cv:NullableDoubleConverter x:Key="TickDefaultPriceConverter" DefaultValue="0"/>
     <cv:TickDisplayConverter x:Key="TickDisplayConverter" x:Shared="False"/>

src/client/spark/view/Shell/ShellModel.cs

@@ -100,7 +100,7 @@
       }
 
       void PerformMEFComposition(string config_source) {
-         Log.Ui.info($"this is v2.2.1 Generating MEF dependencies for the shell" +
+         Log.Ui.info($"Generating MEF dependencies for the shell" +
                $"({typeof(IInitializable)}s, {typeof(IShellModule)}s, " +
                $"{typeof(IShellDaemon)}s, etc.)");
          try {

src/client/spark/view/Utility/MDDisplayHelper.cs

@@ -154,37 +154,48 @@ namespace mbts.ui.spark {
 
    static class MDDisplayHelper {
       /// This delegate should be created by calling <see cref="get_formatter(MarketSizeDisplay)"/>
-      public delegate string SizeFormatter(double merged, double implied, double price);
+      public delegate string SizeFormatter(double merged, double implied, double price, double notional);
       /// The private delegates below use refs bc the <see cref="SizeFormatter"/> returned by <see cref="get_formatter(MarketSizeDisplay)"/>
       /// internally uses several <see cref="_SizeFormatter"/>s to add different text components to the resultant string.
       /// The refs, therefor, avoid making extra copies of the variables.
       /// These are private members, so any code-readability cost is confined to a limited scope.
-      delegate void _SizeFormatter(ref double merged, ref double implied, ref double price, ref StringBuilder sb);
+      delegate void _SizeFormatter(ref double merged, ref double implied, ref double price, ref double notional, ref StringBuilder sb);
       delegate void _SizeFormatterNPS(ref NativePriceSize nps, ref StringBuilder sb);
 
       public static Func<double, double, string> get_value_formatter(
-         MarketSizeDisplay size_display, TickDisplaySettingsImmutable display, TickDisplaySettingsImmutable price_display) {
-         var raw_fmt_size = get_formatter(size_display, display, price_display);
-         return (q, p) => raw_fmt_size(q, 0, p);
+         MarketSizeDisplay size_display, TickDisplaySettingsImmutable display, TickDisplaySettingsImmutable notional_value_display) {
+         var raw_fmt_size = get_formatter(size_display, display, notional_value_display);
+         return (q, p) => raw_fmt_size(q, 0, p, double.NaN);
+      }
+
+      public static Func<LadderPriceLevel, string> get_formatter_lpl(
+         MarketSizeDisplay size_display, TickDisplaySettingsImmutable display, TickDisplaySettingsImmutable notional_value_display) {
+         var raw_fmt_size = get_formatter(size_display, display, notional_value_display);
+         return nps => raw_fmt_size(
+            nps.Volume,
+            0,
+            nps.Price,
+            nps.NotionalValue);
       }
 
       public static Func<NativePriceSize, string> get_formatter_nps(
-         MarketSizeDisplay size_display, TickDisplaySettingsImmutable display, TickDisplaySettingsImmutable price_display) {
-         var raw_fmt_size = get_formatter(size_display, display, price_display);
+         MarketSizeDisplay size_display, TickDisplaySettingsImmutable display, TickDisplaySettingsImmutable notional_value_display) {
+         var raw_fmt_size = get_formatter(size_display, display, notional_value_display);
          return nps => raw_fmt_size(
             nps.implied_size + nps.direct_size,
             nps.implied_size,
-            nps.price);
+            nps.price,
+            nps.notional_size);
       }
 
       public static SizeFormatter get_formatter(
-         MarketSizeDisplay size_display, TickDisplaySettingsImmutable display, TickDisplaySettingsImmutable price_display) {
-         _SizeFormatter implied_absent = _get_formatter(size_display.normal, display, price_display);
+         MarketSizeDisplay size_display, TickDisplaySettingsImmutable display, TickDisplaySettingsImmutable notional_value_display) {
+         _SizeFormatter implied_absent = _get_formatter(size_display.normal, display, notional_value_display);
          _SizeFormatter implied_present = _get_formatter(
-            size_display.implied_present ?? size_display.normal, display, price_display);
-         return (merged, implied, price) => {
+            size_display.implied_present ?? size_display.normal, display, notional_value_display);
+         return (merged, implied, price, notional) => {
             StringBuilder sb = new StringBuilder();
-            (implied > 0 ? implied_present : implied_absent)(ref merged, ref implied, ref price, ref sb);
+            (implied > 0 ? implied_present : implied_absent)(ref merged, ref implied, ref price, ref notional, ref sb);
             return sb.ToString();
          };
       }
@@ -205,13 +216,13 @@ namespace mbts.ui.spark {
       static _SizeFormatter _get_formatter(
          MarketSizeDisplay.SizeFormattingElement[] fmt_elements,
          TickDisplaySettingsImmutable display,
-         TickDisplaySettingsImmutable price_display) {
+         TickDisplaySettingsImmutable notional_value_display) {
          _SizeFormatter[] formatters = fmt_elements.is_null_or_empty() ?
             Array.Empty<_SizeFormatter>() :
-            CollectionHelper.convert_array(fmt_elements, sfe => _get_formatter(sfe, display, price_display));
-         return new _SizeFormatter((ref double m, ref double i, ref double p, ref StringBuilder sb) => {
+            CollectionHelper.convert_array(fmt_elements, sfe => _get_formatter(sfe, display, notional_value_display));
+         return new _SizeFormatter((ref double m, ref double i, ref double p, ref double n, ref StringBuilder sb) => {
             foreach (var f in formatters)
-               f(ref m, ref i, ref p, ref sb);
+               f(ref m, ref i, ref p, ref n, ref sb);
          });
       }
 
@@ -237,7 +248,7 @@ namespace mbts.ui.spark {
       static _SizeFormatter _get_formatter(
          MarketSizeDisplay.SizeFormattingElement fmt_element,
          TickDisplaySettingsImmutable display,
-         TickDisplaySettingsImmutable price_display) =>
+         TickDisplaySettingsImmutable notional_value_display) =>
          fmt_element.type switch {
             MarketSizeDisplay.SizeFormattingElementType.Const =>
                SizeTokenFormatters.char_formatter(fmt_element.data ?? string.Empty),
@@ -248,7 +259,7 @@ namespace mbts.ui.spark {
             MarketSizeDisplay.SizeFormattingElementType.Merged =>
                SizeTokenFormatters.merged_formatter(display),
             MarketSizeDisplay.SizeFormattingElementType.MergedNotional =>
-               SizeTokenFormatters.merged_notional_formatter(price_display),
+               SizeTokenFormatters.merged_notional_formatter(notional_value_display),
             _ => throw ExceptionBuilder.Argument.unexpected_case(
                fmt_element.type, nameof(fmt_element.type)),
          };
@@ -256,24 +267,24 @@ namespace mbts.ui.spark {
       static class SizeTokenFormatters {
          public static _SizeFormatter char_formatter(string data) {
             string local = data.ToString();
-            return (ref double m, ref double i, ref double p, ref StringBuilder sb) =>
+            return (ref double m, ref double i, ref double p, ref double n, ref StringBuilder sb) =>
                sb.Append(local);
          }
          public static _SizeFormatter merged_formatter(TickDisplaySettingsImmutable display) =>
-            (ref double m, ref double i, ref double p, ref StringBuilder sb) =>
+            (ref double m, ref double i, ref double p, ref double n, ref StringBuilder sb) =>
                sb.Append(display.format(m));
 
          public static _SizeFormatter direct_formatter(TickDisplaySettingsImmutable display) =>
-            (ref double m, ref double i, ref double p, ref StringBuilder sb) =>
+            (ref double m, ref double i, ref double p, ref double n, ref StringBuilder sb) =>
                sb.Append(display.format(m - i));
 
          public static _SizeFormatter implied_formatter(TickDisplaySettingsImmutable display) =>
-            (ref double m, ref double i, ref double p, ref StringBuilder sb) =>
+            (ref double m, ref double i, ref double p, ref double n, ref StringBuilder sb) =>
                sb.Append(display.format(i));
 
          public static _SizeFormatter merged_notional_formatter(TickDisplaySettingsImmutable display) =>
-            (ref double m, ref double i, ref double p, ref StringBuilder sb) =>
-               sb.Append(display.format(m * p));
+            (ref double m, ref double i, ref double p, ref double n, ref StringBuilder sb) =>
+               sb.Append(display.format(n));
 
          public static _SizeFormatterNPS char_formatter_nps(string data) {
             string local = data.ToString();
@@ -373,9 +384,10 @@ namespace mbts.ui.spark {
             _display = backing(nameof(display), fmt);
 
             string fmt() {
+               const double SAMPLE_PRICE = 25.0;
                var impl = MDDisplayHelper.get_formatter(
                   owner.create(), Security.Defaults.quantity_display_settingsi, Security.Defaults.price_display_settingsi);
-               return impl(merged, implied, 1);
+               return impl(merged, implied, 1, merged * SAMPLE_PRICE);
             }
          }
 

src/client/spark_api/MarketData/SecurityMD.cs

@@ -11,6 +11,7 @@ namespace mbts {
    using System.Collections.Generic;
    using System.Collections.Immutable;
    using System.Linq;
+   using System.Threading;
 
    public readonly struct SecurityMD : IEquatable<SecurityMD> {
       public static readonly Func<SecurityMD, uint> key_fn = smd => smd.security.Id;
@@ -60,4 +61,128 @@ namespace mbts {
       public static bool operator !=(SecurityMD left, SecurityMD right) =>
          !(left == right);
    }
+
+   static class SecurityMDExtensions {
+      const uint DEPTH_COUNT = 10;
+      [ThreadStatic]
+      static NativePriceSize[] leg1Bids;
+
+      [ThreadStatic]
+      static NativePriceSize[] leg1Asks;
+
+      static SecurityMDExtensions() {
+         leg1Bids = new NativePriceSize[DEPTH_COUNT];
+         leg1Asks = new NativePriceSize[DEPTH_COUNT];
+      }
+
+      public static Book Depth(this SecurityMD security, IReadOnlyDictionary<uint, SecurityMD> securities, ref ulong time) {
+         var bids = new NativePriceSize[DEPTH_COUNT];
+         var asks = new NativePriceSize[DEPTH_COUNT];
+         var bidsOut = new (double, double, double)[(int)DEPTH_COUNT];
+         var asksOut = new (double, double, double)[(int)DEPTH_COUNT];
+         if (!security.security.HasSynthetic) {
+            security.md.get_depth(ref time, ref bids, ref asks, DEPTH_COUNT);
+            for (int i = 0; i < DEPTH_COUNT; i++) {
+               bidsOut[i] = (bids[i].price, bids[i].size, bids[i].notional_size);
+               asksOut[i] = (asks[i].price, asks[i].size, asks[i].notional_size);
+            }
+            return new(bidsOut, asksOut);
+         }
+         Log.assert(security.security.Synthetic.Legs.TryGetFirst(out var leg1));
+         if (!securities.TryGetValue(leg1.Security.Id, out var sec1)) {
+            return Book.Empty;
+         }
+         ulong sec1Time = 0;
+         if (!sec1.md.get_depth(ref sec1Time, ref leg1Bids, ref leg1Asks, DEPTH_COUNT)) {
+            return Book.Empty;
+         }
+         security.md.get_depth(ref time, ref bids, ref asks, DEPTH_COUNT);
+         int leg1BidsLevel = 0;
+         int leg1AsksLevel = 0;
+         double leg1BidsConsumed = 0;
+         double leg1AsksConsumed = 0;
+         double takeQuantity;
+         double notional;
+         for (int i = 0; i < DEPTH_COUNT && !bids[i].is_nan; i++) {
+            takeQuantity = bids[i].size * leg1.Quantity;
+            notional = getNotionalValue(ref leg1Bids, ref leg1BidsLevel, ref leg1BidsConsumed, ref takeQuantity);
+            bidsOut[i] = (bids[i].price, bids[i].size, notional);
+            takeQuantity = asks[i].size * leg1.Quantity;
+            notional = getNotionalValue(ref leg1Asks, ref leg1AsksLevel, ref leg1AsksConsumed, ref takeQuantity);
+            asksOut[i] = (asks[i].price, asks[i].size, notional);
+         }
+         return new(bidsOut, asksOut);
+      }
+
+      public static Book Depth(this SecurityMD security, NativePriceSize[] bids, NativePriceSize[] asks, IReadOnlyDictionary<uint, SecurityMD> securities) {
+         bids ??= new NativePriceSize[NativePriceHelpers.COCONUT_MAX_DEPTH];
+         asks ??= new NativePriceSize[NativePriceHelpers.COCONUT_MAX_DEPTH];
+         var bidsOut = new (double, double, double)[bids.Length];
+         var asksOut = new (double, double, double)[asks.Length];
+         var depthCount = (uint)Math.Min(bids.Length, asks.Length);
+         if (!security.security.HasSynthetic) {
+            for (int i = 0; i < depthCount; i++) {
+               bidsOut[i] = (bids[i].price, bids[i].size, bids[i].notional_size);
+               asksOut[i] = (asks[i].price, asks[i].size, asks[i].notional_size);
+            }
+            return new(bidsOut, asksOut);
+         }
+         Log.assert(security.security.Synthetic.Legs.TryGetFirst(out var leg1));
+         if (!securities.TryGetValue(leg1.Security.Id, out var sec1)) {
+            return Book.Empty;
+         }
+         ulong sec1Time = 0;
+         if (!sec1.md.get_depth(ref sec1Time, ref leg1Bids, ref leg1Asks, depthCount)) {
+            return Book.Empty;
+         }
+         int leg1BidsLevel = 0;
+         int leg1AsksLevel = 0;
+         double leg1BidsConsumed = 0;
+         double leg1AsksConsumed = 0;
+         double takeQuantity;
+         double notional;
+         for (int i = 0; i < depthCount && !bids[i].is_nan; i++) {
+            takeQuantity = bids[i].size;
+            notional = getNotionalValue(ref leg1Bids, ref leg1BidsLevel, ref leg1BidsConsumed, ref takeQuantity);
+            bidsOut[i] = (bids[i].price, bids[i].size, notional);
+            takeQuantity = asks[i].size;
+            notional = getNotionalValue(ref leg1Asks, ref leg1AsksLevel, ref leg1AsksConsumed, ref takeQuantity);
+            asksOut[i] = (asks[i].price, asks[i].size, notional);
+         }
+         return new(bidsOut, asksOut);
+      }
+
+      static double getNotionalValue(ref NativePriceSize[] depth, ref int level, ref double levelConsumed, ref double takeQuantity) {
+         if (level >= depth.Length) {
+            return double.NaN;
+         }
+         double notionalValue = 0;
+         while (takeQuantity > MathUtils.epsilon<double>() && level < depth.Length && !depth[level].is_nan) {
+            var lvlDepth = (double)depth[level].size - levelConsumed;
+            if (lvlDepth <= 0) {
+               level++;
+               levelConsumed = 0;
+               continue;
+            }
+            var lvlTake = Math.Min(takeQuantity, lvlDepth);
+            notionalValue += lvlTake * depth[level].price;
+            levelConsumed += lvlTake;
+            takeQuantity -= lvlDepth;
+         }
+         return notionalValue;
+      }
+
+      public class Book {
+         public Book(IEnumerable<(double Price, double Volume, double NotionalValue)> bids, IEnumerable<(double Price, double Quantity, double NotionalValue)> asks) {
+            Bids = bids;
+            Asks = asks;
+         }
+
+         public static Book Empty =>
+            new(Array.Empty<(double, double, double)>(), Array.Empty<(double, double, double)>());
+
+         public IEnumerable<(double Price, double Volume, double NotionalValue)> Bids { get; }
+         public IEnumerable<(double Price, double Volume, double NotionalValue)> Asks { get; }
+      }
+   }
 }

src/client/spark_api/Orders/OrderRecordTuple.cs

@@ -69,6 +69,20 @@ namespace Spark.Api.Orders {
       public uint security_id => order.security_id;
       public SecurityImmutable? security => order.security;
 
+      // TODO: remove assumption that synthetic quantities are already notional values
+      public double notional_value =>
+         (security?.HasSynthetic ?? false)
+         && security.Value.Synthetic.PriceCalc == SyntheticContract.Types.PriceCalc.NotionalLeg
+            ? order.order.quantity()
+            : order.order.notional_value();
+
+      // TODO: remove assumption that synthetic quantities are already notional values
+      public double filled_notional_value =>
+         (security?.HasSynthetic ?? false)
+         && security.Value.Synthetic.PriceCalc == SyntheticContract.Types.PriceCalc.NotionalLeg
+            ? order.order.filled_quantity()
+            : order.order.filled_notional_value();
+
       public bool Equals(OrderRecordTupleWithRoot other) =>
          order.Equals(other.order) &&
          EqualityComparer.nullable_equals(_root, other._root);

src/client/spark_api/Orders/OrdersExtensions.cs

@@ -48,6 +48,12 @@ namespace Spark.Api.Orders {
       public static double quantity(this OrderRecordImmutable order) =>
          order.HasState ? order.State.quantity() : 0;
 
+      public static double notional_value(this OrderRecordImmutable order) =>
+         order.quantity() * order.ui_price();
+
+      public static double filled_notional_value(this OrderRecordImmutable order) =>
+         order.avg_price() * order.filled_quantity();
+
       public static double working_quantity(this OrderRecordImmutable order) =>
          order.HasState ? order.State.working_quantity() : 0;
 
@@ -487,5 +493,8 @@ namespace Spark.Api.Orders {
             CustomOrderDefinition.CustomOneofCase.Composite => "COMP",
             _ => algo_type.ToString(),
          };
+
+      public static double notional_value(this FillImmutable fill) =>
+         fill.LastShares * fill.LastPrice;
    }
 }

src/client/spark_api/data/SecurityCriteria.cs

@@ -27,6 +27,12 @@ namespace mbts.spark {
       internal static SecurityCriteria from_keys(IEnumerable<uint> keys) =>
          new(new[] { new SecurityCriteriaClause(security_keys: keys.ToArray()) });
 
+      internal static SecurityCriteria from_expirations(params DateTime[] exp_dates) =>
+         from_expirations(exp_dates.AsEnumerable());
+
+      internal static SecurityCriteria from_expirations(IEnumerable<DateTime> expirations) =>
+         new(new[] { SecurityCriteriaClause.from_expirations(expirations) });
+
       internal static SecurityCriteria from_securities(params SecurityImmutable[] securities) =>
          from_securities(securities.AsEnumerable());
 

src/client/spark_api/data/SecurityCriteriaClause.cs

@@ -58,6 +58,14 @@ namespace mbts.spark {
          return new SecurityCriteriaClause(security_keys: keys, pretty_name: pretty_name);
       }
 
+      public static SecurityCriteriaClause from_expirations(IEnumerable<DateTime> expirations) =>
+         expirations is null
+            ? new()
+            : from_expirations(expirations as DateTime[] ?? expirations.ToArray());
+
+      public static SecurityCriteriaClause from_expirations(params DateTime[] expirations) =>
+         new(expirations: expirations);
+
       [ProtoMember(1)]
       public string? pretty_name { get; }
 

src/client/spark_core/utils_api/BTS.cs

@@ -1,5 +1,5 @@
 using System;
-using System.Collections.Generic;
+using System.Reflection;
 using System.Text;
 using System.Threading;
 
@@ -10,8 +10,9 @@ namespace bts
       static int _needs_init_spark_lib = 0;
 
       public static void init(string config_source = null) {
+         var version = Assembly.GetExecutingAssembly().GetName().Version;
          Console.WriteLine(
-            $"Configuring BTS with {(config_source is null ? "defaults" : $"'{config_source}'")}");
+            $"Configuring Spark {version} with {(config_source is null ? "defaults" : $"'{config_source}'")}");
          var args = Environment.GetCommandLineArgs();
          utils.BTSNativeMethods.provide_command_line(args, args.Length);
 

src/client/spark_core/utils_api/NumericExtensions.cs

@@ -65,10 +65,10 @@ namespace bts.utils {
       public static bool epsilon_equals<T>(this T x, T y, T epsilon) where T : IFloatingPoint<T> =>
          MathUtils.epsilon_equals(x, y, epsilon);
 
-      public static bool marginal_equals<T>(T x, T y) where T : IFloatingPoint<T> =>
-         marginal_equals(x, y, MathUtils.epsilon<T>());
+      public static bool MarginalEquals<T>(this T x, T y) where T : IFloatingPoint<T> =>
+         MarginalEquals(x, y, MathUtils.epsilon<T>());
 
-      public static bool marginal_equals<T>(T x, T y, T margin) where T : IFloatingPoint<T> =>
+      public static bool MarginalEquals<T>(this T x, T y, T margin) where T : IFloatingPoint<T> =>
          MathUtils.marginal_equals(x, y, margin);
 
       public static bool is_within_range<T>(

src/lib/api/md/DatabentoMarketDataConfig.proto

@@ -13,6 +13,9 @@ message DatabentoMarketDataConfig {
   double replay_sleep_timer_sec = 50;
   string key = 60;
   string dataset = 70;
-  bool filter_securities_one_year= 80;
+  uint32 filter_securities_upto_n_months= 80;
   bool load_reference_data = 90;
+  uint32 debug_sec = 100;
+  bool only_securities =  110;
+  string bbo_start_time = 120;
 }

src/lib/api/services/AnalyticsService.proto

@@ -41,7 +41,7 @@ message VolSurfaceUsageList {
 }
 
 message ArchivedSecurityTheo {
-  uint32 id = 10;
+  uint32 id = 10[ (bts.api.key) = true ];
   double driver = 20;
   double vol = 30;
   double theo = 40;
@@ -49,7 +49,7 @@ message ArchivedSecurityTheo {
 }
 
 message ArchivedTheoList {
-  uint64 save_time = 10;
+  uint64 save_time = 10[ (bts.api.key) = true ];
   // these are sec_tups that are part of this save
   repeated bts.api.SecurityTuple sec_tups = 20;
   repeated ArchivedSecurityTheo sec_theo = 30;
@@ -195,8 +195,10 @@ service AnalyticsService {
 
   /// theos
   rpc archive_theos(ArchivedTheoList) returns (bts.api.ResultMessage);
+  rpc clean_and_archive_theos(ArchivedTheoList) returns (bts.api.ResultMessage);
   // finds the archived_theo closest to this point
   rpc get_archived_theos(ArchivedTheoRetrieval) returns (ArchivedTheoList);
+  rpc archived_theos(bts.api.VoidMessage) returns (stream ArchivedTheoList);
 
   /// reports
   rpc get_new_report_id(bts.api.VoidMessage)

src/lib/api/services/OrderService.cpp

@@ -0,0 +1,174 @@
+#include "OrderService.h"
+#include "lib/api/services/OrderService.pb.h"
+
+namespace bts::api::orderman {
+
+#define PARSE_STRING(name, dest, it)                                           \
+   it = j.find(#name);                                                         \
+   if (it != j.end()) {                                                        \
+      dest.mutable_##name()->clear();                                          \
+      it->get_to(*dest.mutable_##name());                                      \
+   }
+#define PARSE_VALUE(name, dest, it)                                            \
+   it = j.find(#name);                                                         \
+   if (it != j.end()) {                                                        \
+      dest.set_##name(it->get<decltype(dest.name())>());                       \
+   }
+#define PARSE_STRUCT(name, dest, it)                                           \
+   it = j.find(#name);                                                         \
+   if (it != j.end()) {                                                        \
+      it->get_to(*dest.mutable_##name());                                      \
+   }
+#define PARSE_ARRAY(name, dest, it)                                            \
+   it = j.find(#name);                                                         \
+   if (it != j.end() && it->is_array()) {                                      \
+      c.mutable_##name()->Clear();                                             \
+      it->get_to(*c.mutable_##name());                                         \
+   }
+#define PARSE_ONEOF(name, dest, it)                                            \
+   if (!parsed_oneof) {                                                        \
+      it = j.find(#name);                                                      \
+      if (it != j.end()) {                                                     \
+         it->get_to(*dest.mutable_##name());                                   \
+         parsed_oneof = true;                                                  \
+      }                                                                        \
+   }
+#define PARSE_ONEOFS(for_each)                                                 \
+   {                                                                           \
+      bool parsed_oneof = false;                                               \
+      for_each(PARSE_ONEOF)                                                    \
+   }
+
+#define WRITE_VALUE(src, name) j[#name] = src.name()
+#define WRITE_STRUCT(src, name)                                                \
+   if (src.has_##name()) {                                                     \
+      j[#name] = src.name();                                                   \
+   }
+
+#define WRITE_IF_NOT_ZERO(src, name)                                           \
+   if (src.name() != 0) {                                                      \
+      j[#name] = src.name();                                                   \
+   }
+#define WRITE_IF_NOT_NAN(src, name)                                            \
+   if (!std::isnan(src.name())) {                                              \
+      j[#name] = src.name();                                                   \
+   }
+#define WRITE_IF_NOT_EMPTY(src, name)                                          \
+   if (!src.name().empty()) {                                                  \
+      j[#name] = src.name();                                                   \
+   }
+#define WRITE_TIMESTAMP(src, name)                                             \
+   j[#name] = bts::core::TimeStamp::from_nanos(                                \
+                  src.name().seconds() * 1'000'000'000 + src.name().nanos())   \
+                  .to_iso_string()
+
+#define PARSE_TIMESTAMP(name, dest, it)                                        \
+   it = j.find(#name);                                                         \
+   if (it != j.end()) {                                                        \
+      bts::core::TimeStamp tmp_ts =                                            \
+          bts::core::TimeStamp::from_iso_string(it->get<std::string>());       \
+      tmp_ts.to_protobuf(*dest.mutable_##name());                              \
+   }
+
+void to_json(nlohmann::json &j, const Fill &f) {
+   WRITE_IF_NOT_ZERO(f, order_key);
+   WRITE_IF_NOT_ZERO(f, security_key);
+   WRITE_IF_NOT_EMPTY(f, account);
+   WRITE_IF_NOT_NAN(f, last_price);
+   WRITE_IF_NOT_NAN(f, avg_price);
+   WRITE_VALUE(f, side);
+   WRITE_IF_NOT_ZERO(f, last_shares);
+   WRITE_IF_NOT_EMPTY(f, fill_key);
+   WRITE_TIMESTAMP(f, timestamp);
+   WRITE_IF_NOT_ZERO(f, internal_account_id);
+   WRITE_VALUE(f, fill_kind);
+   WRITE_VALUE(f, fill_status);
+}
+
+void from_json(const nlohmann::json &j, Fill &f) {
+   nlohmann::json::const_iterator it;
+   PARSE_VALUE(order_key, f, it);
+   PARSE_VALUE(security_key, f, it);
+   PARSE_STRING(account, f, it);
+   PARSE_VALUE(last_price, f, it);
+   PARSE_VALUE(avg_price, f, it);
+   PARSE_VALUE(side, f, it);
+   PARSE_VALUE(last_shares, f, it);
+   PARSE_STRING(fill_key, f, it);
+   PARSE_TIMESTAMP(timestamp, f, it);
+   PARSE_VALUE(internal_account_id, f, it);
+   PARSE_VALUE(fill_kind, f, it);
+   PARSE_VALUE(fill_status, f, it);
+}
+
+void to_json(nlohmann::json &j, const NativeOrderState &n) {
+   WRITE_VALUE(n, exchange_order_id);
+   WRITE_VALUE(n, last_msg_seq_num);
+   WRITE_VALUE(n, sub_error_code);
+   WRITE_VALUE(n, native_status);
+   WRITE_VALUE(n, routing_record);
+   WRITE_VALUE(n, native_type);
+   WRITE_VALUE(n, last_transact_time);
+   WRITE_VALUE(n, last_transact_time_unix_ns);
+   WRITE_VALUE(n, stop_px);
+   WRITE_VALUE(n, exchange_order_id_u64);
+}
+
+void from_json(const nlohmann::json &j, NativeOrderState &n) {
+   nlohmann::json::const_iterator it;
+   PARSE_STRING(exchange_order_id, n, it);
+   PARSE_VALUE(last_msg_seq_num, n, it);
+   PARSE_VALUE(sub_error_code, n, it);
+   PARSE_VALUE(native_status, n, it);
+   PARSE_STRUCT(routing_record, n, it);
+   PARSE_VALUE(native_type, n, it);
+   PARSE_STRING(last_transact_time, n, it);
+   PARSE_VALUE(last_transact_time_unix_ns, n, it);
+   PARSE_VALUE(stop_px, n, it);
+   PARSE_VALUE(exchange_order_id_u64, n, it);
+}
+
+void to_json(nlohmann::json &j, const RoutingRecord &r) {
+   WRITE_IF_NOT_EMPTY(r, session);
+   WRITE_IF_NOT_EMPTY(r, account);
+   WRITE_IF_NOT_EMPTY(r, operator_id);
+   WRITE_VALUE(r, cti_code);
+   WRITE_VALUE(r, customer_or_firm);
+   WRITE_IF_NOT_EMPTY(r, smp_id);
+   WRITE_VALUE(r, smp_instruction);
+   WRITE_IF_NOT_EMPTY(r, location);
+   WRITE_IF_NOT_EMPTY(r, cust_order_handing_inst);
+   WRITE_IF_NOT_EMPTY(r, sender_compid);
+   WRITE_IF_NOT_EMPTY(r, clearing_firm);
+   WRITE_IF_NOT_EMPTY(r, customer_account_ref_id);
+   WRITE_IF_NOT_EMPTY(r, account_code);
+   WRITE_IF_NOT_EMPTY(r, company_id);
+   WRITE_IF_NOT_EMPTY(r, authorized_trader);
+   WRITE_IF_NOT_EMPTY(r, authorized_group);
+   WRITE_IF_NOT_EMPTY(r, cust_order_handing_inst);
+   WRITE_IF_NOT_EMPTY(r, order_server_id);
+}
+
+void from_json(const nlohmann::json &j, RoutingRecord &r) {
+   nlohmann::json::const_iterator it;
+   PARSE_STRING(session, r, it);
+   PARSE_STRING(account, r, it);
+   PARSE_STRING(operator_id, r, it);
+   PARSE_VALUE(cti_code, r, it);
+   PARSE_VALUE(customer_or_firm, r, it);
+   PARSE_STRING(smp_id, r, it);
+   PARSE_VALUE(smp_instruction, r, it);
+   PARSE_STRING(location, r, it);
+   PARSE_STRING(cust_order_handling_inst, r, it);
+   PARSE_STRING(sender_compid, r, it);
+   PARSE_STRING(clearing_firm, r, it);
+   PARSE_STRING(customer_account_ref_id, r, it);
+   PARSE_STRING(account_code, r, it);
+   PARSE_STRING(company_id, r, it);
+   PARSE_STRING(authorized_trader, r, it);
+   PARSE_STRING(authorized_group, r, it);
+   PARSE_STRING(cust_order_handing_inst, r, it);
+   PARSE_STRING(order_server_id, r, it);
+}
+
+} // namespace bts::api::orderman

src/lib/api/services/OrderService.h

@@ -1,6 +1,10 @@
 #pragma once
 
 #include "lib/api/services/OrderService.pb.h"
+#include "lib/api/services/native_order_type.h"
+#include "lib/api/services/order_status.h"
+#include "lib/api/services/time_in_force.h"
+#include "lib/api/services/trigger_price_type.h"
 #include "lib/core/EnumMacros.h"
 #include "lib/core/FMT_IMPL.h"
 #include "lib/io/JsonReader.h"
@@ -10,22 +14,70 @@
 
 namespace bts::api::orderman {
 
-PB_ENUM_FROM_JSON(OrderStatus)
-PB_ENUM_TO_JSON(OrderStatus)
+PB_ENUM_FROM_JSON(Fill_FillKind)
+PB_ENUM_TO_JSON(Fill_FillKind)
 
-PB_ENUM_FROM_JSON(TriggerPriceType)
-PB_ENUM_TO_JSON(TriggerPriceType)
+PB_ENUM_FROM_JSON(Fill_FillStatus)
+PB_ENUM_TO_JSON(Fill_FillStatus)
+
+void to_json(nlohmann::json &j, const Fill &f);
+void from_json(const nlohmann::json &j, Fill &f);
+
+PB_ENUM_FROM_JSON(OrderSubType)
+PB_ENUM_TO_JSON(OrderSubType)
+
+PB_ENUM_FROM_JSON(SelfMatchPreventionInstruction)
+PB_ENUM_TO_JSON(SelfMatchPreventionInstruction)
+
+PB_ENUM_FROM_JSON(CustomerOrFirm)
+PB_ENUM_TO_JSON(CustomerOrFirm)
+
+void to_json(nlohmann::json &j, const RoutingRecord &r);
+void from_json(const nlohmann::json &j, RoutingRecord &r);
+
+PB_ENUM_FROM_JSON(NativeOrderStatus)
+PB_ENUM_TO_JSON(NativeOrderStatus)
+
+PB_ENUM_FROM_JSON(NativeOrderState_SubError)
+PB_ENUM_TO_JSON(NativeOrderState_SubError)
+
+void to_json(nlohmann::json &j, const NativeOrderState &n);
+void from_json(const nlohmann::json &j, NativeOrderState &n);
 
 } // namespace bts::api::orderman
 
 FMT_IMPL(
-    bts::api::orderman::OrderStatus,
-    s,
+    bts::api::orderman::Fill_FillKind,
+    k,
     "{}",
-    bts::api::orderman::OrderStatus_Name(s));
+    bts::api::orderman::Fill_FillKind_Name(k));
 
 FMT_IMPL(
-    bts::api::orderman::TriggerPriceType,
+    bts::api::orderman::Fill_FillStatus,
+    s,
+    "{}",
+    bts::api::orderman::Fill_FillStatus(s));
+
+FMT_IMPL(
+    bts::api::orderman::OrderSubType,
     t,
     "{}",
-    bts::api::orderman::TriggerPriceType_Name(t));
+    bts::api::orderman::OrderSubType_Name(t));
+
+FMT_IMPL(
+    bts::api::orderman::SelfMatchPreventionInstruction,
+    s,
+    "{}",
+    bts::api::orderman::SelfMatchPreventionInstruction_Name(s));
+
+FMT_IMPL(
+    bts::api::orderman::NativeOrderStatus,
+    s,
+    "{}",
+    bts::api::orderman::NativeOrderStatus_Name(s));
+
+FMT_IMPL(
+    bts::api::orderman::NativeOrderState_SubError,
+    s,
+    "{}",
+    bts::api::orderman::NativeOrderState_SubError_Name(s));

src/lib/api/services/native_order_type.h

@@ -0,0 +1,57 @@
+#pragma once
+
+#include "lib/api/services/OrderService.pb.h"
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+#include "lib/utils/string_utils.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api::orderman {
+
+/**
+ * Parses NativeOrderType from a string. Unlike `bts::api::orderman::NativeOrderType_Parse`,
+ * this implementation is case insensitive, and makes the 'NATIVE_ORDER_TYPE_' prefix optional.
+ */
+bool native_order_type_parse_smart(
+    const std::string &str, NativeOrderType *result);
+
+PB_ENUM_FROM_JSON(NativeOrderType)
+PB_ENUM_TO_JSON(NativeOrderType)
+
+std::ostream &operator<<(std::ostream &out, NativeOrderType t) noexcept;
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline bool native_order_type_parse_smart(
+    const std::string &str, NativeOrderType *result) {
+   if (NativeOrderType_Parse(str, result)) {
+      return true;
+   }
+   std::string s = bts::system::string_utils::copy_to_upper(str);
+   if (NativeOrderType_Parse(s, result)) {
+      return true;
+   }
+   const char *prefix = "NATIVE_ORDER_TYPE_";
+   if (!s.starts_with(prefix)) {
+      s.insert(0, prefix);
+   }
+   return NativeOrderType_Parse(s, result);
+}
+
+inline std::ostream &operator<<(std::ostream &out, NativeOrderType t) noexcept {
+   out << NativeOrderType_Name(t);
+   return out;
+}
+
+} // namespace bts::api::orderman
+
+FMT_IMPL(
+    bts::api::orderman::NativeOrderType,
+    t,
+    "{}",
+    bts::api::orderman::NativeOrderType_Name(t));

src/lib/api/services/order_status.h

@@ -0,0 +1,56 @@
+#pragma once
+
+#include "lib/api/services/OrderService.pb.h"
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+#include "lib/utils/string_utils.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api::orderman {
+
+/**
+ * Parses OrderStatus from a string. Unlike `bts::api::orderman::OrderStatus_Parse`,
+ * this implementation is case insensitive, and makes the 'ORDER_STATUS_' prefix optional.
+ */
+bool order_status_parse_smart(const std::string &str, OrderStatus *result);
+
+PB_ENUM_FROM_JSON(OrderStatus)
+PB_ENUM_TO_JSON(OrderStatus)
+
+std::ostream &operator<<(std::ostream &out, OrderStatus s) noexcept;
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline bool order_status_parse_smart(
+    const std::string &str, OrderStatus *result) {
+   if (OrderStatus_Parse(str, result)) {
+      return true;
+   }
+   std::string s = bts::system::string_utils::copy_to_upper(str);
+   if (OrderStatus_Parse(s, result)) {
+      return true;
+   }
+   const char *prefix = "ORDER_STATUS_";
+   if (!s.starts_with(prefix)) {
+      s.insert(0, prefix);
+   }
+   return OrderStatus_Parse(s, result);
+}
+
+inline std::ostream &operator<<(std::ostream &out, OrderStatus s) noexcept {
+   out << OrderStatus_Name(s);
+   return out;
+}
+
+} // namespace bts::api::orderman
+
+FMT_IMPL(
+    bts::api::orderman::OrderStatus,
+    s,
+    "{}",
+    bts::api::orderman::OrderStatus_Name(s));

src/lib/api/services/time_in_force.h

@@ -0,0 +1,56 @@
+#pragma once
+
+#include "lib/api/services/OrderService.pb.h"
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+#include "lib/utils/string_utils.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api::orderman {
+
+/**
+ * Parses TimeInForce from a string. Unlike `bts::api::orderman::TimeInForce_Parse`,
+ * this implementation is case insensitive, and makes the 'TIME_IN_FORCE_' prefix optional.
+ */
+bool time_in_force_parse_smart(const std::string &str, TimeInForce *result);
+
+PB_ENUM_FROM_JSON(TimeInForce)
+PB_ENUM_TO_JSON(TimeInForce)
+
+std::ostream &operator<<(std::ostream &out, TimeInForce tif) noexcept;
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline bool time_in_force_parse_smart(
+    const std::string &str, TimeInForce *result) {
+   if (TimeInForce_Parse(str, result)) {
+      return true;
+   }
+   std::string s = bts::system::string_utils::copy_to_upper(str);
+   if (TimeInForce_Parse(s, result)) {
+      return true;
+   }
+   const char *prefix = "TIME_IN_FORCE_";
+   if (!s.starts_with(prefix)) {
+      s.insert(0, prefix);
+   }
+   return TimeInForce_Parse(s, result);
+}
+
+inline std::ostream &operator<<(std::ostream &out, TimeInForce tif) noexcept {
+   out << TimeInForce_Name(tif);
+   return out;
+}
+
+} // namespace bts::api::orderman
+
+FMT_IMPL(
+    bts::api::orderman::TimeInForce,
+    v,
+    "{}",
+    bts::api::orderman::TimeInForce_Name(v));

src/lib/api/services/trigger_price_type.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include "lib/api/services/OrderService.pb.h"
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+#include "lib/utils/string_utils.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api::orderman {
+
+/**
+ * Parses TriggerPriceType from a string. Unlike `bts::api::orderman::TriggerPriceType_Parse`,
+ * this implementation is case insensitive, and makes the 'TRIGGER_PRICE_TYPE_' prefix optional.
+ */
+bool trigger_price_type_parse_smart(
+    const std::string &str, TriggerPriceType *result);
+
+PB_ENUM_FROM_JSON(TriggerPriceType)
+PB_ENUM_TO_JSON(TriggerPriceType)
+
+std::ostream &operator<<(std::ostream &out, TriggerPriceType t) noexcept;
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline bool trigger_price_type_parse_smart(
+    const std::string &str, TriggerPriceType *result) {
+   if (TriggerPriceType_Parse(str, result)) {
+      return true;
+   }
+   std::string s = bts::system::string_utils::copy_to_upper(str);
+   if (TriggerPriceType_Parse(s, result)) {
+      return true;
+   }
+   const char *prefix = "TRIGGER_PRICE_TYPE_";
+   if (!s.starts_with(prefix)) {
+      s.insert(0, prefix);
+   }
+   return TriggerPriceType_Parse(s, result);
+}
+
+inline std::ostream &operator<<(
+    std::ostream &out, TriggerPriceType t) noexcept {
+   out << TriggerPriceType_Name(t);
+   return out;
+}
+
+} // namespace bts::api::orderman
+
+FMT_IMPL(
+    bts::api::orderman::TriggerPriceType,
+    t,
+    "{}",
+    bts::api::orderman::TriggerPriceType_Name(t));

src/lib/cmake/GetLinuxClientSources.cmake

@@ -75,6 +75,7 @@ function(get_cpp_sources dest)
     expression/ParserExceptions.cpp
     expression/Value.cpp
     expression/tests/expression_parser_tests.cpp
+    io/JsonReader.cpp
     logging/LoggerHub.cpp
     logging/Sink.cpp
     logging/logging.cpp

src/lib/cmake/GetWindowsClientSources.cmake

@@ -74,6 +74,7 @@ function(get_cpp_sources dest)
     expression/ParserExceptions.cpp
     expression/Value.cpp
     expression/tests/expression_parser_tests.cpp
+    io/JsonReader.cpp
     logging/LoggerHub.cpp
     logging/Sink.cpp
     logging/logging.cpp

src/lib/collections/concurrent_factory.h

@@ -0,0 +1,98 @@
+#pragma once
+
+#include "lib/utils/LockBox.h"
+#include "lib/utils/bts_assert.h"
+#include "lib/utils/type_name.h"
+
+#include <functional>
+#include <memory>
+
+namespace bts::collections {
+
+template <class KeyT, class ValueT>
+class ConcurrentFactory {
+   utils::LockBox<std::unordered_map<KeyT, std::shared_ptr<ValueT>>> store;
+
+ public:
+   std::shared_ptr<ValueT> add(
+       const KeyT &key,
+       std::function<void(std::shared_ptr<ValueT> &)> callback) {
+      return store.lock([&](auto &map) {
+         auto [it, is_new] = map.try_emplace(key, std::make_shared<ValueT>());
+         bts_assert_msgf(
+             is_new, "{}::add. Duplicate key '{}'",
+             utils::cached_type_name<ConcurrentFactory<KeyT, ValueT>>(), key);
+         callback(it->second);
+         return it->second;
+      });
+   }
+
+   std::size_t erase(const KeyT &key) {
+      return store.lock([&](auto &map) {
+         map.erase(key);
+         return map.size();
+      });
+   }
+
+   std::size_t size() const {
+      return store.lock([&](auto &map) { return map.size(); });
+   }
+
+   template <
+       typename CallbackT = std::function<void(std::shared_ptr<ValueT> &)>>
+   std::shared_ptr<const ValueT> get(const KeyT &key, CallbackT callback) {
+      return store.lock([&](auto &map) {
+         auto [it, is_new] = map.try_emplace(key, std::make_shared<ValueT>());
+         bts_assert_msgf(
+             !is_new, "{}::get. Key not found '{}'",
+             utils::cached_type_name<ConcurrentFactory<KeyT, ValueT>>(), key);
+         callback(it->second);
+         return it->second;
+      });
+   }
+
+   template <
+       typename CallbackT =
+           std::function<void(std::shared_ptr<ValueT> &, bool)>>
+   std::shared_ptr<const ValueT> get_or_add(
+       const KeyT &key, CallbackT callback) {
+      return store.lock([&](auto &map) {
+         auto [it, is_new] = map.try_emplace(key);
+         if (is_new) {
+            it->second = std::make_shared<ValueT>();
+         }
+         callback(it->second, is_new);
+         return it->second;
+      });
+   }
+
+   template <
+       typename CallbackT =
+           std::function<void(std::shared_ptr<ValueT> &, bool)>>
+   std::shared_ptr<const ValueT> replace(const KeyT &key, CallbackT callback) {
+      return store.lock([&](auto &map) {
+         auto [it, is_new] = map.try_emplace(key);
+         bts_assert_msgf(
+             !is_new, "{}::replace. Key not found '{}'",
+             utils::cached_type_name<ConcurrentFactory<KeyT, ValueT>>(), key);
+         it->second = std::make_shared<ValueT>();
+         callback(it->second, is_new);
+         return it->second;
+      });
+   }
+
+   template <
+       typename CallbackT =
+           std::function<void(std::shared_ptr<ValueT> &, bool)>>
+   std::shared_ptr<const ValueT> replace_or_add(
+       const KeyT &key, CallbackT callback) {
+      return store.lock([&](auto &map) {
+         auto [it, is_new] = map.try_emplace(key);
+         it->second = std::make_shared<ValueT>();
+         callback(it->second, is_new);
+         return it->second;
+      });
+   }
+};
+
+} // namespace bts::collections

src/lib/config/profiles/default/okx_exe_cfg.conf

@@ -0,0 +1,17 @@
+OkxExeCfg {
+    value{
+        account : "ACC A"
+        ops_id : "op_id_bts"
+        secret_key : "DAF6CBB09475D2520B88D2070788A976"
+        api_key : "4492a9ab-5784-454c-acc7-38dc2a503ed0"
+        passphrase : "Pakistan@123"
+        ws_url : "wss://wspap.okx.com:8443"
+        rest_url : "https://www.okx.com"
+        ws_ping_timeout_seconds : 10 
+        ws_ping_interval_seconds : 25
+        ws_reconnect_interval_seconds : 5
+        ws_reconnect_max_count : 5
+        simulated_trading : true
+        tdmode : "cross"
+    }
+}

src/lib/config/profiles/default/orderman_cfg.conf

@@ -1,34 +1,34 @@
 OrdermanCfg {
-   rule: "DeploymentType.type == 'dev'"
-   value {
-      executor {
-         thread_name: "oman_orders"
-	 poll: false 
-	 hosted_actor: "MDSource"
-	 hosted_actor: "OrderManager"
-	 hosted_actor: "ExecutionGateway"
-      }
-      executor {
-         thread_name: "oman_slow"
-	 poll: false 
-	 hosted_actor: "*"
-      }
-   }
+  rule: "DeploymentType.type == 'dev'"
+  value {
+    executor {
+      thread_name: "oman_orders"
+      poll: false 
+      hosted_actor: "MDSource"
+      hosted_actor: "OrderManager"
+      hosted_actor: "ExecutionGateway"
+    }
+    executor {
+      thread_name: "oman_slow"
+      poll: false 
+	    hosted_actor: "*"
+    }
+  }
 }
 
 OrdermanCfg {
-   value {
-      executor {
-         thread_name: "oman_orders"
-	 poll: true 
-	 hosted_actor: "MDSource"
-	 hosted_actor: "OrderManager"
-	 hosted_actor: "ExecutionGateway"
-      }
-      executor {
-         thread_name: "oman_slow"
-	 poll: false 
-	 hosted_actor: "*"
-      }
-   }
-}
+  value {
+    executor {
+      thread_name: "oman_orders"
+      poll: true 
+      hosted_actor: "MDSource"
+      hosted_actor: "OrderManager"
+      hosted_actor: "ExecutionGateway"
+    }
+    executor {
+      thread_name: "oman_slow"
+	    poll: false 
+      hosted_actor: "*"
+    }
+  }
+}

src/lib/config/profiles/default/symbols.conf

@@ -37,6 +37,7 @@ SymbolLists {
     list {
       key: "CBOE"
       symbol:"SPX"
+      symbol:"SPXW"
     }
     list {
       key: "BINANCE_US"

src/lib/config/profiles/default/toggles.conf

@@ -114,13 +114,139 @@ VolSurfaceUsage {
      items: {
       option_name: "SPX"
       option_venue: VENUE_CBOE
-      vs_name: "Main"
+      vs_name: "CBOE_skews"
     }
     items: {
       option_name: "SPXW"
       option_venue: VENUE_CBOE
-      vs_name: "Main"
+      vs_name: "CBOE_skews"
     }
+    items: {
+      option_name: "ES"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }     
+     items: {
+      option_name: "E1A"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "E2A"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "E3A"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E4A"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E5A"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E1B"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "E2B"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "E3B"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E4B"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E5B"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E1C"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "E2C"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "E3C"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E4C"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E5C"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E1D"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "E2D"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "E3D"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E4D"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E5D"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "EW1"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "EW2"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "EW3"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "EW4"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+
 
   }
 }

src/lib/config/profiles/default/ui.conf

@@ -39,8 +39,6 @@ LadderKnobs {
     security_timeout_secs: 60
     use_algo_param_grid: true
     max_mbo_orders: 200
-    lighten_market_cells: true
-    mimic_tt: true
     scrollable_tick_window_size: 256
   }
 }

src/lib/config/protos/okx_exe_cfg.proto

@@ -0,0 +1,18 @@
+syntax = "proto3";
+package bts.config;
+message OkxExeCfg {
+    string account = 1;
+    string ops_id = 2;
+    string secret_key = 3;
+    string api_key = 4;
+    string passphrase = 5;
+    string ws_url = 6;
+    string rest_url = 7;
+    uint32 ws_ping_timeout_seconds = 8;
+    uint32 ws_ping_interval_seconds = 9;
+    uint32 ws_reconnect_interval_seconds = 10;
+    uint32 ws_reconnect_max_count = 11;
+    bool simulated_trading = 12;
+    string tdmode = 13;
+
+}

src/lib/config/protos/ui.proto

@@ -82,7 +82,7 @@ message LadderKnobs {
   bool show_scrollbar = 9;
   // DJ wants bid/ask cells with markets (i.e., size > 0) to have a
   // lighter background than those without markets.
-  bool lighten_market_cells = 10;
+  // bool lighten_market_cells = 10;
 
   // For market cell backgrounds, 0 means 'ignore this color setting' (i.e.,
   // employ the ladder's default behavior instead).
@@ -97,9 +97,9 @@ message LadderKnobs {
   //   Pure Red   : 0xFFFF0000
   //   Pure Blue  : 0xFF00FF00
   //   Pure Green : 0xFF0000FF
-  uint32 bid_market_cell_background = 11;
-  uint32 ask_market_cell_background = 12;
-  bool mimic_tt = 13;
+  // uint32 bid_market_cell_background = 11;
+  // uint32 ask_market_cell_background = 12;
+  // bool mimic_tt = 13;
   // Defines the number of price rows rendered by the ladder. The smaller this
   // window is, the cheaper it is to render the ladder grid (redraw prices,
   // sizes, etc.).  However, if the window is too small, scrolling may appear

src/lib/core/EnumMacros.h

@@ -110,6 +110,19 @@
    ENUM_TO_JSON(type)                                                          \
    ENUM_EQUALITY_OPS(type)
 
+#define ENUM_CLASS_DEF_NO_JSON(type, for_all)                                  \
+   class type {                                                                \
+    public:                                                                    \
+      enum Value { for_all(ENUM_DECLARE_VALUE) };                              \
+                                                                               \
+      ENUM_BASIC_MEMBERS(type)                                                 \
+   };                                                                          \
+                                                                               \
+   ENUM_OUTPUT_OPS(type)                                                       \
+   ENUM_NAME_DEF(type, for_all)                                                \
+   ENUM_PARSE_DEF(type, for_all)                                               \
+   ENUM_EQUALITY_OPS(type)
+
 #define ENUM_BITWISE_COMPARISON_OP(enum_type, numeric_type, op)                \
    constexpr enum_type operator op(enum_type lhs, enum_type rhs) noexcept {    \
       return static_cast<enum_type>(static_cast<numeric_type>(lhs)             \
@@ -154,3 +167,4 @@
    inline void to_json(nlohmann::json &j, const enum_type &value) {            \
       j = enum_type##_Name(value);                                             \
    }
+

src/lib/core/TimeStamp.cpp

@@ -1,6 +1,7 @@
 #include "TimeStamp.h"
 #include "lib/utils/datetime.h"
 #include <boost/date_time/posix_time/time_formatters.hpp>
+#include <google/protobuf/timestamp.pb.h>
 
 namespace bts::core {
 
@@ -43,6 +44,12 @@ bool TimeStamp::from_ymd(
    return true;
 }
 
+void TimeStamp::to_protobuf(google::protobuf::Timestamp &ts) const {
+   ts.set_seconds(seconds());
+   bts_assert(nanos() >= seconds() * 1'000'000'000);
+   ts.set_nanos(nanos() - seconds() * 1'000'000'000);
+}
+
 TimeStamp TimeStamp::from_iso_string(const std::string &iso_string) {
    auto ptime = datetime::parse_iso_date_time_string(iso_string);
    return from_ptime(ptime);

src/lib/core/TimeStamp.h

@@ -10,6 +10,7 @@
 #include <boost/date_time/time.hpp>
 #include <cstdint>
 #include <fmt/format.h>
+#include <google/protobuf/timestamp.pb.h>
 #include <limits>
 #include <string>
 
@@ -87,6 +88,7 @@ struct TimeStamp {
    // Gets a TimeStamp representing this date at midnight.
    TimeStamp date() const;
    constexpr boost::posix_time::ptime to_ptime() const;
+   void to_protobuf(google::protobuf::Timestamp &ts) const;
    std::string to_iso_string() const;
    // current epoch ns
    static uint64_t now_ns() {

src/lib/domain/SecurityCommon.h

@@ -10,6 +10,7 @@
 #include <limits>
 #include <nlohmann/json.hpp>
 #include <string>
+#include <vector>
 
 namespace bts::domain {
 
@@ -31,6 +32,7 @@ class SecurityCommon {
    double min_quantity = 0;
    double max_quantity = std::numeric_limits<double>::max();
    double cash_per_point = 1;
+   std::vector<double> price_ticks;
 
    SecurityCommon() {}
    SecurityCommon(const bts::api::Security &s)

src/lib/domain_model/PutOrCall.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#include "lib/core/EnumMacros.h"
+#include "lib/core/FMT_IMPL.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/io/JsonReader.h"
+#include "lib/logging/logging.h"
+#include "lib/utils/string_utils.h"
+
+#include <nlohmann/json.hpp>
+
+namespace bts::api {
+
+/**
+ * Parses PutOrCall from a string. Unlike `bts::api::SecurityType_Parse`, this
+ * implementation is case insensitive, and makes the 'PUTORCALL_' prefix optional.
+ */
+bool put_or_call_parse_smart(const std::string &str, PutOrCall *result);
+
+PB_ENUM_FROM_JSON(PutOrCall)
+PB_ENUM_TO_JSON(PutOrCall)
+
+std::ostream &operator<<(std::ostream &out, const PutOrCall &poc) noexcept;
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline bool put_or_call_parse_smart(const std::string &str, PutOrCall *result) {
+   if (PutOrCall_Parse(str, result)) {
+      return true;
+   }
+   std::string s = bts::system::string_utils::copy_to_upper(str);
+   if (PutOrCall_Parse(s, result)) {
+      return true;
+   }
+   const char *prefix = "PUTORCALL_";
+   if (!s.starts_with(prefix)) {
+      s.insert(0, prefix);
+   }
+   return PutOrCall_Parse(s, result);
+}
+
+inline std::ostream &operator<<(
+    std::ostream &out, const PutOrCall &poc) noexcept {
+   out << PutOrCall_Name(poc);
+   return out;
+}
+
+} // namespace bts::api
+
+FMT_IMPL(bts::api::PutOrCall, poc, "{}", bts::api::PutOrCall_Name(poc));

src/lib/domain_model/Security.cpp

@@ -2,10 +2,13 @@
 #include "lib/config/config.h"
 #include "lib/config/protos/orders_fills_cleanup.pb.h"
 #include "lib/domain_model/Security.pb.h"
+#include "lib/io/JsonMacros.h"
 #include "lib/md/common.h"
+#include "lib/utils/pb_json_macros.h"
 
 #include <boost/date_time/posix_time/posix_time.hpp>
 #include <google/protobuf/text_format.h>
+#include <nlohmann/json.hpp>
 
 namespace bts {
 namespace securities {
@@ -191,3 +194,389 @@ std::string unique_descriptor(const api::Security &security) {
 
 } // namespace securities
 } // namespace bts
+
+namespace bts::api {
+
+using nlohmann::json;
+using std::uint32_t;
+
+void to_json(json &j, const api::ContractQuantityInfo &cqi) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, min_value);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, max_value);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, tick_unit);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, tick_base);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, multiplier);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, max_price);
+}
+
+void from_json(const json &j, api::ContractQuantityInfo &cqi) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, cqi, min_value, it);
+   PB_JSON_READ_VALUE(j, cqi, max_value, it);
+   PB_JSON_READ_VALUE(j, cqi, max_value, it);
+   PB_JSON_READ_VALUE(j, cqi, tick_unit, it);
+   PB_JSON_READ_VALUE(j, cqi, tick_base, it);
+   PB_JSON_READ_VALUE(j, cqi, max_price, it);
+}
+
+void to_json(json &j, const api::ContractSpecification &spec) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, low_tick);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, price_break);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, high_tick);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, cash_per_point);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, price_multiplier);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, main_fraction);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, sub_fraction);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, price_display_format);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, spec, quantity_infos);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, spec, notional_quantity_infos);
+}
+
+void from_json(const json &j, api::ContractSpecification &spec) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, spec, low_tick, it);
+   PB_JSON_READ_VALUE(j, spec, price_break, it);
+   PB_JSON_READ_VALUE(j, spec, high_tick, it);
+   PB_JSON_READ_VALUE(j, spec, cash_per_point, it);
+   PB_JSON_READ_VALUE(j, spec, price_multiplier, it);
+   PB_JSON_READ_VALUE(j, spec, main_fraction, it);
+   PB_JSON_READ_VALUE(j, spec, sub_fraction, it);
+   PB_JSON_READ_VALUE(j, spec, price_display_format, it);
+   PB_JSON_READ_ARRAY(j, spec, quantity_infos, it);
+   PB_JSON_READ_ARRAY(j, spec, notional_quantity_infos, it);
+}
+
+void to_json(json &, const SpotContract &) {}
+void from_json(const json &, SpotContract &) {}
+
+template <class ExpContract>
+static void exp_to_json(json &j, const ExpContract &ec) {
+   j = bts::core::TimeStamp::from_ymd(
+           ec.expiration_year(), ec.expiration_month(), ec.expiration_day())
+           .to_iso_string();
+}
+
+template <class ExpContract>
+static void exp_from_json(const json &j, ExpContract &ec) {
+   const auto dt = bts::core::TimeStamp::from_iso_string(j.get<std::string>());
+   uint32_t y, m, d;
+   dt.to_date(y, m, d);
+   ec.set_expiration_year(y);
+   ec.set_expiration_month(m);
+   ec.set_expiration_day(d);
+}
+
+void to_json(json &j, const FutureContract &f) {
+   exp_to_json(j["expiration"], f);
+}
+
+void from_json(const json &j, FutureContract &f) {
+   const auto it = j.find("expiration");
+   if (it != j.end()) {
+      exp_from_json(*it, f);
+   }
+}
+
+void to_json(json &j, const OptionContract &opt) {
+   exp_to_json(j["expiration"], opt);
+   PB_JSON_WRITE_VALUE(j, opt, strike);
+   PB_JSON_WRITE_VALUE(j, opt, call_put);
+}
+
+void from_json(const json &j, OptionContract &opt) {
+   json::const_iterator it;
+   it = j.find("expiration");
+   if (it != j.end()) {
+      exp_from_json(*it, opt);
+   }
+   PB_JSON_READ_VALUE(j, opt, strike, it);
+   PB_JSON_READ_VALUE(j, opt, call_put, it);
+}
+
+void to_json(json &j, const ContractLeg &leg) {
+   PB_JSON_WRITE_STRUCT(j, leg, security);
+   PB_JSON_WRITE_VALUE(j, leg, quantity);
+   PB_JSON_WRITE_VALUE(j, leg, side);
+   PB_JSON_WRITE_VALUE(j, leg, price_multiplier);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, leg, leg_delta);
+   PB_JSON_WRITE_VALUE(j, leg, price_calc);
+}
+
+void from_json(const json &j, ContractLeg &leg) {
+   json::const_iterator it;
+   PB_JSON_READ_STRUCT(j, leg, security, it);
+   PB_JSON_READ_VALUE(j, leg, quantity, it);
+   PB_JSON_READ_VALUE(j, leg, side, it);
+   PB_JSON_READ_VALUE(j, leg, price_multiplier, it);
+   PB_JSON_READ_VALUE(j, leg, leg_delta, it);
+   PB_JSON_READ_VALUE(j, leg, price_calc, it);
+}
+
+void to_json(json &j, const SpreadContract &spd) {
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, spd, legs);
+}
+
+void from_json(const json &j, SpreadContract &spd) {
+   json::const_iterator it;
+   PB_JSON_READ_ARRAY(j, spd, legs, it);
+}
+
+void to_json(json &j, const SyntheticContract &syn) {
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, syn, name);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, syn, user_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, syn, legs);
+   PB_JSON_WRITE_VALUE(j, syn, price_calc);
+}
+
+void from_json(const json &j, SyntheticContract &syn) {
+   json::const_iterator it;
+   PB_JSON_READ_STRING(j, syn, name, it);
+   PB_JSON_READ_VALUE(j, syn, user_id, it);
+   PB_JSON_READ_ARRAY(j, syn, legs, it);
+   PB_JSON_READ_VALUE(j, syn, price_calc, it);
+}
+
+void to_json(json &j, const GLBXExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, cfi_code);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, cme_symbol);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, market_segment_id);
+}
+
+void from_json(const json &j, GLBXExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+   PB_JSON_READ_STRING(j, data, cfi_code, it);
+   PB_JSON_READ_STRING(j, data, cme_symbol, it);
+   PB_JSON_READ_VALUE(j, data, market_segment_id, it);
+}
+
+void to_json(json &j, const CBOEExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, class_key);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, source);
+}
+
+void from_json(const json &j, CBOEExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+   PB_JSON_READ_VALUE(j, data, class_key, it);
+   PB_JSON_READ_STRING(j, data, source, it);
+}
+
+void to_json(json &j, const CBOECFEExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, class_key);
+}
+
+void from_json(const json &j, CBOECFEExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+   PB_JSON_READ_VALUE(j, data, class_key, it);
+}
+
+void to_json(json &j, const ICEExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, price_denominator);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, deal_denominator);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, settle_denominator);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, market_description);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, cleared_alias);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, product_name);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, hub_alias);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, strip_name);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, mic_code);
+   PB_JSON_WRITE_IF_TRUE(j, data, is_strip);
+}
+
+void from_json(const json &j, ICEExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+   PB_JSON_READ_VALUE(j, data, price_denominator, it);
+   PB_JSON_READ_VALUE(j, data, deal_denominator, it);
+   PB_JSON_READ_VALUE(j, data, settle_denominator, it);
+   PB_JSON_READ_STRING(j, data, market_description, it);
+   PB_JSON_READ_STRING(j, data, cleared_alias, it);
+   PB_JSON_READ_STRING(j, data, product_name, it);
+   PB_JSON_READ_STRING(j, data, hub_alias, it);
+   PB_JSON_READ_STRING(j, data, strip_name, it);
+   PB_JSON_READ_STRING(j, data, mic_code, it);
+   PB_JSON_READ_VALUE(j, data, is_strip, it);
+}
+
+void to_json(json &j, const TAIFEXExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, class_key);
+}
+
+void from_json(const json &j, TAIFEXExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+   PB_JSON_READ_VALUE(j, data, class_key, it);
+}
+
+void to_json(json &j, const DeribitExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+}
+
+void from_json(const json &j, DeribitExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+}
+
+void to_json(json &j, const FTXExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+}
+
+void from_json(const json &j, FTXExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+}
+
+void to_json(json &j, const BinanceExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+}
+
+void from_json(const json &j, BinanceExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+}
+
+void to_json(json &j, const BinanceUsdsFuturesExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+}
+
+void from_json(const json &j, BinanceUsdsFuturesExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+}
+
+void to_json(json &j, const BinanceCoinFuturesExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+}
+
+void from_json(const json &j, BinanceCoinFuturesExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+}
+
+void to_json(json &j, const BinanceUsExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+}
+
+void from_json(const json &j, BinanceUsExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+}
+
+void to_json(json &j, const OkxExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+}
+
+void from_json(const json &j, OkxExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+}
+
+void to_json(json &j, const CoinbaseExchangeData &data) {
+   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
+}
+
+void from_json(const json &j, CoinbaseExchangeData &data) {
+   json::const_iterator it;
+   PB_JSON_READ_VALUE(j, data, exchange_id, it);
+   PB_JSON_READ_STRING(j, data, security_desc, it);
+}
+
+void to_json(json &j, const Security &s) {
+   // clang-format off
+#define CONTRACT_ONEOF(oneof)           \
+   oneof(j, s, Spot, spot)              \
+   oneof(j, s, Future, future)          \
+   oneof(j, s, Option, option)          \
+   oneof(j, s, Spread, spread)          \
+   oneof(j, s, Synthetic, synthetic)
+
+#define EXCHANGE_DATA_ONEOF(oneof)                        \
+   oneof(j, s, Glbx, glbx)                                \
+   oneof(j, s, Cboe, cboe)                                \
+   oneof(j, s, Cfe, cfe)                                  \
+   oneof(j, s, Ice, ice)                                  \
+   oneof(j, s, Deribit, deribit)                          \
+   oneof(j, s, Ftx, ftx)                                  \
+   oneof(j, s, Binance, binance)                          \
+   oneof(j, s, BinanceCoinFutures, binance_coin_futures)  \
+   oneof(j, s, BinanceUsdsFutures, binance_usds_futures)  \
+   oneof(j, s, BinanceUs, binance_us)                     \
+   oneof(j, s, Coinbase, coinbase)                        \
+   oneof(j, s, Okx, okx)                                  \
+   oneof(j, s, Taifex, taifex)
+   // clang-format on
+
+   PB_JSON_WRITE_IF_NOT_ZERO(j, s, id);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, s, symbol);
+   PB_JSON_WRITE_VALUE(j, s, exchange);
+   PB_JSON_WRITE_STRUCT(j, s, contract_spec);
+   PB_JSON_WRITE_IF_NOT_EMPTY(j, s, currency);
+   PB_JSON_WRITE_ONEOF(j, s, Contract, CONTRACT_ONEOF);
+   PB_JSON_WRITE_ONEOF(j, s, ExchangeData, EXCHANGE_DATA_ONEOF);
+}
+
+void from_json(const json &j, Security &s) {
+   json::const_iterator it;
+
+#define SEC_TYPE_ONEOF(oneof)                                                  \
+   oneof(j, s, future, it);                                                    \
+   oneof(j, s, option, it);                                                    \
+   oneof(j, s, spread, it);                                                    \
+   oneof(j, s, synthetic, it)
+
+#define CONTRACT_CASE_ONEOF(oneof)                                             \
+   oneof(j, s, glbx, it);                                                      \
+   oneof(j, s, cboe, it);                                                      \
+   oneof(j, s, cfe, it);                                                       \
+   oneof(j, s, ice, it);                                                       \
+   oneof(j, s, deribit, it);                                                   \
+   oneof(j, s, ftx, it);                                                       \
+   oneof(j, s, binance, it);                                                   \
+   oneof(j, s, binance_usds_futures, it);                                      \
+   oneof(j, s, binance_coin_futures, it);                                      \
+   oneof(j, s, binance_us, it);                                                \
+   oneof(j, s, coinbase, it);                                                  \
+   oneof(j, s, okx, it);                                                       \
+   oneof(j, s, taifex, it);
+
+   PB_JSON_READ_VALUE(j, s, id, it);
+   PB_JSON_READ_STRING(j, s, symbol, it);
+   PB_JSON_READ_STRING(j, s, currency, it);
+   PB_JSON_READ_STRUCT(j, s, contract_spec, it);
+   PB_JSON_READ_ONEOFS(SEC_TYPE_ONEOF);
+   PB_JSON_READ_ONEOFS(CONTRACT_CASE_ONEOF);
+}
+
+} // namespace bts::api

src/lib/domain_model/Security.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "lib/core/FMT_IMPL.h"
+#include "lib/domain_model/PutOrCall.h"
 #include "lib/domain_model/Security.pb.h"
 #include "lib/domain_model/SecurityType.h"
 #include "lib/domain_model/Side.h"
@@ -9,6 +9,7 @@
 #include "lib/utils/bts_assert.h"
 
 #include <cmath>
+#include <nlohmann/json_fwd.hpp>
 
 namespace boost {
 namespace gregorian {
@@ -16,14 +17,83 @@ class date;
 }
 } // namespace boost
 
-namespace bts {
+namespace bts::api {
 
-namespace api {
 class Security;
 class ICEExchangeData;
-} // namespace api
 
-namespace securities {
+void to_json(nlohmann::json &j, const ContractQuantityInfo &c);
+void from_json(const nlohmann::json &j, ContractQuantityInfo &c);
+
+void to_json(nlohmann::json &j, const ContractSpecification &c);
+void from_json(const nlohmann::json &j, ContractSpecification &c);
+
+void to_json(nlohmann::json &j, const ContractLeg &c);
+void from_json(const nlohmann::json &j, ContractLeg &c);
+
+void to_json(nlohmann::json &j, const SpotContract &c);
+void from_json(const nlohmann::json &j, SpotContract &c);
+
+void to_json(nlohmann::json &j, const FutureContract &c);
+void from_json(const nlohmann::json &j, FutureContract &c);
+
+void to_json(nlohmann::json &j, const OptionContract &c);
+void from_json(const nlohmann::json &j, OptionContract &c);
+
+void to_json(nlohmann::json &j, const SpreadContract &c);
+void from_json(const nlohmann::json &j, SpreadContract &c);
+
+void to_json(nlohmann::json &j, const SyntheticContract_PriceCalc &c);
+void from_json(const nlohmann::json &j, SyntheticContract_PriceCalc &c);
+
+void to_json(nlohmann::json &j, const SyntheticContract &c);
+void from_json(const nlohmann::json &j, SyntheticContract &c);
+
+void to_json(nlohmann::json &j, const GLBXExchangeData &data);
+void from_json(const nlohmann::json &j, GLBXExchangeData &data);
+
+void to_json(nlohmann::json &j, const CBOEExchangeData &data);
+void from_json(const nlohmann::json &j, CBOEExchangeData &data);
+
+void to_json(nlohmann::json &j, const CBOECFEExchangeData &data);
+void from_json(const nlohmann::json &j, CBOECFEExchangeData &data);
+
+void to_json(nlohmann::json &j, const ICEExchangeData &data);
+void from_json(const nlohmann::json &j, ICEExchangeData &data);
+
+void to_json(nlohmann::json &j, const DeribitExchangeData &data);
+void from_json(const nlohmann::json &j, DeribitExchangeData &data);
+
+void to_json(nlohmann::json &j, const FTXExchangeData &data);
+void from_json(const nlohmann::json &j, FTXExchangeData &data);
+
+void to_json(nlohmann::json &j, const BinanceExchangeData &data);
+void from_json(const nlohmann::json &j, BinanceExchangeData &data);
+
+void to_json(nlohmann::json &j, const BinanceUsdsFuturesExchangeData &data);
+void from_json(const nlohmann::json &j, BinanceUsdsFuturesExchangeData &data);
+
+void to_json(nlohmann::json &j, const BinanceCoinFuturesExchangeData &data);
+void from_json(const nlohmann::json &j, BinanceCoinFuturesExchangeData &data);
+
+void to_json(nlohmann::json &j, const BinanceUsExchangeData &data);
+void from_json(const nlohmann::json &j, BinanceUsExchangeData &data);
+
+void to_json(nlohmann::json &j, const OkxExchangeData &data);
+void from_json(const nlohmann::json &j, OkxExchangeData &data);
+
+void to_json(nlohmann::json &j, const CoinbaseExchangeData &data);
+void from_json(const nlohmann::json &j, CoinbaseExchangeData &data);
+
+void to_json(nlohmann::json &j, const Security &s);
+void from_json(const nlohmann::json &j, Security &s);
+
+PB_ENUM_FROM_JSON(SyntheticContract_PriceCalc);
+PB_ENUM_TO_JSON(SyntheticContract_PriceCalc);
+
+} // namespace bts::api
+
+namespace bts::securities {
 
 /**
  * Gets a descriptor string for a security (should be unique within a particular exchange).
@@ -60,7 +130,8 @@ inline api::Side other_side(api::Side side) {
       return side;
    }
 }
-template <class V> class TwoSided {
+template <class V>
+class TwoSided {
  public:
    TwoSided() : buy(), sell() {}
 
@@ -104,7 +175,8 @@ inline bool is_worse(api::Side s, Price p1, Price p2) {
    return is_better(s, p2, p1);
 }
 
-template <class C> void apply_to_both_sides(C c) {
+template <class C>
+void apply_to_both_sides(C c) {
    c(api::SIDE_BUY);
    c(api::SIDE_SELL);
 }
@@ -178,8 +250,8 @@ inline Price encode_exchange_price(api::Security const &sec, Price in) {
       return in;
 }
 
-inline double
-raw_round_to_quote_price(double raw_price, double tick_size, api::Side side) {
+inline double raw_round_to_quote_price(
+    double raw_price, double tick_size, api::Side side) {
    double factor = 1000000000;
    auto i_price = llround(factor * raw_price);
    auto i_tick_size = llround(tick_size * factor);
@@ -222,8 +294,8 @@ inline double checked_round_to_quote_price(
    }
 }
 
-inline Price
-round_to_quote_price(api::Security const &sec, Price in, api::Side side) {
+inline Price round_to_quote_price(
+    api::Security const &sec, Price in, api::Side side) {
    auto p = in.value();
    bts_assert_d(!std::isnan(p));
    if (p < -sec.contract_spec().price_break() ||
@@ -287,8 +359,8 @@ inline const std::string &descriptor(
 // Looking to round md::IntPrice? Look in IntPrice.h
 
 // computes price_2 - price_1 in ticks
-inline int
-price_diff_to_ticks(double price_1, double price_2, double tick_size) {
+inline int price_diff_to_ticks(
+    double price_1, double price_2, double tick_size) {
    return static_cast<int>(std::round((price_2 - price_1) / tick_size));
 }
 // returns the difference (in ticks) bw and order and a market.
@@ -312,8 +384,8 @@ inline int order_diff_market(
 // simple double equality can not be used to compare prices.
 // this function deems 2 prices to be equal if they are within
 // 1 % of the tick size
-inline bool
-test_price_equality(double price_1, double price_2, double tick_size) {
+inline bool test_price_equality(
+    double price_1, double price_2, double tick_size) {
    return std::fabs(price_2 - price_1) < 0.01 * tick_size;
 }
 inline bool gr_than_eq(double p1, double p2, double tick_size) {
@@ -362,7 +434,4 @@ inline bool is_spread_of_strips(const bts::api::Security &sec) {
    return false;
 }
 
-} // namespace securities
-} // namespace bts
-
-FMT_IMPL(bts::api::Security, s, "{}", bts::securities::descriptor(s));
+} // namespace bts::securities

src/lib/domain_model/Security.proto

@@ -205,6 +205,7 @@ message SyntheticContract {
 	enum PriceCalc {
 		PRICECALC_STANDARD = 0;
 		PRICECALC_NET_CHANGE = 1;
+        PRICECALC_NOTIONAL_LEG = 2;
 		// RATIO = 2;
 	}
 }

src/lib/domain_model/common.h

@@ -33,8 +33,8 @@ struct CQuantity {
    uint32_t value() const { return static_cast<uint32_t>(qty); }
    double full_value() const { return qty; }
    float full_value_float() const { return static_cast<float>(full_value()); }
-   uint32_t int_part() { return static_cast<uint32_t>(qty); }
-   double fractional_part() { return full_value() - int_part(); }
+   uint32_t int_part() const { return static_cast<uint32_t>(qty); }
+   double fractional_part() const { return full_value() - int_part(); }
    static bool equal_d(double d1, double d2) {
       return std::abs(d1 - d2) <= err;
    }

src/lib/io/JsonMacros.h

@@ -3,18 +3,14 @@
 #define _TO_JSON_FIELD(field) j[#field] = t.field;
 #define _FROM_JSON_FIELD(field) bts::io::JsonReader{&j, #field}.read(t.field);
 
-#define SPARK_TO_JSON_IMPL(type, for_each)                                     \
-   inline void to_json(nlohmann::json &j, const type &t) {                     \
-      FOR_EACH(_TO_JSON_FIELD)                                                 \
-   }
+#define SPARK_TO_JSON_IMPL(type, fields)                                       \
+   void to_json(nlohmann::json &j, const type &t) { fields(_TO_JSON_FIELD) }
 
-#define SPARK_FROM_JSON_IMPL(type, for_each)                                   \
-   inline void from_json(const nlohmann::json &j, type &t) {                   \
-      FOR_EACH(_FROM_JSON_FIELD)                                               \
-   }
+#define SPARK_FROM_JSON_IMPL(type, fields)                                     \
+   void from_json(const nlohmann::json &j, type &t) { fields(_FROM_JSON_FIELD) }
 
 #define OSTREAM_JSON_IMPL(type)                                                \
-   inline std::ostream &operator<<(std::ostream &out, const type &t) {         \
+   std::ostream &operator<<(std::ostream &out, const type &t) {                \
       nlohmann::json j;                                                        \
       to_json(j, t);                                                           \
       out << j.dump(2);                                                        \

src/lib/io/JsonReader.h

@@ -94,7 +94,18 @@ class JsonReader {
     * @return True if the read succeeds, otherwise false
     */
    template <class T>
-   bool read_or(const std::string &name, T &result, T default_value) const;
+   bool read_or(const std::string &name, T &result, T default_value = {}) const;
+
+   template <class T>
+   bool read_optional_or(
+       std::optional<T> &result,
+       std::optional<T> default_value = std::nullopt) const;
+
+   template <class T>
+   bool read_optional_or(
+       const std::string &name,
+       std::optional<T> &result,
+       std::optional<T> default_value = std::nullopt) const;
 
  private:
    const nlohmann::json *json_;
@@ -269,6 +280,33 @@ inline bool JsonReader::read_or(
    return false;
 }
 
+template <class T>
+inline bool JsonReader::read_optional_or(
+    std::optional<T> &result, std::optional<T> default_value) const {
+   if (read_optional(result)) {
+      return true;
+   }
+   result = std::move(default_value);
+   return false;
+}
+
+template <class T>
+inline bool JsonReader::read_optional_or(
+    const std::string &name,
+    std::optional<T> &result,
+    std::optional<T> default_value) const {
+   if (json_ && json_->is_object()) {
+      auto it = json_->find(name);
+      if (it != json_->end()) {
+         if (JsonReader{&it.value()}.read_optional(result)) {
+            return true;
+         }
+      }
+   }
+   result = std::move(default_value);
+   return false;
+}
+
 template <class T>
 bool JsonReader::read_flat(
     const std::vector<nlohmann::json> &items, std::vector<T> &result) {

src/lib/io/http/curl_session.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "lib/logging/logging.h"
+
 #include <atomic>
 #include <curl/curl.h>
 #include <curl/easy.h>
@@ -12,6 +14,7 @@ struct CurlSession {
 
    CurlSession() {
       if (++ref_count == 1) {
+         LOG(Http, Info, "Initializing curl (global)");
          curl_global_init(CURL_GLOBAL_ALL);
       }
       curl = curl_easy_init();
@@ -20,6 +23,7 @@ struct CurlSession {
    ~CurlSession() {
       curl_easy_cleanup(curl);
       if (--ref_count == 0) {
+         LOG(Http, Info, "Cleaning up curl (global)");
          curl_global_cleanup();
       }
    }

src/lib/io/http/http_client.cpp

@@ -34,7 +34,31 @@ static size_t write_headers(
     void *ptr, size_t size, size_t nmemb, IHttpResponse *data) {
    return data->write_headers((char *)ptr, size, nmemb);
 }
-
+// static int curl_debug_callback(CURL *handle, curl_infotype type, char *data, size_t size, void *userptr) {
+//    (void)handle;
+//    (void)type;
+//    (void)userptr;
+//     switch (type) {
+//         case CURLINFO_TEXT:
+//             LOG(Http, Info, "CURLINFO_TEXT: {}", std::string(data, size));
+//             break;
+//         case CURLINFO_HEADER_IN:
+//             LOG(Http, Info, "Header In: {}", std::string(data, size));
+//             break;
+//         case CURLINFO_HEADER_OUT:
+//             LOG(Http, Info, "Header Out: {}", std::string(data, size));
+//             break;
+//         case CURLINFO_DATA_IN:
+//             LOG(Http, Info, "Data In: {} bytes, in: {}", size,std::string(data, size));
+//             break;
+//         case CURLINFO_DATA_OUT:
+//             LOG(Http, Info, "Data Out: {} bytes, out: {}", size, std::string(data, size));
+//             break;
+//         default:
+//             break;
+//     }
+//     return 0;
+// }
 void HttpClient::get(
     string url,
     unordered_map<string, string> headers,
@@ -91,7 +115,12 @@ void HttpClient::send(const HttpRequest &req, IHttpResponse *response) {
    CURL *curl = session.curl;
    CurlHeaders curl_headers{req.headers};
    // Prepare req
-   curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
+   //to debug curl requests
+   // curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, curl_debug_callback);
+   // Disable signals to avoid strange stack corruption related to set/longjmp
+   // https://stackoverflow.com/questions/9191668/error-longjmp-causes-uninitialized-stack-frame
+   curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
+   curl_easy_setopt(curl, CURLOPT_FAILONERROR, 0L);
    curl_easy_setopt(curl, CURLOPT_URL, req.url.c_str());
    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers.data());
    if (response != nullptr) {
@@ -108,6 +137,7 @@ void HttpClient::send(const HttpRequest &req, IHttpResponse *response) {
       curl_easy_setopt(curl, CURLOPT_HTTPPOST, 1L);
       curl_easy_setopt(curl, CURLOPT_POSTFIELDS, req.body.c_str());
       curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, req.body.size());
+      // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
       break;
    case io::http::HttpMethod::PUT:
       curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");

src/lib/io/json_formatter.h

@@ -0,0 +1,54 @@
+#pragma once
+
+#include <fmt/format.h>
+#include <nlohmann/json.hpp>
+#include <string>
+
+namespace bts::io {
+
+/**
+ * This wrapper struct provides a means to format arbitrary objects as json
+ * in templated strings (e.g., LOG macros).
+ */
+template <class T>
+struct JsonFormatter {
+   T value;
+
+   std::string to_string() const { return fmt::format("{}", *this); }
+};
+
+template <class T>
+JsonFormatter<T> json_formatter(T value) {
+   return JsonFormatter<T>{std::move(value)};
+}
+
+} // namespace bts::io
+
+template <class T>
+struct fmt::formatter<bts::io::JsonFormatter<T>> {
+   int indent = -1;
+
+   constexpr auto parse(fmt::format_parse_context &ctx) -> decltype(ctx.end()) {
+      const auto end = ctx.end();
+      auto it = ctx.begin();
+      if (it != end) {
+         indent = 0;
+         do {
+            int x = *it - '0';
+            if (x < 0 || x > 9) {
+               return it;
+            }
+            indent = indent * 10 + x;
+         } while (++it != end);
+      }
+      return it;
+   }
+
+   template <typename FormatContext>
+   auto format(const bts::io::JsonFormatter<T> &jf, FormatContext &ctx) const
+       -> decltype(ctx.out()) {
+      nlohmann::json j;
+      to_json(j, jf.value);
+      return fmt::format_to(ctx.out(), "{}", j.dump(indent));
+   }
+};

src/lib/io/websocket/websocket_client.cpp

@@ -3,14 +3,19 @@
 #include "lib/core/TimeSpan.h"
 #include "lib/core/TimeStamp.h"
 #include "lib/io/websocket/websocket_client_config.h"
+#include "lib/io/websocket/websocket_client_logger.h"
 #include "lib/io/websocket/websocket_client_state.h"
 #include "lib/io/websocket/websocket_close_status.h"
 #include "lib/io/websocket/websocket_error_code.h"
 #include "lib/io/websocket/websocketpp_exception_formatter.h"
 #include "lib/logging/logging.h"
+#include "lib/utils/bts_assert.h"
 #include "lib/utils/formatters/exception_formatter.h"
 
+#include <algorithm>
 #include <atomic>
+#include <boost/asio/io_context.hpp>
+#include <chrono>
 #include <exception>
 #include <fmt/format.h>
 #include <mutex>
@@ -18,6 +23,7 @@
 #include <optional>
 #include <stdexcept>
 #include <string>
+#include <system_error>
 #include <thread>
 #include <websocketpp/close.hpp>
 #include <websocketpp/common/connection_hdl.hpp>
@@ -28,11 +34,51 @@
 #include <websocketpp/config/core_client.hpp>
 #include <websocketpp/roles/client_endpoint.hpp>
 
+namespace websocketpp::config {
+
+/// Client config with asio transport and TLS enabled
+struct bts_asio_tls_client : public core_client {
+   typedef bts_asio_tls_client type;
+   typedef core_client base;
+
+   typedef base::concurrency_type concurrency_type;
+
+   typedef base::request_type request_type;
+   typedef base::response_type response_type;
+
+   typedef base::message_type message_type;
+   typedef base::con_msg_manager_type con_msg_manager_type;
+   typedef base::endpoint_msg_manager_type endpoint_msg_manager_type;
+
+   typedef websocketpp::log::WebsocketClientLogger<
+       websocketpp::log::ElevelAdapter>
+       elog_type;
+   typedef websocketpp::log::WebsocketClientLogger<
+       websocketpp::log::AlevelAdapter>
+       alog_type;
+
+   typedef base::rng_type rng_type;
+
+   struct transport_config : public base::transport_config {
+      typedef type::concurrency_type concurrency_type;
+      typedef type::alog_type alog_type;
+      typedef type::elog_type elog_type;
+      typedef type::request_type request_type;
+      typedef type::response_type response_type;
+      typedef transport::asio::tls_socket::endpoint socket_type;
+   };
+
+   typedef transport::asio::endpoint<transport_config> transport_type;
+};
+
+} // namespace websocketpp::config
+
 namespace bts::websocket {
 
 struct WebsocketClientSnapshot {
    int client_id;
    WebsocketClientState state;
+   int n_reconnects;
    int version;
 };
 
@@ -41,9 +87,10 @@ struct WebsocketClientSnapshot {
 FMT_IMPL(
     bts::websocket::WebsocketClientSnapshot,
     t,
-    "id: {}; state: {}; version: {}",
+    "id: {}; state: {}; n_reconnects: {}; version: {}",
     t.client_id,
     t.state,
+    t.n_reconnects,
     t.version);
 
 namespace bts::websocket {
@@ -61,10 +108,11 @@ using std::size_t;
 using std::string;
 using std::thread;
 using std::uint32_t;
-using websocketpp::config::asio_tls_client;
+using std::chrono::nanoseconds;
+using std::this_thread::sleep_for;
 
 using WebsocketppClient =
-    websocketpp::client<websocketpp::config::asio_tls_client>;
+    websocketpp::client<websocketpp::config::bts_asio_tls_client>;
 using ConnectionHdl = websocketpp::connection_hdl;
 using ConnectionPtr = WebsocketppClient::connection_ptr;
 using MessageTypePtr = websocketpp::config::core_client::message_type::ptr;
@@ -85,9 +133,16 @@ using State = WebsocketClientState;
 #define ERROR(...) LOG(Http, Error, __VA_ARGS__)
 #define ERROR_FN(...) LOG(Http, Error, "WebsocketClient::" __VA_ARGS__)
 
+static bts::core::TimeSpan reconnect_interval(std::uint32_t n_attempts) {
+   std::int64_t secs = 5 * n_attempts;
+   secs = std::clamp(secs, 5L, 300L);
+   return bts::core::TimeSpan::from_seconds(secs);
+}
+
 struct WebsocketClient::PImpl {
    WebsocketClientConfig config;
    WebsocketppClient client{};
+   boost::asio::io_context *io_context = nullptr;
    WebsocketppClient::connection_ptr con;
    optional<OpenHandler> open_handler;
    optional<MessageHandler> message_handler;
@@ -97,8 +152,48 @@ struct WebsocketClient::PImpl {
    optional<thread> io_thread;
    int client_id{client_id_counter++};
    mutex instance_mx;
-   State state;
+   State state = State::IDLE;
    int version = 0;
+   int n_reconnects = 0;
+
+   PImpl(WebsocketClientConfig config, boost::asio::io_context *io_context)
+       : config(std::move(config)), io_context(io_context) {
+      client.get_alog().set_name(this->config.name);
+      client.get_elog().set_name(this->config.name);
+      client.set_access_channels(websocketpp::log::alevel::all);
+      // client.clear_access_channels(websocketpp::log::alevel::frame_payload);
+      client.set_error_channels(websocketpp::log::elevel::all);
+      if (io_context == nullptr) {
+         client.init_asio();
+      } else {
+         client.init_asio(io_context);
+      }
+      client.set_message_handler(
+          [this](auto hdl, auto msg) { on_message(hdl, msg); });
+      client.set_open_handler([this](auto hdl) { on_open(hdl); });
+      client.set_ping_handler([](auto /*hdl*/, auto /*msg*/) {
+         LOG(Http, Info, "Received ping");
+         return true;
+      });
+      client.set_pong_timeout(0);
+      client.set_pong_handler([](auto /*hdl*/, auto /*msg*/) {
+         LOG(Http, Info, "Received pong");
+         return true;
+      });
+      client.set_tls_init_handler([&](auto hdl) { return on_tls_init(hdl); });
+      client.set_close_handler([=, this](auto hdl) {
+         on_close(hdl);
+         if (config.reconnect_on_close) {
+            sleep_for(nanoseconds{reconnect_interval(n_reconnects).nanos()});
+            connect();
+         }
+      });
+      client.set_fail_handler([this](auto hdl) {
+         const auto ec = client.get_con_from_hdl(hdl)->get_ec();
+         ERROR_FN("fail_handler. {}; err: {}", snapshot(), ec.message());
+         on_fail(hdl);
+      });
+   }
 
    ~PImpl() {
       if (io_thread) {
@@ -119,7 +214,7 @@ struct WebsocketClient::PImpl {
    }
 
    constexpr WebsocketClientSnapshot snapshot() const {
-      return {client_id, state, version};
+      return {client_id, state, n_reconnects, version};
    }
 
    void close(websocketpp::close::status::value code, std::string reason) {
@@ -163,8 +258,7 @@ struct WebsocketClient::PImpl {
    /**
     * Connects to the websocket server
     */
-   void connect(
-       const map<string, string> &headers, bool run_in_background = true) {
+   void connect() {
       lock_guard<mutex> lock{instance_mx};
       INFO_FN("connect. {}", snapshot());
       switch (state.value) {
@@ -176,9 +270,6 @@ struct WebsocketClient::PImpl {
       }
       set_state(State::OPENING);
       try {
-         init_logging();
-         init_handlers();
-         // Get connection
          websocketpp::lib::error_code ec;
          con = client.get_connection(config.url, ec);
          if (ec) {
@@ -186,46 +277,27 @@ struct WebsocketClient::PImpl {
             return;
          }
          // Add headers
-         for (const auto &[k, v] : headers) {
+         for (const auto &[k, v] : config.headers) {
             con->append_header(k, v);
          }
          // Connect
          client.connect(con);
-         if (run_in_background) {
-            bts_assert_msgf(
-                !io_thread, "Websocket io_thread already instantiated. {}",
-                snapshot());
-            io_thread = std::thread{[this]() { run(); }};
-            if (config.connect_timeout > core::TimeSpan::zero()) {
-               wait_for_connect(config.connect_timeout);
-            }
-         }
       } catch (const std::exception &ex) {
+         std::string str = ex.what();
          ERROR_FN(
              "connect_no_timeout. Failed to connect. {}; err: {}", snapshot(),
-             ex);
+             str);
          set_state(State::IDLE);
          // Don't wait for connect
          return;
       }
    }
 
-   void wait_for_connect(const TimeSpan timeout) {
-      INFO_FN("wait_for_connect. {}; timeout: {}", snapshot(), timeout);
-      const auto start = TimeStamp::now();
-      while (state == State::OPENING) {
-         const auto elapsed = TimeStamp::now() - start;
-         const auto rem = timeout - elapsed;
-         if (rem < TimeSpan::zero()) {
-            throw std::runtime_error{fmt::format(
-                "Timed out waiting for connection after {} secs. {}",
-                elapsed.total_seconds(), snapshot())};
-         }
-         WARN_FN(
-             "wait_for_connect. Still waiting. {}; timeout: {}; rem: {}",
-             snapshot(), timeout, rem);
-         std::this_thread::sleep_for(std::chrono::seconds(1));
-      }
+   void reconnect() {
+      WARN_FN("reconnect. {}", snapshot());
+      sleep_for(nanoseconds{reconnect_interval(n_reconnects).nanos()});
+      ++n_reconnects;
+      connect();
    }
 
    void run() {
@@ -236,11 +308,16 @@ struct WebsocketClient::PImpl {
          ERROR_FN(
              "io_thread. Websocket io threw websocketpp::exception. {}; "
              "err: {}",
-             snapshot(), ex);
+             snapshot(), std::string(ex.what()));
+             bts_assert_msgf(false, "io_thread. Websocket io threw websocketpp::exception. {}; "
+             "err: {}",
+               snapshot(), std::string(ex.what()));
       } catch (const std::exception &ex) {
          ERROR_FN(
              "io_thread. Websocket io threw std::exception. {}; err: {}",
-             snapshot(), ex);
+             snapshot(), std::string(ex.what()));
+         bts_assert_msgf(false, "io_thread. Websocket io threw std::exception. {}; err: {}",
+             snapshot(), std::string(ex.what()));
       } catch (...) {
          bts_assert_msgf(
              false,
@@ -251,10 +328,33 @@ struct WebsocketClient::PImpl {
       INFO("Websocket 'run' thread end. {}", snapshot());
    }
 
+   void run_one() {
+      try {
+         client.run_one();
+      } catch (const websocketpp::exception &ex) {
+         ERROR_FN(
+             "io_thread (run_one). Websocket io threw websocketpp::exception. "
+             "{}; err: {}",
+             snapshot(), ex);
+      } catch (const std::exception &ex) {
+         ERROR_FN(
+             "io_thread (run_one). Websocket io threw std::exception. {}; err: "
+             "{}",
+             snapshot(), ex);
+      } catch (...) {
+         bts_assert_msgf(
+             false,
+             "io_thread (run_one). Websocket io threw unknown exception. Could "
+             "this have been thrown by a callback delegate? {}",
+             snapshot());
+      }
+   }
+
  private:
    static std::atomic<int> client_id_counter;
 
-   auto on_tls_init(ConnectionHdl /*hdl*/) {
+   std::shared_ptr<boost::asio::ssl::context> on_tls_init(
+       ConnectionHdl /*hdl*/) {
       INFO_FN("on_tls_init. state: {}; version: {}", state, version);
       auto result =
           websocketpp::lib::make_shared<SslContext>(SslContext::sslv23_client);
@@ -262,44 +362,13 @@ struct WebsocketClient::PImpl {
       return result;
    }
 
-   void init_handlers() {
-      INFO_FN("init_handlers. state: {}; version: {}", state, version);
-      bts_assert(!io_thread);
-      client.init_asio();
-      client.set_message_handler(
-          [this](auto hdl, auto msg) { on_message(hdl, msg); });
-      client.set_open_handler([this](auto hdl) { on_open(hdl); });
-      client.set_ping_handler([](auto /*hdl*/, auto /*msg*/) {
-         LOG(Http, Info, "Received ping");
-         return true;
-      });
-      client.set_pong_timeout(0);
-      client.set_pong_handler([](auto /*hdl*/, auto /*msg*/) {
-         LOG(Http, Info, "Received pong");
-         return true;
-      });
-      client.set_tls_init_handler(
-          [this](auto hdl) { return on_tls_init(hdl); });
-      client.set_close_handler([this](auto hdl) { on_close(hdl); });
-      client.set_fail_handler([this](auto hdl) {
-         const auto ec = client.get_con_from_hdl(hdl)->get_ec();
-         ERROR_FN("fail_handler. {}; err: {}", snapshot(), ec.message());
-      });
-   }
-
-   void init_logging() {
-      INFO_FN("init_logging. {}", snapshot());
-      client.set_access_channels(websocketpp::log::alevel::all);
-      // client.clear_access_channels(websocketpp::log::alevel::frame_payload);
-      client.set_error_channels(websocketpp::log::elevel::all);
-   }
-
    void on_open(ConnectionHdl hdl) {
       lock_guard<mutex> lock{instance_mx};
       INFO_FN("on_open. {}", snapshot());
       const auto ec = client.get_con_from_hdl(hdl)->get_ec();
       if (ec) {
          ERROR_FN("on_open. {}; err: {}", snapshot(), ec.message());
+         bts_assert(false);
       }
       switch (state.value) {
       case State::OPENING:
@@ -350,11 +419,34 @@ struct WebsocketClient::PImpl {
       if (close_handler) {
          (*close_handler)(ec);
       }
+      if (config.reconnect_on_close) {
+         reconnect();
+      }
+   }
+
+   void on_fail(ConnectionHdl hdl) {
+      lock_guard<mutex> lock{instance_mx};
+      ERROR_FN("on_fail. {}", snapshot());
+      set_state(State::IDLE);
+      const auto con = client.get_con_from_hdl(hdl);
+      const auto ec = con->get_ec();
+      if (ec) {
+         ERROR_FN(
+             "on_fail. {}; local_code: {}; "
+             "local_reason: {}; remote_code: {}; remote_reason: {}; err: {}",
+             snapshot(), con->get_local_close_code(),
+             con->get_local_close_reason(), con->get_remote_close_code(),
+             con->get_remote_close_reason(), ec.message());
+      }
+      if (config.reconnect_on_fail) {
+         reconnect();
+      }
    }
 };
 
-WebsocketClient::WebsocketClient(WebsocketClientConfig config)
-    : me_(std::make_unique<PImpl>(std::move(config))) {}
+WebsocketClient::WebsocketClient(
+    WebsocketClientConfig config, boost::asio::io_context *io_context)
+    : me_(std::make_unique<PImpl>(std::move(config), io_context)) {}
 
 WebsocketClient::~WebsocketClient() {
    me_->close(websocketpp::close::status::going_away, "");
@@ -383,13 +475,16 @@ WebsocketClient::CloseHandler &WebsocketClient::close_handler() {
    return *me_->close_handler;
 }
 
-void WebsocketClient::connect(
-    const std::map<std::string, std::string> &headers, bool run_in_background) {
-   me_->connect(headers, run_in_background);
-}
+void WebsocketClient::connect() { me_->connect(); }
 
 void WebsocketClient::run() { me_->run(); }
 
+void WebsocketClient::run_one() { me_->run_one(); }
+
+boost::asio::io_context *WebsocketClient::io_context() {
+   return me_->io_context;
+}
+
 websocketpp::lib::error_code WebsocketClient::send_text(const string &text) {
    return me_->send_text(text);
 }
@@ -407,4 +502,6 @@ bool WebsocketClient::is_connected() const {
 
 void WebsocketClient::ping(const string &payload) { me_->con->ping(payload); }
 
+const string &WebsocketClient::url() const { return me_->config.url; }
+
 } // namespace bts::websocket