Merging

.devcontainer/Dockerfile

@@ -90,12 +90,6 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev
 
-# Docker in Docker
-RUN apt-get update && apt-get install -y \
-    docker.io \
-    docker-compose \
-    docker-compose-v2
-
 # Set up default executables (update-alternatives)
 RUN \
    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \

.devcontainer/devcontainer.json

@@ -12,10 +12,17 @@
   // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
   "runArgs": [
     "--shm-size",
-    "8gb"
-  ],
-  "mounts": [
-    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
+    "8gb",
+    "-p",
+    "8000:8000",
+    "-p",
+    "8001:8001",
+    "-p",
+    "8002:8002",
+    "-p",
+    "8003:8003",
+    "-p",
+    "12345:12345"
   ],
   "customizations": {
     "vscode": {

.env

@@ -1,47 +0,0 @@
-# This file defines the global default configuration for docker deployments.
-# It is tracked in source control. For temporary modifications, please create
-# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
-# If a variable is defined in both files, the value in .env.local will be used.
-#
-# About the deployment procedure:
-#   We use Docker Compose to build images and launch containers for various services
-#   that make up our software suite:
-#     - sparkd
-#     - TODO: pubsub
-#     - TODO: md-publisher
-#     - TODO: webapp (DSQA on the web)
-#     - TODO: api-server (websocket based)
-#   From inside a devcontainer, you can launch all services in separate containers
-#   on your machine by running:
-#
-#     tools/deploy.sh
-#
-#   This script builds release versions for each service, packaged (generally) in
-#   self-contained directories. Next, it runs a few "docker compose" commands to
-#   build docker images loaded with the self-contained binaries from the previous
-#   step, and start containers with those images to run each service. To view the
-#   commands that deploy.sh runs internally (without actually running them), run:
-#
-#     tools/deploy.sh --dry-run
-#
-#   For additional information/help, run:
-#
-#     tools/deploy.sh --help
-#
-
-### Sparkd ###
-# The internal image for sparkd to use. For more info, view:
-#   src/server/sparkd/Dockerfile
-SPARKD__IMAGE=sparkd
-# The config dir for this service to use
-SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
-# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
-# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
-# SPARKD__DATA_DIR=/home/bts/sparkdata/data
-SPARKD__SOURCE_DIR=install/sparkd-release-gcc
-SPARKD__CONFIG_PORT=8000
-SPARKD__QUICKBUS_PORT=8001
-SPARKD__SIM_FIX_PORT=8002
-SPARKD__SHM_SIZE=4gb
-
-# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitignore

@@ -59,4 +59,3 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
-.env.local

.vscode/launch.json

@@ -2,10 +2,9 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "Attach Spark",
+      "name": ".NET Core Attach",
       "type": "coreclr",
-      "request": "attach",
-      "processId": "${command:pickProcess}"
+      "request": "attach"
     },
     {
       "name": "DataStreamPublisher",
@@ -327,4 +326,4 @@
       "description": "Test to run",
     },
   ]
-}
+}

compose.yaml

@@ -1,63 +0,0 @@
-services:
-  sparkd:
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
-    build:
-      dockerfile: src/server/sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
-        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
-          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
-          set -a
-          . /home/bts/sparkdata/conf/.env.local
-          set +a
-        fi
-        ./sparkd run"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    ports:
-      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
-      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
-      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
-    deploy:
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
-
-  sparkd-tail:
-    image: bluetradingsystems/linux_dotnet7
-    volumes:
-      - type: volume
-        read_only: true
-        source: sparkd_data
-        target: /home/bts/sparkdata/data/logging.sparkd
-        volume:
-          subpath: logging.sparkd
-    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
-
-  sparkd-ping-test:
-    depends_on:
-      - sparkd
-    profiles:
-      - ping-test
-    build:
-      context: .
-      dockerfile_inline: |
-        FROM bluetradingsystems/linux_dotnet7
-        USER "root"
-        RUN apt-get update && apt-get -y install iputils-ping
-    command: [ "ping", "sparkd" ]
-
-volumes:
-  sparkd_data:
-    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"

docker/images/bluetradingsystems/linux_dotnet7/Dockerfile

@@ -1,38 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y pkg-config
-RUN apt-get update && apt-get install -y locales
-RUN apt-get update && apt-get install -y ca-certificates
-RUN apt-get update && apt-get install -y libssh2-1-dev
-RUN apt-get update && apt-get install -y software-properties-common
-RUN apt-get update && apt-get install -y sudo
-
-# GCC
-ARG gcc_major=13
-RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
-  && apt-get update \
-  && apt-get install -y g++-$gcc_major
-
-RUN apt-get update \
-  && apt-get install -y dotnet7
-
-RUN locale-gen en_US.UTF-8
-
-ARG docker_user="bts"
-ARG docker_uid="1000"
-ARG docker_pass="bts"
-
-RUN \
-  groupadd -g "$docker_uid" "$docker_user" \
-  && useradd \
-  -m \
-  -u "$docker_uid" \
-  -g "$docker_uid" \
-  -G sudo \
-  --password "$(openssl passwd -1 "$docker_pass")" \
-  "$docker_user"
-
-USER bts
-WORKDIR /home/bts

global.json

@@ -0,0 +1,6 @@
+{
+  "sdk": {
+    "version": "6.0.200",
+    "rollForward": "latestMajor"
+  }
+}

requirements.txt

@@ -1,13 +1,10 @@
 astroid==2.7.3
 autopep8==1.5.7
-boto3==1.37.9
-botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
 colorama==0.4.4
 idna==3.4
 isort==5.9.3
-jmespath==1.0.1
 lazy-object-proxy==1.6.0
 mccabe==0.6.1
 platformdirs==2.3.0
@@ -17,11 +14,7 @@ pycodestyle==2.7.0
 pycparser==2.20
 pylint==2.10.2
 PyMeta3==0.5.1
-python-dotenv==1.0.1
-python-dateutil==2.9.0.post0
 requests==2.31.0
-s3transfer==0.11.4
-six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3

src/client/spark/AW2/Bus/AnalyticsParametersManager.cs

@@ -45,8 +45,7 @@ namespace mbts.spark {
             Func<AnalyticsExpirationParameters, bool> is_unexpired = (p) => {
                var st = p.driver_security.security_tuple.to_api_sec_tuple();
                var exp = p.security.expiration;
-               return exp.Date >= DateTime.Now.Date;
-               //return SecurityExpirationHelper.is_unexpired(st, exp);
+               return SecurityExpirationHelper.is_unexpired(st, exp);
             };
             exp_params = exp_params.Where(t => is_unexpired(t));
             // listens to the commit tag and saves it

src/client/spark/AW2/Bus/SingleVolSurfaceInterpGreeksPublisher.cs

@@ -12,8 +12,8 @@ using bts.api;
 using mbts.ui.converters;
 
 namespace mbts.spark {
-   // This class gathers all the inputs needed to calculate InterpGreeks for a single vol surface,
-   // passes it to VolSurfacePrecomputedTheosProducer, and publishes the InterpGreeks to the bus
+   // This class gathers all the inputs needed to calculate InterpGreeks, pass it to 
+   // VolSurfacePrecomputedTheosProducer, and publishes the InterpGreeks to the bus
    // (called SingleVolSurfaceClientPrecomputedTheosPublisher in Edge)
    // It listens to
    // a. vol surface inputs

src/client/spark/AW2/Bus/VolSurfaceBlackboardManager.cs

@@ -20,7 +20,6 @@ using Spark.Core.Protobuf;
 using mbts.ui;
 using bts.config.toggles;
 using System.Diagnostics.CodeAnalysis;
-using BCG;
 // using mbts.bb;
 
 namespace mbts.spark
@@ -93,11 +92,11 @@ namespace mbts.spark
       // in tags
       private readonly ITag<SystemContractData> _in_scdata_tag =
          SparkTags.SystemContractsTag;
-      private readonly ITag<NMSheetSymbol> _archive_ivs_tag = SparkTags.TC2Tags.ArchiveIVsTag;
       private readonly ITag<VolSurfaceSettings> _in_bb_create_tag = SparkTags.VolSurfaceCreateTag;
       private readonly ITag<int> _in_bb_delete_tag = SparkTags.VolSurfaceDeleteTag;
       private readonly ITag<AnalyticsExpirationParametersMap> _in_bb_exp_params_tag =
          SparkTags.ExpirationParametersInputTag;
+      private readonly ITag<ClientArchivedVolRequest> _in_bb_req_vol = SparkTags.RequestArchivedVols;
       // state
       private SystemContractData _contract_data = null;
       private IDictionary<int, SingleVolSurfaceManager> _vs_manager_map =
@@ -107,43 +106,6 @@ namespace mbts.spark
       private bts.api.IBus _bus;
       private List<IDisposable> _subs = new List<IDisposable>();
 
-      private static void add_temp_vsu_entries(IDictionary<string, int> name_id_map,
-                                                 VolSurfaceUsageList ulist) {
-         Action<string, Venue, string> add_entry = (option_name, option_venue, vs_name) => {
-
-            int vs_key;
-            if (name_id_map.TryGetValue(vs_name, out vs_key)) {
-               ulist.UsageList.Add(new VolSurfaceUsageEntry() {
-                  VsId = vs_key,
-                  SymbolName = option_name,
-                  SecTup = new bts.api.SecurityTuple {
-                     Type = SecurityType.Option,
-                     Venue = option_venue,
-                     Symbol = option_name
-                  }
-               });
-            }
-
-         };
-         add_entry("SPX", Venue.Cboe, "CBOE_skews");
-         add_entry("SPXW", Venue.Cboe, "CBOE_skews");
-         add_entry("E1B", Venue.Glbx, "CME_Skews");
-         add_entry("E1C", Venue.Glbx, "CME_Skews");
-         add_entry("E3B", Venue.Glbx, "CME_Skews");
-         add_entry("E3C", Venue.Glbx, "CME_Skews");
-         add_entry("E3D", Venue.Glbx, "CME_Skews");
-         add_entry("E4A", Venue.Glbx, "CME_Skews");
-         add_entry("E4B", Venue.Glbx, "CME_Skews");
-         add_entry("E4C", Venue.Glbx, "CME_Skews");
-         add_entry("E4C", Venue.Glbx, "CME_Skews");
-         add_entry("E4D", Venue.Glbx, "CME_Skews");
-         add_entry("ES", Venue.Glbx, "CME_Skews");
-         add_entry("EW1", Venue.Glbx, "CME_Skews");
-         add_entry("EW2", Venue.Glbx, "CME_Skews");
-         add_entry("EW3", Venue.Glbx, "CME_Skews");
-         add_entry("EW4", Venue.Glbx, "CME_Skews");
-      }
-
       public VolSurfaceBusManager(bts.api.IBus bus,
                                          bts.api.analytics.AnalyticsService analytics_service,
                                          bts.api.orderman.OrderService order_service)
@@ -157,7 +119,7 @@ namespace mbts.spark
          _bb.subscribe(_in_bb_create_tag);
          _bb.subscribe(_in_bb_delete_tag);
          _bb.subscribe(_in_bb_exp_params_tag);
-         _bb.subscribe(_archive_ivs_tag);
+         _bb.subscribe(_in_bb_req_vol);
          _bb.data_callback += _on_bb_update;
          _bb.timer_callback += _on_timer;
          var contracts_throttle = TimeSpan.FromMilliseconds(200);
@@ -165,25 +127,11 @@ namespace mbts.spark
          var vs_sub = _analytics_service.subscribei_volsurfaces()
             .ObserveOn(SynchronizationContext.Current)
             .Subscribe(_on_vss);
-         //var hist_theos_sub_obs = _analytics_service.subscribei_archived_theos_table();
-         //var obs = hist_theos_sub_obs.ObserveOn(SynchronizationContext.Current);//.Subcribe(_on_hist_theos);
-         //obs.Subscribe(tu => {
-         //   Log.Gen.info("Received historical theos");
-         //   _on_hist_theos(tu);
-         //});
-
-         var archived_theos_obs = _analytics_service.subscribei_archived_theos().ObserveOn(SynchronizationContext.Current).Subscribe(tu => {
-            Log.Gen.info("Received historical theos");
-            _on_hist_theos(tu);
-         });
-         
-
          var vs_usage_sub = _analytics_service.subscribei_volsurface_usage()
             .ObserveOn(SynchronizationContext.Current)
             .Subscribe(_on_vs_usage_update);
          _subs.Add(vs_sub);
          _subs.Add(vs_usage_sub);
-         _subs.Add(archived_theos_obs);
          _subs.Add(new AnalyticsParametersManager(bus, _analytics_service));
          _subs.Add(new ResourceAllocator(bus, _bb, contracts_throttle));
          _subs.Add(new SymbolAggregationPublisher<OptionDriverPriceMap>(
@@ -195,7 +143,7 @@ namespace mbts.spark
          _subs.Add(new LiveTheoPublisher(_bb));
          _subs.Add(new InterpTheosServerPublisher(_bb, bus, order_service, contracts_throttle));
          _subs.Add(new VolSurfaceInterpGreeksMultiplexer(_bb));
-         // list of all volsurface. in this callback, we read the usage config, and detemine the id of the named vol surface         
+         // usage         
          var list_sub = _bus.subscribe(SparkTags.VolSurfaceListTag).OnDispatcher().Subscribe(t =>
          {
             VolSurfaceUsageList ulist = new VolSurfaceUsageList();
@@ -206,12 +154,9 @@ namespace mbts.spark
             }
             Log.Ui.info($"VolSurfaceUsageList name_to_ids count: {name_to_ids.Count}");
             var vs_usage = mbts.config.Configuration.get<bts.config.toggles.VolSurfaceUsage>();
-            Log.Ui.info($"VolSurfaceUsageList vs_usage acount: {vs_usage.Items.Count}");            
+            Log.Ui.info($"VolSurfaceUsageList vs_usage acount: {vs_usage.Items.Count}");
             foreach (var u in vs_usage.Items)
             {
-               if (string.Compare(u.OptionName, "SPX", 0) == 0 || string.Compare(u.OptionName, "SPXW", 0) == 0){
-                  continue;
-               }
                if (string.Compare(u.OptionName, "SPX", 0) == 0 || string.Compare(u.OptionName, "SPXW", 0) == 0 && u.OptionVenue == Venue.Unset)
                {
                   u.OptionVenue = Venue.Cboe;
@@ -241,11 +186,10 @@ namespace mbts.spark
                else
                {
                   Log.Gen.error($"Could not find vol surface with name: {vs_name}");
-               }               
+               }
             }
-            add_temp_vsu_entries(name_to_ids, ulist);
             var list_imm = new VolSurfaceUsageListImmutable(ulist);
-            Log.Ui.info("VolSurfaceUsage: " + list_imm.log_string());
+            Log.Ui.info(list_imm.log_string());
             _bb.post(SparkTags.VolSurfaceUsageTag, list_imm);
             _spin_vss(list_imm);
          });
@@ -348,8 +292,6 @@ namespace mbts.spark
 
       void _on_vs_usage_update(Update<VolSurfaceUsageListImmutable, int> update)
       {
-         var state = update.all;
-         Log.Gen.info($"VSBBMan._on_vs_usage_update size: {state.Count()}");
          // TODO
          return;
       }
@@ -368,17 +310,6 @@ namespace mbts.spark
          _init_vss(to_publish.Keys);
       }
 
-      void _on_hist_theos(Update<bts.api.analytics.ArchivedTheoListImmutable, ulong> update) {
-         //var total_state = update.state;
-         Log.Gen.info($"VSBBMan._on_hist_theos update: {update}");
-         foreach (var kv in update.all) {
-            bts.Log.Gen.info($"kv: {kv}");
-            _bb.post(SparkTags.ArchivedTheos, kv);
-            break;
-         }
-         //Log.assert(false);
-      }
-
       private void _init_vss(IEnumerable<int> vs_ids)
       {
          // we consider every vs in the db as "activated" which might prove to be expensive
@@ -431,56 +362,29 @@ namespace mbts.spark
             _bb.post(SparkTags.VolSurfaceNonVolInputTag,
                                           new mbts.analytics.VolSurfaceNonVolInput(exp_param_map.exp_map.ToDictionary()));
          }
-         // this is publishe dby the sheets 
-         else if (_archive_ivs_tag.Equals(tag)){
-            //Log.assert(false);
-            NMSheetSymbol sym = data as NMSheetSymbol;
-            SystemContractData sc_data = _bb.read_state(SparkTags.SystemContractsTag);            
-            var associated_syms = _get_associated_symbols(sym);
-            List<ArchivedSecurityTheo> theo_list = new List<ArchivedSecurityTheo>();
-            foreach (NMSheetSymbol s in associated_syms){
-               ExpStrikeKeyedData<TheoPoint> symbol_theos = _bb.read_state(SparkTags.TC2Tags.LiveTheosTag((s.option_name, s.option_exchange)));
-               foreach (var t in symbol_theos.get_data_linearly()){
-                  Venue v = s.option_exchange;
-                  string symbol = s.option_name;
-                  DateTime exp = t.first;
-                  double strike = t.second;
-                  TheoPoint tp = t.third;
-                  Tuple<uint?, uint?> cp_id = sc_data.get_option_sec_id(v, symbol, exp, strike);                  
-                  if (cp_id != null && cp_id.Item1.HasValue && cp_id.Item2.HasValue){
-                     // call
-                     ArchivedSecurityTheo c_theo = new ArchivedSecurityTheo();
-                     c_theo.Id = cp_id.Item1.Value;
-                     c_theo.Driver = tp.driver;
-                     c_theo.Vol = tp.vol;
-                     c_theo.Theo = tp.call;
-                     c_theo.Delta = tp.cdelta;
-                     theo_list.Add(c_theo);
-                     // put
-                     ArchivedSecurityTheo p_theo = new ArchivedSecurityTheo();
-                     p_theo.Id = cp_id.Item1.Value;
-                     p_theo.Driver = tp.driver;
-                     p_theo.Vol = tp.vol;
-                     p_theo.Theo = tp.put;
-                     p_theo.Delta = tp.pdelta;
-                     theo_list.Add(p_theo);
-                  }
-               }
+         else if (_in_bb_req_vol.Equals(tag))
+         {
+            ClientArchivedVolRequest client_req = _bb.read_state(_in_bb_req_vol);
+            bts.Log.Ui.info($"VolSurfaceBusManager._on_bb_update ClientArchivedVolRequest: {client_req.ToString()}");
+            ArchivedTheoRetrieval ret = client_req.ret;
+            if (ret == null || ret.SecTup == null)
+            {
+               bts.Log.Ui.warn($"VolSurfaceBusManager malformed ArchivedTheoRetrieval");
+               ArchivedTheoList empty = new ArchivedTheoList();
+               _bb.post(SparkTags.ResponseArchivedVols, Tuple.Create(client_req, empty));
+               return;
             }
-            ArchivedTheoList rv_list = new ArchivedTheoList();
-            // we dont care abouyt the save time any more, and dont want it to te the ky of the table anymore
-            rv_list.SaveTime = 0;
-            rv_list.SecTheo.AddRange(theo_list);
-            bts.Log.Gen.info($"VSBBMan archiving {rv_list.SecTheo.Count} entries");
-            _analytics_service.clean_and_archive_theos(rv_list);
-            // send it to the server
+            bts.Log.Ui.info($"VolSurfaceBusManager ArchivedTheoRetrieval id: {client_req.id} save_time: {ret.SaveTime} " +
+                           $"sec_tup: {ret.SecTup.Venue}_{ret.SecTup.Type}_{ret.SecTup.Symbol}");
+            var timer = Stopwatch.StartNew();
+            ArchivedTheoList arch_theos = _analytics_service.get_archived_theos(ret);
+            timer.Stop();
+            bts.Log.Ui.info($"VolSurfaceBusManager ArchivedTheoRetrieval time: {timer.Elapsed.TotalMilliseconds}ms " +
+                            $"n_entries: {arch_theos.SecTheo.Count}");
+            _bb.post(SparkTags.ResponseArchivedVols, Tuple.Create(client_req, arch_theos));
          }
       }
 
-      private List<NMSheetSymbol> _get_associated_symbols(NMSheetSymbol symbol){
-         return new List<NMSheetSymbol>() {new NMSheetSymbol(Venue.Cboe, "SPX"),new NMSheetSymbol(Venue.Cboe, "SPXW")};
-      }
-
       private int _timer_count = 0;
       private void _on_timer()
       {

src/client/spark/AW2/LiveTheoPublisher.cs

@@ -14,11 +14,6 @@ namespace mbts.spark {
    // Publishes theos for all options (on the side, also publishes the theos for TC2).
    // Listens to driver and interp theos, on a change, calls _interp_and_publish
    public class LiveTheoPublisher : IDisposable {
-      public enum Impetus { 
-         DRIVER,
-         SKEW,
-      }
-
       private ThreadedBlackboard _bb;
       private readonly ITag<IReadOnlyDictionary<(string, Venue), OptionTheoInterpolationData>>
          _greeks_tag = SparkTags.AllInterpGreeks;
@@ -35,14 +30,13 @@ namespace mbts.spark {
          _bb.subscribe(_drivers_tag);
          _bb.data_callback += _on_data;
          _bb.timer_callback += _on_timer;
-         _interp_and_publish(Impetus.SKEW);
+         _interp_and_publish();
       }
 
       private void _on_data(object tag, object data) {
          if (tag.Equals(_drivers_tag) ||
             tag.Equals(_greeks_tag)) {
-            Impetus imp = tag.Equals(_drivers_tag) ? Impetus.DRIVER : Impetus.SKEW;
-            _interp_and_publish(imp);
+            _interp_and_publish();
          }
       }
 
@@ -109,7 +103,7 @@ namespace mbts.spark {
          return TheoPoint.getNulledObject();
       }
 
-      private void _publish_tc2_theos(IDictionary<SecurityStrike, TheoPoint> theos, Impetus imp) {
+      private void _publish_tc2_theos(IDictionary<SecurityStrike, TheoPoint> theos) {
          var rv = new Dictionary<(string, Venue), ExpStrikeKeyedData<TheoPoint>>();
          foreach (var kv in theos) {
             SecurityStrike k = kv.Key;
@@ -124,9 +118,6 @@ namespace mbts.spark {
          _override_theos(ref rv);
          foreach (var kv in rv) {
             _bb.post(SparkTags.TC2Tags.LiveTheosTag(kv.Key), kv.Value);
-            if (imp == Impetus.SKEW) {
-               _bb.post(SparkTags.TC2Tags.VolSaveEventTheosTag(kv.Key), kv.Value);
-            }
          }
       }
 
@@ -141,23 +132,19 @@ namespace mbts.spark {
                   double strike = t.second;
                   TheoPoint tp = t.third;
                   var call_sec_md = sc_data.get_option(exchange, sym, exp, strike, PutOrCall.Call);
-                  if (call_sec_md == null)
-                     continue;
                   {
-                     NativePriceSize call_bid, call_ask;
-                     ulong time = 0;
-                     call_sec_md.Value.md.get_bbo(ref time, out call_bid, out call_ask);
-                     double call_mid = (call_bid.price + call_ask.price) / 2;
+                     double bid_price, ask_price;
+                     float bid_sz, ask_sz;
+                     call_sec_md.Value.md.get_bbo(out bid_price, out bid_sz, out ask_price, out ask_sz);
+                     double call_mid = (bid_price + ask_price) / 2;
                      tp.call = call_mid;
                   }
                   {
                      var put_sec_md = sc_data.get_option(exchange, sym, exp, strike, PutOrCall.Put);
-                     if (put_sec_md == null)
-                        continue;
-                     NativePriceSize put_bid, put_ask;
-                     ulong time = 0;
-                     put_sec_md.Value.md.get_bbo(ref time, out put_bid, out put_ask);
-                     double put_mid = (put_bid.price + put_ask.price) / 2;
+                     double bid_price, ask_price;
+                     float bid_sz, ask_sz;
+                     put_sec_md.Value.md.get_bbo(out bid_price, out bid_sz, out ask_price, out ask_sz);
+                     double put_mid = (bid_price + ask_price) / 2;
                      tp.put = put_mid;
                   }
                }
@@ -176,7 +163,11 @@ namespace mbts.spark {
       }
 
 
-      private void _interp_and_publish(Impetus imp) {
+      private void _interp_and_publish() {
+         // int i = 0;
+         // if (i < Int32.MaxValue) {
+         //    return;
+         // }
          var timer = Stopwatch.StartNew();
          var res = new Dictionary<uint, GreeksTuple>();
          var driver_map = _bb.read_state(_drivers_tag);
@@ -218,13 +209,13 @@ namespace mbts.spark {
          }
          // copy and publish
          IReadOnlyDictionary<uint, GreeksTuple> res_ro = res;
-         bts.Log.Gen.info($"LiveTheoPublisher._interp_and_publish Impetus: {imp} posting " +
+         bts.Log.Gen.info($"LiveTheoPublisher._interp_and_publish posting " +
                              $"count: {res_ro.Count}");
          timer.Stop();
          _logger.log(_log_id, _log_id, timer.Elapsed.TotalMilliseconds);
          _bb.post(SparkTags.Greeks, res_ro);
          // oh and those TC2 theos for comptability
-         _publish_tc2_theos(k_theos, imp);
+         _publish_tc2_theos(k_theos);
       }
 
       // If we do any rolls like SP for futures, it will be here

src/client/spark/AW2/Misc/SystemContractData.cs

@@ -3,6 +3,8 @@ using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 
+using Apache.Arrow;
+
 using bts;
 using bts.api;
 using bts.collections;
@@ -161,11 +163,11 @@ namespace mbts.spark {
          if (!sec_md_map.TryGetValue(id.Value, out sec_md)) {
             return null;
          }
-         NativePriceSize bid, ask;
-         ulong time = 0;
-         sec_md.md.get_bbo(ref time, out bid, out ask);
+         double bid_price, ask_price;
+         float bid_sz, ask_sz;
+         sec_md.md.get_bbo(out bid_price, out bid_sz, out ask_price, out ask_sz);
          return Tuple.Create(sec_md.security,
-                             new VolSurfaceMDUnit(bid, ask, sec_md.md.fixing_price));
+                             new VolSurfaceMDUnit(bid_price, bid_sz, ask_price, ask_sz, sec_md.md.fixing_price));
       }
 
       public Tuple<SecurityImmutable, VolSurfaceMDUnit> get_exp_md(SecurityExpiration sec_exp) {

src/client/spark/Analytics/Core/VolSurfaceConstants.cs

@@ -8,7 +8,7 @@ using System.Threading.Tasks;
 
 namespace mbts.analytics {
    public static class VolSurfaceConstants {
-      public static int vol_time_sample_mins = 5;
+      public static int vol_time_sample_mins = 1;
    }
    // TODO: remove this
    public static class VolSurfaceTemp {

src/client/spark/Analytics/Misc/VolSurfaceTimeCalculatorFactory.cs

@@ -21,23 +21,13 @@ namespace mbts.analytics {
       private class TimeCalculator : IVolSurfaceTimeCalculator {
          // TODO, this should be passed in the exp itself
          private Dictionary<string, TimeSpan> utc_time_of_exp_map = new Dictionary<string, TimeSpan>() {
-            // TODO: get it from config
             {"BTC", TimeSpan.FromHours(8)},
-            {"ETH", TimeSpan.FromHours(8)},
-            {"SPXW", TimeSpan.FromHours(20)},
-            {"SPX", new TimeSpan(13, 30, 0)}
+            {"ETH", TimeSpan.FromHours(8)}
          };
          public void compute_vol_time(VolSurfaceInput vs_input, VolSurfaceExpiration exp,
                                       DateTime voltime, double days_shift,
                                       out double yrs, out double days_per_yr) {
-
             days_per_yr = 365;
-            if (vs_input != null && vs_input.surface_settings != null && vs_input.surface_settings.core_settings != null
-               && vs_input.surface_settings.core_settings.time_settings != null) {
-               if (vs_input.surface_settings.core_settings.time_settings.vol_time_measure.account_for_holidays_and_weekends) {
-                  days_per_yr = vs_input.surface_settings.core_settings.time_settings.vol_time_measure.days_per_year;
-               }
-            }
             TimeSpan time_of_exp = TimeSpan.Zero;
             utc_time_of_exp_map.TryGetValue(exp.symbol, out time_of_exp);
             // assumes exp is the utc date of expiration

src/client/spark/Analytics/Pricing/VolSurfaceInterpGreeksProducer.cs

@@ -9,8 +9,8 @@ using bts.api;
 using mbts.spark;
 
 namespace mbts.analytics {
-   // This class computes interp theos/greeks (ie for a number of driver
-   // prices (specified in the vs settings)) for a volsurface
+   // This class computes the theos/greeks for a number of driver
+   // prices (specified in the vs settings).
    // It has public setters that get
    // 1. contracts
    // 2. vs input
@@ -41,8 +41,7 @@ namespace mbts.analytics {
       private IDictionary<VolSurfaceExpiration, Tuple<double, double>>
       _computed_driver_range_map = new Dictionary<VolSurfaceExpiration, Tuple<double, double>>();
       private readonly IEnumerable<double> _default_abs_moves =
-         //new double[] { 300, -200, -125, -100, -75, -50, -25, 0, 25, 50, 75, 100, 125, 200, 300 };
-         new double[] { -100, -50, 0, 50, 100 };
+         new double[] { 300, -200, -125, -100, -75, -50, -25, 0, 25, 50, 75, 100, 125, 200, 300 };
       private readonly double _cushion_percentage = 25.0;
 
       public event
@@ -136,8 +135,8 @@ namespace mbts.analytics {
                      false, true, false, false, false, _filter_map);
                timer.stop();
                vol_compute_time = timer.last_interval_time_ms;
-               Log.Gen.info($"VolSurfacePrecomputedTheosProducer._compute_and_fire volmat input_type: {input_type} vs_id: {_vs_id} time " +
-                           $"{timer.last_interval_time_ms} tid: {System.Threading.Thread.CurrentThread.ManagedThreadId}");
+               Log.Gen.info($"VolSurfacePrecomputedTheosProducer._compute_and_fire volmat time " +
+                           $"{timer.last_interval_time_ms}");
                timer.start();
                IDictionary<VolSurfaceExpiration, IEnumerable<Theos>> exp_theo_map =
                  VolSurfaceTheoCalculator.compute_standard_greeks(synced_vs_input.surface_settings,

src/client/spark/Daemons/LocalMDDaemon.cs

@@ -0,0 +1,243 @@
+using bts.api;
+
+using mbts.co2;
+
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.IO.Pipes;
+using System.Text;
+using System.Threading;
+
+namespace bts {
+   public class LocalMDConsumerHandle : mbts.co2.IMDConsumerHandle {
+      private readonly SecurityImmutable _sec;
+      private readonly LocalMDDaemon.BBO _bbo;
+      // private static NativePriceSize bid_nps = new NativePriceSize(new IntPrice(123.45), Convert.ToSingle(100));
+      // private static NativePriceSize ask_nps = new NativePriceSize(new IntPrice(123.46), Convert.ToSingle(100));
+      public LocalMDConsumerHandle(SecurityImmutable sec, LocalMDDaemon.BBO bbo) {
+         _sec = sec;
+         _bbo = bbo;
+      }
+
+      public UInt32 key => _sec.Id;
+      // The exchange key can change at CBOE. This numeric_id will always be correct
+      public ulong numeric_id => Convert.ToUInt64(_sec.Cboe.ExchangeId);
+
+      public void set_bbo(double bid_price, UInt32 bid_qty, double ask_price, UInt32 ask_qty) {
+         _bbo.bid_price = bid_price;
+         _bbo.bid_qty = bid_qty;
+         _bbo.ask_price = ask_price;
+         _bbo.ask_qty = ask_qty;
+      }
+
+      public bool is_heart_beating => true;
+      public bool has_depth => true;
+
+      // This property exists for specific performance optimization purposes
+      // Most callers should just use get_bbo/get_depth
+      public ulong market_timestamp => 0;
+      public ulong misc_timestamp => 0;
+
+      public ulong total_volume => 0;
+
+      public double settlement_price => Double.NaN;
+      public DateTime settlement_date => DateTime.Now;
+      public double previous_settlement_price => Double.NaN;
+      public DateTime previous_settlement_date => DateTime.Now;
+
+      public double open_price => Double.NaN;
+      public DateTime open_date => DateTime.Now;
+
+      public double close_price => Double.NaN;
+      public DateTime close_date => DateTime.Now;
+      public double session_high => Double.NaN;
+      public double session_low => Double.NaN;
+      public double fixing_price => Double.NaN;
+
+
+      public bool get_bbo(out double bid_price, out float bid_sz, out double ask_price, out float ask_sz) {
+         bid_price = _bbo.bid_price;
+         bid_sz = _bbo.bid_qty;
+         ask_price = _bbo.ask_price;
+         ask_sz = _bbo.ask_qty;
+         return true;
+      }
+
+      public bool get_bbo(ref ulong time, out NativePriceSize bid, out NativePriceSize ask) {
+         bid = new NativePriceSize(new IntPrice(_bbo.bid_price), Convert.ToSingle(_bbo.bid_qty));
+         ask = new NativePriceSize(new IntPrice(_bbo.ask_price), Convert.ToSingle(_bbo.ask_qty));
+         // bid = bid_nps;
+         // ask = ask_nps;
+         return true;
+      }
+
+      public bool get_depth(
+         ref ulong time, ref NativePriceSize[] bids, ref NativePriceSize[] asks, uint depth) {
+         return true;
+      }
+
+      public bool get_last_trade(
+         ref ulong time, out NativePriceSize ps, out DateTime timestamp, out ulong total_volume) {
+         ps = new NativePriceSize();
+         timestamp = DateTime.MinValue;
+         total_volume = 0;
+         return false;
+      }
+
+      public bool get_last_trade(
+         ref ulong time,
+         out NativePriceSize ps,
+         out DateTime timestamp,
+         out ulong total_volume,
+         out bool from_spread
+      ) {
+         ps = new NativePriceSize();
+         timestamp = DateTime.MinValue;
+         total_volume = 0;
+         from_spread = false;
+         return false;
+      }
+
+      public class PriceLevel {
+         public double price;
+         public uint qty;
+
+         public PriceLevel(double price_arg, uint qty_arg) {
+            price = price_arg;
+            qty = qty_arg;
+         }
+
+         public static PriceLevel empty => new(double.NaN, 0);
+
+         public static IntPrice intprice_from_double(double d) =>
+            double.IsNaN(d) ? IntPrice.NaN : new IntPrice((long)d, 0);
+
+         public NativePriceSize get_native_ps() =>
+            new(intprice_from_double(price), qty);
+      }
+   }
+
+   public class LocalMDDaemon {
+
+      public class BBO {
+         public double bid_price = Double.NaN;
+         public UInt32 bid_qty = 0;
+         public double ask_price = Double.NaN;
+         public UInt32 ask_qty = 0;
+         public UInt32 exchange_id = 0;
+         public BBO(uint exchange_id) :
+         this(Double.NaN, 0, Double.NaN, 0, exchange_id) { }
+         public BBO(double bid_price_, uint bid_qty_, double ask_price_, uint ask_qty_, uint exchange_id_) {
+            bid_price = bid_qty_ == 0 ? Double.NaN : bid_price_;
+            bid_qty = bid_qty_;
+            ask_price = ask_qty_ == 0 ? Double.NaN : ask_price_;
+            ask_qty = ask_qty_;
+            exchange_id = exchange_id_;
+         }
+
+         public override string ToString() {
+            return $"bid_price: {bid_price} bid_qty: {bid_qty} ask_price: {ask_price} ask_qty: {ask_qty} exchange_id: {exchange_id}";
+         }
+      }
+      static Process producer_process = null;
+      static void start_or_restart_producer_process(string exePath) {
+         foreach (var process in Process.GetProcessesByName(Path.GetFileNameWithoutExtension(exePath))) {
+            try {
+               bts.Log.Gen.info($"Stopping existing producer process (PID: {process.Id})...");
+               process.Kill();
+               process.WaitForExit();
+            } catch (Exception ex) {
+               bts.Log.Gen.error($"Stopping existing producer process (PID: {process.Id})...ex: {ex.Message}");
+            }
+         }
+         if (producer_process != null && !producer_process.HasExited) {
+            bts.Log.Gen.info("Stopping existing producer process...");
+            producer_process.Kill();
+            producer_process.WaitForExit();
+            producer_process.Dispose();
+         }
+
+         bts.Log.Gen.info("Starting new producer process...");
+
+         producer_process = new Process();
+         producer_process.StartInfo.FileName = exePath;  // Path to the EXE
+         producer_process.StartInfo.UseShellExecute = false;
+         // producer_process.StartInfo.RedirectStandardOutput = true;
+         // producer_process.StartInfo.RedirectStandardError = true;
+         producer_process.Start();
+
+         bts.Log.Gen.info("Producer process started.");
+      }
+      public Action<LocalMDDaemon.BBO> on_bbo = (bbo) => { };
+
+      public LocalMDDaemon() {
+         Log.Gen.info("LocalMDDaemon.cons called, starting the reading thread now");
+         System.Threading.Thread pipe_read_thr = new Thread(_start_reading);
+         pipe_read_thr.IsBackground = true;
+         pipe_read_thr.Start();
+
+      }
+
+      private void _start_reading() {
+         // TODO, fix this
+         string exe_path = @"dist\producer_delayed\producer_delayed.exe";
+         //@"dist\producer\producer.exe";
+
+         // delete pipe if already exists
+         using NamedPipeClientStream old_pipe_client = new NamedPipeClientStream(".", "MyPipe", PipeDirection.In);
+         try {
+            old_pipe_client.Close();
+            old_pipe_client.Dispose();
+         } catch (Exception ex) {
+            Log.Gen.error($"LocalMDDaemon._start_reading ex: {ex}");
+         }
+         start_or_restart_producer_process(exe_path);
+
+         // Connect to the named pipe
+         using NamedPipeClientStream pipe_client = new NamedPipeClientStream(".", "MyPipe", PipeDirection.In);
+
+         System.Threading.Thread.Sleep(100);
+         pipe_client.Connect();
+
+         bts.Log.Gen.info("Connected to pipe. Waiting for messages...");
+
+         byte[] buffer = new byte[1024];
+
+         while (true) {
+            try {
+               int bytes_read = pipe_client.Read(buffer, 0, buffer.Length);
+
+               if (bytes_read == 0)// this means there was an error
+               {
+                  System.Threading.Thread.Sleep(2);
+                  continue;
+               }
+               Func<double, bool> qty_ok = (d) => {
+                  if (d < 0 || d > Convert.ToDouble(UInt32.MaxValue))
+                     return false;
+                  return true;
+               };
+               double double1 = BitConverter.ToDouble(buffer, 0);
+               double double2 = BitConverter.ToDouble(buffer, 8);
+               double double3 = BitConverter.ToDouble(buffer, 16);
+               double double4 = BitConverter.ToDouble(buffer, 24);
+               uint int5 = BitConverter.ToUInt32(buffer, 32);
+               //Log.Gen.info($"Read from pipe {double1} ,  {double2} , {double3} , {double4} , {int5}");
+               if (!qty_ok(double2) || !qty_ok(double4))
+                  continue;
+               LocalMDDaemon.BBO bbo = new BBO(double1, Convert.ToUInt32(double2),
+                                          double3, Convert.ToUInt32(double4), int5);
+               on_bbo(bbo);
+            } catch (IOException) {
+               bts.Log.Gen.error("Pipe disconnected. Attempting to reconnect...");
+               pipe_client.Dispose();
+               System.Threading.Thread.Sleep(1000);
+               pipe_client.Connect(); // Try reconnecting
+            } catch (Exception ex) {
+               bts.Log.Gen.error($"Exception: {ex.Message}");
+            }
+         }
+      }
+   }
+}

src/client/spark/Daemons/LocalMDDaemon2.cs

@@ -0,0 +1,104 @@
+using bts;
+
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.IO.Pipes;
+using System.Text;
+
+class LocalMDDaemon2 {
+   static Process producerProcess = null;
+   static void StartOrRestartProducerProcess(string exePath) {
+      // foreach (var process in Process.GetProcessesByName(Path.GetFileNameWithoutExtension(exePath)))
+      // {
+      //     try
+      //     {
+      //         Console.WriteLine($"Stopping existing producer process (PID: {process.Id})...");
+      //         process.Kill();
+      //         process.WaitForExit();
+      //     }
+      //     catch (Exception ex)
+      //     {
+      //         Console.WriteLine($"Error stopping process: {ex.Message}");
+      //     }
+      // }
+      // if (producerProcess != null && !producerProcess.HasExited)
+      // {
+      //     Console.WriteLine("Stopping existing producer process...");
+      //     producerProcess.Kill();
+      //     producerProcess.WaitForExit();
+      //     producerProcess.Dispose();
+      // }
+
+      Console.WriteLine("Starting new producer process...");
+
+      producerProcess = new Process();
+      producerProcess.StartInfo.FileName = exePath;  // Path to the EXE
+      producerProcess.StartInfo.UseShellExecute = false;
+      // producerProcess.StartInfo.RedirectStandardOutput = true;
+      // producerProcess.StartInfo.RedirectStandardError = true;
+      producerProcess.Start();
+      Console.WriteLine("Producer process started.");
+   }
+
+   public static void start() {
+
+      // string exePath = @".\dist\producer\producer.exe"; //uncomment for live
+      // string exePath = @".\dist\producer_delayed\producer_delayed.exe"; //UNCOMMENT FOR DELAYED
+      string exePath = //@".\dist\producer_replay\producer_replay.exe"; //UNCOMMENT FOR replay
+      @"C:\Users\m_tah\Downloads\HelloWorld\HelloWorld\dist\producer_replay\producer_replay.exe";
+
+
+      // delete pipe if already exists
+      // using NamedPipeClientStream oldpipeClient = new NamedPipeClientStream(".", "MyPipe", PipeDirection.In);
+      // try{
+      //     oldpipeClient.Close();
+      //     oldpipeClient.Dispose();
+      // }catch{
+
+      // }
+      StartOrRestartProducerProcess(exePath);
+
+      // Connect to the named pipe
+      using NamedPipeClientStream pipeClient = new NamedPipeClientStream(".", "MyPipe", PipeDirection.In);
+
+      // Thread.Sleep(2000);
+      pipeClient.Connect();
+
+      Console.WriteLine("Connected to pipe. Waiting for messages...");
+
+      byte[] buffer = new byte[1024];
+      var start = true;
+      while (true) {
+         try {
+            int bytesRead = pipeClient.Read(buffer, 0, buffer.Length);
+
+            if (bytesRead == 0)// this means there was an error
+            {
+               Console.WriteLine("0 bytes recieved ending...");
+               if (start) {
+                  start = false;
+                  continue;
+               }
+               break;
+            }
+
+            double double1 = BitConverter.ToDouble(buffer, 0);
+            double double2 = BitConverter.ToDouble(buffer, 8);
+            double double3 = BitConverter.ToDouble(buffer, 16);
+            double double4 = BitConverter.ToDouble(buffer, 24);
+            uint int5 = BitConverter.ToUInt32(buffer, 32);
+            Log.Gen.info($"Received: {double1}, {double2}, {double3}, {double4}, {int5}");
+
+         } catch (IOException) {
+            Console.WriteLine("Pipe disconnected. Attempting to reconnect...");
+            pipeClient.Dispose();
+            System.Threading.Thread.Sleep(1000);
+            pipeClient.Connect(); // Try reconnecting
+         }
+      }
+
+      // Console.WriteLine("Pipe closed. Exiting...");
+      // producerProcess?.WaitForExit();
+   }
+}

src/client/spark/Daemons/MDDaemon.cs

@@ -136,6 +136,7 @@ namespace bts {
                var (priceDisplay, qtyDisplay) = GetDisplaySettings(smd.security);
                exch_secs[smd.id] = new(smd.security, smd.md, priceDisplay, qtyDisplay);
             }
+            Log.Gen.info($"MDDaemon.initialize posting sec, n: {exch_secs.Count}");
             _bus.post(SparkTags.ServerSecuritiesTag, exch_secs);
          }
 

src/client/spark/Daemons/MergedMDDaemon.cs

@@ -1,10 +1,7 @@
 using BCG;
 
 using bts.api;
-using bts.api.synthetic;
-using bts.collections;
-using bts.md;
-using bts.utils;
+
 
 using mbts;
 using mbts.co2;
@@ -14,78 +11,103 @@ using mbts.ui;
 using Spark.Core.Disposables;
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using System.Composition;
 using System.Linq;
 using System.Reactive.Linq;
 using System.Threading.Tasks;
+using System.Threading;
 
 using SecurityKey = System.UInt32;
 
-namespace bts
-{
+namespace bts {
+
    [Export(typeof(IShellDaemon))]
-   internal class MergedMDDaemon : IShellDaemon, IDisposable
-   {
+   internal class MergedMDDaemon : IShellDaemon, IDisposable {
       private readonly IBus _bus;
+      private LocalMDDaemon _local_md_daemon;
       IDisposable _dispose;
-      Dictionary<uint, SecurityMD> _server_sec_map = new Dictionary<uint, SecurityMD>();
+      ConcurrentDictionary<uint, SecurityMD> _server_sec_map = new ConcurrentDictionary<uint, SecurityMD>();
       Dictionary<uint, SecurityMD> _fake_sec_map = new Dictionary<uint, SecurityMD>();
+      private ConcurrentDictionary<uint, SecurityKey> _exchange_id_sec_id_map = new ConcurrentDictionary<uint, SecurityKey>();
       private readonly IList<IDisposable> _subs = new List<IDisposable>();
+      private SyncQueue<LocalMDDaemon.BBO> _waiting_bbo = new SyncQueue<LocalMDDaemon.BBO>();
+      private readonly Venue _local_md_exchange = Venue.Cboe;
+      private bool _localmd_daemon_inited = false;
+
+      private void _init_local_md_daemon() {
+         if (_localmd_daemon_inited)
+            return;
+         //LocalMDDaemon2.start();
+         _localmd_daemon_inited = true;
+         _local_md_daemon = new LocalMDDaemon();
+         _local_md_daemon.on_bbo += (bbo_pack) => {
+            //bts.Log.Gen.info($"MergedMDDaemon on_bbo, local bbo callback, bbo: {bbo_pack}");
+            _waiting_bbo.enqueue(bbo_pack);
+         };
+         System.Threading.Thread dequeue_bbo_thr = new Thread(_dequeue_bbo_thread_func);
+         dequeue_bbo_thr.IsBackground = true;
+         dequeue_bbo_thr.Priority = ThreadPriority.Normal;
+         dequeue_bbo_thr.Start();
+      }
 
       [ImportingConstructor]
-      public MergedMDDaemon(IBus bus)
-      {
-         //TempTester.test_analytics();
+      public MergedMDDaemon(IBus bus) {
          bts.Log.Gen.info("MergedMDDaemon started");
          _bus = bus;
+
          _subs.Add(_bus.subscribe(SparkTags.ServerSecuritiesTag).OnDispatcher().Subscribe(
             d => _on_server_sec_map(d)));
          _subs.Add(_bus.subscribe(SparkTags.FakeSecuritiesTag).OnDispatcher().Subscribe(
             d => _on_fake_sec_map(d)));
-         // This is done periodically to remove expired contracts
-         // (disabling it right now because we are not sure if deleting securities 
-         // works with the various UI windows)
-         // _subs.Add(_bus.subscribe(SparkTags.SparkTimer).BufferLatest(TimeSpan.FromMinutes(1))
-         //             .Subscribe(t => _merge_and_post()));
-         //_test_md_stream();
-      }
-      // BCG.MDStream _md_stream = null;
-      // private void _test_md_stream()
-      // {
-      //    _bus.subscribe(SparkTags.SparkTimer).Subscribe(t =>
-      //       {
-      //          if (_md_stream == null)
-      //          {
-      //             _md_stream = new BCG.MDStream();
-      //             _md_stream.onChange += _on_md;
-      //          }
-      //          else
-      //          {
-      //             BCG.ViewStreamDatabase.instance.doWork();
-      //          }
-      //       });
-      // }
-
-      private void _on_md(object obj)
-      {
-         BCG.DataPacket dp = obj as DataPacket;
-         bts.Log.Gen.info($"MDStream callback dp: {dp}");
       }
 
-      private void _log_options(IReadOnlyDictionary<uint, SecurityMD> sec_map)
-      {
+      private void _dequeue_bbo_thread_func() {
+         while (true) {
+            System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(0.1));
+            _process_local_bbo_updates();
+         }
+      }
+
+      private void _process_local_bbo_updates() {
+         if (_waiting_bbo.getCount() == 0)
+            return;
+         var bbo_list = _waiting_bbo.dequeue_all();
+         // if (bbo_list.Count > -1)
+         //    return;
+         if (bbo_list.Count >= 10) {
+            Log.Gen.warn($"MergedMDDaemon._process_local_bbo_updates dequeued item count: {bbo_list.Count}");
+         }
+         List<LocalMDDaemon.BBO> unprocessed_bbo_list = new List<LocalMDDaemon.BBO>();
+         foreach (var bbo in bbo_list) {
+            bool processed = false;
+            //bts.Log.Gen.info($"MergedMDDaemon._process_local_bbo_updates read from queue bbo: {bbo}");
+            UInt32 exchange_id = bbo.exchange_id;
+            SecurityKey sec_key;
+            if (_exchange_id_sec_id_map.TryGetValue(exchange_id, out sec_key)) {
+               SecurityMD sec_md;
+               if (_server_sec_map.TryGetValue(sec_key, out sec_md)) {
+                  sec_md.md.set_bbo(bbo.bid_price, bbo.bid_qty, bbo.ask_price, bbo.ask_qty);
+                  processed = true;
+               }
+            }
+            if (!processed) {
+               // TODO: put it back in the queue
+            }
+         }
+      }
+
+      private void _log_options(IReadOnlyDictionary<uint, SecurityMD> sec_map) {
          string rv = "MergedMDDaemon Options:\n";
-         foreach (var kv in sec_map)
-         {
+         foreach (var kv in sec_map) {
             var sec = kv.Value.security;
             var md = kv.Value.md;
             ulong time = 0;
             NativePriceSize bid, ask;
             var has_md = md.get_bbo(ref time, out bid, out ask);
-            if (sec.HasOption)
-            {
+            if (sec.HasOption) {
                rv += $"id: {kv.Key} desc: {sec.security_desc()} sym: {sec.Symbol} " +
                      $"exp: {sec.security_expiration()} " +
                      $"k: {sec.Option.Strike} cp: {sec.Option.CallPut} " +
@@ -96,61 +118,40 @@ namespace bts
          Log.Gen.info(rv);
       }
 
-      private void _on_server_sec_map(IReadOnlyDictionary<uint, SecurityMD> sec_map)
-      {
+      private void _on_server_sec_map(IReadOnlyDictionary<uint, SecurityMD> sec_map) {
          //_log_options(sec_map);
          bts.Log.Gen.info(String.Format("MergedMDDaemon._on_server_sec_map sz: {0}", sec_map.Count()));
-         _server_sec_map = new Dictionary<uint, SecurityMD>(sec_map);
+         foreach (var kv in sec_map) {
+            SecurityMD md = kv.Value;
+            uint sec_id = kv.Key;
+            SecurityMD sec_md = kv.Value;
+            if (kv.Value.security.Exchange == _local_md_exchange) {
+               UInt32 exchange_sec_id = Convert.ToUInt32(kv.Value.security.Cboe.ExchangeId);
+               _exchange_id_sec_id_map[exchange_sec_id] = sec_id;
+               _server_sec_map[sec_id] = new SecurityMD(sec_md.security, new LocalMDConsumerHandle(sec_md.security, new LocalMDDaemon.BBO(exchange_sec_id)));
+            } else {
+               _server_sec_map[sec_id] = sec_md;
+            }
+         }
          _merge_and_post();
+         _init_local_md_daemon();
       }
 
-      private void _on_fake_sec_map(IReadOnlyDictionary<uint, SecurityMD> sec_map)
-      {
+      private void _on_fake_sec_map(IReadOnlyDictionary<uint, SecurityMD> sec_map) {
          bts.Log.Gen.info(String.Format("MergedMDDaemon._on_fake_sec_map sz: {0}", sec_map.Count()));
          _fake_sec_map = new Dictionary<uint, SecurityMD>(sec_map);
          _merge_and_post();
       }
 
-
-
-      private static bool is_active(SecurityMD sec_md) {
-         SecurityImmutable sec = sec_md.security;
-         // temp: to see if spreads work
-         bool fut_or_option = (sec.security_type() == bts.api.SecurityType.Future || sec.security_type() == bts.api.SecurityType.Option);
-         if (!fut_or_option)
-            return true;
-         Func<bool> is_daily_option = () => {
-            bool is_daily = false;
-            if (sec.Exchange == Venue.Cboe) {
-               is_daily = sec.Symbol.ToLower() == "spxw";
-            }
-            if (sec.Exchange == Venue.Glbx) {
-               string symbol = sec.Symbol.ToLower();
-               int unused;
-               is_daily = symbol.Length >= 3 && symbol[0] == 'e' && Int32.TryParse(symbol[1].ToString(), out unused);
-            }
-            return is_daily;
+      private void _merge_and_post() {
+         Func<SecurityMD, bool> is_active = (sec_md) => {
+            var sec = sec_md.security;
+            return SecurityExpirationHelper.is_unexpired(sec);
          };
-         bool is_expired = !SecurityExpirationHelper.is_unexpired(sec);
-         if (is_expired) {
-            return false;
-         }
-         if (sec.security_type() == bts.api.SecurityType.Option) {
-            int days_to_exp = Convert.ToInt32((sec.earliest_expiration() - DateTime.Now).TotalDays);
-            if (is_daily_option()) {
-               return days_to_exp <= 14;
-            } else {
-               return days_to_exp <= 90;
-            }
-         }
-         return true;            
-      }   
-
-      private void _merge_and_post()
-      {
+         // server secs
          var to_pub = _server_sec_map.Where(kv => is_active(kv.Value)).ToDictionary(kv => kv.Key, kv => kv.Value);
-         foreach (var kv in _fake_sec_map)
-         {
+         // add fake secs to the map
+         foreach (var kv in _fake_sec_map) {
             to_pub[kv.Key] = kv.Value;
          }
          bts.Log.Gen.info(String.Format("MergedMDDaemon._merge_and_post sz: {0}", to_pub.Count()));
@@ -162,7 +163,9 @@ namespace bts
          _bus.post(SparkTags.TC2Tags.ExchangeSecurityMapTag, exs_map);
       }
 
-      public void Dispose() =>
+      public void Dispose() {
          Disposable.dispose(ref _dispose);
+         Log.Gen.info("LocalMDDaemon.DIspose called collecting resources now");
+      }
    }
 }

src/client/spark/Modules/Market/MarketView/MarketViewVM.cs

@@ -629,16 +629,16 @@ namespace mbts.spark {
 
       static (string desc, Color fg) _fills_title(ColKey key) =>
          key switch {
-            ColKey.Position => ("Day Fills", BTSColors.DayFills.foreground),
-            ColKey.RawDayPosition => ("Raw Day Fills", BTSColors.DayFills.foreground),
-            _ => ("Fills", BTSColors.UI.default_text),
+            ColKey.Position => ("Day Fills", Colors.Black),
+            ColKey.RawDayPosition => ("Raw Day Fills", Colors.Black),
+            _ => ("Fills", Colors.Black),
          };
 
       static (string desc, Color fg) _orderbook_title(ColKey key) =>
          key switch {
             ColKey.WorkingBids => ((string desc, Color fg))("Working Buy Orders", BTSColors.MarketData.bid),
             ColKey.WorkingAsks => ((string desc, Color fg))("Working Sell Orders", BTSColors.MarketData.ask),
-            _ => ("Working Orders", BTSColors.UI.default_text),
+            _ => ("Working Orders", Colors.Black),
          };
 
       static GridSettings _hide_cols(GridSettings grid, IEnumerable<OrderBookGridView.ColumnKeys> columns) =>

src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookFilterStates.cs

@@ -150,11 +150,7 @@ namespace bts.shell.orderbook {
          pb.add_discrete_values(fs.exchanges, o => o.order.security?.Exchange ?? Venue.Unset);
          pb.add_discrete_values(fs.tifs, o => o.order.order.time_in_force());
          pb.add_discrete_values(fs.order_types, o => AlgoID.from_custom_def(o.order.order.Def.CustomDef));
-         pb.add_discrete_values(
-            Array.ConvertAll(fs.root_algo_types, a => (AlgoID?)a),
-            o => o.root.algo.HasValue
-               ? AlgoID.from_algo_key(o.root.algo.Value.key)
-               : null);
+         pb.add_discrete_values(fs.root_algo_types, o => AlgoID.from_algo_key(o.root.algo?.key ?? default));
          pb.add_discrete_values(fs.order_sub_types, o => o.order.order.sub_type());
          pb.add_discrete_values(fs.sides, o => o.order.order.side());
          pb.add_discrete_values(fs.statuses, o => o.order.order.status());

src/client/spark/Modules/Trading/PNL/VM/PNLVM.cs

@@ -774,17 +774,6 @@ namespace bts.shell.pnl {
             case ColumnKeys.Account: filter.accounts = new[] { g.Name.ToString() }; break;
             case ColumnKeys.Symbol: filter.symbols = new[] { g.Name.ToString() }; break;
             case ColumnKeys.Exchange: filter.exchanges = new[] { (Venue)g.Name }; break;
-            case ColumnKeys.Expiration: {
-                  if (g.Name is DateTime exp) {
-                     if (filter.security_criteria is null) {
-                        filter.security_criteria = SecurityCriteria.from_expirations(exp);
-                     } else {
-                        filter.security_criteria.clauses = ArrayHelper.add(
-                           filter.security_criteria.clauses, SecurityCriteriaClause.from_expirations(exp));
-                     }
-                  }
-                  break;
-               }
             case ColumnKeys.Security:
                if (g.Name is SecurityImmutable sec)
                   filter.security_criteria = SecurityCriteria.from_securities(sec);

src/client/spark/NMSheets/BlackboardStructure/NMSheetSingleOptionData.cs

@@ -237,7 +237,7 @@ namespace BCG {
          PNLRollOverDataUnit u;
          if (_roll_over_data != null && (u = _roll_over_data.getValue(exp, strike, true)) != null) {
             cp_row[(int)NMSheetColumnName.vol_change] = tp.vol - u.vol;
-            cp_row[(int)NMSheetColumnName.straddle_change] = (tp.vol - u.vol) * tp.cvega * 2.0 * 100.0;
+            cp_row[(int)NMSheetColumnName.straddle_change] = ((tp.vol - u.vol) * tp.cvega * 2.0 * 100.0) / tp.driver;
          } else {
             cp_row[(int)NMSheetColumnName.vol_change] = Double.NaN;
             cp_row[(int)NMSheetColumnName.straddle_change] = Double.NaN;

src/client/spark/NMSheets/DataProviders/NMSheetArchivedVolProvider.cs

@@ -14,23 +14,103 @@ using mbts.analytics;
 using bts.api.analytics;
 
 namespace mbts.spark {
+   public class ClientArchivedVolRequest {
+      public readonly int id;
+      public readonly ArchivedTheoRetrieval ret;
+      public ClientArchivedVolRequest(Int32 id, ArchivedTheoRetrieval ret) {
+         this.id = id;
+         this.ret = ret;
+      }
+      private static int _id = 0;
+      public static int get_new_id() {
+         int rv = 0;
+         lock (typeof(ClientArchivedVolRequest)) {
+            rv = ++_id;
+         }
+         return rv;
+      }
+      public override string ToString() {
+         return $"id: {id} sym: {ret.SecTup.Symbol} vol_time: {ret.SaveTime}";
+      }
+   }
    // This is created by the NMSheetSingleOptionMainDataSource - it downloads the
    // archived theos and provides access to it
    public class NMSheetArchivedVolProvider : IDisposable {
       private readonly ThreadedBlackboard _bb;
+      private readonly NMSheetSymbol _sym;
+      private readonly bts.api.SecurityTuple _sec_tup;
       private Action _cb = () => { };
       private readonly ITag<SystemContractData> _sc_tag = SparkTags.SystemContractsTag;
-      private Nullable<ArchivedTheoListImmutable> _cached_data;
+      private readonly ITag<NMSheetSettings> _settings_tag;
+      private readonly ITag<Tuple<ClientArchivedVolRequest, bts.api.analytics.ArchivedTheoList>>
+         _response_tag = SparkTags.ResponseArchivedVols;
+      private bool _waiting_for_reponse = false;
+      private ClientArchivedVolRequest _last_req;
+      bts.api.analytics.ArchivedTheoList _cached_data = null;
 
       public NMSheetArchivedVolProvider(string req_id, ThreadedBlackboard bb,
                                        NMSheetSymbol symbol,
                                        Action cb) {
          bts.Log.Ui.info($"NMSheetArchivedVolProvider req_id: {req_id} symbol: {symbol}");
          _bb = bb;
+         _sym = symbol;
+         _sec_tup = new bts.api.SecurityTuple() {
+            Venue = _sym.option_exchange,
+            Type = SecurityType.Option,
+            Symbol = _sym.option_name
+         };
+         _last_req = new ClientArchivedVolRequest(
+            0,
+            new ArchivedTheoRetrieval() {
+               SaveTime = 0,
+               SecTup = _sec_tup
+            });
          _cb = cb;
+         _settings_tag = SparkTags.TC2Tags.NMSheetSettingsTag(req_id);
          _bb.subscribe(_sc_tag);
-         _bb.subscribe(SparkTags.ArchivedTheos);
-         _bb.data_callback += _on_data;         
+         _bb.subscribe(_settings_tag);
+         _bb.subscribe(_response_tag);
+         _bb.data_callback += _on_data;
+         _check_and_send_request(true);
+      }
+
+      private static UInt64 _get_vol_time(NMSheetSettings settings) {
+         NMSheetSettingLayout.VolDay vol_day = settings.setting_layout.vol_day;
+         TimeSpan vol_time = settings.setting_layout.vol_time;
+         DateTime date = DateTime.Today.Date;
+         switch (vol_day) {
+            case NMSheetSettingLayout.VolDay.DayBeforeYesterday:
+               date = date.Subtract(TimeSpan.FromDays(2));
+               break;
+            case NMSheetSettingLayout.VolDay.Yesterday:
+               date = date.Subtract(TimeSpan.FromDays(1));
+               break;
+            case NMSheetSettingLayout.VolDay.Today:
+               break;
+         }
+         DateTime dt = date.Add(vol_time);
+         var rv = VolSurfaceUtilityFunctions.get_unix_epoch_msecs(dt);
+         bts.Log.Ui.info($"NMSheetArchivedVolProvider._get_vol_time vol_day: {vol_day} " +
+            $"vol_time: {vol_time} dt: {dt} epoch_md: {rv}");
+         return rv;
+      }
+
+      private void _check_and_send_request(bool force = false) {
+         NMSheetSettings settings = _bb.read_state(_settings_tag);
+         if (settings != null && settings.setting_layout != null) {
+            UInt64 vol_time = _get_vol_time(settings);
+            if (force || _last_req.ret.SaveTime != vol_time) {
+               _last_req = new ClientArchivedVolRequest(
+                     ClientArchivedVolRequest.get_new_id(),
+                     new ArchivedTheoRetrieval() {
+                        SaveTime = vol_time,
+                        SecTup = _sec_tup
+                     });
+               _waiting_for_reponse = true;
+               bts.Log.Gen.info($"NMSheetArchivedVolProvider._check_and_send_request req: {_last_req}");
+               _bb.post(SparkTags.RequestArchivedVols, _last_req);
+            }
+         }
       }
 
       private void _compute_and_post() {
@@ -38,22 +118,34 @@ namespace mbts.spark {
       }
 
       private void _on_data(object tag, object data) {
-         // just listen to archihved theos and fire callback
-         if (tag.Equals(SparkTags.ArchivedTheos)) {
-            _cached_data = new Nullable<ArchivedTheoListImmutable>((ArchivedTheoListImmutable)data);
-            _compute_and_post();
-          }
-         else if (_sc_tag.Equals(tag)) {
+         if (_settings_tag.Equals(tag)) {
+            _check_and_send_request();
+         } else if (_response_tag.Equals(tag)) {
+            if (_waiting_for_reponse) {
+               Tuple<ClientArchivedVolRequest, bts.api.analytics.ArchivedTheoList> resp =
+                  _bb.read_state(_response_tag);
+               if (resp != null) {
+                  ClientArchivedVolRequest responsed_req = resp.Item1;
+                  if (responsed_req.id == _last_req.id) {
+                     // got we were looking for
+                     _cached_data = resp.Item2;
+                     _waiting_for_reponse = false;
+                     bts.Log.Gen.info($"NMSheetArchivedVolProvider._on_data recvd data, req: {_last_req}");
+                     _compute_and_post();
+                  }
+               }
+            }
+         } else if (_sc_tag.Equals(tag)) {
             _compute_and_post();
          }
       }
 
       public OptionContractKeyedData<PNLRollOverDataUnit> get_all_data() {
          SystemContractData sc_data = _bb.read_state(_sc_tag);
-         if (sc_data != null && _cached_data != null && _cached_data.HasValue) {
+         if (_cached_data != null && sc_data != null) {
             OptionContractKeyedData<PNLRollOverDataUnit> rv =
                new OptionContractKeyedData<PNLRollOverDataUnit>();
-            foreach (var v in _cached_data.Value.SecTheo) {
+            foreach (var v in _cached_data.SecTheo) {
                var sec_md = sc_data.get_sec_md(v.Id);
                try {
                   if (sec_md != null) {

src/client/spark/NMSheets/DataProviders/NMSheetBBODataSource.cs

@@ -16,7 +16,7 @@ namespace mbts.spark {
       private readonly ThreadedBlackboard _bb;
       HashSet<BCG.NMSheetSymbol> _sym_map;
       private Action _cb = () => { };
-      private readonly TimeSpan fire_period = TimeSpan.FromMilliseconds(300);
+      private readonly TimeSpan fire_period = TimeSpan.FromMilliseconds(333);
 
       public NMSheetBBODataSource(ThreadedBlackboard bb, List<BCG.NMSheetSymbol> syms, Action cb) {
          _bb = bb;

src/client/spark/NMSheets/UIComponents/MainSheet/NMSheetControl.Designer.cs

@@ -1,7 +1,5 @@
-namespace BCG
-{
-   partial class NMSheetControl
-   {
+namespace BCG {
+   partial class NMSheetControl {
       /// <summary> 
       /// Required designer variable.
       /// </summary>
@@ -11,11 +9,9 @@
       /// Clean up any resources being used.
       /// </summary>
       /// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
-      protected override void Dispose(bool disposing)
-      {
+      protected override void Dispose(bool disposing) {
          _logical_dispose(disposing);
-         if (disposing && (components != null))
-         {
+         if (disposing && (components != null)) {
             components.Dispose();
 
          }
@@ -28,8 +24,7 @@
       /// Required method for Designer support - do not modify 
       /// the contents of this method with the code editor.
       /// </summary>
-      private void InitializeComponent()
-      {
+      private void InitializeComponent() {
          this.components = new System.ComponentModel.Container();
          this._menu_panel = new System.Windows.Forms.Panel();
          this._button_minimize_spreads = new System.Windows.Forms.Button();
@@ -72,7 +67,6 @@
          this._toolstripmenuitem_task_manage_templates = new System.Windows.Forms.ToolStripMenuItem();
          this._toolstripseperator_task_eye = new System.Windows.Forms.ToolStripSeparator();
          this._toolstrip_menuitem_save_data_to_file = new System.Windows.Forms.ToolStripMenuItem();
-         this._toolstrip_menuitem_save_ivs = new System.Windows.Forms.ToolStripMenuItem();
          this._toolstripseperator_export_data = new System.Windows.Forms.ToolStripSeparator();
          this._toolstripmenuitem_card = new System.Windows.Forms.ToolStripMenuItem();
          this._toolstripseperator_expirations = new System.Windows.Forms.ToolStripSeparator();
@@ -299,8 +293,7 @@
             this._toolstripmenuitem_atm,
             this._toolseperator_atm,
          };
-         if (_api_tasking_enabled)
-         {
+         if (_api_tasking_enabled) {
             right_click_menu.AddRange(new System.Collections.Generic.List<System.Windows.Forms.ToolStripItem>() {
                this._toolstripmenuitem_task_eye,
                // no quoter
@@ -314,8 +307,7 @@
          //       this._toolstripmenuitem_task_override_size,
          //    });
          // }
-         if (_api_tasking_enabled)
-         {
+         if (_api_tasking_enabled) {
             right_click_menu.AddRange(new System.Collections.Generic.List<System.Windows.Forms.ToolStripItem>() {
                this._toolstripmenuitem_task_eye_switch_all_off,
                // We can do this from the algo studio now
@@ -325,7 +317,6 @@
          right_click_menu.AddRange(new System.Collections.Generic.List<System.Windows.Forms.ToolStripItem>() {
             // this._toolstripseperator_task_eye,
             this._toolstrip_menuitem_save_data_to_file,
-            this._toolstrip_menuitem_save_ivs,
             this._toolstripseperator_export_data,
             this._toolstripmenuitem_card,
             this._toolstripseperator_expirations,
@@ -333,8 +324,7 @@
             this._toolstripmenuitem_minimize_spreads,
             this._toolstripseperator_filter,
          });
-         if (_api_tasking_enabled)
-         {
+         if (_api_tasking_enabled) {
             right_click_menu.AddRange(new System.Collections.Generic.List<System.Windows.Forms.ToolStripItem>() {
                this._toolstripmenuitem_filter,
                this._toolseperator_quotesets
@@ -507,13 +497,6 @@
          this._toolstripseperator_task_eye.Name = "_toolstripseperator_task_eye";
          this._toolstripseperator_task_eye.Size = new System.Drawing.Size(213, 6);
          // 
-         // _toolstrip_menuitem_save_ivs
-         // 
-         this._toolstrip_menuitem_save_ivs.Name = "_toolstrip_menuitem_save_ivs";
-         this._toolstrip_menuitem_save_ivs.Size = new System.Drawing.Size(216, 22);
-         this._toolstrip_menuitem_save_ivs.Text = "Save IVs";
-         //
-         // 
          // _toolstrip_menuitem_save_data_to_file
          // 
          this._toolstrip_menuitem_save_data_to_file.Name = "_toolstrip_menuitem_save_data_to_file";
@@ -598,8 +581,7 @@
             this._toolstripmenuitem_settings_volchange,
             this._toolstripmenuitem_settings_straddle
          };
-         if (_api_tasking_enabled)
-         {
+         if (_api_tasking_enabled) {
             sheet_settings_items.AddRange(new System.Collections.Generic.List<System.Windows.Forms.ToolStripItem>() {
                this._toolseperator_task_ee,
                this._toolstripmenuitem_settings_show_task_ee_data,
@@ -918,7 +900,6 @@
       private System.Windows.Forms.ToolStripSeparator _toolstripseparator_task_size;
       private System.Windows.Forms.ToolStripMenuItem _toolstripmenuitem_task_undo_size_override;
       private System.Windows.Forms.ToolStripMenuItem _toolstrip_menuitem_save_data_to_file;
-      private System.Windows.Forms.ToolStripMenuItem _toolstrip_menuitem_save_ivs;
       private System.Windows.Forms.ToolStripSeparator _toolstripseperator_export_data;
       private System.Windows.Forms.Button _button_theo_acceptance_confirmation;
       private System.Windows.Forms.ToolStripMenuItem _toolstripmenuitem_filter_strikes_with_pos_edge;

src/client/spark/NMSheets/UIComponents/MainSheet/NMSheetControl.cs

@@ -20,8 +20,6 @@ using System.Collections;
 using System.Reactive.Linq;
 using System.Threading;
 using bts.api.orderman;
-using bts;
-using mbts.analytics;
 
 namespace BCG {
    public partial class NMSheetControl : UserControl {
@@ -254,7 +252,7 @@ namespace BCG {
          /// windows timer
          _windows_timer = new System.Windows.Forms.Timer(components);
          _windows_timer.Enabled = true;
-         _windows_timer.Interval = 100;
+         _windows_timer.Interval = 20;
          _windows_timer.Tick += _windows_timer_tick;
          _toolstripmenuitem_card.Visible = false;
          /// status bar
@@ -319,9 +317,6 @@ namespace BCG {
             _set_new_settings(_settings);
             _theo_grid.theo_grid.scroll_to_point(Point.Empty);
          };
-         _toolstrip_menuitem_save_ivs.Click += (s, e) => {
-            _threaded_bb.post(SparkTags.TC2Tags.ArchiveIVsTag, _nmsheet_symbol_list[0]);
-         };
          _toolstripmenuitem_filter_strikes_in_tasks.Click += (s, e) => {
             _settings.setting_filter.filter_ee_task_strikes = !_settings.setting_filter.filter_ee_task_strikes;
             _set_new_settings(_settings);
@@ -617,7 +612,7 @@ namespace BCG {
                cell.col == NMSheetColumnName.put_ask_price || cell.col == NMSheetColumnName.put_ask_size) {
                _launch_order_entry_window(cell, _is_control_down() ? OrderEntryWindow.Ladder : OrderEntryWindow.Ticket);
             } else if (cell.col == NMSheetColumnName.call_theo || cell.col == NMSheetColumnName.put_theo ||
-                         cell.col == NMSheetColumnName.straddle || cell.col == NMSheetColumnName.strike) {
+                       cell.col == NMSheetColumnName.straddle || cell.col == NMSheetColumnName.strike) {
                if (cell.call_selected)
                   _add_spread_leg(cell.expiration, cell.strike, true, false, leg_size, state);
                if (cell.put_selected)
@@ -722,9 +717,9 @@ namespace BCG {
 
       private void _windows_timer_tick(object sender, EventArgs e) {
          _publish_activation_state();
-         // this computes the data and then paints it
+         //_threaded_bb.sec_thread_dowork();
          _check_and_propagate_data();
-         //_poll_theo_acceptance_button();
+         _poll_theo_acceptance_button();
          _refresh_statusbar_data(false);
       }
 
@@ -785,31 +780,21 @@ namespace BCG {
       #region Coalescer NMSheetData interaction
 
       private DateTime _last_data_draw_time = DateTime.MinValue;
-      private const double _min_draw_period_ms = 500;
+      private const double _min_draw_period_ms = 333.3;
       private NMSheetGeometricLayout _last_geo_lo_info = null;
-      private VolSurfaceTimer _vs_timer = new VolSurfaceTimer();
-      // this is where we assemble the data (in _coalescer.get_data())
-      // and then painted in _respond_to_data
       private void _check_and_propagate_data() {
          if (is_visible) {
             if ((DateTime.Now - _last_data_draw_time).Milliseconds >= Math.Max(_settings.setting_grid_internal.grid_paint_throttle_ms, _min_draw_period_ms)) {
-               _vs_timer.start();
                NMSheetData new_sheet_data = _coalescer.get_data();
-               _vs_timer.stop();
-               Log.Gen.info($"NMSheetControl._check_and_propagate_data get_data time: {_vs_timer.last_interval_time_ms} msecs symbols: {_title_prefix} tid: {System.Threading.Thread.CurrentThread.ManagedThreadId}");
                if (new_sheet_data != null) {
                   _last_data_draw_time = DateTime.Now;
                   _sheet_data = new_sheet_data;
-                  _vs_timer.start();
                   _respond_to_data();
-                  _vs_timer.stop();
-                  Log.Gen.info($"NMSheetControl._check_and_propagate_data _respond_to_data time: {_vs_timer.last_interval_time_ms} msecs symbols: {_title_prefix} tid: {System.Threading.Thread.CurrentThread.ManagedThreadId}");
                }
             }
          }
       }
 
-      // this is the painting
       private void _respond_to_data() {
          if (_sheet_data != null) {
             _set_driver_control_display();

src/client/spark/Orders/OrderTicket.xaml

@@ -1,161 +1,161 @@
-<Window
-  x:Class="mbts.ui.spark.OrderTicket"
-  xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
-  xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
-  xmlns:msys="clr-namespace:bts.api;assembly=spark_api"
-  xmlns:u="clr-namespace:mbts.ui"
-  xmlns:cv="clr-namespace:mbts.ui.converters"
-  xmlns:usp="clr-namespace:mbts.ui.spark"
-  xmlns:sys="clr-namespace:System;assembly=mscorlib"
-  u:WindowStyle.SizeToContent="WidthAndHeight"
-  u:WindowStyle.TitleContent="{Binding bbo, Mode=OneWay}"
-  Style="{StaticResource BTSWindowTitleTemplate}"
->
-  <Window.Resources>
-    <ResourceDictionary>
-      <ResourceDictionary.MergedDictionaries>
-        <ResourceDictionary Source="/view/Resources/Converters.xaml"/>
-      </ResourceDictionary.MergedDictionaries>
-      <sys:Double x:Key="HorizontalSpacing">8.0</sys:Double>
-      <sys:Double x:Key="VerticalSpacing">4.0</sys:Double>
-      <sys:Double x:Key="ButtonHeight">22.0</sys:Double>
-      <Style x:Key="SellButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
-         <Setter Property="Background" Value="{u:BTSColor MarketData.ask_dark}" />
-         <Setter Property="Foreground" Value="White" />
-         <Setter Property="MinHeight" Value="40" />
-         <Setter Property="FontWeight" Value="Bold" />
-         <Setter Property="HorizontalAlignment" Value="Stretch" />
-         <Setter Property="VerticalAlignment" Value="Stretch" />
-         <Setter Property="Content">
-            <Setter.Value>
-              <TextBlock VerticalAlignment="Center" Text="Sell" />
-            </Setter.Value>
-         </Setter>
-         <Setter Property="Template">
-            <Setter.Value>
-               <ControlTemplate TargetType="Button">
-                  <Border 
-                        x:Name="Border"
-                        CornerRadius="0" 
-                        BorderThickness="1"
-                        BorderBrush="{u:BTSColor UI.Button.normal_border}"
-                        Background="{u:BTSColor MarketData.ask}"
-                        Padding="2"
-                        Height="Auto"
-                        SnapsToDevicePixels="True">
-                        <ContentPresenter 
-                           HorizontalAlignment="Center"
-                           VerticalAlignment="Center"
-                           RecognizesAccessKey="True"
-                           Name="button_content" 
-                        />
-                     </Border>
-                  <ControlTemplate.Triggers>
-                        <Trigger Property="IsMouseOver" Value="True">
-                           <Setter TargetName="Border" Property="Background" Value="#99c00000" />
-                           <Setter TargetName="Border" Property="BorderBrush" Value="{u:BTSColor UI.Button.mouseover_border}" />
-                        </Trigger>
-                     </ControlTemplate.Triggers>
-               </ControlTemplate>
-            </Setter.Value>
-         </Setter>
-      </Style>
-      <Style x:Key="BuyButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
-         <Setter Property="Background" Value="{u:BTSColor MarketData.bid_dark}" />
-         <Setter Property="Foreground" Value="White" />
-         <Setter Property="MinHeight" Value="40" />
-         <Setter Property="FontWeight" Value="Bold" />
-         <Setter Property="HorizontalAlignment" Value="Stretch" />
-         <Setter Property="VerticalAlignment" Value="Stretch" />
-         <Setter Property="Content">
-            <Setter.Value>
-              <TextBlock VerticalAlignment="Center" Text="Buy" />
-            </Setter.Value>
-         </Setter>
-         <Setter Property="Template">
-            <Setter.Value>
-               <ControlTemplate TargetType="Button">
-                  <Border 
-                     x:Name="Border"
-                     CornerRadius="0" 
-                     BorderThickness="1"
-                     BorderBrush="{u:BTSColor UI.Button.normal_border}"
-                     Background="{u:BTSColor MarketData.bid}"
-                     Padding="2"
-                     Height="Auto"
-                     SnapsToDevicePixels="True">
-                     <ContentPresenter 
-                        HorizontalAlignment="Center"
-                        VerticalAlignment="Center"
-                        RecognizesAccessKey="True"
-                        Name="button_content" 
-                     />
-                  </Border>
-                  <ControlTemplate.Triggers>
-                     <Trigger Property="IsMouseOver" Value="True">
-                        <Setter TargetName="Border" Property="Background" Value="#99167ed3" />
-                        <Setter TargetName="Border" Property="BorderBrush" Value="{u:BTSColor UI.Button.mouseover_border}" />
-                     </Trigger>
-                  </ControlTemplate.Triggers>
-               </ControlTemplate>
-            </Setter.Value>
-         </Setter>
-      </Style>
-      </ResourceDictionary>
-  </Window.Resources>
-  <Window.InputBindings>
-    <KeyBinding Key="Esc" Command="{x:Static SystemCommands.CloseWindowCommand}"/>
-  </Window.InputBindings>
-  <Grid Margin="10">
-    <Grid.RowDefinitions>
-      <RowDefinition Height="Auto"/>
-      <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}"/>
-      <RowDefinition Height="Auto"/>
-    </Grid.RowDefinitions>
-    <Grid.ColumnDefinitions>
-      <ColumnDefinition MinWidth="120"/>
-      <ColumnDefinition Width="{Binding Source={StaticResource HorizontalSpacing}}"/>
-      <ColumnDefinition MinWidth="120"/>
-    </Grid.ColumnDefinitions>
-    <usp:OrderModificationControl Grid.ColumnSpan="3" />
-      <Button Grid.Row="2"                 Style="{StaticResource BuyButton}"  Command="{Binding send_order}" CommandParameter="{u:Enum Buy, Type=msys:Side}" />
-      <Button Grid.Row="2" Grid.Column="2" Style="{StaticResource SellButton}" Command="{Binding send_order}" CommandParameter="{u:Enum Sell, Type=msys:Side}" />
-   </Grid>
-  <u:WindowStyle.TitleContentTemplate>
-    <DataTemplate>
-      <TextBlock Style="{StaticResource TextBlock.Dark}">
-        <TextBlock.Inlines>
-          <Run FontWeight="Bold" Text="{Binding security, Mode=OneWay}" />
-          <Run Foreground="{u:BTSColor MarketData.bid}" FontSize="10">
-            <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}" StringFormat="({0})">
-              <Binding Path="bid_size" Mode="OneWay"/>
-              <Binding Path="DataContext.qty_display" RelativeSource="{RelativeSource AncestorType=usp:OrderTicket}"/>
-            </MultiBinding>
-          </Run>
-          <Run Foreground="{u:BTSColor MarketData.bid}" FontWeight="Bold">
-            <Run.Text>
-              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
-                <Binding Path="bid"/>
-                <Binding Path="security.ContractSpec"/>
-              </MultiBinding>
-            </Run.Text>
-          </Run>
-          <Run Foreground="{u:BTSColor MarketData.ask}" FontWeight="Bold">
-            <Run.Text>
-              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
-                <Binding Path="ask"/>
-                <Binding Path="security.ContractSpec"/>
-              </MultiBinding>
-            </Run.Text>
-          </Run>
-          <Run Foreground="{u:BTSColor MarketData.ask}" FontSize="10">
-            <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}" StringFormat="({0})">
-              <Binding Path="ask_size" Mode="OneWay"/>
-              <Binding Path="DataContext.qty_display" RelativeSource="{RelativeSource AncestorType=usp:OrderTicket}"/>
-            </MultiBinding>
-          </Run>
-        </TextBlock.Inlines>
-      </TextBlock>
-    </DataTemplate>
-  </u:WindowStyle.TitleContentTemplate>
-</Window>
+<Window
+  x:Class="mbts.ui.spark.OrderTicket"
+  xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+  xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+  xmlns:msys="clr-namespace:bts.api;assembly=spark_api"
+  xmlns:u="clr-namespace:mbts.ui"
+  xmlns:cv="clr-namespace:mbts.ui.converters"
+  xmlns:usp="clr-namespace:mbts.ui.spark"
+  xmlns:sys="clr-namespace:System;assembly=mscorlib"
+  u:WindowStyle.SizeToContent="WidthAndHeight"
+  u:WindowStyle.TitleContent="{Binding bbo, Mode=OneWay}"
+  Style="{StaticResource BTSWindowTitleTemplate}"
+>
+  <Window.Resources>
+    <ResourceDictionary>
+      <ResourceDictionary.MergedDictionaries>
+        <ResourceDictionary Source="/view/Resources/Converters.xaml"/>
+      </ResourceDictionary.MergedDictionaries>
+      <sys:Double x:Key="HorizontalSpacing">8.0</sys:Double>
+      <sys:Double x:Key="VerticalSpacing">4.0</sys:Double>
+      <sys:Double x:Key="ButtonHeight">22.0</sys:Double>
+      <Style x:Key="SellButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
+         <Setter Property="Background" Value="{u:BTSColor MarketData.ask}" />
+         <Setter Property="Foreground" Value="White" />
+         <Setter Property="MinHeight" Value="40" />
+         <Setter Property="FontWeight" Value="Bold" />
+         <Setter Property="HorizontalAlignment" Value="Stretch" />
+         <Setter Property="VerticalAlignment" Value="Stretch" />
+         <Setter Property="Content">
+            <Setter.Value>
+              <TextBlock VerticalAlignment="Center" Text="Sell" />
+            </Setter.Value>
+         </Setter>
+         <Setter Property="Template">
+            <Setter.Value>
+               <ControlTemplate TargetType="Button">
+                  <Border 
+                        x:Name="Border"
+                        CornerRadius="0" 
+                        BorderThickness="1"
+                        BorderBrush="{u:BTSColor UI.Button.normal_border}"
+                        Background="{u:BTSColor MarketData.ask}"
+                        Padding="2"
+                        Height="Auto"
+                        SnapsToDevicePixels="True">
+                        <ContentPresenter 
+                           HorizontalAlignment="Center"
+                           VerticalAlignment="Center"
+                           RecognizesAccessKey="True"
+                           Name="button_content" 
+                        />
+                     </Border>
+                  <ControlTemplate.Triggers>
+                        <Trigger Property="IsMouseOver" Value="True">
+                           <Setter TargetName="Border" Property="Background" Value="#99c00000" />
+                           <Setter TargetName="Border" Property="BorderBrush" Value="{u:BTSColor UI.Button.mouseover_border}" />
+                        </Trigger>
+                     </ControlTemplate.Triggers>
+               </ControlTemplate>
+            </Setter.Value>
+         </Setter>
+      </Style>
+      <Style x:Key="BuyButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
+         <Setter Property="Background" Value="{u:BTSColor MarketData.bid}" />
+         <Setter Property="Foreground" Value="White" />
+         <Setter Property="MinHeight" Value="40" />
+         <Setter Property="FontWeight" Value="Bold" />
+         <Setter Property="HorizontalAlignment" Value="Stretch" />
+         <Setter Property="VerticalAlignment" Value="Stretch" />
+         <Setter Property="Content">
+            <Setter.Value>
+              <TextBlock VerticalAlignment="Center" Text="Buy" />
+            </Setter.Value>
+         </Setter>
+         <Setter Property="Template">
+            <Setter.Value>
+               <ControlTemplate TargetType="Button">
+                  <Border 
+                     x:Name="Border"
+                     CornerRadius="0" 
+                     BorderThickness="1"
+                     BorderBrush="{u:BTSColor UI.Button.normal_border}"
+                     Background="{u:BTSColor MarketData.bid}"
+                     Padding="2"
+                     Height="Auto"
+                     SnapsToDevicePixels="True">
+                     <ContentPresenter 
+                        HorizontalAlignment="Center"
+                        VerticalAlignment="Center"
+                        RecognizesAccessKey="True"
+                        Name="button_content" 
+                     />
+                  </Border>
+                  <ControlTemplate.Triggers>
+                     <Trigger Property="IsMouseOver" Value="True">
+                        <Setter TargetName="Border" Property="Background" Value="#99167ed3" />
+                        <Setter TargetName="Border" Property="BorderBrush" Value="{u:BTSColor UI.Button.mouseover_border}" />
+                     </Trigger>
+                  </ControlTemplate.Triggers>
+               </ControlTemplate>
+            </Setter.Value>
+         </Setter>
+      </Style>
+      </ResourceDictionary>
+  </Window.Resources>
+  <Window.InputBindings>
+    <KeyBinding Key="Esc" Command="{x:Static SystemCommands.CloseWindowCommand}"/>
+  </Window.InputBindings>
+  <Grid Margin="10">
+    <Grid.RowDefinitions>
+      <RowDefinition Height="Auto"/>
+      <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}"/>
+      <RowDefinition Height="Auto"/>
+    </Grid.RowDefinitions>
+    <Grid.ColumnDefinitions>
+      <ColumnDefinition MinWidth="120"/>
+      <ColumnDefinition Width="{Binding Source={StaticResource HorizontalSpacing}}"/>
+      <ColumnDefinition MinWidth="120"/>
+    </Grid.ColumnDefinitions>
+    <usp:OrderModificationControl Grid.ColumnSpan="3" />
+      <Button Grid.Row="2"                 Style="{StaticResource BuyButton}"  Command="{Binding send_order}" CommandParameter="{u:Enum Buy, Type=msys:Side}" />
+      <Button Grid.Row="2" Grid.Column="2" Style="{StaticResource SellButton}" Command="{Binding send_order}" CommandParameter="{u:Enum Sell, Type=msys:Side}" />
+   </Grid>
+  <u:WindowStyle.TitleContentTemplate>
+    <DataTemplate>
+      <TextBlock Style="{StaticResource TextBlock.Dark}">
+        <TextBlock.Inlines>
+          <Run FontWeight="Bold" Text="{Binding security, Mode=OneWay}" />
+          <Run Foreground="{u:BTSColor MarketData.bid}" FontSize="10">
+            <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}" StringFormat="({0})">
+              <Binding Path="bid_size" Mode="OneWay"/>
+              <Binding Path="DataContext.qty_display" RelativeSource="{RelativeSource AncestorType=usp:OrderTicket}"/>
+            </MultiBinding>
+          </Run>
+          <Run Foreground="{u:BTSColor MarketData.bid}" FontWeight="Bold">
+            <Run.Text>
+              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
+                <Binding Path="bid"/>
+                <Binding Path="security.ContractSpec"/>
+              </MultiBinding>
+            </Run.Text>
+          </Run>
+          <Run Foreground="{u:BTSColor MarketData.ask}" FontWeight="Bold">
+            <Run.Text>
+              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
+                <Binding Path="ask"/>
+                <Binding Path="security.ContractSpec"/>
+              </MultiBinding>
+            </Run.Text>
+          </Run>
+          <Run Foreground="{u:BTSColor MarketData.ask}" FontSize="10">
+            <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}" StringFormat="({0})">
+              <Binding Path="ask_size" Mode="OneWay"/>
+              <Binding Path="DataContext.qty_display" RelativeSource="{RelativeSource AncestorType=usp:OrderTicket}"/>
+            </MultiBinding>
+          </Run>
+        </TextBlock.Inlines>
+      </TextBlock>
+    </DataTemplate>
+  </u:WindowStyle.TitleContentTemplate>
+</Window>

src/client/spark/Risk/PNL/PNLTradeLogRow.cs

@@ -4,9 +4,6 @@ using bts.api;
 
 using BCG;
 using mbts.co2;
-using System.Xml.Serialization;
-using ZstdSharp.Unsafe;
-using bts.proto;
 
 namespace mbts.analytics {
    // Pairs ITradeLogRow along with the data (theo, md) that is needed to compute
@@ -21,87 +18,20 @@ namespace mbts.analytics {
 
       public SecurityImmutable security { get { return md.security; } }
 
-      // private double _option_pnl(double trade_price, double now_price) {
-      //    double coin_terms = (now_price - trade_price) * tl_row.size;
-      //    double usd_terms = coin_terms * driver;
-      //    return usd_terms;
-      // }
-
-      private bool _is_notional {
-         get {
-            return false;
-         }
+      private double _option_pnl(double trade_price, double now_price) {
+         double coin_terms = (now_price - trade_price) * tl_row.size;
+         double usd_terms = coin_terms * driver;
+         return usd_terms;
       }
 
-      private bool _is_inverse_option_contract {
-         get {
-            return false;
-         }
+      private double _future_pnl(double trade_price, double now_price) {
+         // the qty is in USD, so this is the initial cash flow
+         double init_asset_value = tl_row.size;
+         double asset_size = init_asset_value / tl_row.price;
+         double now_asset_value = asset_size * now_price;
+         return now_asset_value - init_asset_value;
       }
 
-      private double _cpp {
-         get {
-            return 1.0;
-         }
-      }
-
-      private bool _is_reverse_option_contract {
-         get {
-            // check for Deribit options
-            return false;
-         }
-      }
-
-      private double _option_pnl {
-         get {
-            if (_is_notional) {
-               var now_price = theo;
-               double coin_terms = (now_price - tl_row.price) * tl_row.size;
-               double usd_terms = coin_terms * driver;
-               return usd_terms;
-            } else {
-               return _cpp * tl_row.size * (theo - tl_row.price);
-            }
-         }
-      }
-
-      private double _spot_pnl {
-         get {
-            // both are computed the same way
-            return _future_pnl;
-         }
-      }
-
-      private double _future_pnl {
-         get {
-            if (_is_notional) {
-               // TODO:
-               // let it fall through right now
-               // the qty is in USD, so this is the initial cash flow
-               var now_price = theo;
-               double init_asset_value = tl_row.size;
-               double asset_size = init_asset_value / tl_row.price;
-               double now_asset_value = asset_size * now_price;
-               return now_asset_value - init_asset_value;
-            } else {
-               // TODO: is position pnl, compute it wrt settlement price?
-               return tl_row.size * (theo - tl_row.price);
-            }
-         }
-      }
-
-      // private double _future_pnl(double trade_price, double now_price) {
-      //    if (_is_notional) {
-      //       // the qty is in USD, so this is the initial cash flow
-      //       double init_asset_value = tl_row.size;
-      //       double asset_size = init_asset_value / tl_row.price;
-      //       double now_asset_value = asset_size * now_price;
-      //       return now_asset_value - init_asset_value;
-      //    } else {
-      //       return tl_row.size * (theo - tl_row.price);
-      //    }
-      // }
-
 
       public double driver {
          get {
@@ -116,16 +46,9 @@ namespace mbts.analytics {
       public double theo {
          get {
             switch (tl_row.security_type) {
-               // TODO:
                case SecurityType.Option:
-                  if (_is_inverse_option_contract) {
-                     return tl_row.call_put == PutOrCall.Call ?
-                        opt_theo.fwd_normalized_call : opt_theo.fwd_normalized_put;
-                  } else {
-                     return tl_row.call_put == PutOrCall.Call ?
-                        opt_theo.call : opt_theo.put;
-                  }
-               // future and options, just get mid market
+                  return tl_row.call_put == PutOrCall.Call ?
+                     opt_theo.fwd_normalized_call : opt_theo.fwd_normalized_put;
                case SecurityType.Future:
                //return fut_theo.theo;
                case SecurityType.Spot:
@@ -139,22 +62,28 @@ namespace mbts.analytics {
       public double pnl {
          get {
             double rv = Double.NaN;
-            switch (tl_row.security_type) {
-               case SecurityType.Option:
-                  rv = _option_pnl;
-                  break;
-               case SecurityType.Future:
-                  rv = _future_pnl;
-                  break;
-               case SecurityType.Spot:
-                  // = tl_row.size * (theo - tl_row.price);
-                  rv = _spot_pnl;
-                  break;
+            if (is_day_trade) {
+               if (tl_row.security_type == SecurityType.Future) {
+                  rv = _future_pnl(tl_row.price, theo);
+               } else if (tl_row.security_type == SecurityType.Option) {
+                  rv = _option_pnl(tl_row.price, theo);
+               } else if (tl_row.security_type == SecurityType.Spot) {
+                  rv = tl_row.size * (theo - tl_row.price);
+               }
+               return rv;
             }
+            // use settle price ?
+            if (tl_row.security_type == SecurityType.Future) {
+               rv = _future_pnl(tl_row.price, theo);
+            } else if (tl_row.security_type == SecurityType.Option) {
+               rv = _option_pnl(tl_row.price, theo);
+            } else if (tl_row.security_type == SecurityType.Spot) {
+               rv = tl_row.size * tl_row.price - settle_price;
+            }
+
             return rv;
          }
       }
-
       public double settle_price {
          get {
             return md.md == null ? Double.NaN : md.md.settlement_price;

src/client/spark/Security/SecurityExpirationHelper.cs

@@ -36,8 +36,9 @@ namespace mbts {
       }
 
       public static bool is_unexpired(bts.api.SecurityTuple st, DateTime exp_date) {
-         if (st.Venue == Venue.Cboe) {
-            return exp_date.Date >= DateTime.Now.Date;
+         if (st.Venue == Venue.Cboe || st.Venue == Venue.Glbx) {
+            // server is already getting teh righ contracts
+            return true;
          }
          TimeSpan exp_time = get_exp_time(st);
          DateTime full_exp_utc = exp_date.ToUniversalTime().Date.Add(exp_time);
@@ -46,12 +47,8 @@ namespace mbts {
       }
 
       public static bool is_unexpired(bts.api.SecurityImmutable sec) {
-         if (sec.security_tuple().Venue == Venue.Cboe || sec.security_tuple().Venue == Venue.Glbx) {
-            var exp = sec.to_sec_exp().expiration;
-            bool is_unexp = exp.Date >= DateTime.Now.Date;
-//            Log.Gen.info($"SecurityExpirationMapper.is_unexpired exp: {exp} is_unexp: {is_unexp}");
-            return is_unexp;
-         }
+         if (sec.security_tuple().Venue == Venue.Cboe || sec.security_tuple().Venue == Venue.Glbx)
+            return true;
          var sec_type = sec.security_type();
          switch (sec_type) {
             case bts.api.SecurityType.Future:

src/client/spark/core/Bus/SparkTags.cs

@@ -353,8 +353,11 @@ namespace mbts.spark {
 
       public static ITag<Nothing> ShowGlobalAlertsTag { get; } = _mk_tag<Nothing>();
 
-      public static ITag<ArchivedTheoListImmutable>
-      ArchivedTheos { get; } = _mk_tag<ArchivedTheoListImmutable>();
+      public static ITag<ClientArchivedVolRequest> RequestArchivedVols { get; } = _mk_tag<ClientArchivedVolRequest>();
+
+      public static ITag<Tuple<ClientArchivedVolRequest, bts.api.analytics.ArchivedTheoList>>
+      ResponseArchivedVols { get; } =
+         _mk_tag<Tuple<ClientArchivedVolRequest, bts.api.analytics.ArchivedTheoList>>();
 
       static SparkTag<T> _mk_tag<T>([CallerMemberName] string name = null) =>
          new(name ?? throw new ArgumentNullException(nameof(name)));
@@ -370,7 +373,6 @@ namespace mbts.spark {
       public class Nothing { Nothing() { } }
 
       public class TC2Tags {
-         public static ITag<NMSheetSymbol> ArchiveIVsTag {get;} = _mk_tag<NMSheetSymbol>();
          public static ITag<NMSheetAllTaskingData> TaskData { get; } =
             _mk_tag<NMSheetAllTaskingData>();
          // Data request for a sheet 
@@ -384,11 +386,6 @@ namespace mbts.spark {
          public static ITag<ExpStrikeKeyedData<TheoPoint>> LiveTheosTag((string, Venue) symbol) =>
             _mk_tag<ExpStrikeKeyedData<TheoPoint>, (string, Venue)>(symbol);
 
-         // these are the theos at time of the vol save (we use it to extrapolate the theos prices for the spreads)
-         public static ITag<ExpStrikeKeyedData<TheoPoint>> VolSaveEventTheosTag((string, Venue) symbol) =>
-            _mk_tag<ExpStrikeKeyedData<TheoPoint>, (string, Venue)>(symbol);
-
-
          // Aggregated version of the above
          public static ITag<IReadOnlyDictionary<(string, Venue), ExpStrikeKeyedData<TheoPoint>>>
          AllLiveTheosTag { get; } =

src/client/spark/core/FontsAndColors/BTSColors.cs

@@ -216,10 +216,8 @@ namespace mbts.ui {
          public static BTSColor ask_text => BTSCoreColors.red_dark;
          public static BTSColor bid_text => BTSCoreColors.blue;
          public static BTSColor bid => 0xff76b7fa;
-         public static BTSColor bid_dark => 0xff26239a;
          public static BTSColor bid_alt => BTSCoreColors.blue_light.with(0x70);
          public static BTSColor ask => 0xffff6666;
-         public static BTSColor ask_dark => 0xff7a2331;
          public static BTSColor ask_alt => 0x70ffb4b4;
          public static BTSColor last => BTSCoreColors.black;
          public static BTSColor bid_ask_alt_blend => BTSCoreColors.purple_light;
@@ -1040,7 +1038,6 @@ namespace mbts.ui {
       }
       public static class DayFills {
          public static BTSColor background => UI.common_background;
-         public static BTSColor foreground => UI.default_foreground;
       }
       public static class ComboReport {
          public static BTSColor foreground => UI.default_text;

src/client/spark/view/Controls/Order/OrderTicketResources.xaml

@@ -19,7 +19,7 @@
   <sys:Double x:Key="ButtonHeight">22.0</sys:Double>
 
   <Style x:Key="SellButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
-    <Setter Property="Background" Value="{u:BTSBrush MarketData.ask_dark}"/>
+    <Setter Property="Background" Value="{u:BTSColor MarketData.ask}"/>
     <Setter Property="BorderBrush" Value="{u:BTSColor UI.Button.normal_border}"/>
     <Setter Property="Foreground" Value="{u:BTSColor UI.default_text}" />
     <Setter Property="MinHeight" Value="40"/>
@@ -32,7 +32,7 @@
   </Style>
 
   <Style x:Key="BuyButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
-    <Setter Property="Background" Value="{u:BTSBrush MarketData.bid_dark}"/>
+    <Setter Property="Background" Value="{u:BTSColor MarketData.bid}"/>
     <Setter Property="BorderBrush" Value="{u:BTSColor UI.Button.normal_border}"/>
     <Setter Property="Padding" Value="2"/>
     <Setter Property="Foreground" Value="{u:BTSColor UI.default_text}" />

src/client/spark/view/Controls/TickUpDown/PriceEntryCommitControl.xaml

@@ -8,8 +8,7 @@
              xmlns:cv="clr-namespace:mbts.ui.converters"
              xmlns:c="clr-namespace:mbts.ui.controls"
              xmlns:usp="clr-namespace:mbts.ui.spark"
-             d:DesignHeight="100" d:DesignWidth="800" x:Name="_me_price"
-             Foreground="{u:BTSColor UI.input_text}">
+             d:DesignHeight="100" d:DesignWidth="800" x:Name="_me_price">
   <UserControl.Resources>
     <cv:NullableDoubleConverter x:Key="TickDefaultPriceConverter" DefaultValue="0"/>
     <cv:TickDisplayConverter x:Key="TickDisplayConverter" x:Shared="False"/>

src/client/spark_api/MarketData/FakeMDConsumerHandle.cs

@@ -12,6 +12,10 @@ namespace mbts.co2 {
 
       readonly ulong _numeric_key;
 
+      public void set_bbo(double bid_price, UInt32 bid_qty, double ask_price, UInt32 ask_qty) {
+
+      }
+
       public FakeMDConsumerHandle(
          security_key key,
          ulong numeric_key,
@@ -52,6 +56,17 @@ namespace mbts.co2 {
       public double session_low => 500;
       public double fixing_price => 20;
 
+      public bool get_bbo(out double bid_price, out float bid_sz, out double ask_price, out float ask_sz) {
+         NativePriceSize bid;
+         NativePriceSize ask;
+         ulong time = 0;
+         bool rv = get_bbo(ref time, out bid, out ask);
+         bid_price = bid.price;
+         bid_sz = bid.size;
+         ask_price = ask.price;
+         ask_sz = ask.size;
+         return rv;
+      }
 
       public bool get_bbo(ref ulong time, out NativePriceSize bid, out NativePriceSize ask) {
          bool rv = false;

src/client/spark_api/MarketData/IMDConsumerHandle.cs

@@ -7,6 +7,7 @@ namespace mbts.co2 {
       bool is_heart_beating { get; }
       bool has_depth { get; }
       security_key key { get; }
+      public void set_bbo(double bid_price, UInt32 bid_qty, double ask_price, UInt32 ask_qty);
 
       // The exchange key can change at CBOE. This numeric_id will always be correct
       ulong numeric_id { get; }
@@ -32,6 +33,7 @@ namespace mbts.co2 {
       double session_low { get; }
       double fixing_price { get; }
 
+      bool get_bbo(out double bid_price, out float bid_sz, out double ask_price, out float ask_sz);
       bool get_bbo(ref ulong time, out NativePriceSize bid, out NativePriceSize ask);
 
       bool get_depth(

src/client/spark_api/MarketData/IntPrice.cs

@@ -4,13 +4,38 @@ using System.Runtime.InteropServices;
 namespace mbts.co2 {
    [StructLayout(LayoutKind.Explicit)]
    public readonly struct IntPrice : IEquatable<IntPrice> {
-      public static readonly IntPrice NaN = new(0,127);
+      public static readonly IntPrice NaN = new(0, 127);
 
       [FieldOffset(0)]
       readonly long _mantissa;
       [FieldOffset(7)]
       readonly sbyte _exponent;
 
+      private static bool _is_invalid_double(double value) {
+         return (Double.IsNaN(value) || double.IsInfinity(value) ||
+            value >= long.MaxValue || value <= long.MinValue);
+      }
+
+      private static Tuple<long, sbyte> double_to_mant_exp(double v) {
+         if (_is_invalid_double(v)) {
+            return Tuple.Create<long, sbyte>(0, 127);
+         }
+         sbyte exp = -7;
+         double v_m = v * Math.Pow(10, Math.Abs(exp));
+         if (_is_invalid_double(v_m)) {
+            return Tuple.Create<long, sbyte>(0, 127);
+         }
+         long m = Convert.ToInt64(v_m);
+         return Tuple.Create<long, sbyte>(m, exp);
+      }
+
+      public IntPrice(double v) :
+            this(double_to_mant_exp(v)) {
+      }
+
+      public IntPrice(Tuple<long, sbyte> t)
+      : this(t.Item1, t.Item2) { }
+
       public IntPrice(long mantissa, sbyte exponent) {
          _mantissa = mantissa;
          _exponent = exponent;
@@ -38,10 +63,10 @@ namespace mbts.co2 {
       public static bool operator ==(IntPrice a, IntPrice b) => a.Equals(b);
       public static bool operator !=(IntPrice a, IntPrice b) => !a.Equals(b);
 
-      public static bool operator < (IntPrice a, IntPrice b) =>
+      public static bool operator <(IntPrice a, IntPrice b) =>
          a.exponent == b.exponent ? a.mantissa < b.mantissa : a.price() < b.price();
 
-      public static bool operator > (IntPrice a, IntPrice b) =>
+      public static bool operator >(IntPrice a, IntPrice b) =>
          a.exponent == b.exponent ? a.mantissa > b.mantissa : a.price() > b.price();
    }
 }

src/client/spark_api/MarketData/MDConsumerHandle.cs

@@ -11,6 +11,9 @@
    using security_key = System.UInt32;
 
    public unsafe class MDConsumerHandle : IMDConsumerHandle {
+      public void set_bbo(double bid_price, UInt32 bid_qty, double ask_price, UInt32 ask_qty) {
+         throw new NotImplementedException();
+      }
       readonly HotEntry* _hot;
       readonly HotDepthEntry* _hot_depth;
       readonly ColdEntry* _cold;
@@ -74,6 +77,19 @@
       public double session_low => _cold->session_low;
       public double fixing_price => _cold->fixing_price;
 
+      public bool get_bbo(out double bid_price, out float bid_sz, out double ask_price, out float ask_sz) {
+         NativePriceSize bid;
+         NativePriceSize ask;
+         ulong time = 0;
+         bool rv = get_bbo(ref time, out bid, out ask);
+         bid_price = bid.price;
+         bid_sz = bid.size;
+         ask_price = ask.price;
+         ask_sz = ask.size;
+         return rv;
+      }
+
+
       public bool get_bbo(ref ulong time, out NativePriceSize bid, out NativePriceSize ask) {
          var updated = market_timestamp;
          if (updated <= time || updated == TimeStamp.invalid_value) {

src/client/spark_api/MarketData/SecurityMD.cs

@@ -13,9 +13,9 @@ namespace mbts {
    using System.Linq;
    using System.Threading;
 
-   public readonly struct SecurityMD : IEquatable<SecurityMD> {
+   public /*readonly*/ struct SecurityMD : IEquatable<SecurityMD> {
       public static readonly Func<SecurityMD, uint> key_fn = smd => smd.security.Id;
-
+      private IMDConsumerHandle _md;
       public SecurityMD(
          SecurityImmutable security,
          IMDConsumerHandle md,
@@ -23,7 +23,7 @@ namespace mbts {
          TickDisplaySettingsImmutable? quantity_display = null,
          IEnumerable<TickDisplayRuleImmutable>? price_display_rules = null) {
          this.security = security;
-         this.md = md;
+         _md = md;
          this.price_display = price_display ?? Security.Defaults.price_display_settingsi;
          this.quantity_display = quantity_display ??
             Security.Defaults.quantity_display_settingsi;
@@ -38,7 +38,14 @@ namespace mbts {
 
       public SecurityImmutable security { get; }
       public uint id => security.Id;
-      public IMDConsumerHandle md { get; }
+      public IMDConsumerHandle md {
+         get {
+            return _md;
+         }
+         set {
+            _md = value;
+         }
+      }
       public TickDisplaySettingsImmutable price_display { get; }
       public TickDisplaySettingsImmutable quantity_display { get; }
       public ImmutableList<TickDisplayRuleImmutable> price_display_rules { get; }

src/client/spark_api/data/SecurityCriteria.cs

@@ -27,12 +27,6 @@ namespace mbts.spark {
       internal static SecurityCriteria from_keys(IEnumerable<uint> keys) =>
          new(new[] { new SecurityCriteriaClause(security_keys: keys.ToArray()) });
 
-      internal static SecurityCriteria from_expirations(params DateTime[] exp_dates) =>
-         from_expirations(exp_dates.AsEnumerable());
-
-      internal static SecurityCriteria from_expirations(IEnumerable<DateTime> expirations) =>
-         new(new[] { SecurityCriteriaClause.from_expirations(expirations) });
-
       internal static SecurityCriteria from_securities(params SecurityImmutable[] securities) =>
          from_securities(securities.AsEnumerable());
 

src/client/spark_api/data/SecurityCriteriaClause.cs

@@ -58,14 +58,6 @@ namespace mbts.spark {
          return new SecurityCriteriaClause(security_keys: keys, pretty_name: pretty_name);
       }
 
-      public static SecurityCriteriaClause from_expirations(IEnumerable<DateTime> expirations) =>
-         expirations is null
-            ? new()
-            : from_expirations(expirations as DateTime[] ?? expirations.ToArray());
-
-      public static SecurityCriteriaClause from_expirations(params DateTime[] expirations) =>
-         new(expirations: expirations);
-
       [ProtoMember(1)]
       public string? pretty_name { get; }
 

src/lib/api/md/DatabentoMarketDataConfig.proto

@@ -16,6 +16,4 @@ message DatabentoMarketDataConfig {
   uint32 filter_securities_upto_n_months= 80;
   bool load_reference_data = 90;
   uint32 debug_sec = 100;
-  bool only_securities =  110;
-  string bbo_start_time = 120;
 }

src/lib/api/services/AnalyticsService.proto

@@ -41,7 +41,7 @@ message VolSurfaceUsageList {
 }
 
 message ArchivedSecurityTheo {
-  uint32 id = 10[ (bts.api.key) = true ];
+  uint32 id = 10;
   double driver = 20;
   double vol = 30;
   double theo = 40;
@@ -49,7 +49,7 @@ message ArchivedSecurityTheo {
 }
 
 message ArchivedTheoList {
-  uint64 save_time = 10[ (bts.api.key) = true ];
+  uint64 save_time = 10;
   // these are sec_tups that are part of this save
   repeated bts.api.SecurityTuple sec_tups = 20;
   repeated ArchivedSecurityTheo sec_theo = 30;
@@ -195,10 +195,8 @@ service AnalyticsService {
 
   /// theos
   rpc archive_theos(ArchivedTheoList) returns (bts.api.ResultMessage);
-  rpc clean_and_archive_theos(ArchivedTheoList) returns (bts.api.ResultMessage);
   // finds the archived_theo closest to this point
   rpc get_archived_theos(ArchivedTheoRetrieval) returns (ArchivedTheoList);
-  rpc archived_theos(bts.api.VoidMessage) returns (stream ArchivedTheoList);
 
   /// reports
   rpc get_new_report_id(bts.api.VoidMessage)

src/lib/api/services/OrderService.cpp

@@ -1,174 +0,0 @@
-#include "OrderService.h"
-#include "lib/api/services/OrderService.pb.h"
-
-namespace bts::api::orderman {
-
-#define PARSE_STRING(name, dest, it)                                           \
-   it = j.find(#name);                                                         \
-   if (it != j.end()) {                                                        \
-      dest.mutable_##name()->clear();                                          \
-      it->get_to(*dest.mutable_##name());                                      \
-   }
-#define PARSE_VALUE(name, dest, it)                                            \
-   it = j.find(#name);                                                         \
-   if (it != j.end()) {                                                        \
-      dest.set_##name(it->get<decltype(dest.name())>());                       \
-   }
-#define PARSE_STRUCT(name, dest, it)                                           \
-   it = j.find(#name);                                                         \
-   if (it != j.end()) {                                                        \
-      it->get_to(*dest.mutable_##name());                                      \
-   }
-#define PARSE_ARRAY(name, dest, it)                                            \
-   it = j.find(#name);                                                         \
-   if (it != j.end() && it->is_array()) {                                      \
-      c.mutable_##name()->Clear();                                             \
-      it->get_to(*c.mutable_##name());                                         \
-   }
-#define PARSE_ONEOF(name, dest, it)                                            \
-   if (!parsed_oneof) {                                                        \
-      it = j.find(#name);                                                      \
-      if (it != j.end()) {                                                     \
-         it->get_to(*dest.mutable_##name());                                   \
-         parsed_oneof = true;                                                  \
-      }                                                                        \
-   }
-#define PARSE_ONEOFS(for_each)                                                 \
-   {                                                                           \
-      bool parsed_oneof = false;                                               \
-      for_each(PARSE_ONEOF)                                                    \
-   }
-
-#define WRITE_VALUE(src, name) j[#name] = src.name()
-#define WRITE_STRUCT(src, name)                                                \
-   if (src.has_##name()) {                                                     \
-      j[#name] = src.name();                                                   \
-   }
-
-#define WRITE_IF_NOT_ZERO(src, name)                                           \
-   if (src.name() != 0) {                                                      \
-      j[#name] = src.name();                                                   \
-   }
-#define WRITE_IF_NOT_NAN(src, name)                                            \
-   if (!std::isnan(src.name())) {                                              \
-      j[#name] = src.name();                                                   \
-   }
-#define WRITE_IF_NOT_EMPTY(src, name)                                          \
-   if (!src.name().empty()) {                                                  \
-      j[#name] = src.name();                                                   \
-   }
-#define WRITE_TIMESTAMP(src, name)                                             \
-   j[#name] = bts::core::TimeStamp::from_nanos(                                \
-                  src.name().seconds() * 1'000'000'000 + src.name().nanos())   \
-                  .to_iso_string()
-
-#define PARSE_TIMESTAMP(name, dest, it)                                        \
-   it = j.find(#name);                                                         \
-   if (it != j.end()) {                                                        \
-      bts::core::TimeStamp tmp_ts =                                            \
-          bts::core::TimeStamp::from_iso_string(it->get<std::string>());       \
-      tmp_ts.to_protobuf(*dest.mutable_##name());                              \
-   }
-
-void to_json(nlohmann::json &j, const Fill &f) {
-   WRITE_IF_NOT_ZERO(f, order_key);
-   WRITE_IF_NOT_ZERO(f, security_key);
-   WRITE_IF_NOT_EMPTY(f, account);
-   WRITE_IF_NOT_NAN(f, last_price);
-   WRITE_IF_NOT_NAN(f, avg_price);
-   WRITE_VALUE(f, side);
-   WRITE_IF_NOT_ZERO(f, last_shares);
-   WRITE_IF_NOT_EMPTY(f, fill_key);
-   WRITE_TIMESTAMP(f, timestamp);
-   WRITE_IF_NOT_ZERO(f, internal_account_id);
-   WRITE_VALUE(f, fill_kind);
-   WRITE_VALUE(f, fill_status);
-}
-
-void from_json(const nlohmann::json &j, Fill &f) {
-   nlohmann::json::const_iterator it;
-   PARSE_VALUE(order_key, f, it);
-   PARSE_VALUE(security_key, f, it);
-   PARSE_STRING(account, f, it);
-   PARSE_VALUE(last_price, f, it);
-   PARSE_VALUE(avg_price, f, it);
-   PARSE_VALUE(side, f, it);
-   PARSE_VALUE(last_shares, f, it);
-   PARSE_STRING(fill_key, f, it);
-   PARSE_TIMESTAMP(timestamp, f, it);
-   PARSE_VALUE(internal_account_id, f, it);
-   PARSE_VALUE(fill_kind, f, it);
-   PARSE_VALUE(fill_status, f, it);
-}
-
-void to_json(nlohmann::json &j, const NativeOrderState &n) {
-   WRITE_VALUE(n, exchange_order_id);
-   WRITE_VALUE(n, last_msg_seq_num);
-   WRITE_VALUE(n, sub_error_code);
-   WRITE_VALUE(n, native_status);
-   WRITE_VALUE(n, routing_record);
-   WRITE_VALUE(n, native_type);
-   WRITE_VALUE(n, last_transact_time);
-   WRITE_VALUE(n, last_transact_time_unix_ns);
-   WRITE_VALUE(n, stop_px);
-   WRITE_VALUE(n, exchange_order_id_u64);
-}
-
-void from_json(const nlohmann::json &j, NativeOrderState &n) {
-   nlohmann::json::const_iterator it;
-   PARSE_STRING(exchange_order_id, n, it);
-   PARSE_VALUE(last_msg_seq_num, n, it);
-   PARSE_VALUE(sub_error_code, n, it);
-   PARSE_VALUE(native_status, n, it);
-   PARSE_STRUCT(routing_record, n, it);
-   PARSE_VALUE(native_type, n, it);
-   PARSE_STRING(last_transact_time, n, it);
-   PARSE_VALUE(last_transact_time_unix_ns, n, it);
-   PARSE_VALUE(stop_px, n, it);
-   PARSE_VALUE(exchange_order_id_u64, n, it);
-}
-
-void to_json(nlohmann::json &j, const RoutingRecord &r) {
-   WRITE_IF_NOT_EMPTY(r, session);
-   WRITE_IF_NOT_EMPTY(r, account);
-   WRITE_IF_NOT_EMPTY(r, operator_id);
-   WRITE_VALUE(r, cti_code);
-   WRITE_VALUE(r, customer_or_firm);
-   WRITE_IF_NOT_EMPTY(r, smp_id);
-   WRITE_VALUE(r, smp_instruction);
-   WRITE_IF_NOT_EMPTY(r, location);
-   WRITE_IF_NOT_EMPTY(r, cust_order_handing_inst);
-   WRITE_IF_NOT_EMPTY(r, sender_compid);
-   WRITE_IF_NOT_EMPTY(r, clearing_firm);
-   WRITE_IF_NOT_EMPTY(r, customer_account_ref_id);
-   WRITE_IF_NOT_EMPTY(r, account_code);
-   WRITE_IF_NOT_EMPTY(r, company_id);
-   WRITE_IF_NOT_EMPTY(r, authorized_trader);
-   WRITE_IF_NOT_EMPTY(r, authorized_group);
-   WRITE_IF_NOT_EMPTY(r, cust_order_handing_inst);
-   WRITE_IF_NOT_EMPTY(r, order_server_id);
-}
-
-void from_json(const nlohmann::json &j, RoutingRecord &r) {
-   nlohmann::json::const_iterator it;
-   PARSE_STRING(session, r, it);
-   PARSE_STRING(account, r, it);
-   PARSE_STRING(operator_id, r, it);
-   PARSE_VALUE(cti_code, r, it);
-   PARSE_VALUE(customer_or_firm, r, it);
-   PARSE_STRING(smp_id, r, it);
-   PARSE_VALUE(smp_instruction, r, it);
-   PARSE_STRING(location, r, it);
-   PARSE_STRING(cust_order_handling_inst, r, it);
-   PARSE_STRING(sender_compid, r, it);
-   PARSE_STRING(clearing_firm, r, it);
-   PARSE_STRING(customer_account_ref_id, r, it);
-   PARSE_STRING(account_code, r, it);
-   PARSE_STRING(company_id, r, it);
-   PARSE_STRING(authorized_trader, r, it);
-   PARSE_STRING(authorized_group, r, it);
-   PARSE_STRING(cust_order_handing_inst, r, it);
-   PARSE_STRING(order_server_id, r, it);
-}
-
-} // namespace bts::api::orderman

src/lib/api/services/OrderService.h

@@ -1,10 +1,6 @@
 #pragma once
 
 #include "lib/api/services/OrderService.pb.h"
-#include "lib/api/services/native_order_type.h"
-#include "lib/api/services/order_status.h"
-#include "lib/api/services/time_in_force.h"
-#include "lib/api/services/trigger_price_type.h"
 #include "lib/core/EnumMacros.h"
 #include "lib/core/FMT_IMPL.h"
 #include "lib/io/JsonReader.h"
@@ -14,70 +10,22 @@
 
 namespace bts::api::orderman {
 
-PB_ENUM_FROM_JSON(Fill_FillKind)
-PB_ENUM_TO_JSON(Fill_FillKind)
+PB_ENUM_FROM_JSON(OrderStatus)
+PB_ENUM_TO_JSON(OrderStatus)
 
-PB_ENUM_FROM_JSON(Fill_FillStatus)
-PB_ENUM_TO_JSON(Fill_FillStatus)
-
-void to_json(nlohmann::json &j, const Fill &f);
-void from_json(const nlohmann::json &j, Fill &f);
-
-PB_ENUM_FROM_JSON(OrderSubType)
-PB_ENUM_TO_JSON(OrderSubType)
-
-PB_ENUM_FROM_JSON(SelfMatchPreventionInstruction)
-PB_ENUM_TO_JSON(SelfMatchPreventionInstruction)
-
-PB_ENUM_FROM_JSON(CustomerOrFirm)
-PB_ENUM_TO_JSON(CustomerOrFirm)
-
-void to_json(nlohmann::json &j, const RoutingRecord &r);
-void from_json(const nlohmann::json &j, RoutingRecord &r);
-
-PB_ENUM_FROM_JSON(NativeOrderStatus)
-PB_ENUM_TO_JSON(NativeOrderStatus)
-
-PB_ENUM_FROM_JSON(NativeOrderState_SubError)
-PB_ENUM_TO_JSON(NativeOrderState_SubError)
-
-void to_json(nlohmann::json &j, const NativeOrderState &n);
-void from_json(const nlohmann::json &j, NativeOrderState &n);
+PB_ENUM_FROM_JSON(TriggerPriceType)
+PB_ENUM_TO_JSON(TriggerPriceType)
 
 } // namespace bts::api::orderman
 
 FMT_IMPL(
-    bts::api::orderman::Fill_FillKind,
-    k,
-    "{}",
-    bts::api::orderman::Fill_FillKind_Name(k));
-
-FMT_IMPL(
-    bts::api::orderman::Fill_FillStatus,
+    bts::api::orderman::OrderStatus,
     s,
     "{}",
-    bts::api::orderman::Fill_FillStatus(s));
+    bts::api::orderman::OrderStatus_Name(s));
 
 FMT_IMPL(
-    bts::api::orderman::OrderSubType,
+    bts::api::orderman::TriggerPriceType,
     t,
     "{}",
-    bts::api::orderman::OrderSubType_Name(t));
-
-FMT_IMPL(
-    bts::api::orderman::SelfMatchPreventionInstruction,
-    s,
-    "{}",
-    bts::api::orderman::SelfMatchPreventionInstruction_Name(s));
-
-FMT_IMPL(
-    bts::api::orderman::NativeOrderStatus,
-    s,
-    "{}",
-    bts::api::orderman::NativeOrderStatus_Name(s));
-
-FMT_IMPL(
-    bts::api::orderman::NativeOrderState_SubError,
-    s,
-    "{}",
-    bts::api::orderman::NativeOrderState_SubError_Name(s));
+    bts::api::orderman::TriggerPriceType_Name(t));

src/lib/api/services/native_order_type.h

@@ -1,57 +0,0 @@
-#pragma once
-
-#include "lib/api/services/OrderService.pb.h"
-#include "lib/core/EnumMacros.h"
-#include "lib/core/FMT_IMPL.h"
-#include "lib/io/JsonReader.h"
-#include "lib/logging/logging.h"
-#include "lib/utils/string_utils.h"
-
-#include <nlohmann/json.hpp>
-
-namespace bts::api::orderman {
-
-/**
- * Parses NativeOrderType from a string. Unlike `bts::api::orderman::NativeOrderType_Parse`,
- * this implementation is case insensitive, and makes the 'NATIVE_ORDER_TYPE_' prefix optional.
- */
-bool native_order_type_parse_smart(
-    const std::string &str, NativeOrderType *result);
-
-PB_ENUM_FROM_JSON(NativeOrderType)
-PB_ENUM_TO_JSON(NativeOrderType)
-
-std::ostream &operator<<(std::ostream &out, NativeOrderType t) noexcept;
-
-//////////////////////////////////////////////////
-// Definition
-//////////////////////////////////////////////////
-
-inline bool native_order_type_parse_smart(
-    const std::string &str, NativeOrderType *result) {
-   if (NativeOrderType_Parse(str, result)) {
-      return true;
-   }
-   std::string s = bts::system::string_utils::copy_to_upper(str);
-   if (NativeOrderType_Parse(s, result)) {
-      return true;
-   }
-   const char *prefix = "NATIVE_ORDER_TYPE_";
-   if (!s.starts_with(prefix)) {
-      s.insert(0, prefix);
-   }
-   return NativeOrderType_Parse(s, result);
-}
-
-inline std::ostream &operator<<(std::ostream &out, NativeOrderType t) noexcept {
-   out << NativeOrderType_Name(t);
-   return out;
-}
-
-} // namespace bts::api::orderman
-
-FMT_IMPL(
-    bts::api::orderman::NativeOrderType,
-    t,
-    "{}",
-    bts::api::orderman::NativeOrderType_Name(t));

src/lib/api/services/order_status.h

@@ -1,56 +0,0 @@
-#pragma once
-
-#include "lib/api/services/OrderService.pb.h"
-#include "lib/core/EnumMacros.h"
-#include "lib/core/FMT_IMPL.h"
-#include "lib/io/JsonReader.h"
-#include "lib/logging/logging.h"
-#include "lib/utils/string_utils.h"
-
-#include <nlohmann/json.hpp>
-
-namespace bts::api::orderman {
-
-/**
- * Parses OrderStatus from a string. Unlike `bts::api::orderman::OrderStatus_Parse`,
- * this implementation is case insensitive, and makes the 'ORDER_STATUS_' prefix optional.
- */
-bool order_status_parse_smart(const std::string &str, OrderStatus *result);
-
-PB_ENUM_FROM_JSON(OrderStatus)
-PB_ENUM_TO_JSON(OrderStatus)
-
-std::ostream &operator<<(std::ostream &out, OrderStatus s) noexcept;
-
-//////////////////////////////////////////////////
-// Definition
-//////////////////////////////////////////////////
-
-inline bool order_status_parse_smart(
-    const std::string &str, OrderStatus *result) {
-   if (OrderStatus_Parse(str, result)) {
-      return true;
-   }
-   std::string s = bts::system::string_utils::copy_to_upper(str);
-   if (OrderStatus_Parse(s, result)) {
-      return true;
-   }
-   const char *prefix = "ORDER_STATUS_";
-   if (!s.starts_with(prefix)) {
-      s.insert(0, prefix);
-   }
-   return OrderStatus_Parse(s, result);
-}
-
-inline std::ostream &operator<<(std::ostream &out, OrderStatus s) noexcept {
-   out << OrderStatus_Name(s);
-   return out;
-}
-
-} // namespace bts::api::orderman
-
-FMT_IMPL(
-    bts::api::orderman::OrderStatus,
-    s,
-    "{}",
-    bts::api::orderman::OrderStatus_Name(s));

src/lib/api/services/time_in_force.h

@@ -1,56 +0,0 @@
-#pragma once
-
-#include "lib/api/services/OrderService.pb.h"
-#include "lib/core/EnumMacros.h"
-#include "lib/core/FMT_IMPL.h"
-#include "lib/io/JsonReader.h"
-#include "lib/logging/logging.h"
-#include "lib/utils/string_utils.h"
-
-#include <nlohmann/json.hpp>
-
-namespace bts::api::orderman {
-
-/**
- * Parses TimeInForce from a string. Unlike `bts::api::orderman::TimeInForce_Parse`,
- * this implementation is case insensitive, and makes the 'TIME_IN_FORCE_' prefix optional.
- */
-bool time_in_force_parse_smart(const std::string &str, TimeInForce *result);
-
-PB_ENUM_FROM_JSON(TimeInForce)
-PB_ENUM_TO_JSON(TimeInForce)
-
-std::ostream &operator<<(std::ostream &out, TimeInForce tif) noexcept;
-
-//////////////////////////////////////////////////
-// Definition
-//////////////////////////////////////////////////
-
-inline bool time_in_force_parse_smart(
-    const std::string &str, TimeInForce *result) {
-   if (TimeInForce_Parse(str, result)) {
-      return true;
-   }
-   std::string s = bts::system::string_utils::copy_to_upper(str);
-   if (TimeInForce_Parse(s, result)) {
-      return true;
-   }
-   const char *prefix = "TIME_IN_FORCE_";
-   if (!s.starts_with(prefix)) {
-      s.insert(0, prefix);
-   }
-   return TimeInForce_Parse(s, result);
-}
-
-inline std::ostream &operator<<(std::ostream &out, TimeInForce tif) noexcept {
-   out << TimeInForce_Name(tif);
-   return out;
-}
-
-} // namespace bts::api::orderman
-
-FMT_IMPL(
-    bts::api::orderman::TimeInForce,
-    v,
-    "{}",
-    bts::api::orderman::TimeInForce_Name(v));

src/lib/api/services/trigger_price_type.h

@@ -1,58 +0,0 @@
-#pragma once
-
-#include "lib/api/services/OrderService.pb.h"
-#include "lib/core/EnumMacros.h"
-#include "lib/core/FMT_IMPL.h"
-#include "lib/io/JsonReader.h"
-#include "lib/logging/logging.h"
-#include "lib/utils/string_utils.h"
-
-#include <nlohmann/json.hpp>
-
-namespace bts::api::orderman {
-
-/**
- * Parses TriggerPriceType from a string. Unlike `bts::api::orderman::TriggerPriceType_Parse`,
- * this implementation is case insensitive, and makes the 'TRIGGER_PRICE_TYPE_' prefix optional.
- */
-bool trigger_price_type_parse_smart(
-    const std::string &str, TriggerPriceType *result);
-
-PB_ENUM_FROM_JSON(TriggerPriceType)
-PB_ENUM_TO_JSON(TriggerPriceType)
-
-std::ostream &operator<<(std::ostream &out, TriggerPriceType t) noexcept;
-
-//////////////////////////////////////////////////
-// Definition
-//////////////////////////////////////////////////
-
-inline bool trigger_price_type_parse_smart(
-    const std::string &str, TriggerPriceType *result) {
-   if (TriggerPriceType_Parse(str, result)) {
-      return true;
-   }
-   std::string s = bts::system::string_utils::copy_to_upper(str);
-   if (TriggerPriceType_Parse(s, result)) {
-      return true;
-   }
-   const char *prefix = "TRIGGER_PRICE_TYPE_";
-   if (!s.starts_with(prefix)) {
-      s.insert(0, prefix);
-   }
-   return TriggerPriceType_Parse(s, result);
-}
-
-inline std::ostream &operator<<(
-    std::ostream &out, TriggerPriceType t) noexcept {
-   out << TriggerPriceType_Name(t);
-   return out;
-}
-
-} // namespace bts::api::orderman
-
-FMT_IMPL(
-    bts::api::orderman::TriggerPriceType,
-    t,
-    "{}",
-    bts::api::orderman::TriggerPriceType_Name(t));

src/lib/cmake/GetLinuxClientSources.cmake

@@ -75,7 +75,6 @@ function(get_cpp_sources dest)
     expression/ParserExceptions.cpp
     expression/Value.cpp
     expression/tests/expression_parser_tests.cpp
-    io/JsonReader.cpp
     logging/LoggerHub.cpp
     logging/Sink.cpp
     logging/logging.cpp

src/lib/cmake/GetWindowsClientSources.cmake

@@ -74,7 +74,6 @@ function(get_cpp_sources dest)
     expression/ParserExceptions.cpp
     expression/Value.cpp
     expression/tests/expression_parser_tests.cpp
-    io/JsonReader.cpp
     logging/LoggerHub.cpp
     logging/Sink.cpp
     logging/logging.cpp

src/lib/collections/concurrent_factory.h

@@ -1,98 +0,0 @@
-#pragma once
-
-#include "lib/utils/LockBox.h"
-#include "lib/utils/bts_assert.h"
-#include "lib/utils/type_name.h"
-
-#include <functional>
-#include <memory>
-
-namespace bts::collections {
-
-template <class KeyT, class ValueT>
-class ConcurrentFactory {
-   utils::LockBox<std::unordered_map<KeyT, std::shared_ptr<ValueT>>> store;
-
- public:
-   std::shared_ptr<ValueT> add(
-       const KeyT &key,
-       std::function<void(std::shared_ptr<ValueT> &)> callback) {
-      return store.lock([&](auto &map) {
-         auto [it, is_new] = map.try_emplace(key, std::make_shared<ValueT>());
-         bts_assert_msgf(
-             is_new, "{}::add. Duplicate key '{}'",
-             utils::cached_type_name<ConcurrentFactory<KeyT, ValueT>>(), key);
-         callback(it->second);
-         return it->second;
-      });
-   }
-
-   std::size_t erase(const KeyT &key) {
-      return store.lock([&](auto &map) {
-         map.erase(key);
-         return map.size();
-      });
-   }
-
-   std::size_t size() const {
-      return store.lock([&](auto &map) { return map.size(); });
-   }
-
-   template <
-       typename CallbackT = std::function<void(std::shared_ptr<ValueT> &)>>
-   std::shared_ptr<const ValueT> get(const KeyT &key, CallbackT callback) {
-      return store.lock([&](auto &map) {
-         auto [it, is_new] = map.try_emplace(key, std::make_shared<ValueT>());
-         bts_assert_msgf(
-             !is_new, "{}::get. Key not found '{}'",
-             utils::cached_type_name<ConcurrentFactory<KeyT, ValueT>>(), key);
-         callback(it->second);
-         return it->second;
-      });
-   }
-
-   template <
-       typename CallbackT =
-           std::function<void(std::shared_ptr<ValueT> &, bool)>>
-   std::shared_ptr<const ValueT> get_or_add(
-       const KeyT &key, CallbackT callback) {
-      return store.lock([&](auto &map) {
-         auto [it, is_new] = map.try_emplace(key);
-         if (is_new) {
-            it->second = std::make_shared<ValueT>();
-         }
-         callback(it->second, is_new);
-         return it->second;
-      });
-   }
-
-   template <
-       typename CallbackT =
-           std::function<void(std::shared_ptr<ValueT> &, bool)>>
-   std::shared_ptr<const ValueT> replace(const KeyT &key, CallbackT callback) {
-      return store.lock([&](auto &map) {
-         auto [it, is_new] = map.try_emplace(key);
-         bts_assert_msgf(
-             !is_new, "{}::replace. Key not found '{}'",
-             utils::cached_type_name<ConcurrentFactory<KeyT, ValueT>>(), key);
-         it->second = std::make_shared<ValueT>();
-         callback(it->second, is_new);
-         return it->second;
-      });
-   }
-
-   template <
-       typename CallbackT =
-           std::function<void(std::shared_ptr<ValueT> &, bool)>>
-   std::shared_ptr<const ValueT> replace_or_add(
-       const KeyT &key, CallbackT callback) {
-      return store.lock([&](auto &map) {
-         auto [it, is_new] = map.try_emplace(key);
-         it->second = std::make_shared<ValueT>();
-         callback(it->second, is_new);
-         return it->second;
-      });
-   }
-};
-
-} // namespace bts::collections

src/lib/config/profiles/default/orderman_cfg.conf

@@ -1,34 +1,34 @@
 OrdermanCfg {
-  rule: "DeploymentType.type == 'dev'"
-  value {
-    executor {
-      thread_name: "oman_orders"
-      poll: false 
-      hosted_actor: "MDSource"
-      hosted_actor: "OrderManager"
-      hosted_actor: "ExecutionGateway"
-    }
-    executor {
-      thread_name: "oman_slow"
-      poll: false 
-	    hosted_actor: "*"
-    }
-  }
+   rule: "DeploymentType.type == 'dev'"
+   value {
+      executor {
+         thread_name: "oman_orders"
+	 poll: false 
+	 hosted_actor: "MDSource"
+	 hosted_actor: "OrderManager"
+	 hosted_actor: "ExecutionGateway"
+      }
+      executor {
+         thread_name: "oman_slow"
+	 poll: false 
+	 hosted_actor: "*"
+      }
+   }
 }
 
 OrdermanCfg {
-  value {
-    executor {
-      thread_name: "oman_orders"
-      poll: true 
-      hosted_actor: "MDSource"
-      hosted_actor: "OrderManager"
-      hosted_actor: "ExecutionGateway"
-    }
-    executor {
-      thread_name: "oman_slow"
-	    poll: false 
-      hosted_actor: "*"
-    }
-  }
-}
+   value {
+      executor {
+         thread_name: "oman_orders"
+	 poll: true 
+	 hosted_actor: "MDSource"
+	 hosted_actor: "OrderManager"
+	 hosted_actor: "ExecutionGateway"
+      }
+      executor {
+         thread_name: "oman_slow"
+	 poll: false 
+	 hosted_actor: "*"
+      }
+   }
+}

src/lib/config/profiles/default/toggles.conf

@@ -114,139 +114,13 @@ VolSurfaceUsage {
      items: {
       option_name: "SPX"
       option_venue: VENUE_CBOE
-      vs_name: "CBOE_skews"
+      vs_name: "Main"
     }
     items: {
       option_name: "SPXW"
       option_venue: VENUE_CBOE
-      vs_name: "CBOE_skews"
+      vs_name: "Main"
     }
-    items: {
-      option_name: "ES"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }     
-     items: {
-      option_name: "E1A"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-     items: {
-      option_name: "E2A"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-     items: {
-      option_name: "E3A"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-    items: {
-      option_name: "E4A"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-    items: {
-      option_name: "E5A"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-    items: {
-      option_name: "E1B"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-     items: {
-      option_name: "E2B"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-     items: {
-      option_name: "E3B"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-    items: {
-      option_name: "E4B"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-    items: {
-      option_name: "E5B"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-    items: {
-      option_name: "E1C"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-     items: {
-      option_name: "E2C"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-     items: {
-      option_name: "E3C"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-    items: {
-      option_name: "E4C"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-    items: {
-      option_name: "E5C"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-    items: {
-      option_name: "E1D"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-     items: {
-      option_name: "E2D"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-     items: {
-      option_name: "E3D"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-    items: {
-      option_name: "E4D"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-    items: {
-      option_name: "E5D"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-    items: {
-      option_name: "EW1"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-     items: {
-      option_name: "EW2"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-     items: {
-      option_name: "EW3"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-    items: {
-      option_name: "EW4"
-      option_venue: VENUE_GLBX
-      vs_name: "CME_Skews"
-    }
-
 
   }
 }

src/lib/core/EnumMacros.h

@@ -110,19 +110,6 @@
    ENUM_TO_JSON(type)                                                          \
    ENUM_EQUALITY_OPS(type)
 
-#define ENUM_CLASS_DEF_NO_JSON(type, for_all)                                  \
-   class type {                                                                \
-    public:                                                                    \
-      enum Value { for_all(ENUM_DECLARE_VALUE) };                              \
-                                                                               \
-      ENUM_BASIC_MEMBERS(type)                                                 \
-   };                                                                          \
-                                                                               \
-   ENUM_OUTPUT_OPS(type)                                                       \
-   ENUM_NAME_DEF(type, for_all)                                                \
-   ENUM_PARSE_DEF(type, for_all)                                               \
-   ENUM_EQUALITY_OPS(type)
-
 #define ENUM_BITWISE_COMPARISON_OP(enum_type, numeric_type, op)                \
    constexpr enum_type operator op(enum_type lhs, enum_type rhs) noexcept {    \
       return static_cast<enum_type>(static_cast<numeric_type>(lhs)             \
@@ -167,4 +154,3 @@
    inline void to_json(nlohmann::json &j, const enum_type &value) {            \
       j = enum_type##_Name(value);                                             \
    }
-

src/lib/core/TimeStamp.cpp

@@ -1,7 +1,6 @@
 #include "TimeStamp.h"
 #include "lib/utils/datetime.h"
 #include <boost/date_time/posix_time/time_formatters.hpp>
-#include <google/protobuf/timestamp.pb.h>
 
 namespace bts::core {
 
@@ -44,12 +43,6 @@ bool TimeStamp::from_ymd(
    return true;
 }
 
-void TimeStamp::to_protobuf(google::protobuf::Timestamp &ts) const {
-   ts.set_seconds(seconds());
-   bts_assert(nanos() >= seconds() * 1'000'000'000);
-   ts.set_nanos(nanos() - seconds() * 1'000'000'000);
-}
-
 TimeStamp TimeStamp::from_iso_string(const std::string &iso_string) {
    auto ptime = datetime::parse_iso_date_time_string(iso_string);
    return from_ptime(ptime);

src/lib/core/TimeStamp.h

@@ -10,7 +10,6 @@
 #include <boost/date_time/time.hpp>
 #include <cstdint>
 #include <fmt/format.h>
-#include <google/protobuf/timestamp.pb.h>
 #include <limits>
 #include <string>
 
@@ -88,7 +87,6 @@ struct TimeStamp {
    // Gets a TimeStamp representing this date at midnight.
    TimeStamp date() const;
    constexpr boost::posix_time::ptime to_ptime() const;
-   void to_protobuf(google::protobuf::Timestamp &ts) const;
    std::string to_iso_string() const;
    // current epoch ns
    static uint64_t now_ns() {

src/lib/domain/SecurityCommon.h

@@ -10,7 +10,6 @@
 #include <limits>
 #include <nlohmann/json.hpp>
 #include <string>
-#include <vector>
 
 namespace bts::domain {
 
@@ -32,7 +31,6 @@ class SecurityCommon {
    double min_quantity = 0;
    double max_quantity = std::numeric_limits<double>::max();
    double cash_per_point = 1;
-   std::vector<double> price_ticks;
 
    SecurityCommon() {}
    SecurityCommon(const bts::api::Security &s)

src/lib/domain_model/PutOrCall.h

@@ -1,52 +0,0 @@
-#pragma once
-
-#include "lib/core/EnumMacros.h"
-#include "lib/core/FMT_IMPL.h"
-#include "lib/domain_model/Security.pb.h"
-#include "lib/io/JsonReader.h"
-#include "lib/logging/logging.h"
-#include "lib/utils/string_utils.h"
-
-#include <nlohmann/json.hpp>
-
-namespace bts::api {
-
-/**
- * Parses PutOrCall from a string. Unlike `bts::api::SecurityType_Parse`, this
- * implementation is case insensitive, and makes the 'PUTORCALL_' prefix optional.
- */
-bool put_or_call_parse_smart(const std::string &str, PutOrCall *result);
-
-PB_ENUM_FROM_JSON(PutOrCall)
-PB_ENUM_TO_JSON(PutOrCall)
-
-std::ostream &operator<<(std::ostream &out, const PutOrCall &poc) noexcept;
-
-//////////////////////////////////////////////////
-// Definition
-//////////////////////////////////////////////////
-
-inline bool put_or_call_parse_smart(const std::string &str, PutOrCall *result) {
-   if (PutOrCall_Parse(str, result)) {
-      return true;
-   }
-   std::string s = bts::system::string_utils::copy_to_upper(str);
-   if (PutOrCall_Parse(s, result)) {
-      return true;
-   }
-   const char *prefix = "PUTORCALL_";
-   if (!s.starts_with(prefix)) {
-      s.insert(0, prefix);
-   }
-   return PutOrCall_Parse(s, result);
-}
-
-inline std::ostream &operator<<(
-    std::ostream &out, const PutOrCall &poc) noexcept {
-   out << PutOrCall_Name(poc);
-   return out;
-}
-
-} // namespace bts::api
-
-FMT_IMPL(bts::api::PutOrCall, poc, "{}", bts::api::PutOrCall_Name(poc));

src/lib/domain_model/Security.cpp

@@ -2,13 +2,10 @@
 #include "lib/config/config.h"
 #include "lib/config/protos/orders_fills_cleanup.pb.h"
 #include "lib/domain_model/Security.pb.h"
-#include "lib/io/JsonMacros.h"
 #include "lib/md/common.h"
-#include "lib/utils/pb_json_macros.h"
 
 #include <boost/date_time/posix_time/posix_time.hpp>
 #include <google/protobuf/text_format.h>
-#include <nlohmann/json.hpp>
 
 namespace bts {
 namespace securities {
@@ -194,389 +191,3 @@ std::string unique_descriptor(const api::Security &security) {
 
 } // namespace securities
 } // namespace bts
-
-namespace bts::api {
-
-using nlohmann::json;
-using std::uint32_t;
-
-void to_json(json &j, const api::ContractQuantityInfo &cqi) {
-   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, min_value);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, max_value);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, tick_unit);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, tick_base);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, multiplier);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, cqi, max_price);
-}
-
-void from_json(const json &j, api::ContractQuantityInfo &cqi) {
-   json::const_iterator it;
-   PB_JSON_READ_VALUE(j, cqi, min_value, it);
-   PB_JSON_READ_VALUE(j, cqi, max_value, it);
-   PB_JSON_READ_VALUE(j, cqi, max_value, it);
-   PB_JSON_READ_VALUE(j, cqi, tick_unit, it);
-   PB_JSON_READ_VALUE(j, cqi, tick_base, it);
-   PB_JSON_READ_VALUE(j, cqi, max_price, it);
-}
-
-void to_json(json &j, const api::ContractSpecification &spec) {
-   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, low_tick);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, price_break);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, high_tick);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, cash_per_point);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, price_multiplier);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, main_fraction);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, sub_fraction);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, spec, price_display_format);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, spec, quantity_infos);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, spec, notional_quantity_infos);
-}
-
-void from_json(const json &j, api::ContractSpecification &spec) {
-   json::const_iterator it;
-   PB_JSON_READ_VALUE(j, spec, low_tick, it);
-   PB_JSON_READ_VALUE(j, spec, price_break, it);
-   PB_JSON_READ_VALUE(j, spec, high_tick, it);
-   PB_JSON_READ_VALUE(j, spec, cash_per_point, it);
-   PB_JSON_READ_VALUE(j, spec, price_multiplier, it);
-   PB_JSON_READ_VALUE(j, spec, main_fraction, it);
-   PB_JSON_READ_VALUE(j, spec, sub_fraction, it);
-   PB_JSON_READ_VALUE(j, spec, price_display_format, it);
-   PB_JSON_READ_ARRAY(j, spec, quantity_infos, it);
-   PB_JSON_READ_ARRAY(j, spec, notional_quantity_infos, it);
-}
-
-void to_json(json &, const SpotContract &) {}
-void from_json(const json &, SpotContract &) {}
-
-template <class ExpContract>
-static void exp_to_json(json &j, const ExpContract &ec) {
-   j = bts::core::TimeStamp::from_ymd(
-           ec.expiration_year(), ec.expiration_month(), ec.expiration_day())
-           .to_iso_string();
-}
-
-template <class ExpContract>
-static void exp_from_json(const json &j, ExpContract &ec) {
-   const auto dt = bts::core::TimeStamp::from_iso_string(j.get<std::string>());
-   uint32_t y, m, d;
-   dt.to_date(y, m, d);
-   ec.set_expiration_year(y);
-   ec.set_expiration_month(m);
-   ec.set_expiration_day(d);
-}
-
-void to_json(json &j, const FutureContract &f) {
-   exp_to_json(j["expiration"], f);
-}
-
-void from_json(const json &j, FutureContract &f) {
-   const auto it = j.find("expiration");
-   if (it != j.end()) {
-      exp_from_json(*it, f);
-   }
-}
-
-void to_json(json &j, const OptionContract &opt) {
-   exp_to_json(j["expiration"], opt);
-   PB_JSON_WRITE_VALUE(j, opt, strike);
-   PB_JSON_WRITE_VALUE(j, opt, call_put);
-}
-
-void from_json(const json &j, OptionContract &opt) {
-   json::const_iterator it;
-   it = j.find("expiration");
-   if (it != j.end()) {
-      exp_from_json(*it, opt);
-   }
-   PB_JSON_READ_VALUE(j, opt, strike, it);
-   PB_JSON_READ_VALUE(j, opt, call_put, it);
-}
-
-void to_json(json &j, const ContractLeg &leg) {
-   PB_JSON_WRITE_STRUCT(j, leg, security);
-   PB_JSON_WRITE_VALUE(j, leg, quantity);
-   PB_JSON_WRITE_VALUE(j, leg, side);
-   PB_JSON_WRITE_VALUE(j, leg, price_multiplier);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, leg, leg_delta);
-   PB_JSON_WRITE_VALUE(j, leg, price_calc);
-}
-
-void from_json(const json &j, ContractLeg &leg) {
-   json::const_iterator it;
-   PB_JSON_READ_STRUCT(j, leg, security, it);
-   PB_JSON_READ_VALUE(j, leg, quantity, it);
-   PB_JSON_READ_VALUE(j, leg, side, it);
-   PB_JSON_READ_VALUE(j, leg, price_multiplier, it);
-   PB_JSON_READ_VALUE(j, leg, leg_delta, it);
-   PB_JSON_READ_VALUE(j, leg, price_calc, it);
-}
-
-void to_json(json &j, const SpreadContract &spd) {
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, spd, legs);
-}
-
-void from_json(const json &j, SpreadContract &spd) {
-   json::const_iterator it;
-   PB_JSON_READ_ARRAY(j, spd, legs, it);
-}
-
-void to_json(json &j, const SyntheticContract &syn) {
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, syn, name);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, syn, user_id);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, syn, legs);
-   PB_JSON_WRITE_VALUE(j, syn, price_calc);
-}
-
-void from_json(const json &j, SyntheticContract &syn) {
-   json::const_iterator it;
-   PB_JSON_READ_STRING(j, syn, name, it);
-   PB_JSON_READ_VALUE(j, syn, user_id, it);
-   PB_JSON_READ_ARRAY(j, syn, legs, it);
-   PB_JSON_READ_VALUE(j, syn, price_calc, it);
-}
-
-void to_json(json &j, const GLBXExchangeData &data) {
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, cfi_code);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, cme_symbol);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, market_segment_id);
-}
-
-void from_json(const json &j, GLBXExchangeData &data) {
-   json::const_iterator it;
-   PB_JSON_READ_VALUE(j, data, exchange_id, it);
-   PB_JSON_READ_STRING(j, data, security_desc, it);
-   PB_JSON_READ_STRING(j, data, cfi_code, it);
-   PB_JSON_READ_STRING(j, data, cme_symbol, it);
-   PB_JSON_READ_VALUE(j, data, market_segment_id, it);
-}
-
-void to_json(json &j, const CBOEExchangeData &data) {
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, class_key);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, source);
-}
-
-void from_json(const json &j, CBOEExchangeData &data) {
-   json::const_iterator it;
-   PB_JSON_READ_VALUE(j, data, exchange_id, it);
-   PB_JSON_READ_STRING(j, data, security_desc, it);
-   PB_JSON_READ_VALUE(j, data, class_key, it);
-   PB_JSON_READ_STRING(j, data, source, it);
-}
-
-void to_json(json &j, const CBOECFEExchangeData &data) {
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, class_key);
-}
-
-void from_json(const json &j, CBOECFEExchangeData &data) {
-   json::const_iterator it;
-   PB_JSON_READ_VALUE(j, data, exchange_id, it);
-   PB_JSON_READ_STRING(j, data, security_desc, it);
-   PB_JSON_READ_VALUE(j, data, class_key, it);
-}
-
-void to_json(json &j, const ICEExchangeData &data) {
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, price_denominator);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, deal_denominator);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, settle_denominator);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, market_description);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, cleared_alias);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, product_name);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, hub_alias);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, strip_name);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, mic_code);
-   PB_JSON_WRITE_IF_TRUE(j, data, is_strip);
-}
-
-void from_json(const json &j, ICEExchangeData &data) {
-   json::const_iterator it;
-   PB_JSON_READ_VALUE(j, data, exchange_id, it);
-   PB_JSON_READ_STRING(j, data, security_desc, it);
-   PB_JSON_READ_VALUE(j, data, price_denominator, it);
-   PB_JSON_READ_VALUE(j, data, deal_denominator, it);
-   PB_JSON_READ_VALUE(j, data, settle_denominator, it);
-   PB_JSON_READ_STRING(j, data, market_description, it);
-   PB_JSON_READ_STRING(j, data, cleared_alias, it);
-   PB_JSON_READ_STRING(j, data, product_name, it);
-   PB_JSON_READ_STRING(j, data, hub_alias, it);
-   PB_JSON_READ_STRING(j, data, strip_name, it);
-   PB_JSON_READ_STRING(j, data, mic_code, it);
-   PB_JSON_READ_VALUE(j, data, is_strip, it);
-}
-
-void to_json(json &j, const TAIFEXExchangeData &data) {
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, class_key);
-}
-
-void from_json(const json &j, TAIFEXExchangeData &data) {
-   json::const_iterator it;
-   PB_JSON_READ_VALUE(j, data, exchange_id, it);
-   PB_JSON_READ_STRING(j, data, security_desc, it);
-   PB_JSON_READ_VALUE(j, data, class_key, it);
-}
-
-void to_json(json &j, const DeribitExchangeData &data) {
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
-}
-
-void from_json(const json &j, DeribitExchangeData &data) {
-   json::const_iterator it;
-   PB_JSON_READ_VALUE(j, data, exchange_id, it);
-   PB_JSON_READ_STRING(j, data, security_desc, it);
-}
-
-void to_json(json &j, const FTXExchangeData &data) {
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
-}
-
-void from_json(const json &j, FTXExchangeData &data) {
-   json::const_iterator it;
-   PB_JSON_READ_VALUE(j, data, exchange_id, it);
-   PB_JSON_READ_STRING(j, data, security_desc, it);
-}
-
-void to_json(json &j, const BinanceExchangeData &data) {
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
-}
-
-void from_json(const json &j, BinanceExchangeData &data) {
-   json::const_iterator it;
-   PB_JSON_READ_VALUE(j, data, exchange_id, it);
-   PB_JSON_READ_STRING(j, data, security_desc, it);
-}
-
-void to_json(json &j, const BinanceUsdsFuturesExchangeData &data) {
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
-}
-
-void from_json(const json &j, BinanceUsdsFuturesExchangeData &data) {
-   json::const_iterator it;
-   PB_JSON_READ_VALUE(j, data, exchange_id, it);
-   PB_JSON_READ_STRING(j, data, security_desc, it);
-}
-
-void to_json(json &j, const BinanceCoinFuturesExchangeData &data) {
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
-}
-
-void from_json(const json &j, BinanceCoinFuturesExchangeData &data) {
-   json::const_iterator it;
-   PB_JSON_READ_VALUE(j, data, exchange_id, it);
-   PB_JSON_READ_STRING(j, data, security_desc, it);
-}
-
-void to_json(json &j, const BinanceUsExchangeData &data) {
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
-}
-
-void from_json(const json &j, BinanceUsExchangeData &data) {
-   json::const_iterator it;
-   PB_JSON_READ_VALUE(j, data, exchange_id, it);
-   PB_JSON_READ_STRING(j, data, security_desc, it);
-}
-
-void to_json(json &j, const OkxExchangeData &data) {
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
-}
-
-void from_json(const json &j, OkxExchangeData &data) {
-   json::const_iterator it;
-   PB_JSON_READ_VALUE(j, data, exchange_id, it);
-   PB_JSON_READ_STRING(j, data, security_desc, it);
-}
-
-void to_json(json &j, const CoinbaseExchangeData &data) {
-   PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
-}
-
-void from_json(const json &j, CoinbaseExchangeData &data) {
-   json::const_iterator it;
-   PB_JSON_READ_VALUE(j, data, exchange_id, it);
-   PB_JSON_READ_STRING(j, data, security_desc, it);
-}
-
-void to_json(json &j, const Security &s) {
-   // clang-format off
-#define CONTRACT_ONEOF(oneof)           \
-   oneof(j, s, Spot, spot)              \
-   oneof(j, s, Future, future)          \
-   oneof(j, s, Option, option)          \
-   oneof(j, s, Spread, spread)          \
-   oneof(j, s, Synthetic, synthetic)
-
-#define EXCHANGE_DATA_ONEOF(oneof)                        \
-   oneof(j, s, Glbx, glbx)                                \
-   oneof(j, s, Cboe, cboe)                                \
-   oneof(j, s, Cfe, cfe)                                  \
-   oneof(j, s, Ice, ice)                                  \
-   oneof(j, s, Deribit, deribit)                          \
-   oneof(j, s, Ftx, ftx)                                  \
-   oneof(j, s, Binance, binance)                          \
-   oneof(j, s, BinanceCoinFutures, binance_coin_futures)  \
-   oneof(j, s, BinanceUsdsFutures, binance_usds_futures)  \
-   oneof(j, s, BinanceUs, binance_us)                     \
-   oneof(j, s, Coinbase, coinbase)                        \
-   oneof(j, s, Okx, okx)                                  \
-   oneof(j, s, Taifex, taifex)
-   // clang-format on
-
-   PB_JSON_WRITE_IF_NOT_ZERO(j, s, id);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, s, symbol);
-   PB_JSON_WRITE_VALUE(j, s, exchange);
-   PB_JSON_WRITE_STRUCT(j, s, contract_spec);
-   PB_JSON_WRITE_IF_NOT_EMPTY(j, s, currency);
-   PB_JSON_WRITE_ONEOF(j, s, Contract, CONTRACT_ONEOF);
-   PB_JSON_WRITE_ONEOF(j, s, ExchangeData, EXCHANGE_DATA_ONEOF);
-}
-
-void from_json(const json &j, Security &s) {
-   json::const_iterator it;
-
-#define SEC_TYPE_ONEOF(oneof)                                                  \
-   oneof(j, s, future, it);                                                    \
-   oneof(j, s, option, it);                                                    \
-   oneof(j, s, spread, it);                                                    \
-   oneof(j, s, synthetic, it)
-
-#define CONTRACT_CASE_ONEOF(oneof)                                             \
-   oneof(j, s, glbx, it);                                                      \
-   oneof(j, s, cboe, it);                                                      \
-   oneof(j, s, cfe, it);                                                       \
-   oneof(j, s, ice, it);                                                       \
-   oneof(j, s, deribit, it);                                                   \
-   oneof(j, s, ftx, it);                                                       \
-   oneof(j, s, binance, it);                                                   \
-   oneof(j, s, binance_usds_futures, it);                                      \
-   oneof(j, s, binance_coin_futures, it);                                      \
-   oneof(j, s, binance_us, it);                                                \
-   oneof(j, s, coinbase, it);                                                  \
-   oneof(j, s, okx, it);                                                       \
-   oneof(j, s, taifex, it);
-
-   PB_JSON_READ_VALUE(j, s, id, it);
-   PB_JSON_READ_STRING(j, s, symbol, it);
-   PB_JSON_READ_STRING(j, s, currency, it);
-   PB_JSON_READ_STRUCT(j, s, contract_spec, it);
-   PB_JSON_READ_ONEOFS(SEC_TYPE_ONEOF);
-   PB_JSON_READ_ONEOFS(CONTRACT_CASE_ONEOF);
-}
-
-} // namespace bts::api

src/lib/domain_model/Security.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "lib/domain_model/PutOrCall.h"
+#include "lib/core/FMT_IMPL.h"
 #include "lib/domain_model/Security.pb.h"
 #include "lib/domain_model/SecurityType.h"
 #include "lib/domain_model/Side.h"
@@ -9,7 +9,6 @@
 #include "lib/utils/bts_assert.h"
 
 #include <cmath>
-#include <nlohmann/json_fwd.hpp>
 
 namespace boost {
 namespace gregorian {
@@ -17,83 +16,14 @@ class date;
 }
 } // namespace boost
 
-namespace bts::api {
+namespace bts {
 
+namespace api {
 class Security;
 class ICEExchangeData;
+} // namespace api
 
-void to_json(nlohmann::json &j, const ContractQuantityInfo &c);
-void from_json(const nlohmann::json &j, ContractQuantityInfo &c);
-
-void to_json(nlohmann::json &j, const ContractSpecification &c);
-void from_json(const nlohmann::json &j, ContractSpecification &c);
-
-void to_json(nlohmann::json &j, const ContractLeg &c);
-void from_json(const nlohmann::json &j, ContractLeg &c);
-
-void to_json(nlohmann::json &j, const SpotContract &c);
-void from_json(const nlohmann::json &j, SpotContract &c);
-
-void to_json(nlohmann::json &j, const FutureContract &c);
-void from_json(const nlohmann::json &j, FutureContract &c);
-
-void to_json(nlohmann::json &j, const OptionContract &c);
-void from_json(const nlohmann::json &j, OptionContract &c);
-
-void to_json(nlohmann::json &j, const SpreadContract &c);
-void from_json(const nlohmann::json &j, SpreadContract &c);
-
-void to_json(nlohmann::json &j, const SyntheticContract_PriceCalc &c);
-void from_json(const nlohmann::json &j, SyntheticContract_PriceCalc &c);
-
-void to_json(nlohmann::json &j, const SyntheticContract &c);
-void from_json(const nlohmann::json &j, SyntheticContract &c);
-
-void to_json(nlohmann::json &j, const GLBXExchangeData &data);
-void from_json(const nlohmann::json &j, GLBXExchangeData &data);
-
-void to_json(nlohmann::json &j, const CBOEExchangeData &data);
-void from_json(const nlohmann::json &j, CBOEExchangeData &data);
-
-void to_json(nlohmann::json &j, const CBOECFEExchangeData &data);
-void from_json(const nlohmann::json &j, CBOECFEExchangeData &data);
-
-void to_json(nlohmann::json &j, const ICEExchangeData &data);
-void from_json(const nlohmann::json &j, ICEExchangeData &data);
-
-void to_json(nlohmann::json &j, const DeribitExchangeData &data);
-void from_json(const nlohmann::json &j, DeribitExchangeData &data);
-
-void to_json(nlohmann::json &j, const FTXExchangeData &data);
-void from_json(const nlohmann::json &j, FTXExchangeData &data);
-
-void to_json(nlohmann::json &j, const BinanceExchangeData &data);
-void from_json(const nlohmann::json &j, BinanceExchangeData &data);
-
-void to_json(nlohmann::json &j, const BinanceUsdsFuturesExchangeData &data);
-void from_json(const nlohmann::json &j, BinanceUsdsFuturesExchangeData &data);
-
-void to_json(nlohmann::json &j, const BinanceCoinFuturesExchangeData &data);
-void from_json(const nlohmann::json &j, BinanceCoinFuturesExchangeData &data);
-
-void to_json(nlohmann::json &j, const BinanceUsExchangeData &data);
-void from_json(const nlohmann::json &j, BinanceUsExchangeData &data);
-
-void to_json(nlohmann::json &j, const OkxExchangeData &data);
-void from_json(const nlohmann::json &j, OkxExchangeData &data);
-
-void to_json(nlohmann::json &j, const CoinbaseExchangeData &data);
-void from_json(const nlohmann::json &j, CoinbaseExchangeData &data);
-
-void to_json(nlohmann::json &j, const Security &s);
-void from_json(const nlohmann::json &j, Security &s);
-
-PB_ENUM_FROM_JSON(SyntheticContract_PriceCalc);
-PB_ENUM_TO_JSON(SyntheticContract_PriceCalc);
-
-} // namespace bts::api
-
-namespace bts::securities {
+namespace securities {
 
 /**
  * Gets a descriptor string for a security (should be unique within a particular exchange).
@@ -130,8 +60,7 @@ inline api::Side other_side(api::Side side) {
       return side;
    }
 }
-template <class V>
-class TwoSided {
+template <class V> class TwoSided {
  public:
    TwoSided() : buy(), sell() {}
 
@@ -175,8 +104,7 @@ inline bool is_worse(api::Side s, Price p1, Price p2) {
    return is_better(s, p2, p1);
 }
 
-template <class C>
-void apply_to_both_sides(C c) {
+template <class C> void apply_to_both_sides(C c) {
    c(api::SIDE_BUY);
    c(api::SIDE_SELL);
 }
@@ -250,8 +178,8 @@ inline Price encode_exchange_price(api::Security const &sec, Price in) {
       return in;
 }
 
-inline double raw_round_to_quote_price(
-    double raw_price, double tick_size, api::Side side) {
+inline double
+raw_round_to_quote_price(double raw_price, double tick_size, api::Side side) {
    double factor = 1000000000;
    auto i_price = llround(factor * raw_price);
    auto i_tick_size = llround(tick_size * factor);
@@ -294,8 +222,8 @@ inline double checked_round_to_quote_price(
    }
 }
 
-inline Price round_to_quote_price(
-    api::Security const &sec, Price in, api::Side side) {
+inline Price
+round_to_quote_price(api::Security const &sec, Price in, api::Side side) {
    auto p = in.value();
    bts_assert_d(!std::isnan(p));
    if (p < -sec.contract_spec().price_break() ||
@@ -359,8 +287,8 @@ inline const std::string &descriptor(
 // Looking to round md::IntPrice? Look in IntPrice.h
 
 // computes price_2 - price_1 in ticks
-inline int price_diff_to_ticks(
-    double price_1, double price_2, double tick_size) {
+inline int
+price_diff_to_ticks(double price_1, double price_2, double tick_size) {
    return static_cast<int>(std::round((price_2 - price_1) / tick_size));
 }
 // returns the difference (in ticks) bw and order and a market.
@@ -384,8 +312,8 @@ inline int order_diff_market(
 // simple double equality can not be used to compare prices.
 // this function deems 2 prices to be equal if they are within
 // 1 % of the tick size
-inline bool test_price_equality(
-    double price_1, double price_2, double tick_size) {
+inline bool
+test_price_equality(double price_1, double price_2, double tick_size) {
    return std::fabs(price_2 - price_1) < 0.01 * tick_size;
 }
 inline bool gr_than_eq(double p1, double p2, double tick_size) {
@@ -434,4 +362,7 @@ inline bool is_spread_of_strips(const bts::api::Security &sec) {
    return false;
 }
 
-} // namespace bts::securities
+} // namespace securities
+} // namespace bts
+
+FMT_IMPL(bts::api::Security, s, "{}", bts::securities::descriptor(s));

src/lib/domain_model/common.h

@@ -33,8 +33,8 @@ struct CQuantity {
    uint32_t value() const { return static_cast<uint32_t>(qty); }
    double full_value() const { return qty; }
    float full_value_float() const { return static_cast<float>(full_value()); }
-   uint32_t int_part() const { return static_cast<uint32_t>(qty); }
-   double fractional_part() const { return full_value() - int_part(); }
+   uint32_t int_part() { return static_cast<uint32_t>(qty); }
+   double fractional_part() { return full_value() - int_part(); }
    static bool equal_d(double d1, double d2) {
       return std::abs(d1 - d2) <= err;
    }

src/lib/io/JsonMacros.h

@@ -3,14 +3,18 @@
 #define _TO_JSON_FIELD(field) j[#field] = t.field;
 #define _FROM_JSON_FIELD(field) bts::io::JsonReader{&j, #field}.read(t.field);
 
-#define SPARK_TO_JSON_IMPL(type, fields)                                       \
-   void to_json(nlohmann::json &j, const type &t) { fields(_TO_JSON_FIELD) }
+#define SPARK_TO_JSON_IMPL(type, for_each)                                     \
+   inline void to_json(nlohmann::json &j, const type &t) {                     \
+      FOR_EACH(_TO_JSON_FIELD)                                                 \
+   }
 
-#define SPARK_FROM_JSON_IMPL(type, fields)                                     \
-   void from_json(const nlohmann::json &j, type &t) { fields(_FROM_JSON_FIELD) }
+#define SPARK_FROM_JSON_IMPL(type, for_each)                                   \
+   inline void from_json(const nlohmann::json &j, type &t) {                   \
+      FOR_EACH(_FROM_JSON_FIELD)                                               \
+   }
 
 #define OSTREAM_JSON_IMPL(type)                                                \
-   std::ostream &operator<<(std::ostream &out, const type &t) {                \
+   inline std::ostream &operator<<(std::ostream &out, const type &t) {         \
       nlohmann::json j;                                                        \
       to_json(j, t);                                                           \
       out << j.dump(2);                                                        \

src/lib/io/JsonReader.h

@@ -94,18 +94,7 @@ class JsonReader {
     * @return True if the read succeeds, otherwise false
     */
    template <class T>
-   bool read_or(const std::string &name, T &result, T default_value = {}) const;
-
-   template <class T>
-   bool read_optional_or(
-       std::optional<T> &result,
-       std::optional<T> default_value = std::nullopt) const;
-
-   template <class T>
-   bool read_optional_or(
-       const std::string &name,
-       std::optional<T> &result,
-       std::optional<T> default_value = std::nullopt) const;
+   bool read_or(const std::string &name, T &result, T default_value) const;
 
  private:
    const nlohmann::json *json_;
@@ -280,33 +269,6 @@ inline bool JsonReader::read_or(
    return false;
 }
 
-template <class T>
-inline bool JsonReader::read_optional_or(
-    std::optional<T> &result, std::optional<T> default_value) const {
-   if (read_optional(result)) {
-      return true;
-   }
-   result = std::move(default_value);
-   return false;
-}
-
-template <class T>
-inline bool JsonReader::read_optional_or(
-    const std::string &name,
-    std::optional<T> &result,
-    std::optional<T> default_value) const {
-   if (json_ && json_->is_object()) {
-      auto it = json_->find(name);
-      if (it != json_->end()) {
-         if (JsonReader{&it.value()}.read_optional(result)) {
-            return true;
-         }
-      }
-   }
-   result = std::move(default_value);
-   return false;
-}
-
 template <class T>
 bool JsonReader::read_flat(
     const std::vector<nlohmann::json> &items, std::vector<T> &result) {

src/lib/io/http/curl_session.h

@@ -1,7 +1,5 @@
 #pragma once
 
-#include "lib/logging/logging.h"
-
 #include <atomic>
 #include <curl/curl.h>
 #include <curl/easy.h>
@@ -14,7 +12,6 @@ struct CurlSession {
 
    CurlSession() {
       if (++ref_count == 1) {
-         LOG(Http, Info, "Initializing curl (global)");
          curl_global_init(CURL_GLOBAL_ALL);
       }
       curl = curl_easy_init();
@@ -23,7 +20,6 @@ struct CurlSession {
    ~CurlSession() {
       curl_easy_cleanup(curl);
       if (--ref_count == 0) {
-         LOG(Http, Info, "Cleaning up curl (global)");
          curl_global_cleanup();
       }
    }

src/lib/io/http/http_client.cpp

@@ -117,10 +117,7 @@ void HttpClient::send(const HttpRequest &req, IHttpResponse *response) {
    // Prepare req
    //to debug curl requests
    // curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, curl_debug_callback);
-   // Disable signals to avoid strange stack corruption related to set/longjmp
-   // https://stackoverflow.com/questions/9191668/error-longjmp-causes-uninitialized-stack-frame
-   curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
-   curl_easy_setopt(curl, CURLOPT_FAILONERROR, 0L);
+   curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
    curl_easy_setopt(curl, CURLOPT_URL, req.url.c_str());
    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers.data());
    if (response != nullptr) {

src/lib/io/json_formatter.h

@@ -1,54 +0,0 @@
-#pragma once
-
-#include <fmt/format.h>
-#include <nlohmann/json.hpp>
-#include <string>
-
-namespace bts::io {
-
-/**
- * This wrapper struct provides a means to format arbitrary objects as json
- * in templated strings (e.g., LOG macros).
- */
-template <class T>
-struct JsonFormatter {
-   T value;
-
-   std::string to_string() const { return fmt::format("{}", *this); }
-};
-
-template <class T>
-JsonFormatter<T> json_formatter(T value) {
-   return JsonFormatter<T>{std::move(value)};
-}
-
-} // namespace bts::io
-
-template <class T>
-struct fmt::formatter<bts::io::JsonFormatter<T>> {
-   int indent = -1;
-
-   constexpr auto parse(fmt::format_parse_context &ctx) -> decltype(ctx.end()) {
-      const auto end = ctx.end();
-      auto it = ctx.begin();
-      if (it != end) {
-         indent = 0;
-         do {
-            int x = *it - '0';
-            if (x < 0 || x > 9) {
-               return it;
-            }
-            indent = indent * 10 + x;
-         } while (++it != end);
-      }
-      return it;
-   }
-
-   template <typename FormatContext>
-   auto format(const bts::io::JsonFormatter<T> &jf, FormatContext &ctx) const
-       -> decltype(ctx.out()) {
-      nlohmann::json j;
-      to_json(j, jf.value);
-      return fmt::format_to(ctx.out(), "{}", j.dump(indent));
-   }
-};

src/lib/io/websocket/websocket_client.cpp

@@ -3,7 +3,6 @@
 #include "lib/core/TimeSpan.h"
 #include "lib/core/TimeStamp.h"
 #include "lib/io/websocket/websocket_client_config.h"
-#include "lib/io/websocket/websocket_client_logger.h"
 #include "lib/io/websocket/websocket_client_state.h"
 #include "lib/io/websocket/websocket_close_status.h"
 #include "lib/io/websocket/websocket_error_code.h"
@@ -12,10 +11,7 @@
 #include "lib/utils/bts_assert.h"
 #include "lib/utils/formatters/exception_formatter.h"
 
-#include <algorithm>
 #include <atomic>
-#include <boost/asio/io_context.hpp>
-#include <chrono>
 #include <exception>
 #include <fmt/format.h>
 #include <mutex>
@@ -23,7 +19,6 @@
 #include <optional>
 #include <stdexcept>
 #include <string>
-#include <system_error>
 #include <thread>
 #include <websocketpp/close.hpp>
 #include <websocketpp/common/connection_hdl.hpp>
@@ -34,51 +29,11 @@
 #include <websocketpp/config/core_client.hpp>
 #include <websocketpp/roles/client_endpoint.hpp>
 
-namespace websocketpp::config {
-
-/// Client config with asio transport and TLS enabled
-struct bts_asio_tls_client : public core_client {
-   typedef bts_asio_tls_client type;
-   typedef core_client base;
-
-   typedef base::concurrency_type concurrency_type;
-
-   typedef base::request_type request_type;
-   typedef base::response_type response_type;
-
-   typedef base::message_type message_type;
-   typedef base::con_msg_manager_type con_msg_manager_type;
-   typedef base::endpoint_msg_manager_type endpoint_msg_manager_type;
-
-   typedef websocketpp::log::WebsocketClientLogger<
-       websocketpp::log::ElevelAdapter>
-       elog_type;
-   typedef websocketpp::log::WebsocketClientLogger<
-       websocketpp::log::AlevelAdapter>
-       alog_type;
-
-   typedef base::rng_type rng_type;
-
-   struct transport_config : public base::transport_config {
-      typedef type::concurrency_type concurrency_type;
-      typedef type::alog_type alog_type;
-      typedef type::elog_type elog_type;
-      typedef type::request_type request_type;
-      typedef type::response_type response_type;
-      typedef transport::asio::tls_socket::endpoint socket_type;
-   };
-
-   typedef transport::asio::endpoint<transport_config> transport_type;
-};
-
-} // namespace websocketpp::config
-
 namespace bts::websocket {
 
 struct WebsocketClientSnapshot {
    int client_id;
    WebsocketClientState state;
-   int n_reconnects;
    int version;
 };
 
@@ -87,10 +42,9 @@ struct WebsocketClientSnapshot {
 FMT_IMPL(
     bts::websocket::WebsocketClientSnapshot,
     t,
-    "id: {}; state: {}; n_reconnects: {}; version: {}",
+    "id: {}; state: {}; version: {}",
     t.client_id,
     t.state,
-    t.n_reconnects,
     t.version);
 
 namespace bts::websocket {
@@ -108,11 +62,10 @@ using std::size_t;
 using std::string;
 using std::thread;
 using std::uint32_t;
-using std::chrono::nanoseconds;
-using std::this_thread::sleep_for;
+using websocketpp::config::asio_tls_client;
 
 using WebsocketppClient =
-    websocketpp::client<websocketpp::config::bts_asio_tls_client>;
+    websocketpp::client<websocketpp::config::asio_tls_client>;
 using ConnectionHdl = websocketpp::connection_hdl;
 using ConnectionPtr = WebsocketppClient::connection_ptr;
 using MessageTypePtr = websocketpp::config::core_client::message_type::ptr;
@@ -133,16 +86,9 @@ using State = WebsocketClientState;
 #define ERROR(...) LOG(Http, Error, __VA_ARGS__)
 #define ERROR_FN(...) LOG(Http, Error, "WebsocketClient::" __VA_ARGS__)
 
-static bts::core::TimeSpan reconnect_interval(std::uint32_t n_attempts) {
-   std::int64_t secs = 5 * n_attempts;
-   secs = std::clamp(secs, 5L, 300L);
-   return bts::core::TimeSpan::from_seconds(secs);
-}
-
 struct WebsocketClient::PImpl {
    WebsocketClientConfig config;
    WebsocketppClient client{};
-   boost::asio::io_context *io_context = nullptr;
    WebsocketppClient::connection_ptr con;
    optional<OpenHandler> open_handler;
    optional<MessageHandler> message_handler;
@@ -152,48 +98,8 @@ struct WebsocketClient::PImpl {
    optional<thread> io_thread;
    int client_id{client_id_counter++};
    mutex instance_mx;
-   State state = State::IDLE;
+   State state;
    int version = 0;
-   int n_reconnects = 0;
-
-   PImpl(WebsocketClientConfig config, boost::asio::io_context *io_context)
-       : config(std::move(config)), io_context(io_context) {
-      client.get_alog().set_name(this->config.name);
-      client.get_elog().set_name(this->config.name);
-      client.set_access_channels(websocketpp::log::alevel::all);
-      // client.clear_access_channels(websocketpp::log::alevel::frame_payload);
-      client.set_error_channels(websocketpp::log::elevel::all);
-      if (io_context == nullptr) {
-         client.init_asio();
-      } else {
-         client.init_asio(io_context);
-      }
-      client.set_message_handler(
-          [this](auto hdl, auto msg) { on_message(hdl, msg); });
-      client.set_open_handler([this](auto hdl) { on_open(hdl); });
-      client.set_ping_handler([](auto /*hdl*/, auto /*msg*/) {
-         LOG(Http, Info, "Received ping");
-         return true;
-      });
-      client.set_pong_timeout(0);
-      client.set_pong_handler([](auto /*hdl*/, auto /*msg*/) {
-         LOG(Http, Info, "Received pong");
-         return true;
-      });
-      client.set_tls_init_handler([&](auto hdl) { return on_tls_init(hdl); });
-      client.set_close_handler([=, this](auto hdl) {
-         on_close(hdl);
-         if (config.reconnect_on_close) {
-            sleep_for(nanoseconds{reconnect_interval(n_reconnects).nanos()});
-            connect();
-         }
-      });
-      client.set_fail_handler([this](auto hdl) {
-         const auto ec = client.get_con_from_hdl(hdl)->get_ec();
-         ERROR_FN("fail_handler. {}; err: {}", snapshot(), ec.message());
-         on_fail(hdl);
-      });
-   }
 
    ~PImpl() {
       if (io_thread) {
@@ -214,7 +120,7 @@ struct WebsocketClient::PImpl {
    }
 
    constexpr WebsocketClientSnapshot snapshot() const {
-      return {client_id, state, n_reconnects, version};
+      return {client_id, state, version};
    }
 
    void close(websocketpp::close::status::value code, std::string reason) {
@@ -258,7 +164,8 @@ struct WebsocketClient::PImpl {
    /**
     * Connects to the websocket server
     */
-   void connect() {
+   void connect(
+       const map<string, string> &headers, bool run_in_background = true) {
       lock_guard<mutex> lock{instance_mx};
       INFO_FN("connect. {}", snapshot());
       switch (state.value) {
@@ -270,6 +177,9 @@ struct WebsocketClient::PImpl {
       }
       set_state(State::OPENING);
       try {
+         init_logging();
+         init_handlers();
+         // Get connection
          websocketpp::lib::error_code ec;
          con = client.get_connection(config.url, ec);
          if (ec) {
@@ -277,11 +187,20 @@ struct WebsocketClient::PImpl {
             return;
          }
          // Add headers
-         for (const auto &[k, v] : config.headers) {
+         for (const auto &[k, v] : headers) {
             con->append_header(k, v);
          }
          // Connect
          client.connect(con);
+         if (run_in_background) {
+            bts_assert_msgf(
+                !io_thread, "Websocket io_thread already instantiated. {}",
+                snapshot());
+            io_thread = std::thread{[this]() { run(); }};
+            if (config.connect_timeout > core::TimeSpan::zero()) {
+               wait_for_connect(config.connect_timeout);
+            }
+         }
       } catch (const std::exception &ex) {
          std::string str = ex.what();
          ERROR_FN(
@@ -293,11 +212,22 @@ struct WebsocketClient::PImpl {
       }
    }
 
-   void reconnect() {
-      WARN_FN("reconnect. {}", snapshot());
-      sleep_for(nanoseconds{reconnect_interval(n_reconnects).nanos()});
-      ++n_reconnects;
-      connect();
+   void wait_for_connect(const TimeSpan timeout) {
+      INFO_FN("wait_for_connect. {}; timeout: {}", snapshot(), timeout);
+      const auto start = TimeStamp::now();
+      while (state == State::OPENING) {
+         const auto elapsed = TimeStamp::now() - start;
+         const auto rem = timeout - elapsed;
+         if (rem < TimeSpan::zero()) {
+            throw std::runtime_error{fmt::format(
+                "Timed out waiting for connection after {} secs. {}",
+                elapsed.total_seconds(), snapshot())};
+         }
+         WARN_FN(
+             "wait_for_connect. Still waiting. {}; timeout: {}; rem: {}",
+             snapshot(), timeout, rem);
+         std::this_thread::sleep_for(std::chrono::seconds(1));
+      }
    }
 
    void run() {
@@ -328,33 +258,10 @@ struct WebsocketClient::PImpl {
       INFO("Websocket 'run' thread end. {}", snapshot());
    }
 
-   void run_one() {
-      try {
-         client.run_one();
-      } catch (const websocketpp::exception &ex) {
-         ERROR_FN(
-             "io_thread (run_one). Websocket io threw websocketpp::exception. "
-             "{}; err: {}",
-             snapshot(), ex);
-      } catch (const std::exception &ex) {
-         ERROR_FN(
-             "io_thread (run_one). Websocket io threw std::exception. {}; err: "
-             "{}",
-             snapshot(), ex);
-      } catch (...) {
-         bts_assert_msgf(
-             false,
-             "io_thread (run_one). Websocket io threw unknown exception. Could "
-             "this have been thrown by a callback delegate? {}",
-             snapshot());
-      }
-   }
-
  private:
    static std::atomic<int> client_id_counter;
 
-   std::shared_ptr<boost::asio::ssl::context> on_tls_init(
-       ConnectionHdl /*hdl*/) {
+   auto on_tls_init(ConnectionHdl /*hdl*/) {
       INFO_FN("on_tls_init. state: {}; version: {}", state, version);
       auto result =
           websocketpp::lib::make_shared<SslContext>(SslContext::sslv23_client);
@@ -362,6 +269,38 @@ struct WebsocketClient::PImpl {
       return result;
    }
 
+   void init_handlers() {
+      INFO_FN("init_handlers. state: {}; version: {}", state, version);
+      bts_assert(!io_thread);
+      client.init_asio();
+      client.set_message_handler(
+          [this](auto hdl, auto msg) { on_message(hdl, msg); });
+      client.set_open_handler([this](auto hdl) { on_open(hdl); });
+      client.set_ping_handler([](auto /*hdl*/, auto /*msg*/) {
+         LOG(Http, Info, "Received ping");
+         return true;
+      });
+      client.set_pong_timeout(0);
+      client.set_pong_handler([](auto /*hdl*/, auto /*msg*/) {
+         LOG(Http, Info, "Received pong");
+         return true;
+      });
+      client.set_tls_init_handler(
+          [this](auto hdl) { return on_tls_init(hdl); });
+      client.set_close_handler([this](auto hdl) { on_close(hdl); });
+      client.set_fail_handler([this](auto hdl) {
+         const auto ec = client.get_con_from_hdl(hdl)->get_ec();
+         ERROR_FN("fail_handler. {}; err: {}", snapshot(), ec.message());
+      });
+   }
+
+   void init_logging() {
+      INFO_FN("init_logging. {}", snapshot());
+      client.set_access_channels(websocketpp::log::alevel::all);
+      // client.clear_access_channels(websocketpp::log::alevel::frame_payload);
+      client.set_error_channels(websocketpp::log::elevel::all);
+   }
+
    void on_open(ConnectionHdl hdl) {
       lock_guard<mutex> lock{instance_mx};
       INFO_FN("on_open. {}", snapshot());
@@ -419,34 +358,11 @@ struct WebsocketClient::PImpl {
       if (close_handler) {
          (*close_handler)(ec);
       }
-      if (config.reconnect_on_close) {
-         reconnect();
-      }
-   }
-
-   void on_fail(ConnectionHdl hdl) {
-      lock_guard<mutex> lock{instance_mx};
-      ERROR_FN("on_fail. {}", snapshot());
-      set_state(State::IDLE);
-      const auto con = client.get_con_from_hdl(hdl);
-      const auto ec = con->get_ec();
-      if (ec) {
-         ERROR_FN(
-             "on_fail. {}; local_code: {}; "
-             "local_reason: {}; remote_code: {}; remote_reason: {}; err: {}",
-             snapshot(), con->get_local_close_code(),
-             con->get_local_close_reason(), con->get_remote_close_code(),
-             con->get_remote_close_reason(), ec.message());
-      }
-      if (config.reconnect_on_fail) {
-         reconnect();
-      }
    }
 };
 
-WebsocketClient::WebsocketClient(
-    WebsocketClientConfig config, boost::asio::io_context *io_context)
-    : me_(std::make_unique<PImpl>(std::move(config), io_context)) {}
+WebsocketClient::WebsocketClient(WebsocketClientConfig config)
+    : me_(std::make_unique<PImpl>(std::move(config))) {}
 
 WebsocketClient::~WebsocketClient() {
    me_->close(websocketpp::close::status::going_away, "");
@@ -475,16 +391,13 @@ WebsocketClient::CloseHandler &WebsocketClient::close_handler() {
    return *me_->close_handler;
 }
 
-void WebsocketClient::connect() { me_->connect(); }
+void WebsocketClient::connect(
+    const std::map<std::string, std::string> &headers, bool run_in_background) {
+   me_->connect(headers, run_in_background);
+}
 
 void WebsocketClient::run() { me_->run(); }
 
-void WebsocketClient::run_one() { me_->run_one(); }
-
-boost::asio::io_context *WebsocketClient::io_context() {
-   return me_->io_context;
-}
-
 websocketpp::lib::error_code WebsocketClient::send_text(const string &text) {
    return me_->send_text(text);
 }
@@ -502,6 +415,4 @@ bool WebsocketClient::is_connected() const {
 
 void WebsocketClient::ping(const string &payload) { me_->con->ping(payload); }
 
-const string &WebsocketClient::url() const { return me_->config.url; }
-
 } // namespace bts::websocket

src/lib/io/websocket/websocket_client.h

@@ -3,7 +3,6 @@
 #include "lib/io/websocket/websocket_client_config.h"
 #include "lib/utils/Delegate.h"
 
-#include <boost/asio/io_context.hpp>
 #include <memory>
 #include <string>
 #include <websocketpp/config/core_client.hpp>
@@ -13,33 +12,28 @@ namespace bts::websocket {
 
 class WebsocketClient {
  public:
-   WebsocketClient(
-       WebsocketClientConfig config,
-       boost::asio::io_context *io_context = nullptr);
+   explicit WebsocketClient(WebsocketClientConfig config);
    ~WebsocketClient();
 
    using OpenHandler = bts::core::Delegate<void(const std::error_code &)>;
    using MessageHandler = bts::core::Delegate<void(const std::string &)>;
    using CloseHandler = bts::core::Delegate<void(const std::error_code &)>;
 
-   void connect();
+   void connect(
+       const std::map<std::string, std::string> &headers = {},
+       bool run_in_background = true);
    void disconnect();
    bool is_connected() const;
-   boost::asio::io_context *io_context();
    void run();
-   void run_one();
    CloseHandler &open_handler();
    MessageHandler &message_handler();
    CloseHandler &close_handler();
    websocketpp::lib::error_code send_text(const std::string &text);
    void ping(const std::string &payload = "");
 
-   const std::string &url() const;
-
  private:
    struct PImpl;
    std::unique_ptr<PImpl> me_;
 };
 
 } // namespace bts::websocket
-

src/lib/io/websocket/websocket_client_config.h

@@ -5,14 +5,9 @@
 namespace bts::websocket {
 
 struct WebsocketClientConfig {
-   std::string url{};
-   core::TimeSpan connect_timeout = core::TimeSpan::zero();
-   core::TimeSpan pong_timeout = core::TimeSpan::zero();
-   bool reconnect_on_close = false;
-   bool reconnect_on_fail = false;
-   std::map<std::string, std::string> headers{};
-   // name to use in logs
-   std::string name{};
+   std::string url;
+   core::TimeSpan connect_timeout;
+   core::TimeSpan pong_timeout;
 };
 
 } // namespace bts::websocket

src/lib/io/websocket/websocket_client_logger.h

@@ -1,88 +0,0 @@
-#pragma once
-
-#include "lib/logging/Common.h"
-#include "lib/logging/logging.h"
-
-#include <websocketpp/concurrency/basic.hpp>
-#include <websocketpp/logger/basic.hpp>
-#include <websocketpp/logger/levels.hpp>
-
-namespace websocketpp::log {
-
-struct ElevelAdapter {
-   typedef elevel LevelT;
-
-   constexpr bts::log::Level operator()(level channel) {
-      switch (channel) {
-      case elevel::devel:
-         return bts::Diag;
-      case elevel::library:
-      case elevel::warn:
-         return bts::Warn;
-      case elevel::rerror:
-         return bts::Error;
-      case elevel::fatal:
-         return bts::Fatal;
-      default:
-         return bts::Info;
-      }
-   }
-};
-
-struct AlevelAdapter {
-   typedef alevel LevelT;
-
-   constexpr bts::log::Level operator()(level channel) {
-      switch (channel) {
-      case alevel::control:
-      case alevel::frame_header:
-      case alevel::frame_payload:
-      case alevel::message_header:
-      case alevel::message_payload:
-      case alevel::debug_handshake:
-      case alevel::debug_close:
-      case alevel::devel:
-         return bts::Diag;
-      case alevel::disconnect:
-         return bts::Warn;
-      case alevel::fail:
-         return bts::Error;
-      default:
-         return bts::Info;
-      }
-   }
-};
-
-template <typename LevelAdapter>
-class WebsocketClientLogger
-    : public basic<concurrency::basic, typename LevelAdapter::LevelT> {
-   typedef basic<concurrency::basic, typename LevelAdapter::LevelT> base;
-
-   std::string name;
-   LevelAdapter level_adapter;
-
- public:
-   WebsocketClientLogger(
-       level c, channel_type_hint::value h = channel_type_hint::access)
-       : base(c, h) {}
-
-   void set_name(std::string name) {
-      concurrency::basic::scoped_lock_type lock{this->m_lock};
-      this->name = std::move(name);
-   }
-
-   void write(level channel, const std::string &msg) {
-      concurrency::basic::scoped_lock_type lock{this->m_lock};
-      if (!this->dynamic_test(channel)) {
-         return;
-      }
-      LOG(Http, level_adapter(channel), "WebsocketClient-{} {}. {}", name,
-          std::string{LevelAdapter::LevelT::channel_name(channel)}, msg);
-   }
-
-   void write(level channel, const char *msg) {
-      write(channel, std::string{msg});
-   }
-};
-
-} // namespace websocketpp::log

src/lib/md/GeneralMarketDataListener.cpp

@@ -36,6 +36,7 @@
 #include "lib/utils/bts_assert.h"
 
 #include <bits/stdint-uintn.h>
+#include <cstddef>
 #include <google/protobuf/timestamp.pb.h>
 #include <google/protobuf/util/time_util.h>
 #include <memory>
@@ -732,11 +733,13 @@ static bool get_qty_info(
    }
    return true;
 }
-
+bool is_number(const std::string &s) {
+   return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit);
+ }
 void GeneralMarketDataListener::_copy_security_common(
     const bts::domain::Security &from, uint64_t key, FixedSecurity &to) const {
    to.key = key;
-   to.numeric_id = key;
+   to.numeric_id =is_number(from.security_common.exchange_sec_id)?std::stoul(from.security_common.exchange_sec_id):key;
    auto symbol = from.security_common.symbol;
    to.symbol(symbol);
    to.security_desc(from.security_common.desc);
@@ -746,11 +749,6 @@ void GeneralMarketDataListener::_copy_security_common(
    to.display_factor = 0.0;
    to.price_break = 0.0;
    to.low_tick = to.high_tick = from.security_common.min_price_increment;
-   if(from.security_common.price_ticks.size() == 3){
-      to.price_break = from.security_common.price_ticks[0];
-      to.low_tick =  from.security_common.price_ticks[1];
-      to.high_tick=  from.security_common.price_ticks[2];
-   }
    auto &sc = from.security_common;
    double min_qty = sc.min_quantity;
    double qty_incr = sc.min_quantity_increment;

src/lib/md/MDConsumer.cpp

@@ -435,8 +435,7 @@ class SyntheticConsumerInst {
    void add_security(const api::Security &sec) {
       Mutex::scoped_lock lock(_workmutex);
       if (_data.find(sec.id()) == _data.end()) {
-         LOG(md, Info, "MDConsumer::add_security. {}",
-             bts::securities::descriptor(sec));
+         LOG(md, Info, "MDConsumer::add_security. {}", sec);
          auto ptr = to_strategy(make_shared<api::Security>(sec));
          vector<IncompleteStrategy> vec;
          vec.push_back(ptr);

src/lib/md/SecurityCache.cpp

@@ -14,15 +14,6 @@
 namespace bts {
 namespace md {
 
-#define ASSERT_FN(test, fn, ...)                                               \
-   bts_assert_msg(test, "SecurityCache::" fn ". " __VA_ARGS__)
-#define LOG_FN(level, fn, ...)                                                 \
-   LOG(md, level, "SecurityCache::" fn ". " __VA_ARGS__)
-#define DIAG_FN(fn, ...) LOG_FN(Diag, fn, __VA_ARGS__)
-#define INFO_FN(fn, ...) LOG_FN(Info, fn, __VA_ARGS__)
-#define WARN_FN(fn, ...) LOG_FN(Warn, fn, __VA_ARGS__)
-#define ERROR_FN(fn, ...) LOG_FN(Error, fn, __VA_ARGS__)
-
 using std::string;
 using std::unordered_map;
 
@@ -108,9 +99,6 @@ struct SecurityCacheIndex {
       // Insert but don't overwrite existing
       auto result =
           cache.insert(std::make_pair(SecurityKey(security->id()), security));
-      venue_type_ids.get_or_add(security->exchange())
-          .get_or_add(bts::securities::security_type(*security))
-          .try_emplace(bts::securities::descriptor(*security), security->id());
       return result.second;
    }
 
@@ -178,8 +166,8 @@ struct SecurityCacheImpl {
 };
 
 void SecurityCacheImpl::enqueue_update_db(const SecurityPtr &security) {
-   DIAG_FN(
-       "enqueue_update_db", "security: {}", securities::descriptor(*security));
+   LOG(md, Diag, "SecurityCacheImpl::enqueue_update_db security: {}",
+       bts::securities::descriptor(*security));
    update_queue.push(security);
 }
 
@@ -192,13 +180,7 @@ void SecurityCacheImpl::flush() {
          work.push_back(temp);
       }
       if (!work.empty()) {
-         INFO_FN("flush", "Saving securities to db. count: {}", work.size());
          if (db::insert_or_replace(work) == Result::error) {
-            WARN_FN(
-                "flush",
-                "Failed to save securities to db. Will retry on next flush. "
-                "count: {}",
-                work.size());
             // If we fail to insert then retry later
             for (auto &s : work)
                update_queue.push(std::move(s));
@@ -288,18 +270,14 @@ std::vector<api::Security const *> SecurityCache::get_securities() {
 }
 
 bool SecurityCache::add_live_security(const SecurityPtr &security) {
-   ASSERT_FN(security, "add_live_security", "security is null");
-   const bool is_new_sec = _me->cache.lock([&](SecurityCacheIndex &cache) {
+   bts_assert(security);
+   LOG(md, Diag, "SecurityCache::add_live_security security: {}",
+       bts::securities::descriptor(*security));
+   return _me->cache.lock([&](SecurityCacheIndex &cache) {
       const bool is_new_sec = cache.add_live_security(security);
       _me->enqueue_update_db(security);
       return is_new_sec;
    });
-   if (is_new_sec) {
-      INFO_FN(
-          "add_live_security", "new security: {}",
-          bts::securities::descriptor(*security));
-   }
-   return is_new_sec;
 }
 
 void SecurityCache::flush_to_disk() { _me->flush(); }

src/lib/md/SecurityIDRecorder.cpp

@@ -7,6 +7,7 @@
 #include "lib/utils/checked_arithmetic.h"
 
 #include <boost/filesystem.hpp>
+#include <boost/pool/object_pool.hpp>
 #include <memory>
 #include <sqlite/sqlite3.h>
 #include <stdexcept>

src/lib/mercury/broker/BrokerProxy.cpp

@@ -8,10 +8,7 @@ namespace mercury {
 BrokerProxy::BrokerProxy(ILocalBroker &real_broker)
     : _real_broker(real_broker), _id(real_broker.id()), _cycles(0),
       _outgoing(config::get<config::Mercury>().queue_size()),
-      _incoming(config::get<config::Mercury>().queue_size()), _action_pool() {
-   _action_pool.set_max_count(4096 * 10);
-   _action_pool.set_log_freq(100);
-}
+      _incoming(config::get<config::Mercury>().queue_size()), _action_pool() {}
 
 BrokerProxy::~BrokerProxy() {}
 
@@ -99,60 +96,5 @@ void BrokerProxy::schedule_action(std::function<void()> const &action) {
    send(*m);
 }
 
-void BrokerProxy::do_work(trace::PerfLog &log) {
-   _cycles++;
-   _outgoing.consume_all([this, &log](BrokerMsg *m) {
-      switch (m->msg_type) {
-      case BrokerMsgType::ADD_NEIGHBOUR:
-         _real_broker.add_neighbour(*m->neighbour);
-         break;
-      case BrokerMsgType::REMOVE_NEIGHBOUR:
-         _real_broker.remove_neighbour(m->broker_id);
-         break;
-      case BrokerMsgType::ADVERTISE:
-         _real_broker.advertise(m->broker_id, m->stream_id, m->topic);
-         break;
-      case BrokerMsgType::UNADVERTISE:
-         _real_broker.unadvertise(m->broker_id, m->stream_id);
-         break;
-      case BrokerMsgType::SUBSCRIBE:
-         _real_broker.subscribe(m->broker_id, m->sub_id, m->stream_id);
-         break;
-      case BrokerMsgType::UNSUBSCRIBE:
-         _real_broker.unsubscribe(m->broker_id, m->sub_id, m->stream_id);
-         break;
-      case BrokerMsgType::NOTIFY: {
-         LOG(Latency, Diag, "Notify broker {}. Cycles: {}", _id.value(),
-             _cycles);
-         auto scoped_do_work = trace::log_scope(log, {}, "BrokerProxy::notify");
-         _real_broker.notify(
-             m->broker_id, m->stream_id, m->sub_id, m->notification);
-      } break;
-      case BrokerMsgType::LOCAL_SUBSCRIBE:
-         _real_broker.local_subscribe(
-             m->local_sub_id, m->filter, m->local_handler);
-         break;
-      case BrokerMsgType::LOCAL_UNSUBSCRIBE:
-         _real_broker.local_unsubscribe(m->local_sub_id);
-         break;
-      case BrokerMsgType::EXECUTE_ACTION: {
-         auto start = Timer::now();
-         LOG(Latency, Diag, "Executing lambda on broker {}", _id.value());
-         m->action();
-         auto stop = Timer::now();
-         LOG(Latency, Diag,
-             "Finished executing lambda on broker {}. Took: {}us", _id.value(),
-             us_diff(stop, start));
-         break;
-      }
-      default:
-         bts_assert(false);
-         break;
-      }
-      _incoming.drain();
-      _incoming.push(m);
-   });
-}
-
 } // namespace mercury
 } // namespace bts

src/lib/mercury/broker/BrokerProxy.h

@@ -5,8 +5,8 @@
 #include "lib/mercury/topics/Topic.h"
 #include "lib/tracing/PerfLog.h"
 #include "lib/utils/GrowableSPSCQueue.h"
-#include "lib/utils/object_pool.h"
 
+#include <boost/pool/object_pool.hpp>
 #include <memory>
 
 namespace bts {
@@ -80,7 +80,61 @@ class BrokerProxy : public ILocalBroker {
       send(m);
    }
 
-   void do_work(trace::PerfLog &log);
+   void do_work(trace::PerfLog &log) {
+      _cycles++;
+      _outgoing.consume_all([this, &log](BrokerMsg *m) {
+         switch (m->msg_type) {
+         case BrokerMsgType::ADD_NEIGHBOUR:
+            _real_broker.add_neighbour(*m->neighbour);
+            break;
+         case BrokerMsgType::REMOVE_NEIGHBOUR:
+            _real_broker.remove_neighbour(m->broker_id);
+            break;
+         case BrokerMsgType::ADVERTISE:
+            _real_broker.advertise(m->broker_id, m->stream_id, m->topic);
+            break;
+         case BrokerMsgType::UNADVERTISE:
+            _real_broker.unadvertise(m->broker_id, m->stream_id);
+            break;
+         case BrokerMsgType::SUBSCRIBE:
+            _real_broker.subscribe(m->broker_id, m->sub_id, m->stream_id);
+            break;
+         case BrokerMsgType::UNSUBSCRIBE:
+            _real_broker.unsubscribe(m->broker_id, m->sub_id, m->stream_id);
+            break;
+         case BrokerMsgType::NOTIFY: {
+            LOG(Latency, Diag, "Notify broker {}. Cycles: {}", _id.value(),
+                _cycles);
+            auto scoped_do_work =
+                trace::log_scope(log, {}, "BrokerProxy::notify");
+            _real_broker.notify(
+                m->broker_id, m->stream_id, m->sub_id, m->notification);
+         } break;
+         case BrokerMsgType::LOCAL_SUBSCRIBE:
+            _real_broker.local_subscribe(
+                m->local_sub_id, m->filter, m->local_handler);
+            break;
+         case BrokerMsgType::LOCAL_UNSUBSCRIBE:
+            _real_broker.local_unsubscribe(m->local_sub_id);
+            break;
+         case BrokerMsgType::EXECUTE_ACTION: {
+            auto start = Timer::now();
+            LOG(Latency, Diag, "Executing lambda on broker {}", _id.value());
+            m->action();
+            auto stop = Timer::now();
+            LOG(Latency, Diag,
+                "Finished executing lambda on broker {}. Took: {}us",
+                _id.value(), us_diff(stop, start));
+            break;
+         }
+         default:
+            bts_assert(false);
+            break;
+         }
+         _incoming.drain();
+         _incoming.push(m);
+      });
+   }
 
  private:
    ILocalBroker &_real_broker;
@@ -88,7 +142,7 @@ class BrokerProxy : public ILocalBroker {
    uint64_t _cycles;
    utils::GrowableSPSCQueue<BrokerMsg *> _outgoing;
    utils::GrowableSPSCQueue<BrokerMsg *> _incoming;
-   bts::utils::object_pool<BrokerMsg> _action_pool;
+   boost::object_pool<BrokerMsg> _action_pool;
 
    void send(BrokerMsg &m) {
       _outgoing.push(&m);

src/lib/orderman/CreateOrderMsg.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include "lib/api/services/OrderService.h"
 #include "lib/core/FMT_IMPL.h"
 #include "lib/domain_model/Security.h"
 #include "lib/domain_model/common.h"
@@ -25,25 +24,6 @@ struct CreateOrderMsg {
    api::orderman::RoutingRecord const *routing_record;
 };
 
-/**
- * A CreateOrderMsg wrapper with a custom fmt::formatter that doesn't access the
- * raw security pointer. This is mainly used for logging purposes (because the
- * security pointer may be invalid by the time the fmt::formatter is invoked).
- */
-struct CreateOrderMsg_JsonData {
-   CreateOrderMsg_JsonData() = default;
-   CreateOrderMsg_JsonData(CreateOrderMsg source) : source(std::move(source)) {
-      if (this->source.security != nullptr) {
-         sec_name = securities::descriptor(*this->source.security);
-      }
-   }
-
-   CreateOrderMsg source;
-   std::string sec_name;
-};
-
-void to_json(nlohmann::json &j, const CreateOrderMsg &msg);
-
 inline std::ostream &operator<<(std::ostream &out, CreateOrderMsg const &msg) {
    out << "key: " << msg.key.value() << " price: " << msg.price.value()
        << " qty: " << msg.quantity.full_value() << " security: "
@@ -63,29 +43,6 @@ inline std::ostream &operator<<(std::ostream &out, CreateOrderMsg const &msg) {
    return out;
 }
 
-inline void to_json(
-    nlohmann::json &j, const CreateOrderMsg_JsonData &msg_data) {
-   const auto &msg = msg_data.source;
-   j["key"] = msg.key.value();
-   j["price"] = msg.price.value();
-   j["quantity"] = msg.quantity.value();
-   if (msg.display_quantity.value() != 0) {
-      j["display_quantity"] = msg.quantity.value();
-   }
-   if (!msg_data.sec_name.empty()) {
-      j["security"] = msg_data.sec_name;
-   }
-   j["side"] = msg.side;
-   if (msg.stop_px.value() != 0) {
-      j["stop_px"] = msg.stop_px.value();
-   }
-   j["time_in_force"] = msg.time_in_force;
-   j["manual_order_indicator"] = msg.manual_order_indicator;
-   if (msg.routing_record != nullptr) {
-      j["routing_record"] = *msg.routing_record;
-   }
-}
-
 } // namespace bts::orderman
 
 FMT_IMPL_OSTREAM(bts::orderman::CreateOrderMsg);

src/lib/orderman/OrderServiceServer.cpp

@@ -8,7 +8,6 @@
 #include "lib/config/config.h"
 #include "lib/config/protos/jetfix.pb.h"
 #include "lib/config/protos/orderman_cfg.pb.h"
-#include "lib/core/FMT.h"
 #include "lib/domain_model/Security.pb.h"
 #include "lib/domain_model/common.h"
 #include "lib/logging/logging.h"
@@ -30,11 +29,11 @@
 #include "lib/orderman/actors/order_manager/order_gateway_definition.h"
 #include "lib/orderman/events/ExchangePositionEvent.h"
 #include "lib/orderman/events/OrderOpResultEvent.h"
+#include "lib/orderman/gateways/execution_gateway_provider.h"
 #include "lib/orderman/gateways/i_execution_gateway_provider.h"
 #include "lib/orderman/order_factory/DynamicOrderRegistry.h"
 #include "lib/orderman/orderman_utils/ExchangeOpenOrders.h"
 #include "lib/orderman/orders/Order.h"
-#include "lib/os/os.h"
 #include "lib/os/spark_data_directory.h"
 #include "lib/plugin/SparkServices.h"
 #include "lib/plugin/plugin.h"
@@ -42,6 +41,7 @@
 #include "lib/utils/bts_chrono.h"
 #include "lib/utils/thread_manager.h"
 #include "lib/utils/type_name.h"
+#include "lib_binance_execution/binance_execution/execution_gateway_provider.h"
 
 #include <bits/stdint-uintn.h>
 #include <boost/algorithm/string/predicate.hpp>
@@ -73,58 +73,16 @@ struct ActorThread {
    void start(trace::PerfLog &log) {
       if (cfg.poll())
          thread = threading::make_thread(cfg.thread_name(), [this, &log]() {
-            const core::TimeSpan log_interval =
-                core::TimeSpan::from_seconds(30);
-            core::TimeStamp last_log_time{0};
-            core::TimeStamp now;
-            core::TimeSpan elapsed;
-            std::uint32_t last_count = 0;
-            for (;;) {
-               now = core::TimeStamp::now();
-               elapsed = now - last_log_time;
-               if (elapsed > log_interval) {
-                  LOG(OrderMan, Info,
-                      "OrderServiceServer ActorThread work speed is {:.2f} "
-                      "cycles/sec",
-                      static_cast<float>(last_count) / elapsed.total_seconds());
-                  last_log_time = now;
-                  last_count = 0;
-               }
+            for (;;)
                executor.do_work(log);
-               last_count++;
-            }
          });
       else {
          thread = threading::make_thread(cfg.thread_name(), [this, &log]() {
-            // for logging purposes
-            const std::string t_name = bts::os::get_thread_name();
-            const std::string actors_desc = FMT_JOIN(cfg.hosted_actor(), ", ");
-            const core::TimeSpan log_interval =
-                core::TimeSpan::from_seconds(30);
-
             const auto inner_iter = 1000;
-            bts::core::TimeStamp last_log_time{0};
-            core::TimeStamp now;
-            core::TimeSpan elapsed;
-            std::uint32_t last_count = 0;
             while (!done) {
-               now = core::TimeStamp::now();
-               elapsed = now - last_log_time;
-               auto log_elapsed = now - last_log_time;
-               if (log_elapsed > log_interval) {
-                  LOG(OrderMan, Info,
-                      "OrderServiceServer ActorThread {} work speed is {:.2f} "
-                      "cycles/sec ({})",
-                      t_name,
-                      static_cast<float>(last_count) / elapsed.total_seconds(),
-                      actors_desc);
-                  last_log_time = now;
-                  last_count = 0;
-               }
                auto start = Timer::now();
                for (auto i = 0; i < inner_iter; i++)
                   executor.do_work(log);
-               last_count += inner_iter;
                auto stop = Timer::now();
                loop_metric->observe(ns_diff(stop, start) / inner_iter);
                std::this_thread::sleep_for(10ms);

src/lib/orderman/actors/FillStore.cpp

@@ -2,10 +2,7 @@
 #include "lib/api/services/OrderService.pb.h"
 #include "lib/config/config.h"
 #include "lib/config/protos/toggles.pb.h"
-#include "lib/core/TimeStamp.h"
 #include "lib/domain_model/Security.h"
-#include "lib/domain_model/Security.pb.h"
-#include "lib/domain_model/SecurityKey.h"
 #include "lib/logging/logging.h"
 #include "lib/mercury/events/Update.h"
 #include "lib/orderman/actors/SODPosition.h"
@@ -17,88 +14,18 @@
 #include "lib/os/os.h"
 #include "lib/persistence/SqliteStore.h"
 #include "lib/utils/PBUtils.h"
-#include "lib/utils/TimeStamp.h"
 #include "lib/utils/bts_assert.h"
-#include "lib/utils/bts_chrono.h"
 #include "lib/utils/overloaded.h"
 
 #include <boost/date_time/posix_time/posix_time.hpp>
-#include <google/protobuf/timestamp.pb.h>
 #include <string>
 #include <sys/time.h>
 
 namespace bts {
 namespace orderman {
 
-#define LOG_FN(level, fn, ...)                                                 \
-   LOG(OrderMan, level, "FillStore::" fn ". " __VA_ARGS__)
-#define INFO_FN(fn, ...) LOG_FN(Info, fn, __VA_ARGS__)
-#define WARN_FN(fn, ...) LOG_FN(Warn, fn, __VA_ARGS__)
-#define ERROR_FN(fn, ...) LOG_FN(Error, fn, __VA_ARGS__)
-
 auto const fills_ns = "fills";
 
-struct FillAggregator {
-   struct SecPosition {
-    private:
-      Fill agg_fill;
-
-    public:
-      double agg_price = 0, agg_qty = 0;
-      SecPosition(const Fill &fill) : agg_fill(fill) {
-         const auto &fill_key = fill.fill_key();
-         std::string new_fill_key = "aggpos_" + fill_key;
-         agg_fill.set_fill_key(new_fill_key);
-         auto now_ns = bts::core::TimeStamp::now_ns();
-         agg_fill.mutable_timestamp()->set_nanos(now_ns);
-         add_fill(fill);
-      }
-      void add_fill(const Fill &fill) {
-         bts_assert(fill.security_key() == agg_fill.security_key());
-         double mult = fill.side() == api::SIDE_BUY ? 1.0 : -1.0;
-         double fp = fill.avg_price();
-         double fq = mult * fill.last_shares();
-         double new_agg_price =
-             ((agg_price * agg_qty) + (fp * fq)) / (agg_qty + fq);
-         double new_agg_qty = agg_qty + fq;
-         agg_price = new_agg_price;
-         agg_qty = new_agg_qty;
-      }
-
-    public:
-      Fill fill() const {
-         Fill rv = agg_fill;
-         rv.set_avg_price(agg_price);
-         rv.set_last_shares(std::abs(agg_qty));
-         rv.set_side(::bts::api::Side::SIDE_BUY);
-         if (agg_qty < 0) {
-            rv.set_side(::bts::api::Side::SIDE_SELL);
-         }
-         return rv;
-      }
-   };
-   std::map<SecurityKey, SecPosition> pos_map = {};
-
-   void add_fill(const Fill &fill) {
-      SecurityKey k = SecurityKey(fill.security_key());
-      auto it = pos_map.find(k);
-      if (it != pos_map.end()) {
-         SecPosition &sec_pos = it->second;
-         sec_pos.add_fill(fill);
-      } else {
-         pos_map.insert({k, {fill}});
-      }
-   }
-
-   std::vector<Fill> get_aggregated_fills() const {
-      std::vector<Fill> rv = {};
-      for (const auto &kv : pos_map) {
-         rv.push_back(kv.second.fill());
-      }
-      return rv;
-   }
-};
-
 struct FillStore::UnresolvedFills {
    std::vector<api::orderman::Fill> fills;
    api::Security const *security;
@@ -115,8 +42,6 @@ struct FillStore::PImpl {
    std::unordered_map<OrderKey, UnresolvedFills> fills_with_unresolved_accounts;
    mercury::Stream<StoredFillEvent> fill_stream;
    md::SecurityOracle oracle;
-   FillAggregator fill_aggregator = {};
-   bool aggregate_pos = false;
 };
 
 FillStore::FillStore(md::SecurityOracle o) : me(new PImpl(o)) {
@@ -169,83 +94,40 @@ void FillStore::init() {
        os::get_thread_id(), os::get_thread_name());
    LOG(OrderMan, Info, "FillStore::init oracle n_secs: {}",
        me->oracle.get_securities().size());
+   // Subscribe to NewExecutionReportEvent for new fills
+   subscribe<NewExecutionReportEvent>(
+       [this](auto const &ev) { this->on_execution_report(ev); });
+
    // Load stored fills, resolve their securities and publish the lot
+   api::orderman::Fill fill;
    std::vector<StoredFillEvent> stored_batch;
    LOG(persist, Info, "Loading load all fills");
    auto all = me->store->retrieve_all()[fills_ns];
    LOG(persist, Info, "Finished loading all {} fills", all.size());
-   auto aggregate_clean_and_publish = [&]() {
-      api::orderman::Fill fill;
-      for (auto const &kvp : all) {
-         if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
-            me->fill_aggregator.add_fill(fill);
-         } else {
-            LOG(persist, Error,
-                "Could not recover fill {} from storage. "
-                "This should be investigated",
-                kvp.first);
+   SODPosition sod_helper;
+   for (auto const &kvp : all)
+      if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
+         if (sod_helper.is_sod(fill.timestamp())) {
+            auto fill_kind = sod_fill_kind(fill);
+            fill.set_fill_kind(fill_kind);
          }
-      }
-      // 1. go through all fills in aggregator and make StoredFillEvent
-      auto aggregated_fills = me->fill_aggregator.get_aggregated_fills();
-      for (auto &f : aggregated_fills) {
-         auto sec = me->oracle.get_security(SecurityKey(f.security_key()));
+         auto sec = me->oracle.get_security(SecurityKey(fill.security_key()));
          if (sec) {
-            stored_batch.emplace_back(f, *sec);
-         } else {
-            LOG(persist, Error,
-                "Stored fill {} contains unresolved security {}. ",
-                utils::pb::to_string(f), fill.security_key());
-         }
-      }
-      // 2 delete all fills in the DB
-      me->store->remove_all(fills_ns);
-      // 3. add the aggregated fills back
-      for (const auto &e : stored_batch) {
-         const Fill &f = e.fill;
-         me->store->insert(fills_ns, f.fill_key(), utils::pb::to_string(f));
-      }
-   };
-   auto publish_with_sod_marking = [&]() {
-      api::orderman::Fill fill;
-      SODPosition sod_helper;
-      for (auto const &kvp : all) {
-         if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
-            if (sod_helper.is_sod(fill.timestamp())) {
-               auto fill_kind = sod_fill_kind(fill);
-               fill.set_fill_kind(fill_kind);
+            auto fk = kvp.first;
+            me->known_fills.insert(fk);
+            if (!config::get<config::toggles::FilterExpiredFills>().enabled() ||
+                !securities::is_expired(*sec)) {
+               stored_batch.emplace_back(fill, *sec);
             }
-            auto sec =
-                me->oracle.get_security(SecurityKey(fill.security_key()));
-            if (sec) {
-               auto fk = kvp.first;
-               me->known_fills.insert(fk);
-               if (!config::get<config::toggles::FilterExpiredFills>()
-                        .enabled() ||
-                   !securities::is_expired(*sec)) {
-                  stored_batch.emplace_back(fill, *sec);
-               }
-            } else
-               LOG(persist, Error,
-                   "Stored fill {} contains unresolved security {}. ",
-                   kvp.first, fill.security_key());
          } else
             LOG(persist, Error,
-                "Could not recover fill {} from storage. "
-                "This should be investigated",
-                kvp.first);
-      }
-   };
-
-   // Subscribe to NewExecutionReportEvent for new fills
-   subscribe<NewExecutionReportEvent>(
-       [this](auto const &ev) { this->on_execution_report(ev); });
-   if (me->aggregate_pos) {
-      aggregate_clean_and_publish();
-   } else {
-      publish_with_sod_marking();
-   }
-
+                "Stored fill {} contains unresolved security {}. ", kvp.first,
+                fill.security_key());
+      } else
+         LOG(persist, Error,
+             "Could not recover fill {} from storage. "
+             "This should be investigated",
+             kvp.first);
    me->fill_stream = make_stream<StoredFillEvent>();
    LOG(OrderMan, Info,
        "Fill Store published {} fills recoved from disk out of {} "
@@ -278,7 +160,7 @@ void FillStore::on_execution_report(
              ev.data->to_string());
          auto &er = *ev.data;
          auto fk = fmt::format(
-             "{}:{}", er.order_key.value(), to_strn(er.exchange_fill_id));
+             "{}:{}", er.order_key.value(), to_str(er.exchange_fill_id));
          auto is_known = me->known_fills.count(fk);
          if (er.exec_type == api::orderman::EXEC_TYPE_PARTIAL_FILL ||
              er.exec_type == api::orderman::EXEC_TYPE_FILL) {
@@ -294,7 +176,7 @@ void FillStore::on_execution_report(
                   fill.set_fill_kind(api::orderman::Fill::FILL_KIND_STANDARD);
                   fill.set_security_key(er.security->id());
                }
-               fill.set_account(to_strn(er.account_name));
+               fill.set_account(to_str(er.account_name));
                fill.set_last_price(er.last_price.value());
                fill.set_avg_price(er.avg_price.value());
                fill.set_side(er.side);
@@ -325,15 +207,7 @@ void FillStore::on_execution_report(
                auto it = me->known_orders.find(er.order_key);
                if (it != me->known_orders.end()) {
                   auto &order_def = it->second;
-                  if (fill.account() != order_def.account()) {
-                     WARN_FN(
-                         "on_execution_report",
-                         "ExecutionReport account '{}' does not match "
-                         "OrderDefinition account '{}' (order_key: {}). "
-                         "Setting fill account to the latter",
-                         fill.account(), order_def.account(), er.order_key);
-                     fill.set_account(order_def.account());
-                  }
+                  fill.set_account(order_def.account());
                   fill.set_internal_account_id(order_def.account_id());
                } else
                   me->fills_with_unresolved_accounts[er.order_key]
@@ -341,7 +215,6 @@ void FillStore::on_execution_report(
 
                auto fv = utils::pb::to_string(fill);
                me->store->insert(fills_ns, fk, fv);
-               LOG(OrderMan, Info, "Publishing fill. {}", fv);
                publish(me->fill_stream, fill, *er.security);
             } else
                LOG(OrderMan, Info,

src/lib/orderman/actors/OrderStore.cpp

@@ -5,7 +5,6 @@
 #include "lib/config/protos/orders_fills_cleanup.pb.h"
 #include "lib/config/protos/toggles.pb.h"
 #include "lib/logging/logging.h"
-#include "lib/md/ExchangeSecurityKey.h"
 #include "lib/orderman/actors/SODPosition.h"
 #include "lib/orderman/events/OrderDefinitionUpdateEvent.h"
 #include "lib/orderman/events/OrderStateUpdateEvent.h"
@@ -13,7 +12,6 @@
 #include "lib/persistence/SqliteStore.h"
 #include "lib/utils/PBUtils.h"
 
-#include <google/protobuf/timestamp.pb.h>
 #include <unordered_set>
 
 namespace bts {
@@ -40,7 +38,6 @@ struct OrderStore::PImpl {
    bool soft_cleanup_enabled;
    md::SecurityOracle oracle;
    timeval cutoff_time;
-   bool clear_all_orders = true;
 };
 
 OrderStore::OrderStore(md::SecurityOracle o) : me(new PImpl(o)) {}
@@ -66,46 +63,34 @@ void OrderStore::init() {
 
    me->cutoff_time = SODPosition::get_order_cutoff_time(all);
 
-   auto load_orders = [&]() {
-      for (auto const &kvp : all) {
-         if (utils::pb::from_string<api::orderman::OrderRecord>(
-                 kvp.second, &rec)) {
-            auto ok = OrderKey(std::stoi(kvp.first));
-            auto sk = SecurityKey(rec.def().security_key());
-            auto sec = me->oracle.get_security(sk);
+   for (auto const &kvp : all)
+      if (utils::pb::from_string<api::orderman::OrderRecord>(
+              kvp.second, &rec)) {
+         auto ok = OrderKey(std::stoi(kvp.first));
+         auto sk = SecurityKey(rec.def().security_key());
+         auto sec = me->oracle.get_security(sk);
 
-            if (config::get<
-                    config::toggles::FilterOrdersWithDanglingSecurities>()
-                    .enabled() &&
-                sec == nullptr) {
-               LOG(OrderMan, Diag, "Order {} refers to unresolved security {}",
-                   ok.value(), sk.value());
-            } else {
-               if (!_order_cutoff(rec))
-                  stored_batch.emplace_back(rec);
-               me->known_orders[ok] = PImpl::ORInfo{rec, true};
-            }
-         } else
-            LOG(OrderMan, Error,
-                "Could not recover order {} from storage. "
-                "This should be investigated",
-                kvp.first);
-      }
-   };
+         if (config::get<config::toggles::FilterOrdersWithDanglingSecurities>()
+                 .enabled() &&
+             sec == nullptr) {
+            LOG(OrderMan, Diag, "Order {} refers to unresolved security {}",
+                ok.value(), sk.value());
+         } else {
+            if (!_order_cutoff(rec))
+               stored_batch.emplace_back(rec);
+            me->known_orders[ok] = PImpl::ORInfo{rec, true};
+         }
+      } else
+         LOG(OrderMan, Error,
+             "Could not recover order {} from storage. "
+             "This should be investigated",
+             kvp.first);
 
    me->order_stream = make_stream<StoredOrderEvent>();
    LOG(OrderMan, Info,
        "Order Store published {} orders recoved from disk out of {} "
        "total orders",
        stored_batch.size(), all.size());
-
-   if (me->clear_all_orders) {
-      // clear orders
-      me->store->remove_all(orders_ns);
-   } else {
-      load_orders();
-   }
-
    publish_batch(me->order_stream, stored_batch);
 }
 
@@ -129,22 +114,17 @@ void OrderStore::on_order_state_update(
          rec.set_key(ev.key.value());
          rec.set_parent_key(ev.parent_key.value());
          modified_orders.insert(ev.key);
-         const auto now = utils::pb::pb_now();
          if (rec.has_state()) {
-            google::protobuf::Timestamp created;
-            if (rec.state().has_created()) {
-               created = rec.state().created();
-            }
             rec.mutable_state()->CopyFrom(ev.rebuild_state());
-            *rec.mutable_state()->mutable_created() =
-                created.seconds() > 0 || created.nanos() > 0 ? created : now;
+            *rec.mutable_state()->mutable_created() = utils::pb::pb_now();
+            *rec.mutable_state()->mutable_last_change() = rec.state().created();
             // if filled qty went from 0.99 -> 1, it was copying the int part and not clearing the fraction part,
             // so changing MergeFrom to CopyFrom
          } else {
             *rec.mutable_state() = ev.rebuild_state();
-            *rec.mutable_state()->mutable_created() = now;
+            *rec.mutable_state()->mutable_created() = utils::pb::pb_now();
          }
-         *rec.mutable_state()->mutable_last_change() = now;
+         *rec.mutable_state()->mutable_last_change() = utils::pb::pb_now();
          LOG(OrderMan, Info, "OrderStore::on_order_state_update k: {} rec: {}",
              ev.key, utils::pb::to_string(rec));
       }

src/lib/orderman/actors/ServiceBridge.cpp

@@ -37,12 +37,6 @@
 namespace bts {
 namespace orderman {
 
-#define _LOG_FN(level, fn, ...)                                                \
-   LOG(OrderMan, level, "ServiceBridge::" fn ". " __VA_ARGS__)
-#define INFO_FN(fn, ...) _LOG_FN(Info, fn, __VA_ARGS__)
-#define WARN_FN(fn, ...) _LOG_FN(Warn, fn, __VA_ARGS__)
-#define ERROR_FN(fn, ...) _LOG_FN(Error, fn, __VA_ARGS__)
-
 struct ServiceBridge::PImpl {
    PImpl(
        OnReady const &on_ready,
@@ -179,72 +173,56 @@ void ServiceBridge::init() {
 } // namespace orderman
 
 void ServiceBridge::on(mercury::Update<StoredFillEvent> const &update) {
-   if (!update.events) {
-      WARN_FN("on StoredFillEvent", "no event data");
-      return;
+   if (update.events) {
+      std::vector<api::orderman::Fill> fills;
+      fills.reserve(update.events->size());
+      for (auto ev : *(update.events))
+         fills.push_back(ev.fill);
+      me->on_fills(fills);
    }
-   INFO_FN("on StoredFillEvent", "n_events: {}", update.events->size());
-   std::vector<api::orderman::Fill> fills;
-   fills.reserve(update.events->size());
-   for (auto ev : *(update.events))
-      fills.push_back(ev.fill);
-   me->on_fills(fills);
 }
 
 void ServiceBridge::on(mercury::Update<StoredOrderEvent> const &update) {
-   if (!update.events) {
-      WARN_FN("on StoredOrderEvent", "no event data");
-      return;
-   }
-   INFO_FN("on StoredOrderEvent", "n_events: {}", update.events->size());
-   bool was_ready = is_ready();
-   me->have_orders = true;
+   if (update.events) {
+      bool was_ready = is_ready();
+      me->have_orders = true;
 
-   std::vector<api::orderman::OrderRecord> orders;
-   orders.reserve(update.events->size());
-   for (auto ev : *(update.events)) {
-      me->known_orders.insert(OrderKey(ev.order_record.key()));
-      orders.push_back(ev.order_record);
+      std::vector<api::orderman::OrderRecord> orders;
+      orders.reserve(update.events->size());
+      for (auto ev : *(update.events)) {
+         me->known_orders.insert(OrderKey(ev.order_record.key()));
+         orders.push_back(ev.order_record);
+      }
+      me->on_orders(orders);
+      maybe_signal_ready(was_ready);
    }
-   me->on_orders(orders);
-   maybe_signal_ready(was_ready);
 }
 
 void ServiceBridge::on(mercury::Update<OrderManagerReadyEvent> const &update) {
-   if (!update.events) {
-      WARN_FN("on OrderManagerReadyEvent", "no event data");
-      return;
+   if (update.events) {
+      bool was_ready = is_ready();
+      me->order_manager_ready = true;
+      maybe_signal_ready(was_ready);
    }
-   INFO_FN("on OrderManagerReadyEvent", "n_events: {}", update.events->size());
-   bool was_ready = is_ready();
-   me->order_manager_ready = true;
-   maybe_signal_ready(was_ready);
 }
 
 void ServiceBridge::on(mercury::Update<NewAdminDataEvent> const &update) {
-   if (!update.events) {
-      WARN_FN("on NewAdminDataEvent", "no event data");
-      return;
-   }
-   INFO_FN("on NewAdminDataEvent", "n_events: {}", update.events->size());
-   for (auto ev : *(update.events)) {
-      me->order_router.update(*ev.admin_data);
-   }
+   if (update.events)
+      for (auto ev : *(update.events))
+         me->order_router.update(*ev.admin_data);
 }
 
 void ServiceBridge::on(mercury::Update<OrderOpResultEvent> const &update) {
-   if (!update.events) {
-      WARN_FN("on OrderOpResultEvent", "no event data");
-      return;
-   }
-   INFO_FN("on OrderOpResultEvent", "n_events: {}", update.events->size());
-   for (auto ev : *(update.events)) {
-      me->on_order_op_result(ev);
-   }
+   if (update.events)
+      for (auto ev : *(update.events))
+         me->on_order_op_result(ev);
 }
 
 void ServiceBridge::create_order(uint32_t rid, api::orderman::Create const &c) {
-   INFO_FN("create_order", "rid: {}; req: {}", rid, utils::pb::to_string(c));
+
+   LOG(OrderMan, Info, "Received new order from client: {}",
+       utils::pb::to_string(c));
+
    me->service_call_marshaller.schedule_action([this, c, rid]() {
       if (is_ready()) {
          auto &order =
@@ -261,7 +239,9 @@ void ServiceBridge::create_order(uint32_t rid, api::orderman::Create const &c) {
 }
 
 void ServiceBridge::update_order(uint32_t rid, api::orderman::Update const &c) {
-   INFO_FN("update_order", "rid: {}; req: {}", rid, utils::pb::to_string(c));
+   LOG(OrderMan, Info, "ServiceBridge::update_order update: {}",
+       utils::pb::to_string(c));
+
    me->service_call_marshaller.schedule_action([this, c, rid]() {
       auto ok = OrderKey(c.order_key());
       if (is_ready()) {
@@ -279,7 +259,9 @@ void ServiceBridge::update_order(uint32_t rid, api::orderman::Update const &c) {
 }
 
 void ServiceBridge::cancel_order(uint32_t rid, api::orderman::Cancel const &c) {
-   INFO_FN("cancel_order", "rid: {}; req: {}", rid, utils::pb::to_string(c));
+   LOG(OrderMan, Info, "Received order cancel from client: {}",
+       utils::pb::to_string(c));
+
    me->service_call_marshaller.schedule_action([this, c, rid]() {
       auto ok = OrderKey(c.order_key());
       if (is_ready()) {
@@ -297,7 +279,9 @@ void ServiceBridge::cancel_order(uint32_t rid, api::orderman::Cancel const &c) {
 }
 
 void ServiceBridge::detach_order(uint32_t rid, api::orderman::Detach const &c) {
-   INFO_FN("detach_order", "rid: {}; req: {}", rid, utils::pb::to_string(c));
+   LOG(OrderMan, Info, "Received order detach from client: {}",
+       utils::pb::to_string(c));
+
    me->service_call_marshaller.schedule_action([this, c, rid]() {
       auto ok = OrderKey(c.order_key());
       if (is_ready()) {
@@ -316,13 +300,15 @@ void ServiceBridge::detach_order(uint32_t rid, api::orderman::Detach const &c) {
 
 void ServiceBridge::manually_bust_all_fills(
     const api::orderman::ManualBustParameters &c) {
-   INFO_FN("manually_bust_all_fills", "req: {}", utils::pb::to_string(c));
+   LOG(OrderMan, Info, "ServiceBridge::manually_bust_all_fills: {}",
+       utils::pb::to_string(c));
    me->service_call_marshaller.schedule_action(
        [this, c]() { publish(me->manually_bust_fills_pub, c.account()); });
 }
 
 void ServiceBridge::send_rfq(api::orderman::RFQRequest const &rfq) {
-   INFO_FN("send_rfq", "req: {}", utils::pb::to_string(rfq));
+   LOG(OrderMan, Info, "ServiceBridge::send_rfq rfq: {}",
+       utils::pb::to_string(rfq));
    me->service_call_marshaller.schedule_action(
        [this, rfq]() { publish(me->rfq_pub, rfq); });
 }
@@ -412,11 +398,6 @@ std::optional<FilledPositionEvent> make_pos_event(
 
 void ServiceBridge::update_position(
     api::position::Position_dataUpdate const &pos) {
-   INFO_FN(
-       "update_position",
-       "n_added: {}; n_changed: {}; n_removed: {}; n_total: {}",
-       pos.added().size(), pos.changed().size(), pos.removed().size(),
-       pos.all().size());
    std::vector<FilledPositionEvent> evs;
    for (auto const &pu : pos.added()) {
       auto ev = make_pos_event(*pu, me->sec_db);
@@ -449,9 +430,6 @@ bool ServiceBridge::is_ready() const {
 }
 
 void ServiceBridge::maybe_signal_ready(bool old_ready) {
-   INFO_FN(
-       "maybe_signal_ready", "old_ready: {}; is_ready: {}", old_ready,
-       is_ready());
    if (!old_ready && is_ready())
       me->on_ready();
 }

src/lib/orderman/actors/order_manager/OrderManager.cpp

@@ -40,13 +40,6 @@
 namespace bts {
 namespace orderman {
 
-#define _LOG_FN(level, fn, ...)                                                \
-   LOG(OrderMan, level, "OrderManager::" fn ". " __VA_ARGS__)
-#define DIAG_FN(fn, ...) _LOG_FN(Diag, fn, __VA_ARGS__)
-#define INFO_FN(fn, ...) _LOG_FN(Info, fn, __VA_ARGS__)
-#define WARN_FN(fn, ...) _LOG_FN(Warn, fn, __VA_ARGS__)
-#define ERROR_FN(fn, ...) _LOG_FN(Error, fn, __VA_ARGS__)
-
 using namespace mercury;
 
 enum class OrderManager::State {
@@ -87,23 +80,6 @@ struct OrderManager::PImpl {
    mercury::Stream<OrderManagerReadyEvent> pub_ready;
    std::thread background_thread;
    std::atomic<bool> shutdown = false;
-
-   constexpr std::string state_name() {
-      switch (state) {
-      case State::WAIT_ADMIN:
-         return "WAIT_ADMIN";
-      case State::WAIT_SEC:
-         return "WAIT_SEC";
-      case State::WAIT_FILLED_POSITION:
-         return "WAIT_FILLED_POSITION";
-      case State::WAIT_ORDERS:
-         return "WAIT_ORDERS";
-      case State::READY:
-         return "READY";
-      default:
-         return "UNKNOWN";
-      }
-   }
 };
 
 OrderManager::OrderManager(
@@ -132,7 +108,6 @@ OrderManager::~OrderManager() {
 }
 
 void OrderManager::init() {
-   INFO_FN("init", "state: {}", me->state_name());
    subscribe<NewAdminDataEvent>(
        [this](auto const &admin_update) { this->on_admin_data(admin_update); });
    subscribe<AccountPnLUpdateEvent>(
@@ -144,365 +119,301 @@ void OrderManager::init() {
 }
 
 void OrderManager::on_admin_data(const mercury::Update<NewAdminDataEvent> &m) {
-   if (!m.events) {
-      WARN_FN("on_admin_data", "no event data. state: {}", me->state_name());
-      return;
-   }
-   INFO_FN("on_admin_data", "state: {}", me->state_name());
-   if (me->state == State::WAIT_ADMIN) {
-      for (auto &osd : me->services_data.sessions)
-         osd->gw->on_admin_data(m);
+   if (m.events) {
+      if (me->state == State::WAIT_ADMIN) {
+         LOG(OrderMan, Info,
+             "Order manager received admin data. Subscribing to security list "
+             "now ");
+         for (auto &osd : me->services_data.sessions)
+            osd->gw->on_admin_data(m);
 
-      me->state = State::WAIT_SEC;
-      INFO_FN("on_admin_data", "state: {}", me->state_name());
-      subscribe<NewSecuritiesEvent>(
-          [this](auto const &sec_update) { this->on_securities(sec_update); });
-   }
+         me->state = State::WAIT_SEC;
+         subscribe<NewSecuritiesEvent>([this](auto const &sec_update) {
+            this->on_securities(sec_update);
+         });
+      }
 
-   for (auto ev : *m.events) {
-      me->services_data.order_router.update(*ev.admin_data);
-      me->services_data.limiter.update(*ev.admin_data);
+      for (auto ev : *m.events) {
+         me->services_data.order_router.update(*ev.admin_data);
+         me->services_data.limiter.update(*ev.admin_data);
+      }
    }
 }
 
 void OrderManager::on_pnl_update(
     const mercury::Update<AccountPnLUpdateEvent> &u) {
-   if (!u.events) {
-      WARN_FN("on_pnl_update", "no event data. state: {}", me->state_name());
-      return;
-   }
-   // This log is too noisy. It fires regularly on an interval (sometimes when
-   // the pnl hasn't actually changed).
-   // INFO_FN(
-   //     "on_pnl_update", "state: {}; n_events: {}", me->state_name(),
-   //     u.events->size());
-   for (auto ev : *u.events) {
-      me->services_data.limiter.set_day_pnl(ev.account_id, ev.day_pnl);
-   }
+   if (u.events)
+      for (auto ev : *u.events)
+         me->services_data.limiter.set_day_pnl(ev.account_id, ev.day_pnl);
 }
 
 void OrderManager::on_excess_liquidity_update(
     const mercury::Update<ExcessLiquidityUpdateEvent> &u) {
-   if (!u.events) {
-      WARN_FN(
-          "on_excess_liquidity_update", "no event data. state: {}",
-          me->state_name());
-      return;
-   }
-   INFO_FN(
-       "on_excess_liquidity_update", "state: {}; n_events: {}",
-       me->state_name(), u.events->size());
-   for (auto ev : *u.events) {
-      me->services_data.limiter.set_excess_liquidity(
-          ev.account_id, ev.excess_liquidity);
-   }
+   if (u.events)
+      for (auto ev : *u.events)
+         me->services_data.limiter.set_excess_liquidity(
+             ev.account_id, ev.excess_liquidity);
 }
 
 void OrderManager::on_rfq_send(const mercury::Update<RFQEvent> &rfq) {
-   if (!rfq.events) {
-      WARN_FN("on_rfq_send", "no event data. state: {}", me->state_name());
-      return;
-   }
-   INFO_FN(
-       "on_rfq_send", "state: {}; n_events: {}", me->state_name(),
-       rfq.events->size());
-   std::vector<OrderMsg> msgs = {};
-   for (auto ev : *rfq.events) {
-      RFQMsg rfq_msg = RFQMsg(ev);
-      msgs.emplace_back(rfq_msg);
-   }
-   trace::CorID cid = {};
-   for (auto &osd : me->services_data.sessions) {
-      osd->gw->send_msgs(msgs, cid);
+   if (rfq.events) {
+      std::vector<OrderMsg> msgs = {};
+      for (auto ev : *rfq.events) {
+         RFQMsg rfq_msg = RFQMsg(ev);
+         msgs.emplace_back(rfq_msg);
+      }
+      trace::CorID cid = {};
+      LOG(OrderMan, Info, "OrderManager::on_rfq_send n_events: {}",
+          rfq.events->size());
+      for (auto &osd : me->services_data.sessions)
+         osd->gw->send_msgs(msgs, cid);
    }
 }
 
 void OrderManager::on_securities(
     const mercury::Update<NewSecuritiesEvent> &update) {
-   if (!update.events) {
-      WARN_FN("on_securities", "no event data. state: {}", me->state_name());
-      return;
-   }
-   INFO_FN(
-       "on_securities", "state: {}; n_events: {}", me->state_name(),
-       update.events->size());
+
    // forward to execution gateways
    for (auto &osd : me->services_data.sessions)
       osd->gw->on_securities(update);
 
-   // update our security db and accumulate for the sake of the limiter
-   std::vector<api::Security const *> secs;
-   for (auto const &ev : *(update.events)) {
-      auto sec_ptr = ev.security;
-      secs.push_back(sec_ptr);
-   }
-   me->services_data.limiter.add_securities(secs);
+   if (update.events) {
 
-   if (me->state == State::WAIT_SEC) {
-      me->state = State::WAIT_FILLED_POSITION;
-      INFO_FN(
-          "on_securities", "received initial set of {} securities. state: {}",
-          update.events->size(), me->state_name());
-      me->filled_position_sub = subscribe<FilledPositionEvent>(
-          [this](auto const &pos) { this->on_filled_position(pos); });
+      // update our security db and accumulate for the sake of the limiter
+      std::vector<api::Security const *> secs;
+      for (auto const &ev : *(update.events)) {
+         auto sec_ptr = ev.security;
+         secs.push_back(sec_ptr);
+      }
+      me->services_data.limiter.add_securities(secs);
+
+      if (me->state == State::WAIT_SEC) {
+         LOG(OrderMan, Info,
+             "Order manager received initial set of {} "
+             "securities. Waiting for the filled position now",
+             update.events->size());
+         me->state = State::WAIT_FILLED_POSITION;
+         me->filled_position_sub = subscribe<FilledPositionEvent>(
+             [this](auto const &pos) { this->on_filled_position(pos); });
+      }
    }
 }
 
 void OrderManager::on_filled_position(
     const mercury::Update<FilledPositionEvent> &update) {
-   if (!update.events) {
-      WARN_FN(
-          "on_filled_position", "no event data. state: {}", me->state_name());
-      return;
-   }
-   INFO_FN(
-       "on_filled_position", "state: {}; n_events: {}", me->state_name(),
-       update.events->size());
-   if (me->state == State::WAIT_FILLED_POSITION) {
-      me->state = State::WAIT_ORDERS;
-      INFO_FN("on_filled_position", "new_state: {}", me->state_name());
-      me->stored_order_sub = subscribe<StoredOrderEvent>(
-          [this](auto const &orders) { this->on_order_recovery(orders); });
-   }
+   if (update.events) {
+      if (me->state == State::WAIT_FILLED_POSITION) {
+         LOG(OrderMan, Info,
+             "Order manager received initial set of {} "
+             "stored filled position records. Waiting for stored orders now",
+             update.events->size());
+         me->state = State::WAIT_ORDERS;
+         me->stored_order_sub = subscribe<StoredOrderEvent>(
+             [this](auto const &orders) { this->on_order_recovery(orders); });
+      }
 
-   for (auto const &ev : *(update.events)) {
-      me->services_data.position.update(ev);
-      me->services_data.limiter.set_filled_position(
-          *ev.security, ev.account_id, ev.long_quantity, ev.short_quantity);
+      for (auto const &ev : *(update.events)) {
+         me->services_data.position.update(ev);
+         me->services_data.limiter.set_filled_position(
+             *ev.security, ev.account_id, ev.long_quantity, ev.short_quantity);
+      }
    }
 }
 
 void OrderManager::on_order_recovery(
     const mercury::Update<StoredOrderEvent> &update) {
-   if (!update.events) {
-      WARN_FN(
-          "on_order_recovery", "no event data. state: {}", me->state_name());
-      return;
-   }
-   if (me->state != State::WAIT_ORDERS) {
-      WARN_FN(
-          "on_order_recovery",
-          "Expected state WAIT_ORDERS. state: {}; n_events: {}",
-          me->state_name(), update.events->size());
-      return;
-   }
-   // Got the first batch of persisted orders which has to match the orders
-   // stored on disk (since we are the only ones producing new orders)
-   INFO_FN(
-       "on_order_recovery", "state: {}; n_events: {}", me->state_name(),
-       update.events->size());
-   // Don't need to monitor persisted orders anymore, all the new orders we care
-   // about will be generated by us.
-   unsubscribe(me->stored_order_sub);
+   if (update.events)
+      if (me->state == State::WAIT_ORDERS) {
 
-   init_order_services();
+         // Got the first batch of persisted orders which has to match
+         // the orders stored on disk (since we are the only ones
+         // producing new orders)
+         LOG(OrderMan, Info,
+             "Order manager received initial set of {} persisted orders.",
+             update.events->size());
 
-   // recover orders from disk
-   for (auto const &ev : *(update.events)) {
-      auto &order = me->services_data.order_factory.recover(ev.order_record);
-      auto const &def = order.def();
-      publish_order_def_update(OrderKey(ev.key()), def);
-      order.init_order_services(me->services_data);
+         // Don't need to monitor persisted orders anymore, all the
+         // new orders we care about will be generated by us.
+         unsubscribe(me->stored_order_sub);
 
-      auto r = me->orders.emplace(order.key(), &order);
-      bts_assert(r.second);
-      auto const &ostate = ev.order_record.state();
-      order.on_activate(
-          Price(ostate.price()), Quantity(ostate.quantity()), *order._services);
-   }
+         init_order_services();
 
-   // signal that we are ready
-   INFO_FN("on_order_recovery", "OrderManager is ready");
-   publish(me->pub_ready);
-   me->state = State::READY;
+         // recover orders from disk
+         for (auto const &ev : *(update.events)) {
+            auto &order =
+                me->services_data.order_factory.recover(ev.order_record);
+            auto const &def = order.def();
+            publish_order_def_update(OrderKey(ev.key()), def);
+            order.init_order_services(me->services_data);
+
+            auto r = me->orders.emplace(order.key(), &order);
+            bts_assert(r.second);
+            auto const &ostate = ev.order_record.state();
+            order.on_activate(
+                Price(ostate.price()), Quantity(ostate.quantity()),
+                *order._services);
+         }
+
+         // signal that we are ready
+         LOG(OrderMan, Info, "Order manager is ready");
+         publish(me->pub_ready);
+         me->state = State::READY;
+      }
 }
 
 void OrderManager::on_execution_report(
     const mercury::Update<NewExecutionReportEvent> &m) {
-   if (!m.events) {
-      WARN_FN("on_execution_report", "no event data");
-      return;
-   }
-   INFO_FN(
-       "on_execution_report", "state: {}; n_events", me->state_name(),
-       m.events->size());
-   for (auto const &report : *m.events) {
-      if (report.data->leg_security != nullptr) {
-         continue;
-      }
-      auto k = report.data->order_key;
-      auto it = me->orders.find(k);
-      if (it == me->orders.end()) {
-         ERROR_FN(
-             "on_execution_report",
-             "Received an execution report for an unknown order with key {}. "
-             "Order state USED TO be reconstructed from this report but some "
-             "information will be lost, including CustomerOrFirm,CTI code and "
-             "parent order id but now these are IGNORED",
-             k.value());
-         continue;
-         // auto order = &me->services_data.order_factory.recreate_from_exe_report(
-         //     *report.data);
-         // auto const &def = order->def();
-         // publish_order_def_update(OrderKey(k), def);
-         // order->init_order_services(me->services_data);
-         // auto r = me->orders.emplace(order->key(), order);
-         // bts_assert(r.second);
-         // order->on_activate(
-         //     report.data->price, report.data->quantity, *order->_services);
-      }
-   }
+   if (m.events)
+      for (auto const &report : *m.events)
+         if (report.data->leg_security == nullptr) {
+            auto k = report.data->order_key;
+            auto it = me->orders.find(k);
+            if (it == me->orders.end()) {
+               LOG(OrderMan, Error,
+                   "Received an execution report for an unknown order with key "
+                   "{}. Order state USED TO be reconstructed from this report but "
+                   "some "
+                   "information will be lost, including CustomerOrFirm,CTI "
+                   "code "
+                   "and "
+                   "parent order id but now these are IGNORED",
+                   k.value());
+               continue;
+               auto order =
+                   &me->services_data.order_factory.recreate_from_exe_report(
+                       *report.data);
+               auto const &def = order->def();
+               publish_order_def_update(OrderKey(k), def);
+               order->init_order_services(me->services_data);
+               auto r = me->orders.emplace(order->key(), order);
+               bts_assert(r.second);
+               order->on_activate(
+                   report.data->price, report.data->quantity,
+                   *order->_services);
+            }
+         }
 }
 
 void OrderManager::on_activate_order(
     const mercury::Update<ActivateOrderEvent> &m) {
-   if (!m.events) {
-      WARN_FN("on_activate_order", "no event data");
-      return;
-   }
-   INFO_FN(
-       "on_activate_order", "state: {}; n_events: {}", me->state_name(),
-       m.events->size());
-   for (auto const &ev : *m.events) {
-      LOG(OrderMan, Info,
-          "OrderManager::on_activate_order req_id: {} p: {} q: {}", ev.req_id,
-          ev.price, ev.quantity);
-      auto order = ev.order;
-      auto const &def = order->def();
-      publish_order_def_update(order->key(), def);
-      order->init_order_services(me->services_data);
-      auto r = me->orders.emplace(order->key(), order);
-      bts_assert(r.second);
+   if (m.events)
+      for (auto const &ev : *m.events) {
+         LOG(OrderMan, Info,
+             "OrderManager::on_activate_order req_id: {} p: {} q: {}",
+             ev.req_id, ev.price, ev.quantity);
+         auto order = ev.order;
+         auto const &def = order->def();
+         publish_order_def_update(order->key(), def);
+         order->init_order_services(me->services_data);
+         auto r = me->orders.emplace(order->key(), order);
+         bts_assert(r.second);
 
-      auto rid = ev.req_id;
-      auto k = order->key();
-      auto on_response = [this, rid, k](auto const &order_op_result) {
-         publish(
-             me->services_data.report_order_op_result_pub, k, rid,
-             order_op_result);
-      };
-      order->_services->activate(ev.price, ev.quantity, on_response);
-   }
+         auto rid = ev.req_id;
+         auto k = order->key();
+         auto on_response = [this, rid, k](auto const &order_op_result) {
+            publish(
+                me->services_data.report_order_op_result_pub, k, rid,
+                order_op_result);
+         };
+         order->_services->activate(ev.price, ev.quantity, on_response);
+      }
 }
 
 void OrderManager::on_update_order(const mercury::Update<UpdateOrderEvent> &m) {
-   if (!m.events) {
-      WARN_FN("on_update_order", "no event data");
-      return;
-   }
-   INFO_FN(
-       "on_update_order", "state: {}; n_events: {}", me->state_name(),
-       m.events->size());
-   for (auto const &ev : *m.events) {
-      auto k = ev.key();
-      auto it = me->orders.find(k);
-      auto rid = ev.req_id();
+   if (m.events)
+      for (auto const &ev : *m.events) {
+         auto k = ev.key();
+         auto it = me->orders.find(k);
+         auto rid = ev.req_id();
 
-      auto on_order_op_result = [this, rid, k](auto const &order_op_result) {
-         publish(
-             me->services_data.report_order_op_result_pub, k, rid,
-             order_op_result);
-      };
+         auto on_order_op_result = [this, rid, k](auto const &order_op_result) {
+            publish(
+                me->services_data.report_order_op_result_pub, k, rid,
+                order_op_result);
+         };
 
-      if (it != me->orders.end()) {
-         if (ev.is_full_update()) {
-            it->second->_services->update(ev.full_update(), on_order_op_result);
+         if (it != me->orders.end()) {
+            if (ev.is_full_update())
+               it->second->_services->update(
+                   ev.full_update(), on_order_op_result);
+            else
+               it->second->_services->update(
+                   Price(ev.price()), Quantity(ev.quantity()),
+                   on_order_op_result);
          } else {
-            it->second->_services->update(
-                Price(ev.price()), Quantity(ev.quantity()), on_order_op_result);
+            on_order_op_result(OrderOpResult{
+                api::orderman::ORDER_ERROR_UNKNOWN_ORDER, "Unknown order"});
+            LOG(OrderMan, Error,
+                "Received an update request for an unknown order {}. "
+                "This should not happen and likely indicates a serious bug. ",
+                ev.key().value());
          }
-      } else {
-         on_order_op_result(OrderOpResult{
-             api::orderman::ORDER_ERROR_UNKNOWN_ORDER, "Unknown order"});
-         ERROR_FN(
-             "on_update_order",
-             "order not found (id={}). This likely indicates a serious bug.",
-             ev.key().value());
       }
-   }
 }
 
 void OrderManager::on_cancel_order(const mercury::Update<CancelOrderEvent> &m) {
-   if (!m.events) {
-      WARN_FN("on_cancel_order", "no event data");
-      return;
-   }
-   INFO_FN(
-       "on_cancel_order", "state: {}; n_events: {}", me->state_name(),
-       m.events->size());
-   for (auto const &ev : *m.events) {
-      auto k = ev.key();
-      auto it = me->orders.find(k);
-      auto rid = ev.req_id;
+   if (m.events)
+      for (auto const &ev : *m.events) {
+         auto k = ev.key();
+         auto it = me->orders.find(k);
+         auto rid = ev.req_id;
 
-      auto on_order_op_result = [this, rid, k](auto const &order_op_result) {
-         publish(
-             me->services_data.report_order_op_result_pub, k, rid,
-             order_op_result);
-      };
+         auto on_order_op_result = [this, rid, k](auto const &order_op_result) {
+            publish(
+                me->services_data.report_order_op_result_pub, k, rid,
+                order_op_result);
+         };
 
-      if (it != me->orders.end()) {
-         it->second->_services->cancel(on_order_op_result);
-      } else {
-         on_order_op_result(OrderOpResult{
-             api::orderman::ORDER_ERROR_UNKNOWN_ORDER, "Unknown order"});
-         ERROR_FN(
-             "on_cancel_order",
-             "order not found (id={}). This likely indicates a serious bug.",
-             ev.key().value());
+         if (it != me->orders.end())
+            it->second->_services->cancel(on_order_op_result);
+         else {
+            on_order_op_result(OrderOpResult{
+                api::orderman::ORDER_ERROR_UNKNOWN_ORDER, "Unknown order"});
+            LOG(OrderMan, Error,
+                "Received a cancel request for an unknown order {}. "
+                "This should not happen and likely indicates a serious bug. ",
+                ev.key().value());
+         }
       }
-   }
 }
 
 void OrderManager::on_detach_order(const mercury::Update<DetachOrderEvent> &m) {
-   if (!m.events) {
-      WARN_FN("on_detach_order", "no event data");
-      return;
-   }
-   INFO_FN(
-       "on_detach_order", "state: {}; n_events: {}", me->state_name(),
-       m.events->size());
-   for (auto const &ev : *m.events) {
-      auto k = ev.key();
-      auto it = me->orders.find(k);
-      auto rid = ev.req_id;
+   if (m.events)
+      for (auto const &ev : *m.events) {
+         auto k = ev.key();
+         auto it = me->orders.find(k);
+         auto rid = ev.req_id;
 
-      auto on_order_op_result = [this, rid, k](auto const &order_op_result) {
-         publish(
-             me->services_data.report_order_op_result_pub, k, rid,
-             order_op_result);
-      };
+         auto on_order_op_result = [this, rid, k](auto const &order_op_result) {
+            publish(
+                me->services_data.report_order_op_result_pub, k, rid,
+                order_op_result);
+         };
 
-      if (it != me->orders.end()) {
-         it->second->_services->detach(on_order_op_result);
-      } else {
-         on_order_op_result(OrderOpResult{
-             api::orderman::ORDER_ERROR_UNKNOWN_ORDER, "Unknown Order"});
+         if (it != me->orders.end())
+            it->second->_services->detach(on_order_op_result);
+         else {
+            on_order_op_result(OrderOpResult{
+                api::orderman::ORDER_ERROR_UNKNOWN_ORDER, "Unknown Order"});
 
-         ERROR_FN(
-             "on_detach_order",
-             "order not found (id={}). This likely indicates a serious bug.",
-             ev.key().value());
+            LOG(OrderMan, Error,
+                "Received a detach request for an unknown order {}. "
+                "This should not happen and likely indicates a serious bug. ",
+                ev.key().value());
+         }
       }
-   }
 }
 
 void OrderManager::on_gateway_state(
     const mercury::Update<GatewayStatusChangeEvent> &m) {
-   if (!m.events) {
-      WARN_FN("on_gateway_state", "no event data");
-      return;
-   }
-   INFO_FN(
-       "on_gateway_state", "state: {}; n_events: {}", me->state_name(),
-       m.events->size());
-   for (auto const &ev : *m.events) {
-      LOG(OrderMan, Info,
-          " OrderManager::on_gateway_state gw: {} connected: {}", ev.gw_name,
-          ev.is_connected);
-      auto &sr = me->services_data.get_session(ev.gw_name.value());
-      sr.gw_connected = ev.is_connected;
-   }
+   if (m.events)
+      for (auto const &ev : *m.events) {
+         LOG(OrderMan, Info,
+             " OrderManager::on_gateway_state gw: {} connected: {}", ev.gw_name,
+             ev.is_connected);
+         auto &sr = me->services_data.get_session(ev.gw_name.value());
+         sr.gw_connected = ev.is_connected;
+      }
 }
 
 void OrderManager::add_child_order(OrderServices &parent, Order &child) {
@@ -560,12 +471,9 @@ void OrderManager::init_order_services() {
                   return s->session;
                }),
        ", "));
-   INFO_FN(
-       "init_order_services", "state: {}; sessions: {}", me->state_name(),
-       session_names);
+   LOG(OrderMan, Info, "init_order_services. sessions: {}", session_names);
 
    for (auto &s : me->services_data.sessions) {
-      INFO_FN("init_order_services", "Initializing session {}", s->session);
       auto gw_name = GatewayName(s->session);
 
       // Now we are ready to subscribe to the execution gateway
@@ -628,11 +536,9 @@ void OrderManager::init_order_services() {
       // reject evets
       auto s_ptr = s.get();
       s->exe_report_pub = make_stream<NewExecutionReportEvent>(gw_name);
-      s->gw->on_exe_report() =
-          [s_ptr,
-           this](std::shared_ptr<const ExecutionReport> er, auto const &t) {
-             publish(s_ptr->exe_report_pub, std::move(er), t);
-          };
+      s->gw->on_exe_report() = [s_ptr, this](auto const &er, auto const &t) {
+         publish(s_ptr->exe_report_pub, &er, t);
+      };
       s->gw_status_change_pub = make_stream<GatewayStatusChangeEvent>(gw_name);
       s->gw->on_gw_status_change() = [s_ptr, this, gw_name](bool b) {
          publish(s_ptr->gw_status_change_pub, gw_name, b);
@@ -658,10 +564,9 @@ void OrderManager::init_order_services() {
       s->gw->on_cancel_reject() = [s_ptr, this](auto const &cr) {
          publish(s_ptr->cancel_reject_pub, cr);
       };
-      INFO_FN("init_order_services", "Starting session {}", s->session);
       // handlers are ready so we can start processing fix events
       s->gw->start();
-      INFO_FN("init_order_services", "Started session {}", s->session);
+
       reload_fix_buffer(*s);
    }
 
@@ -772,10 +677,9 @@ void OrderManager::after_event() {
          if (queued_ops.size()) {
             auto &corid = osd->current_trace;
             auto now = utils::TimeStamp::now();
-            INFO_FN(
-                "after_event",
-                "channel: {} sn: {} app_arrival: {} now: {} "
-                "delta: {}",
+            LOG(Latency, Info,
+                "entry_type:after_event channel:{} sn:{} app_arrival:{} now:{} "
+                "delta:{}",
                 std::string{corid.channel}, corid.sn,
                 corid.app_arrival.to_uint64(), now.to_uint64(),
                 (now - corid.app_arrival).nanoseconds());
@@ -791,9 +695,9 @@ void OrderManager::reload_fix_buffer(SessionServicesData &osd) {
       osd.buffered_fix_msgs = osd.free_buffers.back();
       osd.free_buffers.pop_back();
    } else {
-      WARN_FN(
-          "reload_fix_buffer",
-          "Allocating new fix buffers in {}. This will impact performance",
+      LOG(OrderMan, Warn,
+          "Ran out of fix buffers in {}.  Will allocate directly. "
+          "This will have an impact on performance",
           osd.cfg.name());
       osd.buffered_fix_msgs = std::make_shared<jetfix::OutFIX>(osd.cfg);
    }

src/lib/orderman/actors/order_manager/OrderServicesData.cpp

@@ -1,6 +1,10 @@
 #include "OrderServicesData.h"
+#include "lib/config/config.h"
+#include "lib/config/protos/jetfix.pb.h"
 #include "lib/md/SecurityOracle.h"
 #include "lib/orderman/actors/order_manager/order_gateway_definition.h"
+#include "lib/orderman/gateways/ExecutionGatewayFactory.h"
+#include "lib/os/spark_data_directory.h"
 #include "lib/tracing/PerfLog.h"
 
 namespace bts {
@@ -22,6 +26,8 @@ OrderServicesData::OrderServicesData(
    for (auto &g : gateways) {
       sessions.push_back(std::make_unique<SessionServicesData>(std::move(g)));
    }
+   // spark::SparkDataDirectory data_dir;
+   // auto binance_j = data_dir.load_config_json("binance_execution");
 }
 
 } // namespace orderman

src/lib/orderman/actors/order_manager/OrderServicesData.h

@@ -34,6 +34,7 @@
 #include "lib/orderman/events/TickerUpdateEvent.h"
 #include "lib/orderman/events/TradeSignalEvent.h"
 #include "lib/orderman/events/UpdateOrderEvent.h"
+#include "lib/orderman/gateways/ExecutionGatewayFactory.h"
 #include "lib/orderman/gateways/IExecutionGateway.h"
 #include "lib/orderman/limiter/Limiter.h"
 #include "lib/orderman/order_factory/OrderFactory.h"

src/lib/orderman/actors/order_manager/session_services_data.h

@@ -10,6 +10,7 @@
 #include "lib/orderman/events/ExchangePositionEvent.h"
 #include "lib/orderman/events/GatewayStatusChangeEvent.h"
 #include "lib/orderman/events/NewExecutionReportEvent.h"
+#include "lib/orderman/gateways/ExecutionGatewayFactory.h"
 #include "lib/orderman/gateways/IExecutionGateway.h"
 #include "lib/utils/PBUtils.h"
 

src/lib/orderman/ccapi/CCAPIDeribitOrderMux.cpp

@@ -20,7 +20,6 @@
 #include "lib/utils/bts_assert.h"
 #include <bits/stdint-uintn.h>
 #include <boost/lockfree/spsc_queue.hpp>
-#include <ccapi_cpp/bts_session_options.h>
 #include <ccapi_cpp/ccapi_element.h>
 #include <ccapi_cpp/ccapi_event.h>
 #include <ccapi_cpp/ccapi_macro.h>
@@ -256,8 +255,7 @@ struct CCAPIDeribitOrderMux::PImpl {
           {CCAPI_DERIBIT_CLIENT_ID, d_conn_config.client_id},
           {CCAPI_DERIBIT_CLIENT_SECRET, d_conn_config.client_secret},
       });
-      session = new ccapi::Session(
-          {}, {ccapi::BtsVenues::DERIBIT}, session_configs, &event_handler);
+      session = new ccapi::Session({}, session_configs, &event_handler);
    }
 };
 
@@ -955,4 +953,4 @@ void CCAPIDeribitOrderMux::_send_buffered_orders(uint32_t sec_id) {
        "n_buffered_orders: {}",
        me->buffered_creates.size());
 }
-} // namespace bts::orderman
+} // namespace bts::orderman

src/lib/orderman/events/NewExecutionReportEvent.h

@@ -20,13 +20,12 @@ class NewExecutionReportEvent : public mercury::Event<NewExecutionReportEvent> {
  public:
    using Tag = GatewayName;
    using SynopsisPolicy = mercury::CacheAll<NewExecutionReportEvent>;
-   std::shared_ptr<const ExecutionReport> data;
+   ExecutionReport const *data;
    trace::CorID trace;
    // used by ContentSubscription
    OrderKey key() const { return data->order_key; }
-   NewExecutionReportEvent(
-       std::shared_ptr<const ExecutionReport> dt, trace::CorID const &trace)
-       : data(std::move(dt)), trace(trace){};
+   NewExecutionReportEvent(const ExecutionReport *dt, trace::CorID const &trace)
+       : data(dt), trace(trace){};
 };
 
 } // namespace orderman

src/lib/orderman/gateways/CCAPIExecutionGateway.cpp

@@ -40,7 +40,7 @@
 #include <bits/stdint-uintn.h>
 //#include <boost/container/small_vector.hpp>
 #include "lib/orderman/ccapi/CCAPIConnectionConfig.h"
-
+#include <boost/pool/object_pool.hpp>
 #include <cmath>
 #include <cstdint>
 #include <optional>
@@ -138,6 +138,7 @@ struct CCAPIExecutionGateway::PImpl {
    OnExchangeOpenOrdersState on_exchange_open_orders;
    // orders are sent out and callbacks received via IEMOrderMuxer
    size_t msg_seq_num = 0;
+   boost::object_pool<ExecutionReport> exe_report_pool;
    std::unordered_map<OrderKey, CreateOrderMsg> order_map = {};
    // this is updated as we get exchange positions
    ExchangePosition exchange_position;
@@ -338,7 +339,8 @@ void CCAPIExecutionGateway::_on_order_cb(const CCAPIOrderState &s) {
    const auto &cr_msg = it->second;
 
    // cant create one on the stack
-   ExecutionReport er;
+   ExecutionReport *er_p = me->exe_report_pool.construct();
+   ExecutionReport &er = *er_p;
    // length of these have to be <= 12
    to_array(cr_msg.routing_record->account(), &er.account_name.value());
    to_array(cr_msg.routing_record->operator_id(), &er.operator_id.value());
@@ -388,7 +390,7 @@ void CCAPIExecutionGateway::_on_order_cb(const CCAPIOrderState &s) {
    LOG(OrderMan, Info,
        "CCAPIExecutionGateway::_process_cb_and_fire_er firing er: {}",
        er.to_string());
-   me->on_exe_report(std::make_shared<ExecutionReport>(std::move(er)), {});
+   me->on_exe_report(er, {});
    // _fire_leg_er(em_cb, er);
 }
 

src/lib/orderman/gateways/ExecutionGatewayFactory.cpp

@@ -0,0 +1,59 @@
+#include "ExecutionGatewayFactory.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/md/SecurityOracle.h"
+#include "lib/orderman/gateways/CCAPIExecutionGateway.h"
+// #include "lib/orderman/gateways/EMExecutionGateway.h"
+#include "lib/orderman/gateways/FakeExecutionGateway.h"
+#include "lib/orderman/gateways/GeneralExecutionGateway.h"
+#include "lib/orderman/gateways/GLBXExecutionGateway.h"
+#include "lib/orderman/gateways/GLBXiLink3ExecutionGateway.h"
+#include "lib/orderman/gateways/ICEExecutionGateway.h"
+#include "lib/orderman/gateways/IExecutionGenericAdapter.h"
+#include "lib/orderman/gateways/OkxExecutionGateway.h"
+#include "lib/orderman/gateways/OnixFIXAdapter.h"
+#include "lib/orderman/gateways/sim/SimExecutionGatewayFactory.h"
+#include "lib/orderman/gateways/sim/SimExecutionGatewayUniqueWrapper.h"
+#include "lib/tracing/PerfLog.h"
+#include "lib/utils/PBUtils.h"
+#include "lib/utils/ValueWrapper.h"
+#include <memory>
+#include <variant>
+
+namespace bts::orderman {
+
+std::unique_ptr<IExecutionGateway> ExecutionGatewayFactory::create_gateway(
+    const config::JetFIX::Connection &conn,
+    trace::PerfLog &pl,
+    md::SecurityOracle o) {
+   LOG(OrderMan, Info, "ExecutionGatewayFactory::create_gateway conn: {}",
+       utils::pb::to_string(conn));
+   if (SimExecutionGatewayFactory::is_sim_execution_on(conn.venue())) {
+      return std::make_unique<SimExecutionGatewayUniqueWrapper>(conn.venue());
+   }
+   LOG(OrderMan, Info, "ExecutionGatewayFactory::create_gateway NOT SIM conn: {}",
+       utils::pb::to_string(conn));
+   switch(conn.venue()){
+      case bts::api::VENUE_ICE:
+      case bts::api::VENUE_SIMICE:
+         return std::make_unique<ICEExecutionGateway>(conn, pl, o);
+      case bts::api::VENUE_GLBX:{
+         if (conn.use_ilink3())
+            return std::make_unique<ilink3::GLBXiLink3ExecutionGateway>( conn, pl, o);
+         return std::make_unique<GLBXExecutionGateway>(conn, pl, o);
+      }
+      case bts::api::VENUE_DERIBIT:
+         return std::make_unique<CCAPIExecutionGateway>(conn.venue(), o);
+      case bts::api::VENUE_CBOE:
+         {
+             std::unique_ptr<IExecutionGenericAdapter> adapter = std::make_unique<OnixFIXAdapter>(o);
+            return std::make_unique<GeneralExecutionGateway>(conn.venue(),std::move(adapter),o);
+         }
+      case bts::api::VENUE_OKX:
+         {  return std::make_unique<OkxExecutionGateway>(o);
+         }
+      default:
+         return std::make_unique<FakeExecutionGateway>(o);
+   }
+}
+
+} // namespace bts::orderman

src/lib/orderman/gateways/ExecutionGatewayFactory.h

@@ -0,0 +1,25 @@
+#pragma once
+#include "lib/config/protos/jetfix.pb.h"
+#include <memory>
+
+namespace bts::trace {
+class PerfLog;
+}
+
+namespace bts::md {
+class SecurityOracle;
+}
+
+namespace bts::orderman {
+
+class IExecutionGateway;
+
+class ExecutionGatewayFactory {
+ public:
+   static std::unique_ptr<IExecutionGateway> create_gateway(
+       const config::JetFIX::Connection &,
+       trace::PerfLog &,
+       md::SecurityOracle);
+};
+
+} // namespace bts::orderman

src/lib/orderman/gateways/FakeExecutionGateway.cpp

@@ -25,6 +25,7 @@
 #include "server/sparkd/md/platforms/emumba/IEMOrderMuxer.h"
 
 #include <atomic>
+#include <boost/pool/object_pool.hpp>
 #include <cstdint>
 #include <set>
 #include <string>
@@ -53,7 +54,7 @@ struct FakeExecutionGateway::PImpl {
    md::SecurityOracle sec_oracle;
    std::vector<OrderMsg> buffered_msgs = {};
    unsigned long exchange_oid = 0;
-   std::vector<std::shared_ptr<ExecutionReport>> exe_reports = {};
+   std::vector<ExecutionReport> exe_reports = {};
    IEMOrderMuxer *_order_mux = nullptr;
 };
 
@@ -83,9 +84,7 @@ void FakeExecutionGateway::dispatch_incoming_fix_messages() {
              "FakeExecutionGateway::dispatch_incoming_fix_message key: {} sec: "
              "{} p: {} q: {}",
              cr_msg.key, cr_msg.security->id(), cr_msg.price, cr_msg.quantity);
-         me->exe_reports.push_back(std::make_shared<ExecutionReport>());
-         std::shared_ptr<ExecutionReport> &er_p = me->exe_reports.back();
-         ExecutionReport &er = *er_p;
+         ExecutionReport er = {};
          std::string account_name = "ACCA1";
          std::array<char, 12> an_array = {};
          size_t i = 0;
@@ -116,18 +115,19 @@ void FakeExecutionGateway::dispatch_incoming_fix_messages() {
          er.msg = "Message Placed";
          er.exchange = api::VENUE_SIM;
          trace::CorID cid = {};
+         me->exe_reports.push_back(er);
+         const auto &er_pheap = me->exe_reports.back();
          LOG(OrderMan, Info,
              "FakeExecutionGateway::dispatch_incoming_fix_messages er: {}",
-             er.to_string());
-         me->on_exe_report(er_p, cid);
+             er_pheap.to_string());
+         me->on_exe_report(er_pheap, cid);
       } else if (std::holds_alternative<CancelOrderMsg>(m)) {
          CancelOrderMsg cxl_m = std::get<CancelOrderMsg>(m);
-         for (std::shared_ptr<ExecutionReport> &er_p : me->exe_reports) {
-            auto &er = *er_p;
+         for (auto &er : me->exe_reports) {
             if (er.order_key == cxl_m.key) {
                er.order_status = api::orderman::NATIVE_ORDER_STATUS_CANCELED;
                er.msg = "Cancelled";
-               me->on_exe_report(er_p, {});
+               me->on_exe_report(er, {});
                LOG(OrderMan, Info,
                    "FakeExecutionGateway::dispatch_incoming_fix_messages er: "
                    "{}",
@@ -138,14 +138,13 @@ void FakeExecutionGateway::dispatch_incoming_fix_messages() {
       } else if (std::holds_alternative<CancelReplaceOrderMsg>(m)) {
          CancelReplaceOrderMsg cxl_r_msg = std::get<CancelReplaceOrderMsg>(m);
          CreateOrderMsg cr_msg = cxl_r_msg.create_fields;
-         for (std::shared_ptr<ExecutionReport> &er_p : me->exe_reports) {
-            auto &er = *er_p;
+         for (auto &er : me->exe_reports) {
             if (er.order_key == cr_msg.key) {
                er.order_status = api::orderman::NATIVE_ORDER_STATUS_REPLACED;
                er.msg = "Replaced";
                er.price = cr_msg.price;
                er.quantity = cr_msg.quantity;
-               me->on_exe_report(er_p, {});
+               me->on_exe_report(er, {});
                return;
             }
          }
@@ -197,4 +196,4 @@ OnExchangeOpenOrdersState &FakeExecutionGateway::on_exchange_open_orders() {
 }
 
 } // namespace orderman
-} // namespace bts
+} // namespace bts

src/lib/orderman/gateways/GLBXExecutionGateway.cpp

@@ -13,6 +13,8 @@
 #include "lib/utils/TimeStamp.h"
 #include "lib/utils/bts_chrono.h"
 
+#include <boost/pool/object_pool.hpp>
+
 namespace bts {
 namespace orderman {
 
@@ -29,11 +31,12 @@ struct GLBXExecutionGateway::PImpl {
        config::JetFIX::Connection const &cfg,
        trace::PerfLog &l,
        md::SecurityOracle o)
-       : gw(cfg, l), state(State::WAITING_FOR_SECURITIES),
+       : gw(cfg, l), exe_report_pool(), state(State::WAITING_FOR_SECURITIES),
          securities(), securities_by_descriptor(), gw_name(cfg.name()),
          cfg(cfg), perf_log(l), sec_oracle(o) {}
 
    JFGw gw;
+   boost::object_pool<ExecutionReport> exe_report_pool;
    State state;
    std::unordered_map<SecurityKey::Value, api::Security const *> securities;
    std::unordered_map<std::string, api::Security const *>
@@ -206,8 +209,8 @@ void GLBXExecutionGateway::on_exe_report(
       if (msg.has<fix::Text>())
          text_msg = msg.get<fix::Text>();
 
-      ExecutionReport er;
-      ExecutionReport *data = &er;
+      // allocate from our pool
+      auto data = me->exe_report_pool.construct();
 
       data->exec_type = api::orderman::ExecType(msg.get<fix::ExecType>());
       data->msg_seq_num = msg.get<fix::MsgSeqNum>();
@@ -287,8 +290,7 @@ void GLBXExecutionGateway::on_exe_report(
       data->cfg = &me->cfg;
       data->msg = text_msg;
       data->exchange = api::Venue::VENUE_GLBX;
-      me->on_exe_report(
-          std::make_shared<ExecutionReport>(std::move(er)), msg.trace());
+      me->on_exe_report(*data, msg.trace());
    }
 }
 

src/lib/orderman/gateways/GLBXiLink3ExecutionGateway.cpp

@@ -12,7 +12,6 @@
 #include "lib/orderman/ilink3/IL3Reactor.h"
 #include "lib/orderman/ilink3/IL3SecDB.h"
 #include "lib/orderman/ilink3/SessionStore.h"
-#include "lib/orderman/orderman_utils/ExecutionReport.h"
 #include "lib/orderman/orderman_utils/fix_order_codec.h"
 #include "lib/utils/PBUtils.h"
 #include <cstddef>
@@ -150,11 +149,8 @@ void GLBXiLink3ExecutionGateway::connect(jetfix::SessionName const &sn) {
    if (!me->has_replayed) {
       me->has_replayed = true;
       me->replay_messages = me->session_store.value().replay();
-      for (ExecutionReport &er : me->replay_messages) {
-         me->on_exe_report(
-             std::make_shared<ExecutionReport>(std::move(er)), {});
-      }
-      me->replay_messages.clear();
+      for (auto const &er : me->replay_messages)
+         me->on_exe_report(er, {});
    }
 
    c.raw_message_logger =
@@ -173,7 +169,7 @@ void GLBXiLink3ExecutionGateway::connect(jetfix::SessionName const &sn) {
 
    me->gw->subscribe_execution_reports([this](auto er) {
       bts_assert(me->state == State::READY || me->state == State::CONNECTING);
-      me->on_exe_report(std::move(er), {});
+      me->on_exe_report(*er, {});
    });
 
    me->gw->subscribe_session(

src/lib/orderman/gateways/ICEExecutionGateway.cpp

@@ -18,6 +18,7 @@
 #include "lib/utils/helper_macros.h"
 
 #include <atomic>
+#include <boost/pool/object_pool.hpp>
 #include <set>
 
 namespace bts {
@@ -38,12 +39,13 @@ struct ICEExecutionGateway::PImpl {
        const config::JetFIX::Connection &cfg,
        trace::PerfLog &l,
        md::SecurityOracle o)
-       : gw(cfg, l), state(State::WAITING_FOR_SECURITIES), securities(),
-         securities_by_exchange_id(), gw_name(cfg.name()), cfg(cfg),
-         perf_log(l), admin_data(), nusers(-1), connection_time(),
+       : gw(cfg, l), exe_report_pool(), state(State::WAITING_FOR_SECURITIES),
+         securities(), securities_by_exchange_id(), gw_name(cfg.name()),
+         cfg(cfg), perf_log(l), admin_data(), nusers(-1), connection_time(),
          exec_id_to_memo(), sec_oracle(o) {}
 
    JFGw gw;
+   boost::object_pool<ExecutionReport> exe_report_pool;
    State state;
    std::unordered_map<SecurityKey::Value, api::Security const *> securities;
    std::unordered_map<int64_t, api::Security const *> securities_by_exchange_id;
@@ -371,8 +373,7 @@ void ICEExecutionGateway::on_exe_report(
                      std::to_string(msg.get<fix::DeleteReason>());
 
       // allocate from our pool
-      ExecutionReport er;
-      ExecutionReport *data = &er;
+      auto data = me->exe_report_pool.construct();
 
       if (msg.has<fix::Memo>())
          data->memo = msg.get<fix::Memo>();
@@ -466,8 +467,7 @@ void ICEExecutionGateway::on_exe_report(
       data->cfg = &me->cfg;
       data->msg = text_msg;
       data->exchange = api::Venue::VENUE_ICE;
-      me->on_exe_report(
-          std::make_shared<ExecutionReport>(std::move(er)), msg.trace());
+      me->on_exe_report(*data, msg.trace());
    }
 }