Merge branch 'development.cboe-prod' of bitbucket.org:bluetradesys/spark into development.cboe-prod

It was black in a couple places before, making it hard to read.

.devcontainer/Dockerfile

@@ -90,6 +90,12 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev
 
+# Docker in Docker
+RUN apt-get update && apt-get install -y \
+    docker.io \
+    docker-compose \
+    docker-compose-v2
+
 # Set up default executables (update-alternatives)
 RUN \
    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \

.devcontainer/devcontainer.json

@@ -12,17 +12,10 @@
   // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
   "runArgs": [
     "--shm-size",
-    "8gb",
-    "-p",
-    "8000:8000",
-    "-p",
-    "8001:8001",
-    "-p",
-    "8002:8002",
-    "-p",
-    "8003:8003",
-    "-p",
-    "12345:12345"
+    "8gb"
+  ],
+  "mounts": [
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
   ],
   "customizations": {
     "vscode": {

.env

@@ -0,0 +1,47 @@
+# This file defines the global default configuration for docker deployments.
+# It is tracked in source control. For temporary modifications, please create
+# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
+# If a variable is defined in both files, the value in .env.local will be used.
+#
+# About the deployment procedure:
+#   We use Docker Compose to build images and launch containers for various services
+#   that make up our software suite:
+#     - sparkd
+#     - TODO: pubsub
+#     - TODO: md-publisher
+#     - TODO: webapp (DSQA on the web)
+#     - TODO: api-server (websocket based)
+#   From inside a devcontainer, you can launch all services in separate containers
+#   on your machine by running:
+#
+#     tools/deploy.sh
+#
+#   This script builds release versions for each service, packaged (generally) in
+#   self-contained directories. Next, it runs a few "docker compose" commands to
+#   build docker images loaded with the self-contained binaries from the previous
+#   step, and start containers with those images to run each service. To view the
+#   commands that deploy.sh runs internally (without actually running them), run:
+#
+#     tools/deploy.sh --dry-run
+#
+#   For additional information/help, run:
+#
+#     tools/deploy.sh --help
+#
+
+### Sparkd ###
+# The internal image for sparkd to use. For more info, view:
+#   src/server/sparkd/Dockerfile
+SPARKD__IMAGE=sparkd
+# The config dir for this service to use
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
+# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
+# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
+# SPARKD__DATA_DIR=/home/bts/sparkdata/data
+SPARKD__SOURCE_DIR=install/sparkd-release-gcc
+SPARKD__CONFIG_PORT=8000
+SPARKD__QUICKBUS_PORT=8001
+SPARKD__SIM_FIX_PORT=8002
+SPARKD__SHM_SIZE=4gb
+
+# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitignore

@@ -59,3 +59,4 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
+.env.local

.vscode/launch.json

@@ -2,9 +2,10 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": ".NET Core Attach",
+      "name": "Attach Spark",
       "type": "coreclr",
-      "request": "attach"
+      "request": "attach",
+      "processId": "${command:pickProcess}"
     },
     {
       "name": "DataStreamPublisher",
@@ -326,4 +327,4 @@
       "description": "Test to run",
     },
   ]
-}
+}

compose.yaml

@@ -0,0 +1,63 @@
+services:
+  sparkd:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+    build:
+      dockerfile: src/server/sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+
+  sparkd-tail:
+    image: bluetradingsystems/linux_dotnet7
+    volumes:
+      - type: volume
+        read_only: true
+        source: sparkd_data
+        target: /home/bts/sparkdata/data/logging.sparkd
+        volume:
+          subpath: logging.sparkd
+    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
+
+  sparkd-ping-test:
+    depends_on:
+      - sparkd
+    profiles:
+      - ping-test
+    build:
+      context: .
+      dockerfile_inline: |
+        FROM bluetradingsystems/linux_dotnet7
+        USER "root"
+        RUN apt-get update && apt-get -y install iputils-ping
+    command: [ "ping", "sparkd" ]
+
+volumes:
+  sparkd_data:
+    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"

docker/images/bluetradingsystems/linux_dotnet7/Dockerfile

@@ -0,0 +1,38 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y pkg-config
+RUN apt-get update && apt-get install -y locales
+RUN apt-get update && apt-get install -y ca-certificates
+RUN apt-get update && apt-get install -y libssh2-1-dev
+RUN apt-get update && apt-get install -y software-properties-common
+RUN apt-get update && apt-get install -y sudo
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+  && apt-get update \
+  && apt-get install -y g++-$gcc_major
+
+RUN apt-get update \
+  && apt-get install -y dotnet7
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+  groupadd -g "$docker_uid" "$docker_user" \
+  && useradd \
+  -m \
+  -u "$docker_uid" \
+  -g "$docker_uid" \
+  -G sudo \
+  --password "$(openssl passwd -1 "$docker_pass")" \
+  "$docker_user"
+
+USER bts
+WORKDIR /home/bts

global.json

@@ -1,6 +0,0 @@
-{
-  "sdk": {
-    "version": "6.0.200",
-    "rollForward": "latestMajor"
-  }
-}

requirements.txt

@@ -1,10 +1,13 @@
 astroid==2.7.3
 autopep8==1.5.7
+boto3==1.37.9
+botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
 colorama==0.4.4
 idna==3.4
 isort==5.9.3
+jmespath==1.0.1
 lazy-object-proxy==1.6.0
 mccabe==0.6.1
 platformdirs==2.3.0
@@ -14,7 +17,11 @@ pycodestyle==2.7.0
 pycparser==2.20
 pylint==2.10.2
 PyMeta3==0.5.1
+python-dotenv==1.0.1
+python-dateutil==2.9.0.post0
 requests==2.31.0
+s3transfer==0.11.4
+six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3

src/client/spark/AW2/Bus/AnalyticsParametersManager.cs

@@ -45,7 +45,8 @@ namespace mbts.spark {
             Func<AnalyticsExpirationParameters, bool> is_unexpired = (p) => {
                var st = p.driver_security.security_tuple.to_api_sec_tuple();
                var exp = p.security.expiration;
-               return SecurityExpirationHelper.is_unexpired(st, exp);
+               return exp.Date >= DateTime.Now.Date;
+               //return SecurityExpirationHelper.is_unexpired(st, exp);
             };
             exp_params = exp_params.Where(t => is_unexpired(t));
             // listens to the commit tag and saves it

src/client/spark/AW2/Bus/SingleVolSurfaceInterpGreeksPublisher.cs

@@ -12,8 +12,8 @@ using bts.api;
 using mbts.ui.converters;
 
 namespace mbts.spark {
-   // This class gathers all the inputs needed to calculate InterpGreeks, pass it to 
-   // VolSurfacePrecomputedTheosProducer, and publishes the InterpGreeks to the bus
+   // This class gathers all the inputs needed to calculate InterpGreeks for a single vol surface,
+   // passes it to VolSurfacePrecomputedTheosProducer, and publishes the InterpGreeks to the bus
    // (called SingleVolSurfaceClientPrecomputedTheosPublisher in Edge)
    // It listens to
    // a. vol surface inputs

src/client/spark/AW2/Bus/VolSurfaceBlackboardManager.cs

@@ -20,6 +20,7 @@ using Spark.Core.Protobuf;
 using mbts.ui;
 using bts.config.toggles;
 using System.Diagnostics.CodeAnalysis;
+using BCG;
 // using mbts.bb;
 
 namespace mbts.spark
@@ -92,11 +93,11 @@ namespace mbts.spark
       // in tags
       private readonly ITag<SystemContractData> _in_scdata_tag =
          SparkTags.SystemContractsTag;
+      private readonly ITag<NMSheetSymbol> _archive_ivs_tag = SparkTags.TC2Tags.ArchiveIVsTag;
       private readonly ITag<VolSurfaceSettings> _in_bb_create_tag = SparkTags.VolSurfaceCreateTag;
       private readonly ITag<int> _in_bb_delete_tag = SparkTags.VolSurfaceDeleteTag;
       private readonly ITag<AnalyticsExpirationParametersMap> _in_bb_exp_params_tag =
          SparkTags.ExpirationParametersInputTag;
-      private readonly ITag<ClientArchivedVolRequest> _in_bb_req_vol = SparkTags.RequestArchivedVols;
       // state
       private SystemContractData _contract_data = null;
       private IDictionary<int, SingleVolSurfaceManager> _vs_manager_map =
@@ -106,6 +107,43 @@ namespace mbts.spark
       private bts.api.IBus _bus;
       private List<IDisposable> _subs = new List<IDisposable>();
 
+      private static void add_temp_vsu_entries(IDictionary<string, int> name_id_map,
+                                                 VolSurfaceUsageList ulist) {
+         Action<string, Venue, string> add_entry = (option_name, option_venue, vs_name) => {
+
+            int vs_key;
+            if (name_id_map.TryGetValue(vs_name, out vs_key)) {
+               ulist.UsageList.Add(new VolSurfaceUsageEntry() {
+                  VsId = vs_key,
+                  SymbolName = option_name,
+                  SecTup = new bts.api.SecurityTuple {
+                     Type = SecurityType.Option,
+                     Venue = option_venue,
+                     Symbol = option_name
+                  }
+               });
+            }
+
+         };
+         add_entry("SPX", Venue.Cboe, "CBOE_skews");
+         add_entry("SPXW", Venue.Cboe, "CBOE_skews");
+         add_entry("E1B", Venue.Glbx, "CME_Skews");
+         add_entry("E1C", Venue.Glbx, "CME_Skews");
+         add_entry("E3B", Venue.Glbx, "CME_Skews");
+         add_entry("E3C", Venue.Glbx, "CME_Skews");
+         add_entry("E3D", Venue.Glbx, "CME_Skews");
+         add_entry("E4A", Venue.Glbx, "CME_Skews");
+         add_entry("E4B", Venue.Glbx, "CME_Skews");
+         add_entry("E4C", Venue.Glbx, "CME_Skews");
+         add_entry("E4C", Venue.Glbx, "CME_Skews");
+         add_entry("E4D", Venue.Glbx, "CME_Skews");
+         add_entry("ES", Venue.Glbx, "CME_Skews");
+         add_entry("EW1", Venue.Glbx, "CME_Skews");
+         add_entry("EW2", Venue.Glbx, "CME_Skews");
+         add_entry("EW3", Venue.Glbx, "CME_Skews");
+         add_entry("EW4", Venue.Glbx, "CME_Skews");
+      }
+
       public VolSurfaceBusManager(bts.api.IBus bus,
                                          bts.api.analytics.AnalyticsService analytics_service,
                                          bts.api.orderman.OrderService order_service)
@@ -119,7 +157,7 @@ namespace mbts.spark
          _bb.subscribe(_in_bb_create_tag);
          _bb.subscribe(_in_bb_delete_tag);
          _bb.subscribe(_in_bb_exp_params_tag);
-         _bb.subscribe(_in_bb_req_vol);
+         _bb.subscribe(_archive_ivs_tag);
          _bb.data_callback += _on_bb_update;
          _bb.timer_callback += _on_timer;
          var contracts_throttle = TimeSpan.FromMilliseconds(200);
@@ -127,11 +165,25 @@ namespace mbts.spark
          var vs_sub = _analytics_service.subscribei_volsurfaces()
             .ObserveOn(SynchronizationContext.Current)
             .Subscribe(_on_vss);
+         //var hist_theos_sub_obs = _analytics_service.subscribei_archived_theos_table();
+         //var obs = hist_theos_sub_obs.ObserveOn(SynchronizationContext.Current);//.Subcribe(_on_hist_theos);
+         //obs.Subscribe(tu => {
+         //   Log.Gen.info("Received historical theos");
+         //   _on_hist_theos(tu);
+         //});
+
+         var archived_theos_obs = _analytics_service.subscribei_archived_theos().ObserveOn(SynchronizationContext.Current).Subscribe(tu => {
+            Log.Gen.info("Received historical theos");
+            _on_hist_theos(tu);
+         });
+         
+
          var vs_usage_sub = _analytics_service.subscribei_volsurface_usage()
             .ObserveOn(SynchronizationContext.Current)
             .Subscribe(_on_vs_usage_update);
          _subs.Add(vs_sub);
          _subs.Add(vs_usage_sub);
+         _subs.Add(archived_theos_obs);
          _subs.Add(new AnalyticsParametersManager(bus, _analytics_service));
          _subs.Add(new ResourceAllocator(bus, _bb, contracts_throttle));
          _subs.Add(new SymbolAggregationPublisher<OptionDriverPriceMap>(
@@ -143,7 +195,7 @@ namespace mbts.spark
          _subs.Add(new LiveTheoPublisher(_bb));
          _subs.Add(new InterpTheosServerPublisher(_bb, bus, order_service, contracts_throttle));
          _subs.Add(new VolSurfaceInterpGreeksMultiplexer(_bb));
-         // usage         
+         // list of all volsurface. in this callback, we read the usage config, and detemine the id of the named vol surface         
          var list_sub = _bus.subscribe(SparkTags.VolSurfaceListTag).OnDispatcher().Subscribe(t =>
          {
             VolSurfaceUsageList ulist = new VolSurfaceUsageList();
@@ -154,9 +206,12 @@ namespace mbts.spark
             }
             Log.Ui.info($"VolSurfaceUsageList name_to_ids count: {name_to_ids.Count}");
             var vs_usage = mbts.config.Configuration.get<bts.config.toggles.VolSurfaceUsage>();
-            Log.Ui.info($"VolSurfaceUsageList vs_usage acount: {vs_usage.Items.Count}");
+            Log.Ui.info($"VolSurfaceUsageList vs_usage acount: {vs_usage.Items.Count}");            
             foreach (var u in vs_usage.Items)
             {
+               if (string.Compare(u.OptionName, "SPX", 0) == 0 || string.Compare(u.OptionName, "SPXW", 0) == 0){
+                  continue;
+               }
                if (string.Compare(u.OptionName, "SPX", 0) == 0 || string.Compare(u.OptionName, "SPXW", 0) == 0 && u.OptionVenue == Venue.Unset)
                {
                   u.OptionVenue = Venue.Cboe;
@@ -186,10 +241,11 @@ namespace mbts.spark
                else
                {
                   Log.Gen.error($"Could not find vol surface with name: {vs_name}");
-               }
+               }               
             }
+            add_temp_vsu_entries(name_to_ids, ulist);
             var list_imm = new VolSurfaceUsageListImmutable(ulist);
-            Log.Ui.info(list_imm.log_string());
+            Log.Ui.info("VolSurfaceUsage: " + list_imm.log_string());
             _bb.post(SparkTags.VolSurfaceUsageTag, list_imm);
             _spin_vss(list_imm);
          });
@@ -292,6 +348,8 @@ namespace mbts.spark
 
       void _on_vs_usage_update(Update<VolSurfaceUsageListImmutable, int> update)
       {
+         var state = update.all;
+         Log.Gen.info($"VSBBMan._on_vs_usage_update size: {state.Count()}");
          // TODO
          return;
       }
@@ -310,6 +368,17 @@ namespace mbts.spark
          _init_vss(to_publish.Keys);
       }
 
+      void _on_hist_theos(Update<bts.api.analytics.ArchivedTheoListImmutable, ulong> update) {
+         //var total_state = update.state;
+         Log.Gen.info($"VSBBMan._on_hist_theos update: {update}");
+         foreach (var kv in update.all) {
+            bts.Log.Gen.info($"kv: {kv}");
+            _bb.post(SparkTags.ArchivedTheos, kv);
+            break;
+         }
+         //Log.assert(false);
+      }
+
       private void _init_vss(IEnumerable<int> vs_ids)
       {
          // we consider every vs in the db as "activated" which might prove to be expensive
@@ -362,29 +431,56 @@ namespace mbts.spark
             _bb.post(SparkTags.VolSurfaceNonVolInputTag,
                                           new mbts.analytics.VolSurfaceNonVolInput(exp_param_map.exp_map.ToDictionary()));
          }
-         else if (_in_bb_req_vol.Equals(tag))
-         {
-            ClientArchivedVolRequest client_req = _bb.read_state(_in_bb_req_vol);
-            bts.Log.Ui.info($"VolSurfaceBusManager._on_bb_update ClientArchivedVolRequest: {client_req.ToString()}");
-            ArchivedTheoRetrieval ret = client_req.ret;
-            if (ret == null || ret.SecTup == null)
-            {
-               bts.Log.Ui.warn($"VolSurfaceBusManager malformed ArchivedTheoRetrieval");
-               ArchivedTheoList empty = new ArchivedTheoList();
-               _bb.post(SparkTags.ResponseArchivedVols, Tuple.Create(client_req, empty));
-               return;
+         // this is publishe dby the sheets 
+         else if (_archive_ivs_tag.Equals(tag)){
+            //Log.assert(false);
+            NMSheetSymbol sym = data as NMSheetSymbol;
+            SystemContractData sc_data = _bb.read_state(SparkTags.SystemContractsTag);            
+            var associated_syms = _get_associated_symbols(sym);
+            List<ArchivedSecurityTheo> theo_list = new List<ArchivedSecurityTheo>();
+            foreach (NMSheetSymbol s in associated_syms){
+               ExpStrikeKeyedData<TheoPoint> symbol_theos = _bb.read_state(SparkTags.TC2Tags.LiveTheosTag((s.option_name, s.option_exchange)));
+               foreach (var t in symbol_theos.get_data_linearly()){
+                  Venue v = s.option_exchange;
+                  string symbol = s.option_name;
+                  DateTime exp = t.first;
+                  double strike = t.second;
+                  TheoPoint tp = t.third;
+                  Tuple<uint?, uint?> cp_id = sc_data.get_option_sec_id(v, symbol, exp, strike);                  
+                  if (cp_id != null && cp_id.Item1.HasValue && cp_id.Item2.HasValue){
+                     // call
+                     ArchivedSecurityTheo c_theo = new ArchivedSecurityTheo();
+                     c_theo.Id = cp_id.Item1.Value;
+                     c_theo.Driver = tp.driver;
+                     c_theo.Vol = tp.vol;
+                     c_theo.Theo = tp.call;
+                     c_theo.Delta = tp.cdelta;
+                     theo_list.Add(c_theo);
+                     // put
+                     ArchivedSecurityTheo p_theo = new ArchivedSecurityTheo();
+                     p_theo.Id = cp_id.Item1.Value;
+                     p_theo.Driver = tp.driver;
+                     p_theo.Vol = tp.vol;
+                     p_theo.Theo = tp.put;
+                     p_theo.Delta = tp.pdelta;
+                     theo_list.Add(p_theo);
+                  }
+               }
             }
-            bts.Log.Ui.info($"VolSurfaceBusManager ArchivedTheoRetrieval id: {client_req.id} save_time: {ret.SaveTime} " +
-                           $"sec_tup: {ret.SecTup.Venue}_{ret.SecTup.Type}_{ret.SecTup.Symbol}");
-            var timer = Stopwatch.StartNew();
-            ArchivedTheoList arch_theos = _analytics_service.get_archived_theos(ret);
-            timer.Stop();
-            bts.Log.Ui.info($"VolSurfaceBusManager ArchivedTheoRetrieval time: {timer.Elapsed.TotalMilliseconds}ms " +
-                            $"n_entries: {arch_theos.SecTheo.Count}");
-            _bb.post(SparkTags.ResponseArchivedVols, Tuple.Create(client_req, arch_theos));
+            ArchivedTheoList rv_list = new ArchivedTheoList();
+            // we dont care abouyt the save time any more, and dont want it to te the ky of the table anymore
+            rv_list.SaveTime = 0;
+            rv_list.SecTheo.AddRange(theo_list);
+            bts.Log.Gen.info($"VSBBMan archiving {rv_list.SecTheo.Count} entries");
+            _analytics_service.clean_and_archive_theos(rv_list);
+            // send it to the server
          }
       }
 
+      private List<NMSheetSymbol> _get_associated_symbols(NMSheetSymbol symbol){
+         return new List<NMSheetSymbol>() {new NMSheetSymbol(Venue.Cboe, "SPX"),new NMSheetSymbol(Venue.Cboe, "SPXW")};
+      }
+
       private int _timer_count = 0;
       private void _on_timer()
       {

src/client/spark/AW2/LiveTheoPublisher.cs

@@ -14,6 +14,11 @@ namespace mbts.spark {
    // Publishes theos for all options (on the side, also publishes the theos for TC2).
    // Listens to driver and interp theos, on a change, calls _interp_and_publish
    public class LiveTheoPublisher : IDisposable {
+      public enum Impetus { 
+         DRIVER,
+         SKEW,
+      }
+
       private ThreadedBlackboard _bb;
       private readonly ITag<IReadOnlyDictionary<(string, Venue), OptionTheoInterpolationData>>
          _greeks_tag = SparkTags.AllInterpGreeks;
@@ -30,13 +35,14 @@ namespace mbts.spark {
          _bb.subscribe(_drivers_tag);
          _bb.data_callback += _on_data;
          _bb.timer_callback += _on_timer;
-         _interp_and_publish();
+         _interp_and_publish(Impetus.SKEW);
       }
 
       private void _on_data(object tag, object data) {
          if (tag.Equals(_drivers_tag) ||
             tag.Equals(_greeks_tag)) {
-            _interp_and_publish();
+            Impetus imp = tag.Equals(_drivers_tag) ? Impetus.DRIVER : Impetus.SKEW;
+            _interp_and_publish(imp);
          }
       }
 
@@ -103,7 +109,7 @@ namespace mbts.spark {
          return TheoPoint.getNulledObject();
       }
 
-      private void _publish_tc2_theos(IDictionary<SecurityStrike, TheoPoint> theos) {
+      private void _publish_tc2_theos(IDictionary<SecurityStrike, TheoPoint> theos, Impetus imp) {
          var rv = new Dictionary<(string, Venue), ExpStrikeKeyedData<TheoPoint>>();
          foreach (var kv in theos) {
             SecurityStrike k = kv.Key;
@@ -118,6 +124,9 @@ namespace mbts.spark {
          _override_theos(ref rv);
          foreach (var kv in rv) {
             _bb.post(SparkTags.TC2Tags.LiveTheosTag(kv.Key), kv.Value);
+            if (imp == Impetus.SKEW) {
+               _bb.post(SparkTags.TC2Tags.VolSaveEventTheosTag(kv.Key), kv.Value);
+            }
          }
       }
 
@@ -132,6 +141,8 @@ namespace mbts.spark {
                   double strike = t.second;
                   TheoPoint tp = t.third;
                   var call_sec_md = sc_data.get_option(exchange, sym, exp, strike, PutOrCall.Call);
+                  if (call_sec_md == null)
+                     continue;
                   {
                      NativePriceSize call_bid, call_ask;
                      ulong time = 0;
@@ -141,6 +152,8 @@ namespace mbts.spark {
                   }
                   {
                      var put_sec_md = sc_data.get_option(exchange, sym, exp, strike, PutOrCall.Put);
+                     if (put_sec_md == null)
+                        continue;
                      NativePriceSize put_bid, put_ask;
                      ulong time = 0;
                      put_sec_md.Value.md.get_bbo(ref time, out put_bid, out put_ask);
@@ -163,7 +176,7 @@ namespace mbts.spark {
       }
 
 
-      private void _interp_and_publish() {
+      private void _interp_and_publish(Impetus imp) {
          var timer = Stopwatch.StartNew();
          var res = new Dictionary<uint, GreeksTuple>();
          var driver_map = _bb.read_state(_drivers_tag);
@@ -205,13 +218,13 @@ namespace mbts.spark {
          }
          // copy and publish
          IReadOnlyDictionary<uint, GreeksTuple> res_ro = res;
-         bts.Log.Gen.info($"LiveTheoPublisher._interp_and_publish posting " +
+         bts.Log.Gen.info($"LiveTheoPublisher._interp_and_publish Impetus: {imp} posting " +
                              $"count: {res_ro.Count}");
          timer.Stop();
          _logger.log(_log_id, _log_id, timer.Elapsed.TotalMilliseconds);
          _bb.post(SparkTags.Greeks, res_ro);
          // oh and those TC2 theos for comptability
-         _publish_tc2_theos(k_theos);
+         _publish_tc2_theos(k_theos, imp);
       }
 
       // If we do any rolls like SP for futures, it will be here

src/client/spark/Analytics/Core/VolSurfaceConstants.cs

@@ -8,7 +8,7 @@ using System.Threading.Tasks;
 
 namespace mbts.analytics {
    public static class VolSurfaceConstants {
-      public static int vol_time_sample_mins = 1;
+      public static int vol_time_sample_mins = 5;
    }
    // TODO: remove this
    public static class VolSurfaceTemp {

src/client/spark/Analytics/Misc/VolSurfaceTimeCalculatorFactory.cs

@@ -21,13 +21,23 @@ namespace mbts.analytics {
       private class TimeCalculator : IVolSurfaceTimeCalculator {
          // TODO, this should be passed in the exp itself
          private Dictionary<string, TimeSpan> utc_time_of_exp_map = new Dictionary<string, TimeSpan>() {
+            // TODO: get it from config
             {"BTC", TimeSpan.FromHours(8)},
-            {"ETH", TimeSpan.FromHours(8)}
+            {"ETH", TimeSpan.FromHours(8)},
+            {"SPXW", TimeSpan.FromHours(20)},
+            {"SPX", new TimeSpan(13, 30, 0)}
          };
          public void compute_vol_time(VolSurfaceInput vs_input, VolSurfaceExpiration exp,
                                       DateTime voltime, double days_shift,
                                       out double yrs, out double days_per_yr) {
+
             days_per_yr = 365;
+            if (vs_input != null && vs_input.surface_settings != null && vs_input.surface_settings.core_settings != null
+               && vs_input.surface_settings.core_settings.time_settings != null) {
+               if (vs_input.surface_settings.core_settings.time_settings.vol_time_measure.account_for_holidays_and_weekends) {
+                  days_per_yr = vs_input.surface_settings.core_settings.time_settings.vol_time_measure.days_per_year;
+               }
+            }
             TimeSpan time_of_exp = TimeSpan.Zero;
             utc_time_of_exp_map.TryGetValue(exp.symbol, out time_of_exp);
             // assumes exp is the utc date of expiration

src/client/spark/Analytics/Pricing/VolSurfaceInterpGreeksProducer.cs

@@ -9,8 +9,8 @@ using bts.api;
 using mbts.spark;
 
 namespace mbts.analytics {
-   // This class computes the theos/greeks for a number of driver
-   // prices (specified in the vs settings).
+   // This class computes interp theos/greeks (ie for a number of driver
+   // prices (specified in the vs settings)) for a volsurface
    // It has public setters that get
    // 1. contracts
    // 2. vs input
@@ -41,7 +41,8 @@ namespace mbts.analytics {
       private IDictionary<VolSurfaceExpiration, Tuple<double, double>>
       _computed_driver_range_map = new Dictionary<VolSurfaceExpiration, Tuple<double, double>>();
       private readonly IEnumerable<double> _default_abs_moves =
-         new double[] { 300, -200, -125, -100, -75, -50, -25, 0, 25, 50, 75, 100, 125, 200, 300 };
+         //new double[] { 300, -200, -125, -100, -75, -50, -25, 0, 25, 50, 75, 100, 125, 200, 300 };
+         new double[] { -100, -50, 0, 50, 100 };
       private readonly double _cushion_percentage = 25.0;
 
       public event
@@ -135,8 +136,8 @@ namespace mbts.analytics {
                      false, true, false, false, false, _filter_map);
                timer.stop();
                vol_compute_time = timer.last_interval_time_ms;
-               Log.Gen.info($"VolSurfacePrecomputedTheosProducer._compute_and_fire volmat time " +
-                           $"{timer.last_interval_time_ms}");
+               Log.Gen.info($"VolSurfacePrecomputedTheosProducer._compute_and_fire volmat input_type: {input_type} vs_id: {_vs_id} time " +
+                           $"{timer.last_interval_time_ms} tid: {System.Threading.Thread.CurrentThread.ManagedThreadId}");
                timer.start();
                IDictionary<VolSurfaceExpiration, IEnumerable<Theos>> exp_theo_map =
                  VolSurfaceTheoCalculator.compute_standard_greeks(synced_vs_input.surface_settings,

src/client/spark/Daemons/MergedMDDaemon.cs

@@ -111,13 +111,43 @@ namespace bts
          _merge_and_post();
       }
 
+
+
+      private static bool is_active(SecurityMD sec_md) {
+         SecurityImmutable sec = sec_md.security;
+         // temp: to see if spreads work
+         bool fut_or_option = (sec.security_type() == bts.api.SecurityType.Future || sec.security_type() == bts.api.SecurityType.Option);
+         if (!fut_or_option)
+            return true;
+         Func<bool> is_daily_option = () => {
+            bool is_daily = false;
+            if (sec.Exchange == Venue.Cboe) {
+               is_daily = sec.Symbol.ToLower() == "spxw";
+            }
+            if (sec.Exchange == Venue.Glbx) {
+               string symbol = sec.Symbol.ToLower();
+               int unused;
+               is_daily = symbol.Length >= 3 && symbol[0] == 'e' && Int32.TryParse(symbol[1].ToString(), out unused);
+            }
+            return is_daily;
+         };
+         bool is_expired = !SecurityExpirationHelper.is_unexpired(sec);
+         if (is_expired) {
+            return false;
+         }
+         if (sec.security_type() == bts.api.SecurityType.Option) {
+            int days_to_exp = Convert.ToInt32((sec.earliest_expiration() - DateTime.Now).TotalDays);
+            if (is_daily_option()) {
+               return days_to_exp <= 14;
+            } else {
+               return days_to_exp <= 90;
+            }
+         }
+         return true;            
+      }   
+
       private void _merge_and_post()
       {
-         Func<SecurityMD, bool> is_active = (sec_md) =>
-         {
-            var sec = sec_md.security;
-            return SecurityExpirationHelper.is_unexpired(sec);
-         };
          var to_pub = _server_sec_map.Where(kv => is_active(kv.Value)).ToDictionary(kv => kv.Key, kv => kv.Value);
          foreach (var kv in _fake_sec_map)
          {

src/client/spark/Modules/Market/MarketView/MarketViewVM.cs

@@ -629,16 +629,16 @@ namespace mbts.spark {
 
       static (string desc, Color fg) _fills_title(ColKey key) =>
          key switch {
-            ColKey.Position => ("Day Fills", Colors.Black),
-            ColKey.RawDayPosition => ("Raw Day Fills", Colors.Black),
-            _ => ("Fills", Colors.Black),
+            ColKey.Position => ("Day Fills", BTSColors.DayFills.foreground),
+            ColKey.RawDayPosition => ("Raw Day Fills", BTSColors.DayFills.foreground),
+            _ => ("Fills", BTSColors.UI.default_text),
          };
 
       static (string desc, Color fg) _orderbook_title(ColKey key) =>
          key switch {
             ColKey.WorkingBids => ((string desc, Color fg))("Working Buy Orders", BTSColors.MarketData.bid),
             ColKey.WorkingAsks => ((string desc, Color fg))("Working Sell Orders", BTSColors.MarketData.ask),
-            _ => ("Working Orders", Colors.Black),
+            _ => ("Working Orders", BTSColors.UI.default_text),
          };
 
       static GridSettings _hide_cols(GridSettings grid, IEnumerable<OrderBookGridView.ColumnKeys> columns) =>

src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookFilterStates.cs

@@ -150,7 +150,11 @@ namespace bts.shell.orderbook {
          pb.add_discrete_values(fs.exchanges, o => o.order.security?.Exchange ?? Venue.Unset);
          pb.add_discrete_values(fs.tifs, o => o.order.order.time_in_force());
          pb.add_discrete_values(fs.order_types, o => AlgoID.from_custom_def(o.order.order.Def.CustomDef));
-         pb.add_discrete_values(fs.root_algo_types, o => AlgoID.from_algo_key(o.root.algo?.key ?? default));
+         pb.add_discrete_values(
+            Array.ConvertAll(fs.root_algo_types, a => (AlgoID?)a),
+            o => o.root.algo.HasValue
+               ? AlgoID.from_algo_key(o.root.algo.Value.key)
+               : null);
          pb.add_discrete_values(fs.order_sub_types, o => o.order.order.sub_type());
          pb.add_discrete_values(fs.sides, o => o.order.order.side());
          pb.add_discrete_values(fs.statuses, o => o.order.order.status());

src/client/spark/Modules/Trading/PNL/VM/PNLVM.cs

@@ -774,6 +774,17 @@ namespace bts.shell.pnl {
             case ColumnKeys.Account: filter.accounts = new[] { g.Name.ToString() }; break;
             case ColumnKeys.Symbol: filter.symbols = new[] { g.Name.ToString() }; break;
             case ColumnKeys.Exchange: filter.exchanges = new[] { (Venue)g.Name }; break;
+            case ColumnKeys.Expiration: {
+                  if (g.Name is DateTime exp) {
+                     if (filter.security_criteria is null) {
+                        filter.security_criteria = SecurityCriteria.from_expirations(exp);
+                     } else {
+                        filter.security_criteria.clauses = ArrayHelper.add(
+                           filter.security_criteria.clauses, SecurityCriteriaClause.from_expirations(exp));
+                     }
+                  }
+                  break;
+               }
             case ColumnKeys.Security:
                if (g.Name is SecurityImmutable sec)
                   filter.security_criteria = SecurityCriteria.from_securities(sec);

src/client/spark/NMSheets/BlackboardStructure/NMSheetSingleOptionData.cs

@@ -237,7 +237,7 @@ namespace BCG {
          PNLRollOverDataUnit u;
          if (_roll_over_data != null && (u = _roll_over_data.getValue(exp, strike, true)) != null) {
             cp_row[(int)NMSheetColumnName.vol_change] = tp.vol - u.vol;
-            cp_row[(int)NMSheetColumnName.straddle_change] = ((tp.vol - u.vol) * tp.cvega * 2.0 * 100.0) / tp.driver;
+            cp_row[(int)NMSheetColumnName.straddle_change] = (tp.vol - u.vol) * tp.cvega * 2.0 * 100.0;
          } else {
             cp_row[(int)NMSheetColumnName.vol_change] = Double.NaN;
             cp_row[(int)NMSheetColumnName.straddle_change] = Double.NaN;

src/client/spark/NMSheets/DataProviders/NMSheetArchivedVolProvider.cs

@@ -14,103 +14,23 @@ using mbts.analytics;
 using bts.api.analytics;
 
 namespace mbts.spark {
-   public class ClientArchivedVolRequest {
-      public readonly int id;
-      public readonly ArchivedTheoRetrieval ret;
-      public ClientArchivedVolRequest(Int32 id, ArchivedTheoRetrieval ret) {
-         this.id = id;
-         this.ret = ret;
-      }
-      private static int _id = 0;
-      public static int get_new_id() {
-         int rv = 0;
-         lock (typeof(ClientArchivedVolRequest)) {
-            rv = ++_id;
-         }
-         return rv;
-      }
-      public override string ToString() {
-         return $"id: {id} sym: {ret.SecTup.Symbol} vol_time: {ret.SaveTime}";
-      }
-   }
    // This is created by the NMSheetSingleOptionMainDataSource - it downloads the
    // archived theos and provides access to it
    public class NMSheetArchivedVolProvider : IDisposable {
       private readonly ThreadedBlackboard _bb;
-      private readonly NMSheetSymbol _sym;
-      private readonly bts.api.SecurityTuple _sec_tup;
       private Action _cb = () => { };
       private readonly ITag<SystemContractData> _sc_tag = SparkTags.SystemContractsTag;
-      private readonly ITag<NMSheetSettings> _settings_tag;
-      private readonly ITag<Tuple<ClientArchivedVolRequest, bts.api.analytics.ArchivedTheoList>>
-         _response_tag = SparkTags.ResponseArchivedVols;
-      private bool _waiting_for_reponse = false;
-      private ClientArchivedVolRequest _last_req;
-      bts.api.analytics.ArchivedTheoList _cached_data = null;
+      private Nullable<ArchivedTheoListImmutable> _cached_data;
 
       public NMSheetArchivedVolProvider(string req_id, ThreadedBlackboard bb,
                                        NMSheetSymbol symbol,
                                        Action cb) {
          bts.Log.Ui.info($"NMSheetArchivedVolProvider req_id: {req_id} symbol: {symbol}");
          _bb = bb;
-         _sym = symbol;
-         _sec_tup = new bts.api.SecurityTuple() {
-            Venue = _sym.option_exchange,
-            Type = SecurityType.Option,
-            Symbol = _sym.option_name
-         };
-         _last_req = new ClientArchivedVolRequest(
-            0,
-            new ArchivedTheoRetrieval() {
-               SaveTime = 0,
-               SecTup = _sec_tup
-            });
          _cb = cb;
-         _settings_tag = SparkTags.TC2Tags.NMSheetSettingsTag(req_id);
          _bb.subscribe(_sc_tag);
-         _bb.subscribe(_settings_tag);
-         _bb.subscribe(_response_tag);
-         _bb.data_callback += _on_data;
-         _check_and_send_request(true);
-      }
-
-      private static UInt64 _get_vol_time(NMSheetSettings settings) {
-         NMSheetSettingLayout.VolDay vol_day = settings.setting_layout.vol_day;
-         TimeSpan vol_time = settings.setting_layout.vol_time;
-         DateTime date = DateTime.Today.Date;
-         switch (vol_day) {
-            case NMSheetSettingLayout.VolDay.DayBeforeYesterday:
-               date = date.Subtract(TimeSpan.FromDays(2));
-               break;
-            case NMSheetSettingLayout.VolDay.Yesterday:
-               date = date.Subtract(TimeSpan.FromDays(1));
-               break;
-            case NMSheetSettingLayout.VolDay.Today:
-               break;
-         }
-         DateTime dt = date.Add(vol_time);
-         var rv = VolSurfaceUtilityFunctions.get_unix_epoch_msecs(dt);
-         bts.Log.Ui.info($"NMSheetArchivedVolProvider._get_vol_time vol_day: {vol_day} " +
-            $"vol_time: {vol_time} dt: {dt} epoch_md: {rv}");
-         return rv;
-      }
-
-      private void _check_and_send_request(bool force = false) {
-         NMSheetSettings settings = _bb.read_state(_settings_tag);
-         if (settings != null && settings.setting_layout != null) {
-            UInt64 vol_time = _get_vol_time(settings);
-            if (force || _last_req.ret.SaveTime != vol_time) {
-               _last_req = new ClientArchivedVolRequest(
-                     ClientArchivedVolRequest.get_new_id(),
-                     new ArchivedTheoRetrieval() {
-                        SaveTime = vol_time,
-                        SecTup = _sec_tup
-                     });
-               _waiting_for_reponse = true;
-               bts.Log.Gen.info($"NMSheetArchivedVolProvider._check_and_send_request req: {_last_req}");
-               _bb.post(SparkTags.RequestArchivedVols, _last_req);
-            }
-         }
+         _bb.subscribe(SparkTags.ArchivedTheos);
+         _bb.data_callback += _on_data;         
       }
 
       private void _compute_and_post() {
@@ -118,34 +38,22 @@ namespace mbts.spark {
       }
 
       private void _on_data(object tag, object data) {
-         if (_settings_tag.Equals(tag)) {
-            _check_and_send_request();
-         } else if (_response_tag.Equals(tag)) {
-            if (_waiting_for_reponse) {
-               Tuple<ClientArchivedVolRequest, bts.api.analytics.ArchivedTheoList> resp =
-                  _bb.read_state(_response_tag);
-               if (resp != null) {
-                  ClientArchivedVolRequest responsed_req = resp.Item1;
-                  if (responsed_req.id == _last_req.id) {
-                     // got we were looking for
-                     _cached_data = resp.Item2;
-                     _waiting_for_reponse = false;
-                     bts.Log.Gen.info($"NMSheetArchivedVolProvider._on_data recvd data, req: {_last_req}");
-                     _compute_and_post();
-                  }
-               }
-            }
-         } else if (_sc_tag.Equals(tag)) {
+         // just listen to archihved theos and fire callback
+         if (tag.Equals(SparkTags.ArchivedTheos)) {
+            _cached_data = new Nullable<ArchivedTheoListImmutable>((ArchivedTheoListImmutable)data);
+            _compute_and_post();
+          }
+         else if (_sc_tag.Equals(tag)) {
             _compute_and_post();
          }
       }
 
       public OptionContractKeyedData<PNLRollOverDataUnit> get_all_data() {
          SystemContractData sc_data = _bb.read_state(_sc_tag);
-         if (_cached_data != null && sc_data != null) {
+         if (sc_data != null && _cached_data != null && _cached_data.HasValue) {
             OptionContractKeyedData<PNLRollOverDataUnit> rv =
                new OptionContractKeyedData<PNLRollOverDataUnit>();
-            foreach (var v in _cached_data.SecTheo) {
+            foreach (var v in _cached_data.Value.SecTheo) {
                var sec_md = sc_data.get_sec_md(v.Id);
                try {
                   if (sec_md != null) {

src/client/spark/NMSheets/UIComponents/MainSheet/NMSheetControl.Designer.cs

@@ -1,5 +1,7 @@
-namespace BCG {
-   partial class NMSheetControl {
+namespace BCG
+{
+   partial class NMSheetControl
+   {
       /// <summary> 
       /// Required designer variable.
       /// </summary>
@@ -9,9 +11,11 @@
       /// Clean up any resources being used.
       /// </summary>
       /// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
-      protected override void Dispose(bool disposing) {
+      protected override void Dispose(bool disposing)
+      {
          _logical_dispose(disposing);
-         if (disposing && (components != null)) {
+         if (disposing && (components != null))
+         {
             components.Dispose();
 
          }
@@ -24,7 +28,8 @@
       /// Required method for Designer support - do not modify 
       /// the contents of this method with the code editor.
       /// </summary>
-      private void InitializeComponent() {
+      private void InitializeComponent()
+      {
          this.components = new System.ComponentModel.Container();
          this._menu_panel = new System.Windows.Forms.Panel();
          this._button_minimize_spreads = new System.Windows.Forms.Button();
@@ -67,6 +72,7 @@
          this._toolstripmenuitem_task_manage_templates = new System.Windows.Forms.ToolStripMenuItem();
          this._toolstripseperator_task_eye = new System.Windows.Forms.ToolStripSeparator();
          this._toolstrip_menuitem_save_data_to_file = new System.Windows.Forms.ToolStripMenuItem();
+         this._toolstrip_menuitem_save_ivs = new System.Windows.Forms.ToolStripMenuItem();
          this._toolstripseperator_export_data = new System.Windows.Forms.ToolStripSeparator();
          this._toolstripmenuitem_card = new System.Windows.Forms.ToolStripMenuItem();
          this._toolstripseperator_expirations = new System.Windows.Forms.ToolStripSeparator();
@@ -293,7 +299,8 @@
             this._toolstripmenuitem_atm,
             this._toolseperator_atm,
          };
-         if (_api_tasking_enabled) {
+         if (_api_tasking_enabled)
+         {
             right_click_menu.AddRange(new System.Collections.Generic.List<System.Windows.Forms.ToolStripItem>() {
                this._toolstripmenuitem_task_eye,
                // no quoter
@@ -307,7 +314,8 @@
          //       this._toolstripmenuitem_task_override_size,
          //    });
          // }
-         if (_api_tasking_enabled) {
+         if (_api_tasking_enabled)
+         {
             right_click_menu.AddRange(new System.Collections.Generic.List<System.Windows.Forms.ToolStripItem>() {
                this._toolstripmenuitem_task_eye_switch_all_off,
                // We can do this from the algo studio now
@@ -317,6 +325,7 @@
          right_click_menu.AddRange(new System.Collections.Generic.List<System.Windows.Forms.ToolStripItem>() {
             // this._toolstripseperator_task_eye,
             this._toolstrip_menuitem_save_data_to_file,
+            this._toolstrip_menuitem_save_ivs,
             this._toolstripseperator_export_data,
             this._toolstripmenuitem_card,
             this._toolstripseperator_expirations,
@@ -324,7 +333,8 @@
             this._toolstripmenuitem_minimize_spreads,
             this._toolstripseperator_filter,
          });
-         if (_api_tasking_enabled) {
+         if (_api_tasking_enabled)
+         {
             right_click_menu.AddRange(new System.Collections.Generic.List<System.Windows.Forms.ToolStripItem>() {
                this._toolstripmenuitem_filter,
                this._toolseperator_quotesets
@@ -497,6 +507,13 @@
          this._toolstripseperator_task_eye.Name = "_toolstripseperator_task_eye";
          this._toolstripseperator_task_eye.Size = new System.Drawing.Size(213, 6);
          // 
+         // _toolstrip_menuitem_save_ivs
+         // 
+         this._toolstrip_menuitem_save_ivs.Name = "_toolstrip_menuitem_save_ivs";
+         this._toolstrip_menuitem_save_ivs.Size = new System.Drawing.Size(216, 22);
+         this._toolstrip_menuitem_save_ivs.Text = "Save IVs";
+         //
+         // 
          // _toolstrip_menuitem_save_data_to_file
          // 
          this._toolstrip_menuitem_save_data_to_file.Name = "_toolstrip_menuitem_save_data_to_file";
@@ -581,7 +598,8 @@
             this._toolstripmenuitem_settings_volchange,
             this._toolstripmenuitem_settings_straddle
          };
-         if (_api_tasking_enabled) {
+         if (_api_tasking_enabled)
+         {
             sheet_settings_items.AddRange(new System.Collections.Generic.List<System.Windows.Forms.ToolStripItem>() {
                this._toolseperator_task_ee,
                this._toolstripmenuitem_settings_show_task_ee_data,
@@ -900,6 +918,7 @@
       private System.Windows.Forms.ToolStripSeparator _toolstripseparator_task_size;
       private System.Windows.Forms.ToolStripMenuItem _toolstripmenuitem_task_undo_size_override;
       private System.Windows.Forms.ToolStripMenuItem _toolstrip_menuitem_save_data_to_file;
+      private System.Windows.Forms.ToolStripMenuItem _toolstrip_menuitem_save_ivs;
       private System.Windows.Forms.ToolStripSeparator _toolstripseperator_export_data;
       private System.Windows.Forms.Button _button_theo_acceptance_confirmation;
       private System.Windows.Forms.ToolStripMenuItem _toolstripmenuitem_filter_strikes_with_pos_edge;

src/client/spark/NMSheets/UIComponents/MainSheet/NMSheetControl.cs

@@ -20,6 +20,8 @@ using System.Collections;
 using System.Reactive.Linq;
 using System.Threading;
 using bts.api.orderman;
+using bts;
+using mbts.analytics;
 
 namespace BCG {
    public partial class NMSheetControl : UserControl {
@@ -252,7 +254,7 @@ namespace BCG {
          /// windows timer
          _windows_timer = new System.Windows.Forms.Timer(components);
          _windows_timer.Enabled = true;
-         _windows_timer.Interval = 20;
+         _windows_timer.Interval = 100;
          _windows_timer.Tick += _windows_timer_tick;
          _toolstripmenuitem_card.Visible = false;
          /// status bar
@@ -317,6 +319,9 @@ namespace BCG {
             _set_new_settings(_settings);
             _theo_grid.theo_grid.scroll_to_point(Point.Empty);
          };
+         _toolstrip_menuitem_save_ivs.Click += (s, e) => {
+            _threaded_bb.post(SparkTags.TC2Tags.ArchiveIVsTag, _nmsheet_symbol_list[0]);
+         };
          _toolstripmenuitem_filter_strikes_in_tasks.Click += (s, e) => {
             _settings.setting_filter.filter_ee_task_strikes = !_settings.setting_filter.filter_ee_task_strikes;
             _set_new_settings(_settings);
@@ -612,7 +617,7 @@ namespace BCG {
                cell.col == NMSheetColumnName.put_ask_price || cell.col == NMSheetColumnName.put_ask_size) {
                _launch_order_entry_window(cell, _is_control_down() ? OrderEntryWindow.Ladder : OrderEntryWindow.Ticket);
             } else if (cell.col == NMSheetColumnName.call_theo || cell.col == NMSheetColumnName.put_theo ||
-                       cell.col == NMSheetColumnName.straddle || cell.col == NMSheetColumnName.strike) {
+                         cell.col == NMSheetColumnName.straddle || cell.col == NMSheetColumnName.strike) {
                if (cell.call_selected)
                   _add_spread_leg(cell.expiration, cell.strike, true, false, leg_size, state);
                if (cell.put_selected)
@@ -717,9 +722,9 @@ namespace BCG {
 
       private void _windows_timer_tick(object sender, EventArgs e) {
          _publish_activation_state();
-         //_threaded_bb.sec_thread_dowork();
+         // this computes the data and then paints it
          _check_and_propagate_data();
-         _poll_theo_acceptance_button();
+         //_poll_theo_acceptance_button();
          _refresh_statusbar_data(false);
       }
 
@@ -780,21 +785,31 @@ namespace BCG {
       #region Coalescer NMSheetData interaction
 
       private DateTime _last_data_draw_time = DateTime.MinValue;
-      private const double _min_draw_period_ms = 333.3;
+      private const double _min_draw_period_ms = 500;
       private NMSheetGeometricLayout _last_geo_lo_info = null;
+      private VolSurfaceTimer _vs_timer = new VolSurfaceTimer();
+      // this is where we assemble the data (in _coalescer.get_data())
+      // and then painted in _respond_to_data
       private void _check_and_propagate_data() {
          if (is_visible) {
             if ((DateTime.Now - _last_data_draw_time).Milliseconds >= Math.Max(_settings.setting_grid_internal.grid_paint_throttle_ms, _min_draw_period_ms)) {
+               _vs_timer.start();
                NMSheetData new_sheet_data = _coalescer.get_data();
+               _vs_timer.stop();
+               Log.Gen.info($"NMSheetControl._check_and_propagate_data get_data time: {_vs_timer.last_interval_time_ms} msecs symbols: {_title_prefix} tid: {System.Threading.Thread.CurrentThread.ManagedThreadId}");
                if (new_sheet_data != null) {
                   _last_data_draw_time = DateTime.Now;
                   _sheet_data = new_sheet_data;
+                  _vs_timer.start();
                   _respond_to_data();
+                  _vs_timer.stop();
+                  Log.Gen.info($"NMSheetControl._check_and_propagate_data _respond_to_data time: {_vs_timer.last_interval_time_ms} msecs symbols: {_title_prefix} tid: {System.Threading.Thread.CurrentThread.ManagedThreadId}");
                }
             }
          }
       }
 
+      // this is the painting
       private void _respond_to_data() {
          if (_sheet_data != null) {
             _set_driver_control_display();

src/client/spark/Security/SecurityExpirationHelper.cs

@@ -36,9 +36,8 @@ namespace mbts {
       }
 
       public static bool is_unexpired(bts.api.SecurityTuple st, DateTime exp_date) {
-         if (st.Venue == Venue.Cboe || st.Venue == Venue.Glbx) {
-            // server is already getting teh righ contracts
-            return true;
+         if (st.Venue == Venue.Cboe) {
+            return exp_date.Date >= DateTime.Now.Date;
          }
          TimeSpan exp_time = get_exp_time(st);
          DateTime full_exp_utc = exp_date.ToUniversalTime().Date.Add(exp_time);
@@ -47,8 +46,12 @@ namespace mbts {
       }
 
       public static bool is_unexpired(bts.api.SecurityImmutable sec) {
-         if (sec.security_tuple().Venue == Venue.Cboe || sec.security_tuple().Venue == Venue.Glbx)
-            return true;
+         if (sec.security_tuple().Venue == Venue.Cboe || sec.security_tuple().Venue == Venue.Glbx) {
+            var exp = sec.to_sec_exp().expiration;
+            bool is_unexp = exp.Date >= DateTime.Now.Date;
+//            Log.Gen.info($"SecurityExpirationMapper.is_unexpired exp: {exp} is_unexp: {is_unexp}");
+            return is_unexp;
+         }
          var sec_type = sec.security_type();
          switch (sec_type) {
             case bts.api.SecurityType.Future:

src/client/spark/core/Bus/SparkTags.cs

@@ -353,11 +353,8 @@ namespace mbts.spark {
 
       public static ITag<Nothing> ShowGlobalAlertsTag { get; } = _mk_tag<Nothing>();
 
-      public static ITag<ClientArchivedVolRequest> RequestArchivedVols { get; } = _mk_tag<ClientArchivedVolRequest>();
-
-      public static ITag<Tuple<ClientArchivedVolRequest, bts.api.analytics.ArchivedTheoList>>
-      ResponseArchivedVols { get; } =
-         _mk_tag<Tuple<ClientArchivedVolRequest, bts.api.analytics.ArchivedTheoList>>();
+      public static ITag<ArchivedTheoListImmutable>
+      ArchivedTheos { get; } = _mk_tag<ArchivedTheoListImmutable>();
 
       static SparkTag<T> _mk_tag<T>([CallerMemberName] string name = null) =>
          new(name ?? throw new ArgumentNullException(nameof(name)));
@@ -373,6 +370,7 @@ namespace mbts.spark {
       public class Nothing { Nothing() { } }
 
       public class TC2Tags {
+         public static ITag<NMSheetSymbol> ArchiveIVsTag {get;} = _mk_tag<NMSheetSymbol>();
          public static ITag<NMSheetAllTaskingData> TaskData { get; } =
             _mk_tag<NMSheetAllTaskingData>();
          // Data request for a sheet 
@@ -386,6 +384,11 @@ namespace mbts.spark {
          public static ITag<ExpStrikeKeyedData<TheoPoint>> LiveTheosTag((string, Venue) symbol) =>
             _mk_tag<ExpStrikeKeyedData<TheoPoint>, (string, Venue)>(symbol);
 
+         // these are the theos at time of the vol save (we use it to extrapolate the theos prices for the spreads)
+         public static ITag<ExpStrikeKeyedData<TheoPoint>> VolSaveEventTheosTag((string, Venue) symbol) =>
+            _mk_tag<ExpStrikeKeyedData<TheoPoint>, (string, Venue)>(symbol);
+
+
          // Aggregated version of the above
          public static ITag<IReadOnlyDictionary<(string, Venue), ExpStrikeKeyedData<TheoPoint>>>
          AllLiveTheosTag { get; } =

src/client/spark/core/FontsAndColors/BTSColors.cs

@@ -1040,6 +1040,7 @@ namespace mbts.ui {
       }
       public static class DayFills {
          public static BTSColor background => UI.common_background;
+         public static BTSColor foreground => UI.default_foreground;
       }
       public static class ComboReport {
          public static BTSColor foreground => UI.default_text;

src/client/spark/view/Controls/TickUpDown/PriceEntryCommitControl.xaml

@@ -8,7 +8,8 @@
              xmlns:cv="clr-namespace:mbts.ui.converters"
              xmlns:c="clr-namespace:mbts.ui.controls"
              xmlns:usp="clr-namespace:mbts.ui.spark"
-             d:DesignHeight="100" d:DesignWidth="800" x:Name="_me_price">
+             d:DesignHeight="100" d:DesignWidth="800" x:Name="_me_price"
+             Foreground="{u:BTSColor UI.input_text}">
   <UserControl.Resources>
     <cv:NullableDoubleConverter x:Key="TickDefaultPriceConverter" DefaultValue="0"/>
     <cv:TickDisplayConverter x:Key="TickDisplayConverter" x:Shared="False"/>

src/client/spark_api/data/SecurityCriteria.cs

@@ -27,6 +27,12 @@ namespace mbts.spark {
       internal static SecurityCriteria from_keys(IEnumerable<uint> keys) =>
          new(new[] { new SecurityCriteriaClause(security_keys: keys.ToArray()) });
 
+      internal static SecurityCriteria from_expirations(params DateTime[] exp_dates) =>
+         from_expirations(exp_dates.AsEnumerable());
+
+      internal static SecurityCriteria from_expirations(IEnumerable<DateTime> expirations) =>
+         new(new[] { SecurityCriteriaClause.from_expirations(expirations) });
+
       internal static SecurityCriteria from_securities(params SecurityImmutable[] securities) =>
          from_securities(securities.AsEnumerable());
 

src/client/spark_api/data/SecurityCriteriaClause.cs

@@ -58,6 +58,14 @@ namespace mbts.spark {
          return new SecurityCriteriaClause(security_keys: keys, pretty_name: pretty_name);
       }
 
+      public static SecurityCriteriaClause from_expirations(IEnumerable<DateTime> expirations) =>
+         expirations is null
+            ? new()
+            : from_expirations(expirations as DateTime[] ?? expirations.ToArray());
+
+      public static SecurityCriteriaClause from_expirations(params DateTime[] expirations) =>
+         new(expirations: expirations);
+
       [ProtoMember(1)]
       public string? pretty_name { get; }
 

src/lib/api/services/AnalyticsService.proto

@@ -41,7 +41,7 @@ message VolSurfaceUsageList {
 }
 
 message ArchivedSecurityTheo {
-  uint32 id = 10;
+  uint32 id = 10[ (bts.api.key) = true ];
   double driver = 20;
   double vol = 30;
   double theo = 40;
@@ -49,7 +49,7 @@ message ArchivedSecurityTheo {
 }
 
 message ArchivedTheoList {
-  uint64 save_time = 10;
+  uint64 save_time = 10[ (bts.api.key) = true ];
   // these are sec_tups that are part of this save
   repeated bts.api.SecurityTuple sec_tups = 20;
   repeated ArchivedSecurityTheo sec_theo = 30;
@@ -195,8 +195,10 @@ service AnalyticsService {
 
   /// theos
   rpc archive_theos(ArchivedTheoList) returns (bts.api.ResultMessage);
+  rpc clean_and_archive_theos(ArchivedTheoList) returns (bts.api.ResultMessage);
   // finds the archived_theo closest to this point
   rpc get_archived_theos(ArchivedTheoRetrieval) returns (ArchivedTheoList);
+  rpc archived_theos(bts.api.VoidMessage) returns (stream ArchivedTheoList);
 
   /// reports
   rpc get_new_report_id(bts.api.VoidMessage)

src/lib/config/profiles/default/toggles.conf

@@ -114,13 +114,139 @@ VolSurfaceUsage {
      items: {
       option_name: "SPX"
       option_venue: VENUE_CBOE
-      vs_name: "Main"
+      vs_name: "CBOE_skews"
     }
     items: {
       option_name: "SPXW"
       option_venue: VENUE_CBOE
-      vs_name: "Main"
+      vs_name: "CBOE_skews"
     }
+    items: {
+      option_name: "ES"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }     
+     items: {
+      option_name: "E1A"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "E2A"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "E3A"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E4A"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E5A"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E1B"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "E2B"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "E3B"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E4B"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E5B"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E1C"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "E2C"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "E3C"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E4C"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E5C"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E1D"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "E2D"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "E3D"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E4D"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "E5D"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "EW1"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "EW2"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+     items: {
+      option_name: "EW3"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+    items: {
+      option_name: "EW4"
+      option_venue: VENUE_GLBX
+      vs_name: "CME_Skews"
+    }
+
 
   }
 }

src/lib/domain/SecurityCommon.h

@@ -10,6 +10,7 @@
 #include <limits>
 #include <nlohmann/json.hpp>
 #include <string>
+#include <vector>
 
 namespace bts::domain {
 
@@ -31,6 +32,7 @@ class SecurityCommon {
    double min_quantity = 0;
    double max_quantity = std::numeric_limits<double>::max();
    double cash_per_point = 1;
+   std::vector<double> price_ticks;
 
    SecurityCommon() {}
    SecurityCommon(const bts::api::Security &s)

src/lib/domain_model/common.h

@@ -33,8 +33,8 @@ struct CQuantity {
    uint32_t value() const { return static_cast<uint32_t>(qty); }
    double full_value() const { return qty; }
    float full_value_float() const { return static_cast<float>(full_value()); }
-   uint32_t int_part() { return static_cast<uint32_t>(qty); }
-   double fractional_part() { return full_value() - int_part(); }
+   uint32_t int_part() const { return static_cast<uint32_t>(qty); }
+   double fractional_part() const { return full_value() - int_part(); }
    static bool equal_d(double d1, double d2) {
       return std::abs(d1 - d2) <= err;
    }

src/lib/md/GeneralMarketDataListener.cpp

@@ -746,6 +746,11 @@ void GeneralMarketDataListener::_copy_security_common(
    to.display_factor = 0.0;
    to.price_break = 0.0;
    to.low_tick = to.high_tick = from.security_common.min_price_increment;
+   if(from.security_common.price_ticks.size() == 3){
+      to.price_break = from.security_common.price_ticks[0];
+      to.low_tick =  from.security_common.price_ticks[1];
+      to.high_tick=  from.security_common.price_ticks[2];
+   }
    auto &sc = from.security_common;
    double min_qty = sc.min_quantity;
    double qty_incr = sc.min_quantity_increment;

src/lib/orderman/actors/FillStore.cpp

@@ -2,7 +2,10 @@
 #include "lib/api/services/OrderService.pb.h"
 #include "lib/config/config.h"
 #include "lib/config/protos/toggles.pb.h"
+#include "lib/core/TimeStamp.h"
 #include "lib/domain_model/Security.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/domain_model/SecurityKey.h"
 #include "lib/logging/logging.h"
 #include "lib/mercury/events/Update.h"
 #include "lib/orderman/actors/SODPosition.h"
@@ -14,10 +17,13 @@
 #include "lib/os/os.h"
 #include "lib/persistence/SqliteStore.h"
 #include "lib/utils/PBUtils.h"
+#include "lib/utils/TimeStamp.h"
 #include "lib/utils/bts_assert.h"
+#include "lib/utils/bts_chrono.h"
 #include "lib/utils/overloaded.h"
 
 #include <boost/date_time/posix_time/posix_time.hpp>
+#include <google/protobuf/timestamp.pb.h>
 #include <string>
 #include <sys/time.h>
 
@@ -32,6 +38,67 @@ namespace orderman {
 
 auto const fills_ns = "fills";
 
+struct FillAggregator {
+   struct SecPosition {
+    private:
+      Fill agg_fill;
+
+    public:
+      double agg_price = 0, agg_qty = 0;
+      SecPosition(const Fill &fill) : agg_fill(fill) {
+         const auto &fill_key = fill.fill_key();
+         std::string new_fill_key = "aggpos_" + fill_key;
+         agg_fill.set_fill_key(new_fill_key);
+         auto now_ns = bts::core::TimeStamp::now_ns();
+         agg_fill.mutable_timestamp()->set_nanos(now_ns);
+         add_fill(fill);
+      }
+      void add_fill(const Fill &fill) {
+         bts_assert(fill.security_key() == agg_fill.security_key());
+         double mult = fill.side() == api::SIDE_BUY ? 1.0 : -1.0;
+         double fp = fill.avg_price();
+         double fq = mult * fill.last_shares();
+         double new_agg_price =
+             ((agg_price * agg_qty) + (fp * fq)) / (agg_qty + fq);
+         double new_agg_qty = agg_qty + fq;
+         agg_price = new_agg_price;
+         agg_qty = new_agg_qty;
+      }
+
+    public:
+      Fill fill() const {
+         Fill rv = agg_fill;
+         rv.set_avg_price(agg_price);
+         rv.set_last_shares(std::abs(agg_qty));
+         rv.set_side(::bts::api::Side::SIDE_BUY);
+         if (agg_qty < 0) {
+            rv.set_side(::bts::api::Side::SIDE_SELL);
+         }
+         return rv;
+      }
+   };
+   std::map<SecurityKey, SecPosition> pos_map = {};
+
+   void add_fill(const Fill &fill) {
+      SecurityKey k = SecurityKey(fill.security_key());
+      auto it = pos_map.find(k);
+      if (it != pos_map.end()) {
+         SecPosition &sec_pos = it->second;
+         sec_pos.add_fill(fill);
+      } else {
+         pos_map.insert({k, {fill}});
+      }
+   }
+
+   std::vector<Fill> get_aggregated_fills() const {
+      std::vector<Fill> rv = {};
+      for (const auto &kv : pos_map) {
+         rv.push_back(kv.second.fill());
+      }
+      return rv;
+   }
+};
+
 struct FillStore::UnresolvedFills {
    std::vector<api::orderman::Fill> fills;
    api::Security const *security;
@@ -48,6 +115,8 @@ struct FillStore::PImpl {
    std::unordered_map<OrderKey, UnresolvedFills> fills_with_unresolved_accounts;
    mercury::Stream<StoredFillEvent> fill_stream;
    md::SecurityOracle oracle;
+   FillAggregator fill_aggregator = {};
+   bool aggregate_pos = false;
 };
 
 FillStore::FillStore(md::SecurityOracle o) : me(new PImpl(o)) {
@@ -100,40 +169,83 @@ void FillStore::init() {
        os::get_thread_id(), os::get_thread_name());
    LOG(OrderMan, Info, "FillStore::init oracle n_secs: {}",
        me->oracle.get_securities().size());
-   // Subscribe to NewExecutionReportEvent for new fills
-   subscribe<NewExecutionReportEvent>(
-       [this](auto const &ev) { this->on_execution_report(ev); });
-
    // Load stored fills, resolve their securities and publish the lot
-   api::orderman::Fill fill;
    std::vector<StoredFillEvent> stored_batch;
    LOG(persist, Info, "Loading load all fills");
    auto all = me->store->retrieve_all()[fills_ns];
    LOG(persist, Info, "Finished loading all {} fills", all.size());
-   SODPosition sod_helper;
-   for (auto const &kvp : all)
-      if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
-         if (sod_helper.is_sod(fill.timestamp())) {
-            auto fill_kind = sod_fill_kind(fill);
-            fill.set_fill_kind(fill_kind);
+   auto aggregate_clean_and_publish = [&]() {
+      api::orderman::Fill fill;
+      for (auto const &kvp : all) {
+         if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
+            me->fill_aggregator.add_fill(fill);
+         } else {
+            LOG(persist, Error,
+                "Could not recover fill {} from storage. "
+                "This should be investigated",
+                kvp.first);
          }
-         auto sec = me->oracle.get_security(SecurityKey(fill.security_key()));
+      }
+      // 1. go through all fills in aggregator and make StoredFillEvent
+      auto aggregated_fills = me->fill_aggregator.get_aggregated_fills();
+      for (auto &f : aggregated_fills) {
+         auto sec = me->oracle.get_security(SecurityKey(f.security_key()));
          if (sec) {
-            auto fk = kvp.first;
-            me->known_fills.insert(fk);
-            if (!config::get<config::toggles::FilterExpiredFills>().enabled() ||
-                !securities::is_expired(*sec)) {
-               stored_batch.emplace_back(fill, *sec);
+            stored_batch.emplace_back(f, *sec);
+         } else {
+            LOG(persist, Error,
+                "Stored fill {} contains unresolved security {}. ",
+                utils::pb::to_string(f), fill.security_key());
+         }
+      }
+      // 2 delete all fills in the DB
+      me->store->remove_all(fills_ns);
+      // 3. add the aggregated fills back
+      for (const auto &e : stored_batch) {
+         const Fill &f = e.fill;
+         me->store->insert(fills_ns, f.fill_key(), utils::pb::to_string(f));
+      }
+   };
+   auto publish_with_sod_marking = [&]() {
+      api::orderman::Fill fill;
+      SODPosition sod_helper;
+      for (auto const &kvp : all) {
+         if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
+            if (sod_helper.is_sod(fill.timestamp())) {
+               auto fill_kind = sod_fill_kind(fill);
+               fill.set_fill_kind(fill_kind);
             }
+            auto sec =
+                me->oracle.get_security(SecurityKey(fill.security_key()));
+            if (sec) {
+               auto fk = kvp.first;
+               me->known_fills.insert(fk);
+               if (!config::get<config::toggles::FilterExpiredFills>()
+                        .enabled() ||
+                   !securities::is_expired(*sec)) {
+                  stored_batch.emplace_back(fill, *sec);
+               }
+            } else
+               LOG(persist, Error,
+                   "Stored fill {} contains unresolved security {}. ",
+                   kvp.first, fill.security_key());
          } else
             LOG(persist, Error,
-                "Stored fill {} contains unresolved security {}. ", kvp.first,
-                fill.security_key());
-      } else
-         LOG(persist, Error,
-             "Could not recover fill {} from storage. "
-             "This should be investigated",
-             kvp.first);
+                "Could not recover fill {} from storage. "
+                "This should be investigated",
+                kvp.first);
+      }
+   };
+
+   // Subscribe to NewExecutionReportEvent for new fills
+   subscribe<NewExecutionReportEvent>(
+       [this](auto const &ev) { this->on_execution_report(ev); });
+   if (me->aggregate_pos) {
+      aggregate_clean_and_publish();
+   } else {
+      publish_with_sod_marking();
+   }
+
    me->fill_stream = make_stream<StoredFillEvent>();
    LOG(OrderMan, Info,
        "Fill Store published {} fills recoved from disk out of {} "

src/lib/orderman/actors/OrderStore.cpp

@@ -5,6 +5,7 @@
 #include "lib/config/protos/orders_fills_cleanup.pb.h"
 #include "lib/config/protos/toggles.pb.h"
 #include "lib/logging/logging.h"
+#include "lib/md/ExchangeSecurityKey.h"
 #include "lib/orderman/actors/SODPosition.h"
 #include "lib/orderman/events/OrderDefinitionUpdateEvent.h"
 #include "lib/orderman/events/OrderStateUpdateEvent.h"
@@ -12,6 +13,7 @@
 #include "lib/persistence/SqliteStore.h"
 #include "lib/utils/PBUtils.h"
 
+#include <google/protobuf/timestamp.pb.h>
 #include <unordered_set>
 
 namespace bts {
@@ -38,6 +40,7 @@ struct OrderStore::PImpl {
    bool soft_cleanup_enabled;
    md::SecurityOracle oracle;
    timeval cutoff_time;
+   bool clear_all_orders = true;
 };
 
 OrderStore::OrderStore(md::SecurityOracle o) : me(new PImpl(o)) {}
@@ -63,34 +66,46 @@ void OrderStore::init() {
 
    me->cutoff_time = SODPosition::get_order_cutoff_time(all);
 
-   for (auto const &kvp : all)
-      if (utils::pb::from_string<api::orderman::OrderRecord>(
-              kvp.second, &rec)) {
-         auto ok = OrderKey(std::stoi(kvp.first));
-         auto sk = SecurityKey(rec.def().security_key());
-         auto sec = me->oracle.get_security(sk);
+   auto load_orders = [&]() {
+      for (auto const &kvp : all) {
+         if (utils::pb::from_string<api::orderman::OrderRecord>(
+                 kvp.second, &rec)) {
+            auto ok = OrderKey(std::stoi(kvp.first));
+            auto sk = SecurityKey(rec.def().security_key());
+            auto sec = me->oracle.get_security(sk);
 
-         if (config::get<config::toggles::FilterOrdersWithDanglingSecurities>()
-                 .enabled() &&
-             sec == nullptr) {
-            LOG(OrderMan, Diag, "Order {} refers to unresolved security {}",
-                ok.value(), sk.value());
-         } else {
-            if (!_order_cutoff(rec))
-               stored_batch.emplace_back(rec);
-            me->known_orders[ok] = PImpl::ORInfo{rec, true};
-         }
-      } else
-         LOG(OrderMan, Error,
-             "Could not recover order {} from storage. "
-             "This should be investigated",
-             kvp.first);
+            if (config::get<
+                    config::toggles::FilterOrdersWithDanglingSecurities>()
+                    .enabled() &&
+                sec == nullptr) {
+               LOG(OrderMan, Diag, "Order {} refers to unresolved security {}",
+                   ok.value(), sk.value());
+            } else {
+               if (!_order_cutoff(rec))
+                  stored_batch.emplace_back(rec);
+               me->known_orders[ok] = PImpl::ORInfo{rec, true};
+            }
+         } else
+            LOG(OrderMan, Error,
+                "Could not recover order {} from storage. "
+                "This should be investigated",
+                kvp.first);
+      }
+   };
 
    me->order_stream = make_stream<StoredOrderEvent>();
    LOG(OrderMan, Info,
        "Order Store published {} orders recoved from disk out of {} "
        "total orders",
        stored_batch.size(), all.size());
+
+   if (me->clear_all_orders) {
+      // clear orders
+      me->store->remove_all(orders_ns);
+   } else {
+      load_orders();
+   }
+
    publish_batch(me->order_stream, stored_batch);
 }
 
@@ -114,17 +129,22 @@ void OrderStore::on_order_state_update(
          rec.set_key(ev.key.value());
          rec.set_parent_key(ev.parent_key.value());
          modified_orders.insert(ev.key);
+         const auto now = utils::pb::pb_now();
          if (rec.has_state()) {
+            google::protobuf::Timestamp created;
+            if (rec.state().has_created()) {
+               created = rec.state().created();
+            }
             rec.mutable_state()->CopyFrom(ev.rebuild_state());
-            *rec.mutable_state()->mutable_created() = utils::pb::pb_now();
-            *rec.mutable_state()->mutable_last_change() = rec.state().created();
+            *rec.mutable_state()->mutable_created() =
+                created.seconds() > 0 || created.nanos() > 0 ? created : now;
             // if filled qty went from 0.99 -> 1, it was copying the int part and not clearing the fraction part,
             // so changing MergeFrom to CopyFrom
          } else {
             *rec.mutable_state() = ev.rebuild_state();
-            *rec.mutable_state()->mutable_created() = utils::pb::pb_now();
+            *rec.mutable_state()->mutable_created() = now;
          }
-         *rec.mutable_state()->mutable_last_change() = utils::pb::pb_now();
+         *rec.mutable_state()->mutable_last_change() = now;
          LOG(OrderMan, Info, "OrderStore::on_order_state_update k: {} rec: {}",
              ev.key, utils::pb::to_string(rec));
       }

src/lib/orderman/gateways/OkxExecutionGateway.cpp

@@ -148,7 +148,7 @@ struct OkxExecutionGateway::PImpl {
           std::min(account_name.size(), account_array.size()),
           account_array.begin());
       report.account_name = AccountName(account_array);
-
+      report.leg_security = nullptr;
       to_array(ops_id, &report.operator_id.value());
 
       report.order_key =
@@ -548,6 +548,7 @@ void OkxExecutionGateway::launch_new_order(const CreateOrderMsg &msg) {
       er.quantity = msg.quantity;
       er.filled_quantity = 0;
       er.last_shares = 0;
+      er.leg_security = nullptr;
       er.avg_price = Price(0);
       ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now());
       er.transact_time = now;

src/lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp

@@ -134,15 +134,14 @@ bool SpreaderUtils::resolve_synthetic_spread(
                          bts::api::orderman::TimeInForce::TIME_IN_FORCE_IOC);
                   else
                      quoter_order_def.set_time_in_force(
-                         bts::api::orderman::TimeInForce::TIME_IN_FORCE_DAY);
+                         order_def.time_in_force());
                   // hedger order def
                   api::orderman::OrderDefinition hedger_order_def =
                       quoter_order_def;
                   hedger_order_def.mutable_custom_def()
                       ->mutable_limit()
                       ->MergeFrom(api::orderman::LimitOrderDefinition());
-                  hedger_order_def.set_time_in_force(
-                      bts::api::orderman::TimeInForce::TIME_IN_FORCE_DAY);
+                  quoter_order_def.set_time_in_force(order_def.time_in_force());
                   hedger_order_def.set_sub_type(
                       bts::api::orderman::OrderSubType::
                           ORDER_SUB_TYPE_HEDGING_LEG);

src/lib/os/os.cpp

@@ -638,7 +638,7 @@ int num_cores() {
 #endif
 }
 
-std::string expand_env(std::string name, bool assert_vars_exist = true) {
+std::string expand_env(std::string name, bool assert_vars_exist) {
    static const std::regex env_re{R"--(\$\{([^}]+)\})--"};
    std::smatch match;
    while (std::regex_search(name, match, env_re)) {
@@ -656,6 +656,26 @@ std::string expand_env(std::string name, bool assert_vars_exist = true) {
    return name;
 }
 
+bool expand_env_in_place(std::string &name, bool assert_vars_exist) {
+   static const std::regex env_re{R"--(\$\{([^}]+)\})--"};
+   std::smatch match;
+   bool changed = false;
+   while (std::regex_search(name, match, env_re)) {
+      const auto from = match[0];
+      const auto var_name = match[1].str();
+      const auto var = std::getenv(var_name.c_str());
+      if (!var) {
+         bts_assert_msgf(
+             !assert_vars_exist, "Environment variable '{}' is undefined",
+             name);
+         continue;
+      }
+      name.replace(from.first, from.second, var);
+      changed = true;
+   }
+   return changed;
+}
+
 std::optional<std::string> try_expand_env(const std::string &name) {
    static const std::regex env_re{R"--(\$\{([^}]+)\})--"};
    std::smatch match;

src/lib/os/os.h

@@ -90,6 +90,9 @@ boost::filesystem::path app_data_path();
 boost::filesystem::path app_data_path(
     const boost::filesystem::path &relative_path, bool mkdirs = true);
 
+std::string expand_env(std::string name, bool assert_vars_exist = true);
+bool expand_env_in_place(std::string &name, bool assert_vars_exist = true);
+
 std::string username();
 std::string find_newest_file(
     const std::string &directory,

src/lib/utils/lazy.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <mutex>
+
+namespace bts::utils {
+
+template <typename T>
+class Lazy {
+ public:
+   template <typename... Args>
+   explicit Lazy(Args &&...args)
+       : create_([=]() mutable { return std::make_unique<T>(args...); }) {}
+
+   T &get() {
+      std::call_once(init_flag_, [this]() { instance_ = create_(); });
+      return *instance_;
+   }
+
+   static Lazy<T> from_delegate(std::function<T()> create);
+
+ private:
+   explicit Lazy(std::function<T()> create)
+       : create_([&, create]() { return std::make_unique<T>(create()); }) {}
+
+   std::once_flag init_flag_;
+   std::function<std::unique_ptr<T>()> create_;
+   std::unique_ptr<T> instance_;
+};
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+template <typename T>
+Lazy<T> Lazy<T>::from_delegate(std::function<T()> create) {
+   return Lazy<T>(create);
+}
+
+} // namespace bts::utils

src/lib_binance_execution/binance_execution/cross_margin/rest_api_client.cpp

@@ -2,6 +2,7 @@
 #include "lib/io/http/http_client.h"
 #include "lib/io/http/http_response.h"
 #include "lib/utils/crypto/hmac.h"
+#include "lib/utils/type_name.h"
 
 namespace binance_execution::cross_margin {
 
@@ -18,16 +19,24 @@ using std::string;
 #define ERROR_FN(fn, ...) _LOG_FN(bts::Error, fn, __VA_ARGS__)
 
 template <std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_string(std::string &str) {
+static void shrink_str(std::string &str, const char *name) {
    if (str.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_str",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, str.size(), DEFAULT_SIZE, MAX_SIZE);
       str.resize(DEFAULT_SIZE);
       str.shrink_to_fit();
    }
 }
 
 template <class T, std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_vec(std::vector<T> &vec) {
+static void shrink_vec(std::vector<T> &vec, const char *name) {
    if (vec.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_vec",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, vec.size(), DEFAULT_SIZE, MAX_SIZE);
       vec.resize(DEFAULT_SIZE);
       vec.shrink_to_fit();
    }
@@ -57,7 +66,8 @@ void RestApiClient::request_(
    // Use thread_local vars to reduce memory allocations. Shrink strings
    // to reduce memory footprint from previous requests.
    thread_local bts::http::HttpRequest http_req;
-   shrink_string<4096, 128>(http_req.body);
+   shrink_str<4096, 128>(
+       http_req.body, bts::utils::cached_type_name<RequestT>());
    http_req.reset();
    // Prepare request
    http_req.url = url_;
@@ -66,7 +76,9 @@ void RestApiClient::request_(
    http_req.method = method;
    http_req.headers["X-MBX-APIKEY"] = api_key_;
    thread_local std::string qps;
-   shrink_string<256, 16>(qps);
+   static const std::string query_params_name =
+       fmt::format("{}_params", bts::utils::cached_type_name<RequestT>());
+   shrink_str<256, 16>(qps, query_params_name.c_str());
    qps.clear();
    to_query_params(qps, req);
    if (!qps.empty()) {
@@ -78,7 +90,8 @@ void RestApiClient::request_(
    }
    // Send it
    thread_local HttpResponse http_res;
-   shrink_string<4096, 128>(http_res.body);
+   shrink_str<4096, 128>(
+       http_res.body, bts::utils::cached_type_name<ResponseT>());
    http_res.reset();
    HttpClient::send(http_req, &http_res);
    res.http_code = http_res.code;

src/lib_binance_execution/binance_execution/gateway_config.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "lib/io/JsonReader.h"
+#include "lib/os/os.h"
 
 #include <nlohmann/json.hpp>
 #include <string>
@@ -15,6 +16,15 @@ struct GatewayConfig {
    std::string ws_endpoint;
    std::string ws_stream_endpoint;
    bool use_cross_margin_api = false;
+
+   void expand_env() {
+      bts::os::expand_env_in_place(api_key);
+      bts::os::expand_env_in_place(secret_key);
+      bts::os::expand_env_in_place(account);
+      bts::os::expand_env_in_place(rest_endpoint);
+      bts::os::expand_env_in_place(ws_endpoint);
+      bts::os::expand_env_in_place(ws_stream_endpoint);
+   }
 };
 
 inline void to_json(nlohmann::json &j, const GatewayConfig &gw) {

src/lib_binance_execution/binance_execution/listen_key_requestor.cpp

@@ -1,19 +1,31 @@
 #include "listen_key_requestor.h"
 #include "lib/core/TimeStamp.h"
-#include "lib/io/json_formatter.h"
 
+#include <chrono>
 #include <string>
 #include <thread>
 
 namespace binance_execution {
 
+#define _LOG_FN(level, fn, ...)                                                \
+   LOG(OrderMan, level, "ListenKeyRequestor::" fn ". " __VA_ARGS__)
+#define DIAG_FN(fn, ...) _LOG_FN(bts::Diag, fn, __VA_ARGS__)
+#define INFO_FN(fn, ...) _LOG_FN(bts::Info, fn, __VA_ARGS__)
+#define WARN_FN(fn, ...) _LOG_FN(bts::Warn, fn, __VA_ARGS__)
+#define ERROR_FN(fn, ...) _LOG_FN(bts::Error, fn, __VA_ARGS__)
+
 using bts::core::Delegate;
 using bts::core::TimeStamp;
-using bts::io::json_formatter;
 using std::optional;
 using std::string;
 using std::thread;
 
+static bts::core::TimeSpan retry_interval(std::uint32_t n_attempts) {
+   std::int64_t secs = 5 * n_attempts;
+   secs = std::clamp(secs, 5L, 300L);
+   return bts::core::TimeSpan::from_seconds(secs);
+}
+
 struct ListenKeyRequestor::PImpl {
    std::function<RestApiResult<ListenKeyResponse>(const ListenKeyRequest &)>
        get_listen_key;
@@ -42,6 +54,9 @@ struct ListenKeyRequestor::PImpl {
    }
 
    void run() {
+      static constexpr const std::uint32_t MAX_RETRIES = 30;
+      // Counts consecutive failed attempts (resets to 0 on successful request)
+      std::uint32_t retry_count = 0;
       while (is_running) {
          static const auto refresh_period =
              bts::core::TimeSpan::from_seconds(60 * 30);
@@ -49,10 +64,19 @@ struct ListenKeyRequestor::PImpl {
          if (listen_key.empty()) {
             const auto &res = get_listen_key({});
             if (!res.ok()) {
+               bts_assert(retry_count < MAX_RETRIES);
+               retry_count++;
+               const auto sleep_t = retry_interval(retry_count);
+               WARN_FN(
+                   "run",
+                   "get_listen_key request failed ({}). Will retry in {:.2f} "
+                   "secs",
+                   retry_count, sleep_t.total_seconds());
+               std::this_thread::sleep_for(
+                   std::chrono::milliseconds(sleep_t.millis()));
                continue;
             }
-            bts_assert_msgf(
-                res.ok(), "Failed to get listen key. {}", json_formatter(res));
+            retry_count = 0;
             listen_key = res.response->listenKey;
             bts_assert(!listen_key.empty());
             listen_key_timestamp = now;
@@ -60,9 +84,20 @@ struct ListenKeyRequestor::PImpl {
          }
          if ((now - listen_key_timestamp) > refresh_period) {
             const auto &res = refresh_listen_key({listen_key});
-            bts_assert_msgf(
-                res.ok(), "Failed to refresh listen key. {}",
-                json_formatter(res));
+            if (!res.ok()) {
+               bts_assert(retry_count < MAX_RETRIES);
+               retry_count++;
+               const auto sleep_t = retry_interval(retry_count);
+               WARN_FN(
+                   "run",
+                   "refresh_listen_key request failed ({}). Will retry in "
+                   "{:.2f} secs",
+                   retry_count, sleep_t.total_seconds());
+               std::this_thread::sleep_for(
+                   std::chrono::milliseconds(sleep_t.millis()));
+               continue;
+            }
+            retry_count = 0;
             listen_key_timestamp = now;
          }
          std::this_thread::sleep_for(std::chrono::seconds(2));

src/lib_binance_execution/binance_execution/spots/rest_api_client.cpp

@@ -4,6 +4,7 @@
 #include "lib/io/http/http_response.h"
 #include "lib/io/json_formatter.h"
 #include "lib/utils/crypto/hmac.h"
+#include "lib/utils/type_name.h"
 
 namespace binance_execution::spots {
 
@@ -20,16 +21,24 @@ using std::string;
 #define ERROR_FN(fn, ...) _LOG_FN(bts::Error, fn, __VA_ARGS__)
 
 template <std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_string(std::string &str) {
+static void shrink_str(std::string &str, const char *name) {
    if (str.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_str",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, str.size(), DEFAULT_SIZE, MAX_SIZE);
       str.resize(DEFAULT_SIZE);
       str.shrink_to_fit();
    }
 }
 
 template <class T, std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_vec(std::vector<T> &vec) {
+static void shrink_vec(std::vector<T> &vec, const char *name) {
    if (vec.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_vec",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, vec.size(), DEFAULT_SIZE, MAX_SIZE);
       vec.resize(DEFAULT_SIZE);
       vec.shrink_to_fit();
    }
@@ -59,7 +68,8 @@ void RestApiClient::request_(
    // Use thread_local vars to reduce memory allocations. Shrink strings
    // to reduce memory footprint from previous requests.
    thread_local bts::http::HttpRequest http_req;
-   shrink_string<4096, 128>(http_req.body);
+   shrink_str<4096, 128>(
+       http_req.body, bts::utils::cached_type_name<RequestT>());
    http_req.reset();
    // Prepare request
    http_req.url = url_;
@@ -68,7 +78,9 @@ void RestApiClient::request_(
    http_req.method = method;
    http_req.headers["X-MBX-APIKEY"] = api_key_;
    thread_local std::string qps;
-   shrink_string<256, 16>(qps);
+   static const std::string query_params_name =
+       fmt::format("{}_params", bts::utils::cached_type_name<RequestT>());
+   shrink_str<256, 16>(qps, query_params_name.c_str());
    qps.clear();
    to_query_params(qps, req);
    if (!qps.empty()) {
@@ -80,7 +92,8 @@ void RestApiClient::request_(
    }
    // Send it
    thread_local HttpResponse http_res;
-   shrink_string<4096, 128>(http_res.body);
+   shrink_str<4096, 128>(
+       http_res.body, bts::utils::cached_type_name<ResponseT>());
    http_res.reset();
    HttpClient::send(http_req, &http_res);
    res.http_code = http_res.code;

src/lib_binance_execution/binance_execution/usds_futures/rest_api_client.cpp

@@ -2,6 +2,7 @@
 #include "lib/io/http/http_client.h"
 #include "lib/io/http/http_response.h"
 #include "lib/utils/crypto/hmac.h"
+#include "lib/utils/type_name.h"
 
 namespace binance_execution::usds_futures {
 
@@ -18,16 +19,24 @@ using std::string;
 #define ERROR_FN(fn, ...) _LOG_FN(bts::Error, fn, __VA_ARGS__)
 
 template <std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_string(std::string &str) {
+static void shrink_str(std::string &str, const char *name) {
    if (str.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_str",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, str.size(), DEFAULT_SIZE, MAX_SIZE);
       str.resize(DEFAULT_SIZE);
       str.shrink_to_fit();
    }
 }
 
 template <class T, std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_vec(std::vector<T> &vec) {
+static void shrink_vec(std::vector<T> &vec, const char *name) {
    if (vec.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_vec",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, vec.size(), DEFAULT_SIZE, MAX_SIZE);
       vec.resize(DEFAULT_SIZE);
       vec.shrink_to_fit();
    }
@@ -57,7 +66,8 @@ void RestApiClient::request_(
    // Use thread_local vars to reduce memory allocations. Shrink strings
    // to reduce memory footprint from previous requests.
    thread_local bts::http::HttpRequest http_req;
-   shrink_string<4096, 128>(http_req.body);
+   shrink_str<4096, 128>(
+       http_req.body, bts::utils::cached_type_name<RequestT>());
    http_req.reset();
    // Prepare request
    http_req.url = url_;
@@ -66,7 +76,9 @@ void RestApiClient::request_(
    http_req.method = method;
    http_req.headers["X-MBX-APIKEY"] = api_key_;
    thread_local std::string qps;
-   shrink_string<256, 16>(qps);
+   static const std::string query_params_name =
+       fmt::format("{}_params", bts::utils::cached_type_name<RequestT>());
+   shrink_str<256, 16>(qps, query_params_name.c_str());
    qps.clear();
    to_query_params(qps, req);
    if (!qps.empty()) {
@@ -78,7 +90,8 @@ void RestApiClient::request_(
    }
    // Send it
    thread_local HttpResponse http_res;
-   shrink_string<4096, 128>(http_res.body);
+   shrink_str<4096, 128>(
+       http_res.body, bts::utils::cached_type_name<ResponseT>());
    http_res.reset();
    HttpClient::send(http_req, &http_res);
    res.http_code = http_res.code;

src/server/sparkd/Dockerfile

@@ -0,0 +1,25 @@
+ARG IMAGE="sparkd"
+ARG CONFIG_DIR="src/server/sparkd/configurations/dev"
+ARG SOURCE_DIR="install/sparkd-release-gcc"
+
+FROM bluetradingsystems/linux_dotnet7 AS sparkd_base
+ARG CONFIG_DIR
+ARG SOURCE_DIR
+
+USER "root"
+RUN apt-get update && apt-get install -y liblttng-ust-dev
+RUN apt-get update && apt-get install -y libpcap0.8-dev
+RUN apt-get update && apt-get install -y python3.10-dev
+
+USER "bts"
+COPY --chown=bts:bts "${SOURCE_DIR}" "."
+COPY --chown=bts:bts "${CONFIG_DIR}" "/home/bts/sparkdata/conf"
+WORKDIR "/home/bts/bin"
+
+FROM sparkd_base AS sparkd
+
+FROM sparkd_base AS sparkd_md_replay
+ARG MD_RECORDING_DIR="src/server/sparkd/.md_recording"
+COPY --chown=bts:bts "${MD_RECORDING_DIR}" "/home/bts/sparkdata/data/md_recording"
+
+FROM ${IMAGE}

src/server/sparkd/analytics/AnalyticsServiceServer.cpp

@@ -106,6 +106,8 @@ struct AnalyticsServiceServer::PImpl {
    AnalyticsServiceBase::Exp_parametersStatefulPublisher exp_params_publisher;
    AnalyticsServiceBase::Volsurface_usageStatefulPublisher vs_usage_publisher;
    AnalyticsServiceBase::ReportsStatefulPublisher reports_publisher;
+   AnalyticsServiceBase::Archived_theosStatefulPublisher
+       archived_theos_publisher;
    PImpl(
        bts::api::ServiceContext &context,
        const AnalyticsServiceBase::VolsurfacesStatefulPublisher &vs_publisher,
@@ -113,7 +115,9 @@ struct AnalyticsServiceServer::PImpl {
            &exp_params_publisher,
        const AnalyticsServiceBase::Volsurface_usageStatefulPublisher
            &vs_usage_publisher,
-       const AnalyticsServiceBase::ReportsStatefulPublisher &reports_publisher)
+       const AnalyticsServiceBase::ReportsStatefulPublisher &reports_publisher,
+       const AnalyticsServiceBase::Archived_theosStatefulPublisher
+           &archived_theos_publisher_)
        : context(context),
          vs_store(std::make_unique<persistence::SqliteStore>("volsurfaces")),
          exp_params_store(std::make_unique<persistence::SqliteStore>(
@@ -125,9 +129,17 @@ struct AnalyticsServiceServer::PImpl {
              std::make_unique<persistence::SqliteStore>("archived_theos")),
          vs_publisher(vs_publisher), exp_params_publisher(exp_params_publisher),
          vs_usage_publisher(vs_usage_publisher),
-         reports_publisher(reports_publisher) {}
+         reports_publisher(reports_publisher),
+         archived_theos_publisher(archived_theos_publisher_) {}
 };
 
+void AnalyticsServiceServer::_publish_archived_theos(
+    const api::analytics::ArchivedTheoList &l) {
+   // remove all rows
+   me->archived_theos_publisher.add_or_update(l);
+   me->archived_theos_publisher.flush();
+}
+
 AnalyticsServiceServer::AnalyticsServiceServer(
     bts::api::ServiceContext &context)
     : AnalyticsServiceBase(context),
@@ -140,7 +152,8 @@ AnalyticsServiceServer::AnalyticsServiceServer(
           // make_volsurfaces_stateful_publisher(),
           make_exp_parameters_stateful_publisher(),
           make_volsurface_usage_stateful_publisher(),
-          make_reports_stateful_publisher())) {
+          make_reports_stateful_publisher(),
+          make_archived_theos_stateful_publisher())) {
    // publishing all vol surfaces
    LOG(Gen, Info, "Starting AnalyticsServiceServer");
    _clean_up_vss();
@@ -206,6 +219,28 @@ AnalyticsServiceServer::AnalyticsServiceServer(
       }
       me->reports_publisher.flush();
    }
+   // archived theos
+   {
+      size_t n = 0;
+      // ns_table is ns vs {key, val}[]
+      auto ns_table = me->theos_store->retrieve_all();
+      LOG(OrderMan, Info,
+          "AnalyticsServiceServer::cons loaded archived_theos n: {}",
+          ns_table.size());
+      bts_assert(ns_table.size() == 0 || ns_table.size() == 1);
+      for (const auto &ns_vals : ns_table) {
+         for (const auto &kvp : ns_vals.second) {
+            ArchivedTheoList theo_list;
+            utils::pb::from_string(kvp.second, &theo_list);
+            _publish_archived_theos(theo_list);
+            n += theo_list.sec_theo().size();
+         }
+      }
+      LOG(Gen, Info,
+          "AnalyticsServiceServer::AnalyticsServiceServer publishing theos n: "
+          "{}",
+          n);
+   }
 }
 
 void AnalyticsServiceServer::get_new_volsurface_id(
@@ -452,6 +487,31 @@ void AnalyticsServiceServer::archive_theos(
    fire_cb(resp, true);
 }
 
+void AnalyticsServiceServer::clean_and_archive_theos(
+    const bts::api::analytics::ArchivedTheoList &theo_list,
+    const api::Response<::bts::api::ResultMessage> &resp) {
+   LOG(Gen, Info,
+       "AnalyticsServiceServer::clean_and_archive_theos theo_list count: {}",
+       theo_list.sec_theo_size());
+   if (theo_list.sec_theo().empty()) {
+      std::string err_msg = "Empty archived theo list, call failed";
+      LOG(Gen, Error, "AnalyticsServiceServer::clean_and_archive_theos {}",
+          err_msg);
+      fire_cb(resp, false, &err_msg);
+      return;
+   }
+   // for (const auto &t : theo_list.sec_theo()) {
+   //    me->archived_theos_publisher.add_or_update(t);
+   // }
+   me->theos_store->remove_all(me->archived_theos_ns);
+   const std::string &ns = me->archived_theos_ns;
+   const std::string key = "dont care"; //std::to_string(theo_list.save_time());
+   std::string value = utils::pb::to_string(theo_list);
+   me->theos_store->insert(ns, key, value);
+   _publish_archived_theos(theo_list);
+   fire_cb(resp, true);
+}
+
 void AnalyticsServiceServer::get_archived_theos(
     const bts::api::analytics::ArchivedTheoRetrieval &ret,
     const bts::api::Response<bts::api::analytics::ArchivedTheoList> &resp) {

src/server/sparkd/analytics/AnalyticsServiceServer.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "lib/api/services/AnalyticsService.pb.h"
 #include "lib/api/services/AnalyticsService.server.pb.h"
 
 namespace bts {
@@ -17,6 +18,10 @@ class AnalyticsServiceServer
    struct PImpl;
    std::unique_ptr<PImpl> me;
 
+ private:
+   void _publish_archived_theos(
+       const api::analytics::ArchivedTheoList &total_state);
+
  public:
    AnalyticsServiceServer(bts::api::ServiceContext &context);
 
@@ -62,6 +67,10 @@ class AnalyticsServiceServer
        const bts::api::analytics::ArchivedTheoList &theos,
        const api::Response<::bts::api::ResultMessage> &reply) override;
 
+   void clean_and_archive_theos(
+       const bts::api::analytics::ArchivedTheoList &theos,
+       const api::Response<::bts::api::ResultMessage> &reply) override;
+
    void get_archived_theos(
        const bts::api::analytics::ArchivedTheoRetrieval &theo_ret,
        const bts::api::Response<bts::api::analytics::ArchivedTheoList> &reply)

src/server/sparkd/cmds/Run.cpp

@@ -85,7 +85,7 @@ static OrderServiceServer create_order_service_server(
 static void execute() {
    using namespace bts;
    LOG(Gen, Warn,
-       "------------------------------------ Sparkd v2.2.5.4 launched "
+       "------------------------------------ Sparkd v2.3 launched "
        "--------------------------------");
 
    // Initializes curl for the application, and eventual cleanup in destructor

src/server/sparkd/cmds/md/cmds/DumpSecurity.cpp

@@ -1,32 +1,48 @@
 #include "DumpSecurity.h"
-#include "lib/config/protos/market_data.pb.h"
 #include "lib/core/TableFormatter.h"
 #include "lib/domain_model/Security.h"
 #include "lib/domain_model/Security.pb.h"
 #include "lib/domain_model/SecurityType.h"
-#include "lib/md/Diag/ShmDumper.h"
 #include "lib/md/SecurityDB.h"
 #include "lib/md/SymbolLoader.h"
-#include "lib/md/UserSymbol.h"
+#include "lib/utils/lazy.h"
+
 #include <unordered_set>
 #include <vector>
 
 namespace bts::cmds::md::cmds {
 
 using bts::api::Security;
+using bts::api::security_type_parse_smart;
+using bts::api::SecurityType;
 using bts::api::SecurityType_Name;
+using bts::api::Venue;
+using bts::api::venue_parse_smart;
+using bts::core::TableFormatter;
+using bts::md::securitytype_of;
 using bts::md::SymbolLoader;
+using bts::md::db::load_metadata;
+using bts::md::db::load_securities;
 using bts::securities::security_type;
+using bts::utils::Lazy;
+using std::function;
 using std::make_tuple;
+using std::string;
+using std::unordered_set;
+using std::vector;
 
 template <class EnumT>
 using EnumNameMap = std::map<std::string, EnumT>;
 
-void print_security(const api::Security &security) {
+void print_security(
+    const Security &security,
+    Lazy<vector<api::synthetic::Metadata>> &metadata) {
+   static const char *SEPARATOR =
+       "\n**************************************************\n";
+   std::cout << SEPARATOR;
    security.PrintDebugString();
-   if (bts::md::securitytype_of(security) == api::SECURITYTYPE_SYNTHETIC) {
-      auto metadata = bts::md::db::load_metadata();
-      for (const auto &m : metadata) {
+   if (securitytype_of(security) == api::SECURITYTYPE_SYNTHETIC) {
+      for (const auto &m : metadata.get()) {
          if (m.security_key() == security.id()) {
             m.PrintDebugString();
             break;
@@ -35,17 +51,20 @@ void print_security(const api::Security &security) {
    }
 }
 
-void print_securities(std::function<bool(const api::Security &)> predicate) {
+void print_securities(function<bool(const Security &)> predicate) {
+   auto metadata =
+       Lazy<vector<api::synthetic::Metadata>>::from_delegate(load_metadata);
    const auto symbols = SymbolLoader::load();
-   bts::md::db::load_securities(symbols, [&](const api::Security &security) {
-      if (predicate(security))
-         print_security(security);
+   load_securities(symbols, [&](const Security &security) {
+      if (predicate(security)) {
+         print_security(security, metadata);
+      }
    });
 }
 
-void print_db_securities(std::function<bool(const api::Security &)> predicate) {
-   std::vector<bts::api::Security> secs;
-   const bts::core::TableFormatter::Schema<bts::api::Security> schema{
+void print_db_securities(function<bool(const Security &)> predicate) {
+   vector<Security> secs;
+   const TableFormatter::Schema<Security> schema{
        make_tuple(
            "id", 12, [](const Security &s) { return std::to_string(s.id()); }),
        make_tuple(
@@ -62,22 +81,67 @@ void print_db_securities(std::function<bool(const api::Security &)> predicate) {
            "name", 32,
            [](const Security &s) { return bts::securities::descriptor(s); }),
    };
-   auto on_sec = [&](const bts::api::Security &sec) {
+   auto on_sec = [&](const Security &sec) {
       if (!predicate(sec)) {
          return;
       }
       secs.push_back(sec);
    };
-   bts::md::db::load_securities([](auto) { return true; }, on_sec);
-   bts::core::TableFormatter::format(std::cout, secs, schema);
+   load_securities([](auto) { return true; }, on_sec);
+   TableFormatter::format(std::cout, secs, schema);
+}
+
+template <class T, class ParseT = function<bool(const string &, T *)>>
+static vector<T> parse_enums(const vector<string> &names, ParseT parse) {
+   vector<T> rv;
+   for (const auto &t : names) {
+      if (!parse(t, &rv.emplace_back())) {
+         rv.pop_back();
+      }
+   }
+   return rv;
 }
 
 struct DumpSecurity::Args {
    bool all;
    bool db = false;
-   std::vector<std::string> types;
-   std::vector<std::string> venues;
-   std::vector<uint32_t> security_ids;
+   vector<string> types;
+   vector<string> venues;
+   vector<uint32_t> security_ids;
+
+   function<bool(const Security &)> get_filter() {
+      if (all) {
+         return [](const auto &) { return true; };
+      }
+      auto types =
+          parse_enums<SecurityType>(this->types, security_type_parse_smart);
+      auto venues = parse_enums<Venue>(this->venues, venue_parse_smart);
+      unordered_set<uint32_t> ids;
+      for (auto id : security_ids) {
+         ids.insert(id);
+      }
+      return [&, types = std::move(types), venues = std::move(venues),
+              ids = std::move(ids)](const auto &s) {
+         if (!ids.empty()) {
+            return ids.find(s.id()) != ids.end();
+         }
+         if (!types.empty()) {
+            auto st = security_type(s);
+            if (!std::any_of(
+                    types.begin(), types.end(),
+                    [st](SecurityType _st) { return _st == st; })) {
+               return false;
+            }
+         }
+         if (!venues.empty() &&
+             !std::any_of(
+                 venues.begin(), venues.end(),
+                 [v = s.exchange()](Venue _v) { return _v == v; })) {
+            return false;
+         }
+         return true;
+      };
+   }
 };
 
 DumpSecurity::DumpSecurity()
@@ -94,44 +158,11 @@ DumpSecurity::DumpSecurity()
        "Print information about specific securities");
 
    callback([&] {
-      if (_args->all) {
-         print_securities([](const auto &) { return true; });
-      } else if (_args->db) {
-         std::vector<api::SecurityType> types;
-         for (const auto &t : _args->types) {
-            if (!api::security_type_parse_smart(t, &types.emplace_back())) {
-               types.pop_back();
-            }
-         }
-         std::vector<api::Venue> venues;
-         for (const auto &t : _args->venues) {
-            if (!api::venue_parse_smart(t, &venues.emplace_back())) {
-               venues.pop_back();
-            }
-         }
-         print_db_securities([&](const auto &s) {
-            if (!types.empty()) {
-               auto st = bts::securities::security_type(s);
-               if (!std::any_of(
-                       types.begin(), types.end(),
-                       [st](api::SecurityType _st) { return _st == st; })) {
-                  return false;
-               }
-            }
-            if (!venues.empty() &&
-                !std::any_of(
-                    venues.begin(), venues.end(),
-                    [v = s.exchange()](api::Venue _v) { return _v == v; })) {
-               return false;
-            }
-            return true;
-         });
+      auto filter = _args->get_filter();
+      if (_args->db) {
+         print_db_securities(filter);
       } else {
-         std::unordered_set<uint32_t> ids;
-         for (auto id : _args->security_ids)
-            ids.insert(id);
-         print_securities(
-             [&](const auto &s) { return ids.find(s.id()) != ids.end(); });
+         print_securities(filter);
       }
    });
 }

src/server/sparkd/configurations/alphaworx_prod/.env

@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/alphaworx_prod
+SPARKD__SHM_SIZE=8gb

src/server/sparkd/configurations/alphaworx_prod/README.txt

@@ -0,0 +1,6 @@
+* Connects to Binance and Okx PRODUCTION sites for market data and execution
+* Api keys NOT included
+* Configured for Alphaworx
+
+To retrieve api keys, run "python init_env.py". It should dump them to a file
+called  ".env.local".

src/server/sparkd/configurations/alphaworx_prod/binance_1.json

@@ -0,0 +1,9 @@
+{
+  "api_key": "${BINANCE_API_KEY}",
+  "secret_key": "${BINANCE_SECRET_KEY}",
+  "account": "BINANCE_PROD",
+  "rest_endpoint": "https://api.binance.com",
+  "ws_endpoint": "wss://ws-api.binance.com:443/ws-api/v3",
+  "ws_stream_endpoint": "wss://stream.binance.com:9443",
+  "use_cross_margin_api": true
+}

src/server/sparkd/configurations/alphaworx_prod/binance_usds_futures_1.json

@@ -0,0 +1,9 @@
+{
+  "api_key": "${BINANCE_API_KEY}",
+  "secret_key": "${BINANCE_SECRET_KEY}",
+  "account": "BINANCE_PROD",
+  "rest_endpoint": "https://fapi.binance.com",
+  "ws_endpoint": "wss://ws-fapi.binance.com/ws-fapi/v1",
+  "ws_stream_endpoint": "wss://fstream.binance.com",
+  "use_cross_margin_api": false
+}

src/server/sparkd/configurations/alphaworx_prod/init_env.py

@@ -0,0 +1,88 @@
+import botocore
+import json
+import os
+
+from boto3.session import Session
+from botocore.exceptions import ClientError
+from fileinput import FileInput
+
+
+def update_env_file(path, env_vars: dict):
+    """Update or add vars in a .env file in place."""
+    if not os.path.exists(path):
+        with open(path, "w") as file:
+            for k, v in env_vars.items():
+                file.write(f"{k}={v}\n")
+        return
+
+    with FileInput(path, inplace=True) as file:
+        for line in file:
+            if not line.startswith("#"):
+                stripped: str = line.lstrip()
+                key_len = stripped.find("=")
+                if key_len >= 0:
+                    k = stripped[0:key_len]
+                    if k in env_vars:
+                        print(f"{k}={env_vars[k]}")
+                        del env_vars[k]
+                        continue
+            print(line, end="")
+
+    if len(env_vars) > 0:
+        with open(path, "a") as file:
+            for k, v in env_vars.items():
+                file.write(f"{k}={v}\n")
+
+
+def get_secret():
+    secret_name = "alphaworx-binance-api-keys"
+    region_name = "ap-northeast-1"
+    profile_name = "AdministratorAccess-849458505639"
+    session = Session(profile_name=profile_name)
+    client = session.client(
+        service_name="secretsmanager",
+        region_name=region_name,
+    )
+
+    def on_token_error(e: Exception):
+        print(f"""
+Failed to get secret value. {e}
+Verify you are logged in to aws, then try again.
+
+  aws sso login --profile {profile_name}
+
+If this error persists, verify you have access to these api keys.
+For more help or information, talk to John.
+""")
+
+    try:
+        get_secret_value_response = client.get_secret_value(
+            SecretId=secret_name)
+    except ClientError as e:
+        print(f"Failed to get secret value.\n{e}")
+        raise e
+    except botocore.exceptions.SSOTokenLoadError as tl_err:
+        on_token_error(tl_err)
+        return None
+    except botocore.exceptions.TokenRetrievalError as tr_err:
+        on_token_error(tr_err)
+        return None
+    return get_secret_value_response["SecretString"]
+
+
+def _run():
+    this_dir = os.path.dirname(__file__)
+    secret_str = get_secret()
+    if secret_str is None:
+        exit(1)
+    secret = json.loads(secret_str)
+    env_vars = {}
+    if "api-key" in secret:
+        env_vars["BINANCE_API_KEY"] = secret["api-key"]
+    if "secret-key" in secret:
+        env_vars["BINANCE_SECRET_KEY"] = secret["secret-key"]
+    update_env_file(f"{this_dir}/.env.local", env_vars)
+
+
+if __name__ == "__main__":
+    _run()

src/server/sparkd/configurations/alphaworx_prod/spark.conf

@@ -0,0 +1,121 @@
+QuickbusCfg {
+  value {
+    node {
+      reconnect_period_secs: 1
+      hb_period_msecs: 15000
+      max_missed_hbs: 5
+      parent_host: ""
+      parent_port: 0
+      my_port: 0
+    }
+    client {
+      reconnect_period_secs: 1
+      hb_period_msecs: 15000
+      max_missed_hbs: 5
+      parent_host: @SystemConfig.config_ip
+      parent_port: @Ports.qbus_port
+      my_port: 0
+    }
+  }
+}
+
+JetFIX {
+  value {
+    connection {
+      name: "binance_1"
+      venue: VENUE_BINANCE
+    }
+    connection {
+      name: "binance_usds_futures_1"
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    connection {
+      name: "okx_1"
+      venue: VENUE_OKX
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    binance {
+      num_depth_levels: 5
+      url_rest_base: "https://api.binance.com"
+      url_ws_base: "wss://stream.binance.com:9443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: @SymbolLists.list[Binance].symbol
+          name_regexps: "USDC|USDT$"
+        }
+      }
+    }
+    binance_usds_futures {
+      num_depth_levels: 5
+      url_rest_base: "https://fapi.binance.com"
+      url_ws_base: "wss://fstream.binance.com"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
+        }
+      }
+    }
+    okx {
+      num_depth_levels: 5
+      url_rest_base: "https://aws.okx.com"
+      url_ws_base: "wss://wsaws.okx.com:8443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[Okx].symbol
+        }
+      }
+      is_application_ping_pong_enabled: true
+      application_ping_interval_ms: 20000
+      application_pong_timeout_ms: 10000
+      is_protocol_ping_pong_enabled: true
+      protocol_ping_interval_ms: 60000
+      protocol_pong_timeout_ms: 30000
+    }
+  }
+}
+
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_BINANCE
+    }
+    entries {
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    entries {
+      venue: VENUE_GLBX
+    }
+    entries {
+      venue: VENUE_OKX
+    }
+    entries {
+      venue: VENUE_TAIFEX
+    }
+  }
+}
+
+OkxExeCfg {
+  value {
+    account: "prod"
+    ops_id: "op_id_bts"
+    secret_key: "240038D7A28D7F9461247CFCB8E43710"
+    api_key: "dc188532-c74c-4d40-affe-3c6bf4d0ca9a"
+    passphrase: ":,7i%tA%ivfj%SCbX?Ut"
+    ws_url: "wss://wsaws.okx.com:8443"
+    rest_url: "https://www.okx.com"
+    ws_ping_timeout_seconds: 10 
+    ws_ping_interval_seconds: 25
+    ws_reconnect_interval_seconds: 5
+    ws_reconnect_max_count: 5
+	  simulated_trading: false
+    tdmode: "cross"
+  }
+}

src/server/sparkd/configurations/alphaworx_prod/symbols.conf

@@ -0,0 +1,70 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "AAVE" 
+      symbol: "ACT"
+      symbol: "ADA"
+      symbol: "ALGO"
+      symbol: "AR"
+      symbol: "ATOM"
+      symbol: "AVAX"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "HIVE"
+      symbol: "IO"
+      symbol: "PAXG"
+      symbol: "SPELL"
+      symbol: "WBETH"
+      symbol: "WIF"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "ADA"
+      symbol: "HIVE"
+      symbol: "SPELL"
+      symbol: "IO"
+      symbol: "UNI"
+    }
+    list {
+      key: "Okx"
+      symbol: "AAVE"
+      symbol: "ACT"
+      symbol: "ADA"
+      symbol: "ALGO"
+      symbol: "AR"
+      symbol: "ATOM"
+      symbol: "AVAX"
+      symbol: "BETH"
+      symbol: "BTC"
+      symbol: "CRV"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "ETHFI"
+      symbol: "UNI"
+      symbol: "USDC"
+      symbol: "USDT"
+      symbol: "WIF"
+      symbol: "XAUT"
+      symbol: "XRP"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE"
+      spot: @SymbolLists.list[Binance].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    }
+    venues {
+      name: "VENUE_OKX"
+      spot: @SymbolLists.list[Okx].symbol
+      future: @SymbolLists.list[Okx].symbol
+    }
+  }
+}

src/server/sparkd/configurations/binance_dev/.env

@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/binance_dev
+SPARKD__SHM_SIZE=8gb

src/server/sparkd/configurations/binance_dev/README.txt

@@ -0,0 +1 @@
+* Connects to Binance test nets for market data and execution

src/server/sparkd/configurations/binance_dev/binance_1.json

@@ -1,7 +1,7 @@
 {
   "api_key": "QGwn9OVcpRTU76gOLV0gmQyVAJUmXW6w7H6y9tH8ctanzVnpZN1TP27liObE1Wsd",
   "secret_key": "7ohKg6X3DcsZMGaJmngC84EglsCFse9flMl8aKisa5FUY18LjGCEEdm0KEZljk9T",
-  "account": "binance_dev",
+  "account": "BINANCE_DEV",
   "rest_endpoint": "https://testnet.binance.vision",
   "ws_endpoint": "wss://testnet.binance.vision/ws-api/v3",
   "ws_stream_endpoint": "wss://stream.testnet.binance.vision:9443"

src/server/sparkd/configurations/binance_dev/binance_usds_futures_1.json

@@ -1,7 +1,7 @@
 {
   "api_key": "b838c108cd2648e34a4f088a7fc635a9cbccf3c2faa6bcaabcb532ffacc087b0",
   "secret_key": "a68f0ab27403c42fa0306b6341b2ddd51d99ac42916f849404155449457d0492",
-  "account": "binance_dev",
+  "account": "BINANCE_DEV",
   "rest_endpoint": "https://testnet.binancefuture.com",
   "ws_endpoint": "wss://testnet.binancefuture.com/ws-fapi/v1",
   "ws_stream_endpoint": "wss://stream.binancefuture.com"

src/server/sparkd/configurations/binance_dev/spark.conf

@@ -1,26 +1,11 @@
-ExchangeSubAccounts {
-  value {
-    accounts{
-      exchange_subaccount: "ZmIKL6fb"
-      bts_account_name: "MAIN"     
-    }
-    accounts{
-      exchange_subaccount: "v1iSzZrY"
-      bts_account_name: "SMALL"
-    }
-  }
-}
-
 JetFIX {
   value {
     connection {
       name: "binance_1"
-      max_order_msgs_per_window: 300
       venue: VENUE_BINANCE
     }
     connection {
       name: "binance_usds_futures_1"
-      max_order_msgs_per_window: 300
       venue: VENUE_BINANCE_USDS_FUTURES
     }
   }
@@ -46,15 +31,11 @@ Logging {
     initial_queue_size: 120001
     max_queue_growth: 4
     console {
-      default_level: "Warn"
+      default_level: "Info"
       subsystem {
         name: "OrderMan"
         level: "Diag"
       }
-      subsystem {
-        name: "Http"
-        level: "Diag"
-      }
     }
     file {
       default_level: "Info"
@@ -90,17 +71,12 @@ MarketDataConfig {
       security_criteria {
         clauses {
           security_types: SECURITY_TYPE_SPOT
-          symbols: "AAVE"
-          symbols: "AVAX"
-          symbols: "BNB"
-          symbols: "BTC"
-          symbols: "DOGE"
-          symbols: "EIGEN"
-          symbols: "ETH"
+          symbols: @SymbolLists.list[Binance].symbol
           name_regexps: "USDC|USDT$"
         }
       }
     }
+
     binance_usds_futures {
       num_depth_levels: 5
       # url_rest_base: "https://fapi.binance.com"
@@ -111,44 +87,13 @@ MarketDataConfig {
       security_criteria {
         clauses {
           security_types: SECURITY_TYPE_FUTURE
-          symbols: "AAVE"
-          symbols: "AVAX"
-          symbols: "BNB"
-          symbols: "BTC"
-          symbols: "DOGE"
-          symbols: "EIGEN"
-          symbols: "ETH"
+          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
         }
       }
     }
   }
 }
 
-Symbols {
-  value {
-    venues {
-      name: "VENUE_BINANCE"
-      spot: "AAVE"
-      spot: "AVAX"
-      spot: "BNB"
-      spot: "BTC"
-      spot: "DOGE"
-      spot: "EIGEN"
-      spot: "ETH"
-    }
-    venues {
-      name: "VENUE_BINANCE_USDS_FUTURES"
-      future: "AAVE"
-      future: "AVAX"
-      future: "BNB"
-      future: "BTC"
-      future: "DOGE"
-      future: "EIGEN"
-      future: "ETH"
-    }
-  }
-}
-
 NotSimExecution {
   value {
     entries {

src/server/sparkd/configurations/binance_dev/symbols.conf

@@ -0,0 +1,37 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE"
+      spot: @SymbolLists.list[Binance].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    }
+  }
+}

src/server/sparkd/configurations/dev/.env

@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
+SPARKD__SHM_SIZE=8gb

src/server/sparkd/configurations/dev/README.txt

@@ -0,0 +1 @@
+* Connects to Binance and Okx test nets for market data and execution

src/server/sparkd/configurations/dev/binance_1.json

@@ -0,0 +1,8 @@
+{
+  "api_key": "QGwn9OVcpRTU76gOLV0gmQyVAJUmXW6w7H6y9tH8ctanzVnpZN1TP27liObE1Wsd",
+  "secret_key": "7ohKg6X3DcsZMGaJmngC84EglsCFse9flMl8aKisa5FUY18LjGCEEdm0KEZljk9T",
+  "account": "BINANCE_DEV",
+  "rest_endpoint": "https://testnet.binance.vision",
+  "ws_endpoint": "wss://testnet.binance.vision/ws-api/v3",
+  "ws_stream_endpoint": "wss://stream.testnet.binance.vision:9443"
+}

src/server/sparkd/configurations/dev/binance_usds_futures_1.json

@@ -0,0 +1,8 @@
+{
+  "api_key": "b838c108cd2648e34a4f088a7fc635a9cbccf3c2faa6bcaabcb532ffacc087b0",
+  "secret_key": "a68f0ab27403c42fa0306b6341b2ddd51d99ac42916f849404155449457d0492",
+  "account": "BINANCE_DEV",
+  "rest_endpoint": "https://testnet.binancefuture.com",
+  "ws_endpoint": "wss://testnet.binancefuture.com/ws-fapi/v1",
+  "ws_stream_endpoint": "wss://stream.binancefuture.com"
+}

src/server/sparkd/configurations/dev/spark.conf

@@ -0,0 +1,135 @@
+JetFIX {
+  value {
+    connection {
+      name: "binance_1"
+      venue: VENUE_BINANCE
+    }
+    connection {
+      name: "binance_usds_futures_1"
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    connection {
+      name: "okx_1"
+      venue: VENUE_OKX
+    }
+  }
+}
+
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "OrderMan"
+        level: "Diag"
+      }
+    }
+    file {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Diag"
+      }
+      subsystem {
+        name: "OrderMan"
+        level: "Diag"
+      }
+      subsystem {
+        name: "Http"
+        level: "Diag"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    binance {
+      num_depth_levels: 5
+      # url_rest_base: "wss://ws-api.binance.com:9443"
+      # According to Binance, the "apiN" urls offer performance at the cost of stability.
+      # https://developers.binance.com/docs/binance-spot-api-docs/rest-api/general-api-information
+      # url_rest_base: "https://api1.binance.com:9443"
+      # url_rest_base: "https://api.binance.com"
+      url_rest_base: "https://testnet.binance.vision"
+      # url_ws_base: "wss://ws-api.binance.com"
+      url_ws_base: "wss://testnet.binance.vision"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: @SymbolLists.list[Binance].symbol
+          name_regexps: "USDC|USDT$"
+        }
+      }
+    }
+    binance_usds_futures {
+      num_depth_levels: 5
+      # url_rest_base: "https://fapi.binance.com"
+      # url_ws_base: "wss://fstream.binance.com"
+      url_rest_base: "https://testnet.binancefuture.com"
+      url_ws_base: "wss://stream.binancefuture.com"
+
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
+        }
+      }
+    }
+    okx {
+      num_depth_levels: 5
+      url_rest_base: "https://aws.okx.com"
+      url_ws_base: "wss://wsaws.okx.com:8443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[Okx].symbol
+        }
+      }
+      is_application_ping_pong_enabled: true                                                                                                                                            
+      application_ping_interval_ms: 20000
+      application_pong_timeout_ms: 10000
+      is_protocol_ping_pong_enabled: true
+      protocol_ping_interval_ms: 60000
+      protocol_pong_timeout_ms: 30000
+    }
+  }
+}
+
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_BINANCE
+    }
+    entries {
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    entries {
+      venue: VENUE_GLBX
+    }
+    entries {
+      venue: VENUE_OKX
+    }
+    entries {
+      venue: VENUE_TAIFEX
+    }
+  }
+}

src/server/sparkd/configurations/dev/symbols.conf

@@ -0,0 +1,63 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+    list {
+      key: "Okx"
+      symbol: "AAVE"
+      symbol: "ACT"
+      symbol: "ADA"
+      symbol: "ALGO"
+      symbol: "AR"
+      symbol: "ATOM"
+      symbol: "AVAX"
+      symbol: "BETH"
+      symbol: "BTC"
+      symbol: "CRV"
+      symbol: "EIGEN"
+      symbol: "ETHFI"
+      symbol: "UNI"
+      symbol: "WIF"
+      symbol: "XAUT"
+      symbol: "XRP"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE"
+      spot: @SymbolLists.list[Binance].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    }
+    venues {
+      name: "VENUE_OKX"
+      spot: @SymbolLists.list[Okx].symbol
+      future: @SymbolLists.list[Okx].symbol
+    }
+  }
+}

src/server/sparkd/configurations/sim_dev/.env

@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/sim_dev
+SPARKD__SHM_SIZE=8gb

src/server/sparkd/configurations/sim_dev/README.txt

@@ -0,0 +1,2 @@
+* Connects to Binance and Okx test nets for market data
+* Execution is SIM

src/server/sparkd/configurations/sim_dev/spark.conf

@@ -0,0 +1,77 @@
+JetFIX {
+  value {
+    connection {
+      name: "binance_1"
+      venue: VENUE_BINANCE
+    }
+    connection {
+      name: "binance_usds_futures_1"
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    connection {
+      name: "okx_1"
+      venue: VENUE_OKX
+    }
+  }
+}
+
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+MarketDataConfig {
+  value {
+    binance {
+      num_depth_levels: 5
+      url_rest_base: "https://testnet.binance.vision"
+      url_ws_base: "wss://testnet.binance.vision"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: @SymbolLists.list[Binance].symbol
+          name_regexps: "USDC|USDT$"
+        }
+      }
+    }
+    binance_usds_futures {
+      num_depth_levels: 5
+      url_rest_base: "https://testnet.binancefuture.com"
+      url_ws_base: "wss://stream.binancefuture.com"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
+        }
+      }
+    }
+    okx {
+      num_depth_levels: 5
+      url_rest_base: "https://aws.okx.com"
+      url_ws_base: "wss://wsaws.okx.com:8443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[Okx].symbol
+        }
+      }
+      is_application_ping_pong_enabled: true                                                                                                                                            
+      application_ping_interval_ms: 20000
+      application_pong_timeout_ms: 10000
+      is_protocol_ping_pong_enabled: true
+      protocol_ping_interval_ms: 60000
+      protocol_pong_timeout_ms: 30000
+    }
+  }
+}

src/server/sparkd/configurations/sim_dev/symbols.conf

@@ -0,0 +1,63 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+    list {
+      key: "Okx"
+      symbol: "AAVE"
+      symbol: "ACT"
+      symbol: "ADA"
+      symbol: "ALGO"
+      symbol: "AR"
+      symbol: "ATOM"
+      symbol: "AVAX"
+      symbol: "BETH"
+      symbol: "BTC"
+      symbol: "CRV"
+      symbol: "EIGEN"
+      symbol: "ETHFI"
+      symbol: "UNI"
+      symbol: "WIF"
+      symbol: "XAUT"
+      symbol: "XRP"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE"
+      spot: @SymbolLists.list[Binance].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    }
+    venues {
+      name: "VENUE_OKX"
+      spot: @SymbolLists.list[Okx].symbol
+      future: @SymbolLists.list[Okx].symbol
+    }
+  }
+}

src/server/sparkd/execution/execution_gateway_provider.cpp

@@ -88,8 +88,9 @@ struct ExecutionGatewayProvider::PImpl {
          return std::make_unique<OkxExecutionGateway>(o);
       }
       case api::VENUE_BINANCE: {
-         const auto cfg =
+         auto cfg =
              parse_config_json<binance_execution::GatewayConfig>(conn.name());
+         cfg.expand_env();
          if (cfg.use_cross_margin_api) {
             return std::make_unique<
                 binance_execution::cross_margin::CrossMarginGateway>(
@@ -101,6 +102,7 @@ struct ExecutionGatewayProvider::PImpl {
       case api::VENUE_BINANCE_USDS_FUTURES: {
          auto cfg =
              parse_config_json<binance_execution::GatewayConfig>(conn.name());
+         cfg.expand_env();
          if (cfg.use_cross_margin_api) {
             WARN_FN(
                 "create_gateway(BinanceUsdsFutures)",

src/server/sparkd/md/databento/DatabentoDataHandler.cpp

@@ -30,9 +30,11 @@
 #include <cassert>
 #include <chrono>
 #include <cstddef>
+#include <cstdint>
 #include <databento/constants.hpp>
 #include <databento/historical.hpp>
 #include <databento/live.hpp>
+#include <filesystem>
 #include <optional>
 #include <ratio>
 #include <string>
@@ -77,6 +79,11 @@ void update_file(
     const uint time_to_run,
     std::string file_path) {
 
+   // Delete the existing file if it exists
+   LOG(md, Info, "Removing old file");
+   std::filesystem::remove(file_path);
+   LOG(md, Info, "Starting securities sub");
+
    auto client =
        databento::LiveBuilder{}
            .SetKey(key)
@@ -172,7 +179,10 @@ void DatabentoDatahandler::run(
                          .debug_sec();
    auto sleep_duration_updated = sleep_duration;
    std::vector<std::vector<std::tuple<uint64_t, std::string>>> all_times;
-   std::unordered_map<uint32_t, bts::domain::SecurityID> sec_id_map;
+   std::unordered_map<uint32_t, bts::domain::SecurityID>
+       exchnangeid_to_systemid;
+   std::unordered_map<std::string, bts::domain::SecurityID> symbol_to_systemid;
+
    if (!use_file) {
       sleep_duration_updated = std::chrono::duration<
           long double>::zero(); // Set to zero for live data
@@ -199,8 +209,32 @@ void DatabentoDatahandler::run(
       }
 
       auto duration = months * 32 * 24 * 60 * 60;
-      return expiration_time.seconds() <
-             bts::core::TimeStamp::now().seconds() + duration;
+      return expiration_time.seconds() >
+                 (bts::core::TimeStamp::now().seconds() - 60 * 60 * 24) &&
+             expiration_time.seconds() <
+                 bts::core::TimeStamp::now().seconds() + duration;
+   };
+   auto update_maps = [&](const databento::Record &record) {
+      const auto &msg = record.Get<databento::InstrumentDefMsg>();
+      LOG(md, Info, "got key for {} ", std::string(msg.RawSymbol()));
+      const auto exp_nanos = msg.expiration;
+      std::string exp_string = databento::ToIso8601(exp_nanos);
+      const auto expiration_time =
+          bts::core::TimeStamp::from_iso_string(exp_string);
+      if (!sec_filter(
+              expiration_time, msg.Underlying(), venue,
+              msg.strike_price / 1000000000)) {
+         return databento::KeepGoing::Continue;
+      }
+
+      if (!symbol_to_systemid.contains(msg.RawSymbol())) {
+         return databento::KeepGoing::Continue;
+      }
+      auto systemid = symbol_to_systemid[msg.RawSymbol()];
+      LOG(md, Info, "Updating map for {} to {} ", std::string(msg.RawSymbol()),
+          msg.hd.instrument_id);
+      exchnangeid_to_systemid.insert({msg.hd.instrument_id, systemid});
+      return databento::KeepGoing::Continue;
    };
    auto process_security = [&](const databento::Record &record) -> bool {
       const auto &msg = record.Get<databento::InstrumentDefMsg>();
@@ -242,6 +276,17 @@ void DatabentoDatahandler::run(
          sec.security.emplace<bts::domain::OptionSecurity>(
              expiration_time, bts::domain::Price(msg.strike_price, -9),
              call_put);
+         if (sec.venue() == api::domain::VENUE_CBOE) {
+            if (sec.security_common.symbol == "SPX" ||
+                sec.security_common.symbol == "SPXW") {
+               sec.security_common.min_quantity_increment = 1;
+               sec.security_common.min_quantity = 1;
+               sec.security_common.price_ticks.push_back(3.0);
+               sec.security_common.price_ticks.push_back(0.05);
+               sec.security_common.price_ticks.push_back(0.1);
+               sec.security_common.cash_per_point = 100;
+            }
+         }
          break;
       }
       default:
@@ -249,27 +294,28 @@ void DatabentoDatahandler::run(
              false, "Only futures and options supported on databento");
       }
       *sec.security_common.id = sec_id_store->get_or_add(
-          venue, sec_type, std::to_string(msg.hd.instrument_id));
+          venue, sec_type,sec.security_common.desc );
       listener->on_securities({&sec});
-      sec_id_map.insert({msg.hd.instrument_id, sec.security_common.id});
+      // exchnangeid_to_systemid.insert({msg.hd.instrument_id, sec.security_common.id});
+      symbol_to_systemid.insert({msg.RawSymbol(), sec.security_common.id});
       return true;
    };
-   auto debug_log = [&](const databento::Mbp1Msg &msg) {
+   auto debug_log = [&](const databento::Cmbp1Msg &msg) {
       if (DEBUG_SEC_ID != msg.hd.instrument_id)
          return;
       const double denom = 1000000000;
       // auto delta = std::to_string(std::chrono::duration<double, std::micro>(msg.ts_in_delta).count());
-      auto seq_num = msg.sequence;
+      // auto seq_num = msg.sequence;
       auto ap = msg.levels[0].ask_px / denom;
       auto bp = msg.levels[0].bid_px / denom;
       auto as = msg.levels[0].ask_sz;
       auto bs = msg.levels[0].bid_sz;
-      LOG(md, Sys, "ID: {}, seq_num: {}, B_P: {}, B_S: {}, A_P: {}, A_S: {}",
-          msg.hd.instrument_id, seq_num, bp, bs, ap, as);
+      LOG(md, Sys, "ID: {},  B_P: {}, B_S: {}, A_P: {}, A_S: {}",
+          msg.hd.instrument_id, bp, bs, ap, as);
    };
 
    auto process_bbo = [&](const databento::Record &record) {
-      const auto &msg = record.Get<databento::Mbp1Msg>();
+      const auto &msg = record.Get<databento::Cmbp1Msg>();
       debug_log(msg);
 
       // std::vector<std::tuple<uint64_t,std::string>>& times = all_times.emplace_back();
@@ -285,9 +331,10 @@ void DatabentoDatahandler::run(
       // prev = current;
       // current = bts::datetime::unix_nano_now();
       // times.push_back({current-prev,"MISC to check log time"});
-      if (!sec_id_map.contains(msg.hd.instrument_id))
+      if (!exchnangeid_to_systemid.contains(msg.hd.instrument_id))
          return databento::KeepGoing::Continue;
-      bts::domain::SecurityID sec_id = sec_id_map.at(msg.hd.instrument_id);
+      bts::domain::SecurityID sec_id =
+          exchnangeid_to_systemid.at(msg.hd.instrument_id);
 
       // prev = current;
       // current = bts::datetime::unix_nano_now();
@@ -297,7 +344,34 @@ void DatabentoDatahandler::run(
       // prev = current;
       // current = bts::datetime::unix_nano_now();
       // times.push_back({current-prev,"Sleep done"});
+      if (msg.action == 'T') {
+         domain::TradeRecord trade;
+         trade.trade_price = bts::domain::Price(msg.price, -9);
+         trade.trade_quantity = bts::domain::Quantity(msg.size);
+         trade.sec_id = sec_id;
+         switch (msg.side) {
+         case databento::Side::Ask:
+            trade.agg_side = domain::AggressorSide::Sell;
+            break;
+         case databento::Side::Bid:
+            trade.agg_side = domain::AggressorSide::Buy;
+            break;
+         case databento::Side::None:
+            trade.agg_side = domain::AggressorSide::None;
+            break;
+         }
+         trade.time_stamp = bts::core::TimeStamp::from_iso_string(
+             databento::ToIso8601(msg.ts_recv));
+         std::vector<const domain::TradeRecord *> trade_ptrs;
+         trade_ptrs.push_back(&trade);
+         listener->on_trades(trade_ptrs);
 
+         return databento::KeepGoing::Continue;
+      }
+      if (msg.action != 'A') {
+         LOG(md, Error, "Got with action neither A or T action:{}", msg.action);
+         return databento::KeepGoing::Continue;
+      }
       domain::Book book;
       book.set_security_id(sec_id);
       // prev = current;
@@ -333,40 +407,41 @@ void DatabentoDatahandler::run(
           Continue; //only added so the same functions work with the threaded builder, not really need in the blocking version
    };
 
-   auto process_trades = [&](const databento::Record &record) {
-      const auto &msg = record.Get<databento::TradeMsg>();
-      // The security type to is needed to get the id. Since the sec type isn't known,
-      // check for an id using both types (OPTION and FUTURE). This is safe to do since
-      // options and futures never have common names (instrument ids).
-      if (!sec_id_map.contains(msg.hd.instrument_id))
-         return databento::KeepGoing::Continue;
-      bts::domain::SecurityID sec_id = sec_id_map.at(msg.hd.instrument_id);
+   // auto process_trades = [&](const databento::Record &record) {
+   //    const auto &msg = record.Get<databento::TradeMsg>();
+   //    // The security type to is needed to get the id. Since the sec type isn't known,
+   //    // check for an id using both types (OPTION and FUTURE). This is safe to do since
+   //    // options and futures never have common names (instrument ids).
+   //    if (!exchnangeid_to_systemid.contains(msg.hd.instrument_id))
+   //       return databento::KeepGoing::Continue;
+   //    bts::domain::SecurityID sec_id =
+   //        exchnangeid_to_systemid.at(msg.hd.instrument_id);
 
-      // sleep_for(sleep_duration_updated);
-      domain::TradeRecord trade;
-      trade.trade_price = bts::domain::Price(msg.price, -9);
-      trade.trade_quantity = bts::domain::Quantity(msg.size);
-      trade.sec_id = sec_id;
-      switch (msg.side) {
-      case databento::Side::Ask:
-         trade.agg_side = domain::AggressorSide::Sell;
-         break;
-      case databento::Side::Bid:
-         trade.agg_side = domain::AggressorSide::Buy;
-         break;
-      case databento::Side::None:
-         trade.agg_side = domain::AggressorSide::None;
-         break;
-      }
-      trade.time_stamp = bts::core::TimeStamp::from_iso_string(
-          databento::ToIso8601(msg.ts_recv));
-      std::vector<const domain::TradeRecord *> trade_ptrs;
-      trade_ptrs.push_back(&trade);
-      listener->on_trades(trade_ptrs);
+   //    // sleep_for(sleep_duration_updated);
+   //    domain::TradeRecord trade;
+   //    trade.trade_price = bts::domain::Price(msg.price, -9);
+   //    trade.trade_quantity = bts::domain::Quantity(msg.size);
+   //    trade.sec_id = sec_id;
+   //    switch (msg.side) {
+   //    case databento::Side::Ask:
+   //       trade.agg_side = domain::AggressorSide::Sell;
+   //       break;
+   //    case databento::Side::Bid:
+   //       trade.agg_side = domain::AggressorSide::Buy;
+   //       break;
+   //    case databento::Side::None:
+   //       trade.agg_side = domain::AggressorSide::None;
+   //       break;
+   //    }
+   //    trade.time_stamp = bts::core::TimeStamp::from_iso_string(
+   //        databento::ToIso8601(msg.ts_recv));
+   //    std::vector<const domain::TradeRecord *> trade_ptrs;
+   //    trade_ptrs.push_back(&trade);
+   //    listener->on_trades(trade_ptrs);
 
-      return databento::KeepGoing::
-          Continue; //only added so the same functions work with the threaded builder, not really need in the blocking version
-   };
+   //    return databento::KeepGoing::
+   //        Continue; //only added so the same functions work with the threaded builder, not really need in the blocking version
+   // };
    if (use_file) {
       auto client = databento::HistoricalBuilder{}.SetKey(key).Build();
       databento::DbnFileStore dbn_file_store(securities_path);
@@ -390,8 +465,9 @@ void DatabentoDatahandler::run(
       }
       //ocassionaly run security writer to update security dbn
       if (!boost::filesystem::exists(securities_path) || load_reference_data) {
-         update_file(key, dataset, symbols, 15 * 60, securities_path);
-      }
+         LOG(md, Info, "Reloading securities file");
+         update_file(key, dataset, symbols, 3 * 60, securities_path);
+      } 
 
       //regular flow
       //read from security dbn, filter contracts that we need and subscribe to those in batches of two thousand
@@ -434,36 +510,44 @@ void DatabentoDatahandler::run(
       // size_t size = all_sec.size();
       LOG(md, Info, "total {} secs loaded", total_count);
 
-
       bool first = true;
       auto client = databento::LiveBlocking{
           databento::ILogReceiver::Default(),           key, dataset, true,
           databento::VersionUpgradePolicy::UpgradeToV2, {}};
       std::string start = config::get<bts::api::md::MarketDataConfig>()
-          .cboe()
-          .at(0)
-          .databento_settings()
-          .bbo_start_time();
-      if(start != ""){
-         for (std::vector vec : all_sec) {
-            client.Subscribe(
-                vec, databento::Schema::Mbp1, databento::SType::RawSymbol, start);
-            client.Subscribe(
-                vec, databento::Schema::Trades, databento::SType::RawSymbol,start);
-         }   
-      }else{
-         for (std::vector vec : all_sec) {
-            client.Subscribe(
-                vec, databento::Schema::Mbp1, databento::SType::RawSymbol);
-            client.Subscribe(
-                vec, databento::Schema::Trades, databento::SType::RawSymbol);
-         } 
+                              .cboe()
+                              .at(0)
+                              .databento_settings()
+                              .bbo_start_time();
+      if (total_count == 0) {
+         LOG(md, Error, "NO SECURITIES FOR DATABENTO SUBSCRIPTION");
+         return;
       }
+      if (start != "") {
+         for (std::vector vec : all_sec) {
+            client.Subscribe(
+                vec, databento::Schema::Cmbp1, databento::SType::RawSymbol,
+                start);
+            // client.Subscribe(
+            //     vec, databento::Schema::Trades, databento::SType::RawSymbol,
+            //     start);
+         }
+      } else {
+         for (std::vector vec : all_sec) {
+            client.Subscribe(
+                vec, databento::Schema::Cmbp1, databento::SType::RawSymbol);
+            // client.Subscribe(
+            //     vec, databento::Schema::Trades, databento::SType::RawSymbol);
+         }
+      }
+      client.Subscribe(
+          symbols, databento::Schema::Definition, databento::SType::Parent,
+          databento::UnixNanos{});
       bool only_securities = config::get<bts::api::md::MarketDataConfig>()
-                         .cboe()
-                         .at(0)
-                         .databento_settings()
-                         .only_securities();
+                                 .cboe()
+                                 .at(0)
+                                 .databento_settings()
+                                 .only_securities();
       client.Start();
       while (true) {
          try {
@@ -476,21 +560,21 @@ void DatabentoDatahandler::run(
                LOG(md, Error, "Timed out waiting for data");
                break;
             }
-            if(only_securities)
+            if (only_securities)
                continue;
-            if (record->Holds<databento::Mbp1Msg>()) {
+            if (record->Holds<databento::Cmbp1Msg>()) {
                process_bbo(*record);
             }
             if (all_times.size() == 1001) {
                log_time_statistics(all_times);
                client.Stop();
             }
-            //  if (record->Holds<databento::InstrumentDefMsg>()) {
-            //      process_security(*record);
-            //  }
-            if (record->Holds<databento::TradeMsg>()) {
-               process_trades(*record);
+            if (record->Holds<databento::InstrumentDefMsg>()) {
+               update_maps(*record);
             }
+            // if (record->Holds<databento::TradeMsg>()) {
+            //    process_trades(*record);
+            // }
          } catch (const databento::TcpError &error) {
             std::string err = error.what();
             LOG(md, Error, "Caught TCP error: {} \nAttempting reconnect\n",
@@ -503,7 +587,11 @@ void DatabentoDatahandler::run(
                 "Caught DbnResponseError what: {} ,msg:{} \nShutting down\n",
                 msg, what);
             client.Stop();
-            // bts_assert(false);
+            bts_assert_msgf(
+                false,
+                "Caught DbnResponseError what: {} ,msg:{} \nShutting down\n",
+                msg, what);
+            bts_assert(false);
          } catch (const databento::Exception &excp) {
             std::string err = excp.Message();
             std::string err2 = excp.what();
@@ -512,13 +600,18 @@ void DatabentoDatahandler::run(
                 "\nShutting down\n",
                 err, err2);
             client.Stop();
-            // bts_assert(false);
+            bts_assert_msgf(
+                false,
+                "Caught unknown dadtabento::exception what: {}, msg: {} "
+                "\nShutting down\n",
+                err, err2);
          } catch (const std::exception &exc) {
             std::string err = exc.what();
             LOG(md, Error, "Caught unknown exception: {} \nShutting down\n",
                 err);
             client.Stop();
-            // bts_assert(false);
+            bts_assert_msgf(
+                false, "Caught unknown exception: {} \nShutting down\n", err);
          }
       }
    }

src/server/sparkd/md/recording/MDRecorder.cpp

@@ -143,6 +143,7 @@ struct MDRecorder::PImpl {
    uintptr_t mdc_id = 0;
    bts::md::MDClient &md_client;
    std::map<SecurityKey, MDRecordingSecEntry> handles;
+   std::optional<std::thread> run_thread;
    MDBatchWriter batch_writer{settings.batch_size,std::vector<bts::md::recording::MDBatchWriter::DBWriteFunction>{DbWriters::writeToSQL}};
    uint64_t start_ns;
    PImpl(
@@ -160,6 +161,14 @@ struct MDRecorder::PImpl {
                  1000 * 1000 * 1000,
              100000, 1000000000,recording_config.batch_size()}),
          md_client(md_client),start_ns(bts::core::TimeStamp::now_ns()) {}
+
+   ~PImpl() {
+      active = false;
+      if (run_thread && run_thread->joinable()) {
+         run_thread->join();
+      }
+      run_thread.reset();
+   }
 };
 
 MDRecorder::MDRecorder(
@@ -171,7 +180,8 @@ MDRecorder::MDRecorder(
    me->file.open(file_name, std::ios::out | std::ios::binary | std::ios::trunc);
    LOG(md, Info, "MDRecorder::MDRecorder venue: {} file: {}",
        api::Venue_Name(venue), file_name);
-   bts::threading::make_thread("MDRecorder", [&]() { _poll_md(); }).detach();
+   me->run_thread.emplace(
+       bts::threading::make_thread("MDRecorder", [&]() { _poll_md(); }));
 }
 
 MDRecorder::~MDRecorder() {}

src/server/sparkd/spark.conf

@@ -0,0 +1,42 @@
+MarketDataConfig {
+  value {
+    binance_us {
+      num_depth_levels: 5
+      url_rest_base: "https://api.binance.us"
+      url_ws_base: "wss://stream.binance.us:9443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: "BTC"
+          symbols: "ETH"
+        }
+      }
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE_US"
+      spot: "BTC"
+      spot: "ETH"
+    }
+  }
+}
+
+Logging {
+  value {
+    console {
+      default_level: "Info"
+    }
+    syslog {
+      default_level: "Info"
+    }
+    file {
+      default_level: "Info"
+    }
+    initial_queue_size: 120001
+    max_queue_growth: 4
+  }
+}

tools/aws/install_aws_cli.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+this_dir="$(dirname "${BASH_SOURCE[0]}")"
+work_dir="$this_dir/.tmp"
+# Clean previous install
+rm -rf "$work_dir"
+mkdir -p "$work_dir"
+cd "$work_dir"
+# Download installer and run it
+curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+  unzip awscliv2.zip && \
+  sudo ./aws/install
+cd "$this_dir"
+# Clean up
+rm -rf "$work_dir"

tools/deploy.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+tools_path="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+eval "$tools_path/ensure_venv.sh" 1> /dev/stdout 2> /dev/stderr
+eval "$tools_path/venv_exec.sh python $tools_path/py/deploy.py $@" 1> /dev/stdout 2> /dev/stderr

tools/py/deploy.py

@@ -0,0 +1,131 @@
+import argparse
+import dotenv
+import subprocess
+
+import publish_sparkd
+
+from core.argparse_utils import add_toggle_opt
+from core.file_utils import SparkPaths
+
+
+def _get_arg_parser():
+    parser = argparse.ArgumentParser(
+        description=
+        "Deploy sparkd, pubsub, md-publisher, webapp, and api server")
+    parser.add_argument(
+        "--cmake-jobs",
+        type=int,
+        help="The number of cmake parallel jobs",
+    )
+    parser.add_argument(
+        "--profile",
+        help="The configuration profile dirs to use, relative to each service")
+    parser.add_argument("--compose-file",
+                        help="The docker compose (yaml) file to use")
+    parser.add_argument(
+        "--clean",
+        action="store_true",
+        help="Perform a clean build (remove artifacts from previous build)",
+    )
+    parser.add_argument("--env-files",
+                        action="append",
+                        help=".env files to pass to docker compose")
+    parser.add_argument(
+        "--verbosity",
+        metavar="{0-5}",
+        choices=range(1, 6),
+        type=int,
+        help="The output verbosity level",
+    )
+    add_toggle_opt(parser,
+                   "build",
+                   help="Whether or not to build the services first")
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Preview the build commands without running them",
+    )
+    # TODO: add more services (api-server, pubsub, dsqa web server, etc.)
+    all_services = ["sparkd"]
+    parser.add_argument("services", nargs="*")
+    parser.set_defaults(
+        configuration="release",
+        services=all_services,
+        build=True,
+        clean=False,
+        profile="dev",
+        verbosity=2,
+    )
+    return parser
+
+
+def _get_env_files(services, profile):
+    root = SparkPaths.root()
+    files = []
+
+    def add(file):
+        if file.exists():
+            files.append(file)
+
+    add(root.joinpath(".env"))
+    add(root.joinpath(".env.local"))
+    if profile is not None:
+        if "sparkd" in services:
+            profile_dir=root.joinpath("src/server/sparkd/configurations", profile)
+            add(profile_dir.joinpath(".env"))
+            add(profile_dir.joinpath(".env.local"))
+    return files
+
+
+def deploy(args):
+    def run(cmd):
+        print(cmd)
+        if args.dry_run:
+            return
+        subprocess.run(cmd, shell=True).check_returncode()
+
+    services: list[str] = args.services or []
+    if args.build:
+        build_all = len(services) == 0
+        if build_all or "sparkd" in services:
+            publish_sparkd.publish(
+                publish_sparkd.PublishArgs(
+                    configuration="release",
+                    cmake_jobs=args.cmake_jobs,
+                    clean=args.clean,
+                    verbosity=args.verbosity,
+                    dry_run=args.dry_run,
+                ))
+    service_args = " ".join(services)
+    compose_cmd = "sudo docker compose"
+    if args.compose_file:
+        compose_cmd += f" -f {args.compose_file}"
+    env_files = _get_env_files(args.services, args.profile)
+    if len(env_files) > 0:
+        compose_cmd += " " + " ".join([f"--env-file {f}" for f in env_files])
+        for ef in env_files:
+            if args.verbosity > 2:
+                print(f"Loading env file {ef}")
+                if args.verbosity > 3:
+                    env_vars = dotenv.dotenv_values(ef)
+                    for k, v in env_vars.items():
+                        print(f"    {k}={v}")
+            dotenv.load_dotenv(ef)
+    run(f"{compose_cmd} down {service_args}")
+    run(f"{compose_cmd} build {service_args}")
+    run(f"{compose_cmd} up {service_args} -d")
+    run(f"{compose_cmd} logs {service_args} --follow")
+
+
+def _main():
+    parser = _get_arg_parser()
+    args = parser.parse_args()
+    deploy(args)
+
+
+if __name__ == "__main__":
+    try:
+        _main()
+    except Exception as e:
+        print(f"deploy.py failed: {e}")
+        exit(1)

tools/sparkd_data.sh

@@ -0,0 +1,217 @@
+#!/usr/bin/env bash
+
+help_text="\
+Manage files in sparkd data volumes
+
+Usage: sparkd_data.sh [OPTIONS] [COMMAND] [...]
+
+Commands:
+  get               Copy files FROM data volume
+  set               Copy files TO data volume
+  shell             Start shell in data volume
+
+Options:
+  -h,--help         Print this message and exit
+
+"
+
+cp_help_text="\
+Copy files to/from sparkd data volume
+
+Usage: sparkd_data.sh <get|set> [OPTIONS] [POSITIONALS]
+
+Positionals:
+  SOURCE            Source file
+  DEST              Dest file
+
+Options:
+  --clean           Recreate the container that mounts to the volume (even if it already exists)
+  --source          Source file
+  --dest            Dest file
+  --config-port     The config port for the volume to manage
+  -h,--help         Print this message and exit
+"
+
+shell_help_text="\
+Start shell in sparkd data volume
+
+Usage: sparkd_data.sh shell [OPTIONS]
+
+Options:
+  --clean           Recreate the container that mounts to the volume (even if it already exists)
+  --config-port     The config port for the volume to manage
+  -h,--help         Print this message and exit
+"
+
+print_help_and_exit() {
+  echo "$1"
+  exit "${2:-0}"
+}
+
+bounce_container() {
+  if [ "$#" -ne 3 ]; then
+    echo "bounce_container invalid args '$@'."
+    echo "Usage: bounce_container <clean:0|1> <container_name> <volume_name>"
+    return 1
+  fi
+  clean="$1"
+  cont_name="$2"
+  vol_name="$3"
+  image_name="bluetradingsystems/linux_dotnet7"
+  if [ -n "$(sudo docker container ls -a -q --filter "name=$cont_name")" ]; then
+    if [[ $clean -eq 1 ]]; then
+      sudo docker stop $cont_name
+      sudo docker rm $cont_name > /dev/null 2>&1
+      sudo docker run --name $cont_name --rm -it -d -v $vol_name:/home/bts/data $image_name sh
+    fi
+  else
+    sudo docker run --name $cont_name --rm -it -d -v $vol_name:/home/bts/data $image_name sh
+  fi
+}
+
+get_container_name() {
+  if [ "$#" -ne 1 ]; then
+    echo "get_container_name invalid args '$@'."
+    echo "Usage: container_name <config_port>"
+    return 1
+  fi
+  echo "sparkd_data_man_${1}"
+}
+
+get_volume_name() {
+  if [ "$#" -ne 1 ]; then
+    echo "get_volume_name invalid args '$@'."
+    echo "Usage: get_volume_name <config_port>"
+    return 1
+  fi
+  echo "sparkd_${1}"
+}
+
+cp_file() {
+  config_port=${SPARKD__CONFIG_PORT:-8000}
+  is_get=${1:-1}
+  shift 1
+  source=""
+  dest=""
+  clean=0
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --clean)
+        clean=1
+        shift 1
+        ;;
+      --source)
+        if [[ -z "$2" ]]; then
+          echo "Source (--source) must be non-empty"
+          print_help_and_exit "$cp_help_text" 1
+        fi
+        source="$2"
+        shift 2
+        ;;
+      --dest)
+        if [[ -z "$2" ]]; then
+          echo "Dest (--Dest) must be non-empty"
+          print_help_and_exit "$cp_help_text" 1
+        fi
+        dest="$2"
+        shift 2
+        ;;
+      --config-port)
+        if [[ -z "$2" ]]; then
+          echo "Config port (--config-port) must be non-empty"
+          print_help_and_exit "$cp_help_text" 1
+        fi
+        config_port="$2"
+        shift 2
+        ;;
+      -h)
+        ;&
+      --help)
+        print_help_and_exit "$cp_help_text" 0
+        ;;
+      *)
+        if [[ -z "$source" ]]; then
+          source="$1"
+        elif [[ -z "$dest" ]]; then
+          dest="$1"
+        else
+          echo "Invalid positional argument '$1'"
+          print_help_and_exit "$cp_help_text" 1
+        fi
+        shift 1
+        ;;
+    esac
+  done
+
+  if [ -z "$source" ]; then
+    echo -e "Error: No source specified to copy\n"
+    print_help_and_exit "$cp_help_text" 1
+  fi
+
+  if [ -z "$dest" ]; then
+    dest="."
+  fi
+
+  cont_name="$(get_container_name "$config_port")"
+  vol_name="$(get_volume_name "$config_port")"
+  bounce_container "$clean" "$cont_name" "$vol_name"
+  if [[ $is_get -eq 1 ]]; then
+    sudo docker cp $cont_name:/home/bts/data/$source $dest
+  else
+    sudo docker cp $source $cont_name:/home/bts/data/$dest
+  fi
+}
+
+shell() {
+  clean=0
+  config_port=${SPARKD__CONFIG_PORT:-8000}
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --clean)
+        clean=1
+        shift 1
+        ;;
+      --config-port)
+        if [[ -z "$2" ]]; then
+          echo "Config port (--config-port) must be non-empty"
+          print_help_and_exit "$shell_help_text" 1
+        fi
+        config_port="$2"
+        shift 2
+        ;;
+      -h)
+        ;&
+      --help)
+        print_help_and_exit "$shell_help_text" 0
+        ;;
+      *)
+        echo "Invalid positional argument '$1'"
+        print_help_and_exit "$shell_help_text" 1
+        ;;
+    esac
+  done
+
+  cont_name="$(get_container_name "$config_port")"
+  vol_name="$(get_volume_name "$config_port")"
+  bounce_container "$clean" "$cont_name" "$vol_name"
+  sudo docker exec -it -w /home/bts/data $cont_name bash
+}
+
+case "$1" in
+  get)
+    shift 1
+    cp_file 1 $@
+    ;;
+  set)
+    shift 1
+    cp_file 0 $@
+    ;;
+  shell)
+    shift 1
+    shell $@
+    ;;
+  *)
+    echo "Invalid argument '$1'"
+    print_help_and_exit "$help_text" 1
+    ;;
+esac