ui: Use dark theme foreground in mini orderbook, fills popups

It was black in a couple places before, making it hard to read.
fix: Check for null in orderbook root algo type filter
2025-03-10 14:47:58 -04:00 · 2025-03-10 14:47:58 -04:00 · 2025-03-10 14:47:58 -04:00 · 2025-03-10 17:22:08 +00:00 · 2025-03-10 15:42:12 +00:00 · 2025-03-10 14:53:07 +00:00
61 changed files with 1842 additions and 233 deletions
@@ -90,6 +90,12 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev

+# Docker in Docker
+RUN apt-get update && apt-get install -y \
+    docker.io \
+    docker-compose \
+    docker-compose-v2
+
 # Set up default executables (update-alternatives)
 RUN \
   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
@@ -12,17 +12,10 @@
  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
  "runArgs": [
    "--shm-size",
-    "8gb",
-    "-p",
-    "8000:8000",
-    "-p",
-    "8001:8001",
-    "-p",
-    "8002:8002",
-    "-p",
-    "8003:8003",
-    "-p",
-    "12345:12345"
+    "8gb"
+  ],
+  "mounts": [
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
  ],
  "customizations": {
    "vscode": {
@@ -0,0 +1,47 @@
+# This file defines the global default configuration for docker deployments.
+# It is tracked in source control. For temporary modifications, please create
+# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
+# If a variable is defined in both files, the value in .env.local will be used.
+#
+# About the deployment procedure:
+#   We use Docker Compose to build images and launch containers for various services
+#   that make up our software suite:
+#     - sparkd
+#     - TODO: pubsub
+#     - TODO: md-publisher
+#     - TODO: webapp (DSQA on the web)
+#     - TODO: api-server (websocket based)
+#   From inside a devcontainer, you can launch all services in separate containers
+#   on your machine by running:
+#
+#     tools/deploy.sh
+#
+#   This script builds release versions for each service, packaged (generally) in
+#   self-contained directories. Next, it runs a few "docker compose" commands to
+#   build docker images loaded with the self-contained binaries from the previous
+#   step, and start containers with those images to run each service. To view the
+#   commands that deploy.sh runs internally (without actually running them), run:
+#
+#     tools/deploy.sh --dry-run
+#
+#   For additional information/help, run:
+#
+#     tools/deploy.sh --help
+#
+
+### Sparkd ###
+# The internal image for sparkd to use. For more info, view:
+#   src/server/sparkd/Dockerfile
+SPARKD__IMAGE=sparkd
+# The config dir for this service to use
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
+# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
+# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
+# SPARKD__DATA_DIR=/home/bts/sparkdata/data
+SPARKD__SOURCE_DIR=install/sparkd-release-gcc
+SPARKD__CONFIG_PORT=8000
+SPARKD__QUICKBUS_PORT=8001
+SPARKD__SIM_FIX_PORT=8002
+SPARKD__SHM_SIZE=4gb
+
+# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
@@ -59,3 +59,4 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
+.env.local
@@ -0,0 +1,63 @@
+services:
+  sparkd:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+    build:
+      dockerfile: src/server/sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+
+  sparkd-tail:
+    image: bluetradingsystems/linux_dotnet7
+    volumes:
+      - type: volume
+        read_only: true
+        source: sparkd_data
+        target: /home/bts/sparkdata/data/logging.sparkd
+        volume:
+          subpath: logging.sparkd
+    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
+
+  sparkd-ping-test:
+    depends_on:
+      - sparkd
+    profiles:
+      - ping-test
+    build:
+      context: .
+      dockerfile_inline: |
+        FROM bluetradingsystems/linux_dotnet7
+        USER "root"
+        RUN apt-get update && apt-get -y install iputils-ping
+    command: [ "ping", "sparkd" ]
+
+volumes:
+  sparkd_data:
+    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
@@ -0,0 +1,38 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y pkg-config
+RUN apt-get update && apt-get install -y locales
+RUN apt-get update && apt-get install -y ca-certificates
+RUN apt-get update && apt-get install -y libssh2-1-dev
+RUN apt-get update && apt-get install -y software-properties-common
+RUN apt-get update && apt-get install -y sudo
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+  && apt-get update \
+  && apt-get install -y g++-$gcc_major
+
+RUN apt-get update \
+  && apt-get install -y dotnet7
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+  groupadd -g "$docker_uid" "$docker_user" \
+  && useradd \
+  -m \
+  -u "$docker_uid" \
+  -g "$docker_uid" \
+  -G sudo \
+  --password "$(openssl passwd -1 "$docker_pass")" \
+  "$docker_user"
+
+USER bts
+WORKDIR /home/bts
@@ -1,10 +1,13 @@
 astroid==2.7.3
 autopep8==1.5.7
+boto3==1.37.9
+botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
 colorama==0.4.4
 idna==3.4
 isort==5.9.3
+jmespath==1.0.1
 lazy-object-proxy==1.6.0
 mccabe==0.6.1
 platformdirs==2.3.0
@@ -14,7 +17,11 @@ pycodestyle==2.7.0
 pycparser==2.20
 pylint==2.10.2
 PyMeta3==0.5.1
+python-dotenv==1.0.1
+python-dateutil==2.9.0.post0
 requests==2.31.0
+s3transfer==0.11.4
+six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3
@@ -629,16 +629,16 @@ namespace mbts.spark {

      static (string desc, Color fg) _fills_title(ColKey key) =>
         key switch {
-            ColKey.Position => ("Day Fills", Colors.Black),
-            ColKey.RawDayPosition => ("Raw Day Fills", Colors.Black),
-            _ => ("Fills", Colors.Black),
+            ColKey.Position => ("Day Fills", BTSColors.DayFills.foreground),
+            ColKey.RawDayPosition => ("Raw Day Fills", BTSColors.DayFills.foreground),
+            _ => ("Fills", BTSColors.UI.default_text),
         };

      static (string desc, Color fg) _orderbook_title(ColKey key) =>
         key switch {
            ColKey.WorkingBids => ((string desc, Color fg))("Working Buy Orders", BTSColors.MarketData.bid),
            ColKey.WorkingAsks => ((string desc, Color fg))("Working Sell Orders", BTSColors.MarketData.ask),
-            _ => ("Working Orders", Colors.Black),
+            _ => ("Working Orders", BTSColors.UI.default_text),
         };

      static GridSettings _hide_cols(GridSettings grid, IEnumerable<OrderBookGridView.ColumnKeys> columns) =>
@@ -150,7 +150,11 @@ namespace bts.shell.orderbook {
         pb.add_discrete_values(fs.exchanges, o => o.order.security?.Exchange ?? Venue.Unset);
         pb.add_discrete_values(fs.tifs, o => o.order.order.time_in_force());
         pb.add_discrete_values(fs.order_types, o => AlgoID.from_custom_def(o.order.order.Def.CustomDef));
-         pb.add_discrete_values(fs.root_algo_types, o => AlgoID.from_algo_key(o.root.algo?.key ?? default));
+         pb.add_discrete_values(
+            Array.ConvertAll(fs.root_algo_types, a => (AlgoID?)a),
+            o => o.root.algo.HasValue
+               ? AlgoID.from_algo_key(o.root.algo.Value.key)
+               : null);
         pb.add_discrete_values(fs.order_sub_types, o => o.order.order.sub_type());
         pb.add_discrete_values(fs.sides, o => o.order.order.side());
         pb.add_discrete_values(fs.statuses, o => o.order.order.status());
@@ -774,6 +774,17 @@ namespace bts.shell.pnl {
            case ColumnKeys.Account: filter.accounts = new[] { g.Name.ToString() }; break;
            case ColumnKeys.Symbol: filter.symbols = new[] { g.Name.ToString() }; break;
            case ColumnKeys.Exchange: filter.exchanges = new[] { (Venue)g.Name }; break;
+            case ColumnKeys.Expiration: {
+                  if (g.Name is DateTime exp) {
+                     if (filter.security_criteria is null) {
+                        filter.security_criteria = SecurityCriteria.from_expirations(exp);
+                     } else {
+                        filter.security_criteria.clauses = ArrayHelper.add(
+                           filter.security_criteria.clauses, SecurityCriteriaClause.from_expirations(exp));
+                     }
+                  }
+                  break;
+               }
            case ColumnKeys.Security:
               if (g.Name is SecurityImmutable sec)
                  filter.security_criteria = SecurityCriteria.from_securities(sec);
@@ -1040,6 +1040,7 @@ namespace mbts.ui {
      }
      public static class DayFills {
         public static BTSColor background => UI.common_background;
+         public static BTSColor foreground => UI.default_foreground;
      }
      public static class ComboReport {
         public static BTSColor foreground => UI.default_text;
@@ -8,7 +8,8 @@
             xmlns:cv="clr-namespace:mbts.ui.converters"
             xmlns:c="clr-namespace:mbts.ui.controls"
             xmlns:usp="clr-namespace:mbts.ui.spark"
-             d:DesignHeight="100" d:DesignWidth="800" x:Name="_me_price">
+             d:DesignHeight="100" d:DesignWidth="800" x:Name="_me_price"
+             Foreground="{u:BTSColor UI.input_text}">
  <UserControl.Resources>
    <cv:NullableDoubleConverter x:Key="TickDefaultPriceConverter" DefaultValue="0"/>
    <cv:TickDisplayConverter x:Key="TickDisplayConverter" x:Shared="False"/>
@@ -27,6 +27,12 @@ namespace mbts.spark {
      internal static SecurityCriteria from_keys(IEnumerable<uint> keys) =>
         new(new[] { new SecurityCriteriaClause(security_keys: keys.ToArray()) });

+      internal static SecurityCriteria from_expirations(params DateTime[] exp_dates) =>
+         from_expirations(exp_dates.AsEnumerable());
+
+      internal static SecurityCriteria from_expirations(IEnumerable<DateTime> expirations) =>
+         new(new[] { SecurityCriteriaClause.from_expirations(expirations) });
+
      internal static SecurityCriteria from_securities(params SecurityImmutable[] securities) =>
         from_securities(securities.AsEnumerable());

@@ -58,6 +58,14 @@ namespace mbts.spark {
         return new SecurityCriteriaClause(security_keys: keys, pretty_name: pretty_name);
      }

+      public static SecurityCriteriaClause from_expirations(IEnumerable<DateTime> expirations) =>
+         expirations is null
+            ? new()
+            : from_expirations(expirations as DateTime[] ?? expirations.ToArray());
+
+      public static SecurityCriteriaClause from_expirations(params DateTime[] expirations) =>
+         new(expirations: expirations);
+
      [ProtoMember(1)]
      public string? pretty_name { get; }

@@ -33,8 +33,8 @@ struct CQuantity {
   uint32_t value() const { return static_cast<uint32_t>(qty); }
   double full_value() const { return qty; }
   float full_value_float() const { return static_cast<float>(full_value()); }
-   uint32_t int_part() { return static_cast<uint32_t>(qty); }
-   double fractional_part() { return full_value() - int_part(); }
+   uint32_t int_part() const { return static_cast<uint32_t>(qty); }
+   double fractional_part() const { return full_value() - int_part(); }
   static bool equal_d(double d1, double d2) {
      return std::abs(d1 - d2) <= err;
   }
@@ -290,7 +290,7 @@ struct GeneralMarketDataListener::PImpl {
   // if this is set to true, then the security key passed on in the
   // on_securities function is used. If this is not set to true, then a new key
   // is generated
-   const bool use_given_sec_key = true;
+   const bool use_given_sec_key = false;
   const std::string cc_name_prefix = "cc_security";
   size_t n_bk_updates = 0;
   bts::persistence::IDGenerator &id_gen =
@@ -2,7 +2,10 @@
 #include "lib/api/services/OrderService.pb.h"
 #include "lib/config/config.h"
 #include "lib/config/protos/toggles.pb.h"
+#include "lib/core/TimeStamp.h"
 #include "lib/domain_model/Security.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/domain_model/SecurityKey.h"
 #include "lib/logging/logging.h"
 #include "lib/mercury/events/Update.h"
 #include "lib/orderman/actors/SODPosition.h"
@@ -14,10 +17,13 @@
 #include "lib/os/os.h"
 #include "lib/persistence/SqliteStore.h"
 #include "lib/utils/PBUtils.h"
+#include "lib/utils/TimeStamp.h"
 #include "lib/utils/bts_assert.h"
+#include "lib/utils/bts_chrono.h"
 #include "lib/utils/overloaded.h"

 #include <boost/date_time/posix_time/posix_time.hpp>
+#include <google/protobuf/timestamp.pb.h>
 #include <string>
 #include <sys/time.h>

@@ -32,6 +38,67 @@ namespace orderman {

 auto const fills_ns = "fills";

+struct FillAggregator {
+   struct SecPosition {
+    private:
+      Fill agg_fill;
+
+    public:
+      double agg_price = 0, agg_qty = 0;
+      SecPosition(const Fill &fill) : agg_fill(fill) {
+         const auto &fill_key = fill.fill_key();
+         std::string new_fill_key = "aggpos_" + fill_key;
+         agg_fill.set_fill_key(new_fill_key);
+         auto now_ns = bts::core::TimeStamp::now_ns();
+         agg_fill.mutable_timestamp()->set_nanos(now_ns);
+         add_fill(fill);
+      }
+      void add_fill(const Fill &fill) {
+         bts_assert(fill.security_key() == agg_fill.security_key());
+         double mult = fill.side() == api::SIDE_BUY ? 1.0 : -1.0;
+         double fp = fill.avg_price();
+         double fq = mult * fill.last_shares();
+         double new_agg_price =
+             ((agg_price * agg_qty) + (fp * fq)) / (agg_qty + fq);
+         double new_agg_qty = agg_qty + fq;
+         agg_price = new_agg_price;
+         agg_qty = new_agg_qty;
+      }
+
+    public:
+      Fill fill() const {
+         Fill rv = agg_fill;
+         rv.set_avg_price(agg_price);
+         rv.set_last_shares(std::abs(agg_qty));
+         rv.set_side(::bts::api::Side::SIDE_BUY);
+         if (agg_qty < 0) {
+            rv.set_side(::bts::api::Side::SIDE_SELL);
+         }
+         return rv;
+      }
+   };
+   std::map<SecurityKey, SecPosition> pos_map = {};
+
+   void add_fill(const Fill &fill) {
+      SecurityKey k = SecurityKey(fill.security_key());
+      auto it = pos_map.find(k);
+      if (it != pos_map.end()) {
+         SecPosition &sec_pos = it->second;
+         sec_pos.add_fill(fill);
+      } else {
+         pos_map.insert({k, {fill}});
+      }
+   }
+
+   std::vector<Fill> get_aggregated_fills() const {
+      std::vector<Fill> rv = {};
+      for (const auto &kv : pos_map) {
+         rv.push_back(kv.second.fill());
+      }
+      return rv;
+   }
+};
+
 struct FillStore::UnresolvedFills {
   std::vector<api::orderman::Fill> fills;
   api::Security const *security;
@@ -48,6 +115,8 @@ struct FillStore::PImpl {
   std::unordered_map<OrderKey, UnresolvedFills> fills_with_unresolved_accounts;
   mercury::Stream<StoredFillEvent> fill_stream;
   md::SecurityOracle oracle;
+   FillAggregator fill_aggregator = {};
+   bool aggregate_pos = false;
 };

 FillStore::FillStore(md::SecurityOracle o) : me(new PImpl(o)) {
@@ -100,40 +169,83 @@ void FillStore::init() {
       os::get_thread_id(), os::get_thread_name());
   LOG(OrderMan, Info, "FillStore::init oracle n_secs: {}",
       me->oracle.get_securities().size());
-   // Subscribe to NewExecutionReportEvent for new fills
-   subscribe<NewExecutionReportEvent>(
-       [this](auto const &ev) { this->on_execution_report(ev); });
-
   // Load stored fills, resolve their securities and publish the lot
-   api::orderman::Fill fill;
   std::vector<StoredFillEvent> stored_batch;
   LOG(persist, Info, "Loading load all fills");
   auto all = me->store->retrieve_all()[fills_ns];
   LOG(persist, Info, "Finished loading all {} fills", all.size());
-   SODPosition sod_helper;
-   for (auto const &kvp : all)
-      if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
-         if (sod_helper.is_sod(fill.timestamp())) {
-            auto fill_kind = sod_fill_kind(fill);
-            fill.set_fill_kind(fill_kind);
+   auto aggregate_clean_and_publish = [&]() {
+      api::orderman::Fill fill;
+      for (auto const &kvp : all) {
+         if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
+            me->fill_aggregator.add_fill(fill);
+         } else {
+            LOG(persist, Error,
+                "Could not recover fill {} from storage. "
+                "This should be investigated",
+                kvp.first);
         }
-         auto sec = me->oracle.get_security(SecurityKey(fill.security_key()));
+      }
+      // 1. go through all fills in aggregator and make StoredFillEvent
+      auto aggregated_fills = me->fill_aggregator.get_aggregated_fills();
+      for (auto &f : aggregated_fills) {
+         auto sec = me->oracle.get_security(SecurityKey(f.security_key()));
         if (sec) {
-            auto fk = kvp.first;
-            me->known_fills.insert(fk);
-            if (!config::get<config::toggles::FilterExpiredFills>().enabled() ||
-                !securities::is_expired(*sec)) {
-               stored_batch.emplace_back(fill, *sec);
+            stored_batch.emplace_back(f, *sec);
+         } else {
+            LOG(persist, Error,
+                "Stored fill {} contains unresolved security {}. ",
+                utils::pb::to_string(f), fill.security_key());
+         }
+      }
+      // 2 delete all fills in the DB
+      me->store->remove_all(fills_ns);
+      // 3. add the aggregated fills back
+      for (const auto &e : stored_batch) {
+         const Fill &f = e.fill;
+         me->store->insert(fills_ns, f.fill_key(), utils::pb::to_string(f));
+      }
+   };
+   auto publish_with_sod_marking = [&]() {
+      api::orderman::Fill fill;
+      SODPosition sod_helper;
+      for (auto const &kvp : all) {
+         if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
+            if (sod_helper.is_sod(fill.timestamp())) {
+               auto fill_kind = sod_fill_kind(fill);
+               fill.set_fill_kind(fill_kind);
            }
+            auto sec =
+                me->oracle.get_security(SecurityKey(fill.security_key()));
+            if (sec) {
+               auto fk = kvp.first;
+               me->known_fills.insert(fk);
+               if (!config::get<config::toggles::FilterExpiredFills>()
+                        .enabled() ||
+                   !securities::is_expired(*sec)) {
+                  stored_batch.emplace_back(fill, *sec);
+               }
+            } else
+               LOG(persist, Error,
+                   "Stored fill {} contains unresolved security {}. ",
+                   kvp.first, fill.security_key());
         } else
            LOG(persist, Error,
-                "Stored fill {} contains unresolved security {}. ", kvp.first,
-                fill.security_key());
-      } else
-         LOG(persist, Error,
-             "Could not recover fill {} from storage. "
-             "This should be investigated",
-             kvp.first);
+                "Could not recover fill {} from storage. "
+                "This should be investigated",
+                kvp.first);
+      }
+   };
+
+   // Subscribe to NewExecutionReportEvent for new fills
+   subscribe<NewExecutionReportEvent>(
+       [this](auto const &ev) { this->on_execution_report(ev); });
+   if (me->aggregate_pos) {
+      aggregate_clean_and_publish();
+   } else {
+      publish_with_sod_marking();
+   }
+
   me->fill_stream = make_stream<StoredFillEvent>();
   LOG(OrderMan, Info,
       "Fill Store published {} fills recoved from disk out of {} "
@@ -5,6 +5,7 @@
 #include "lib/config/protos/orders_fills_cleanup.pb.h"
 #include "lib/config/protos/toggles.pb.h"
 #include "lib/logging/logging.h"
+#include "lib/md/ExchangeSecurityKey.h"
 #include "lib/orderman/actors/SODPosition.h"
 #include "lib/orderman/events/OrderDefinitionUpdateEvent.h"
 #include "lib/orderman/events/OrderStateUpdateEvent.h"
@@ -12,6 +13,7 @@
 #include "lib/persistence/SqliteStore.h"
 #include "lib/utils/PBUtils.h"

+#include <google/protobuf/timestamp.pb.h>
 #include <unordered_set>

 namespace bts {
@@ -38,6 +40,7 @@ struct OrderStore::PImpl {
   bool soft_cleanup_enabled;
   md::SecurityOracle oracle;
   timeval cutoff_time;
+   bool clear_all_orders = true;
 };

 OrderStore::OrderStore(md::SecurityOracle o) : me(new PImpl(o)) {}
@@ -63,34 +66,46 @@ void OrderStore::init() {

   me->cutoff_time = SODPosition::get_order_cutoff_time(all);

-   for (auto const &kvp : all)
-      if (utils::pb::from_string<api::orderman::OrderRecord>(
-              kvp.second, &rec)) {
-         auto ok = OrderKey(std::stoi(kvp.first));
-         auto sk = SecurityKey(rec.def().security_key());
-         auto sec = me->oracle.get_security(sk);
+   auto load_orders = [&]() {
+      for (auto const &kvp : all) {
+         if (utils::pb::from_string<api::orderman::OrderRecord>(
+                 kvp.second, &rec)) {
+            auto ok = OrderKey(std::stoi(kvp.first));
+            auto sk = SecurityKey(rec.def().security_key());
+            auto sec = me->oracle.get_security(sk);

-         if (config::get<config::toggles::FilterOrdersWithDanglingSecurities>()
-                 .enabled() &&
-             sec == nullptr) {
-            LOG(OrderMan, Diag, "Order {} refers to unresolved security {}",
-                ok.value(), sk.value());
-         } else {
-            if (!_order_cutoff(rec))
-               stored_batch.emplace_back(rec);
-            me->known_orders[ok] = PImpl::ORInfo{rec, true};
-         }
-      } else
-         LOG(OrderMan, Error,
-             "Could not recover order {} from storage. "
-             "This should be investigated",
-             kvp.first);
+            if (config::get<
+                    config::toggles::FilterOrdersWithDanglingSecurities>()
+                    .enabled() &&
+                sec == nullptr) {
+               LOG(OrderMan, Diag, "Order {} refers to unresolved security {}",
+                   ok.value(), sk.value());
+            } else {
+               if (!_order_cutoff(rec))
+                  stored_batch.emplace_back(rec);
+               me->known_orders[ok] = PImpl::ORInfo{rec, true};
+            }
+         } else
+            LOG(OrderMan, Error,
+                "Could not recover order {} from storage. "
+                "This should be investigated",
+                kvp.first);
+      }
+   };

   me->order_stream = make_stream<StoredOrderEvent>();
   LOG(OrderMan, Info,
       "Order Store published {} orders recoved from disk out of {} "
       "total orders",
       stored_batch.size(), all.size());
+
+   if (me->clear_all_orders) {
+      // clear orders
+      me->store->remove_all(orders_ns);
+   } else {
+      load_orders();
+   }
+
   publish_batch(me->order_stream, stored_batch);
 }

@@ -114,17 +129,22 @@ void OrderStore::on_order_state_update(
         rec.set_key(ev.key.value());
         rec.set_parent_key(ev.parent_key.value());
         modified_orders.insert(ev.key);
+         const auto now = utils::pb::pb_now();
         if (rec.has_state()) {
+            google::protobuf::Timestamp created;
+            if (rec.state().has_created()) {
+               created = rec.state().created();
+            }
            rec.mutable_state()->CopyFrom(ev.rebuild_state());
-            *rec.mutable_state()->mutable_created() = utils::pb::pb_now();
-            *rec.mutable_state()->mutable_last_change() = rec.state().created();
+            *rec.mutable_state()->mutable_created() =
+                created.seconds() > 0 || created.nanos() > 0 ? created : now;
            // if filled qty went from 0.99 -> 1, it was copying the int part and not clearing the fraction part,
            // so changing MergeFrom to CopyFrom
         } else {
            *rec.mutable_state() = ev.rebuild_state();
-            *rec.mutable_state()->mutable_created() = utils::pb::pb_now();
+            *rec.mutable_state()->mutable_created() = now;
         }
-         *rec.mutable_state()->mutable_last_change() = utils::pb::pb_now();
+         *rec.mutable_state()->mutable_last_change() = now;
         LOG(OrderMan, Info, "OrderStore::on_order_state_update k: {} rec: {}",
             ev.key, utils::pb::to_string(rec));
      }
@@ -248,7 +248,7 @@ OrderOpResult AlgoWrapperOrder::on_activate(
      me->set_price_quantity(p, q);
      for (auto &ct : me->child_templates)
         ct->builder.activate(*me->services);
-      if (q.value() == 0) {
+      if (q.full_value() == 0) {
         ReportStateGuard _(*this);
         terminate(ORDER_STATUS_COMPLETED);
      } else if (is_attaching())
@@ -292,8 +292,11 @@ OrderOpResult AlgoWrapperOrder::on_update(Update const &update) {
   }
   case Status::WORKING: {
      ReportStateGuard _(*this);
+      const double frac_qty = update.has_fractional_quantity()
+                                  ? update.fractional_quantity().fraction()
+                                  : 0;
      me->set_price_quantity(
-          Price(update.price()), Quantity(update.quantity()));
+          Price(update.price()), Quantity(update.quantity(), frac_qty));
      if (update.has_custom()) {
         *me->definition.mutable_custom_def() = update.custom();
         me->services->report_definition(me->definition);
@@ -313,6 +316,7 @@ OrderOpResult AlgoWrapperOrder::on_update(Price p, Quantity q) {
   Update up;
   up.set_price(p.value());
   up.set_quantity(q.value());
+   up.mutable_fractional_quantity()->set_fraction(q.fractional_part());
   return on_update(up);
 }

@@ -396,11 +400,17 @@ std::vector<api::Security const *> AlgoWrapperOrder::known_securities() const {
 Price AlgoWrapperOrder::price() const { return Price(me->state().price()); }

 Quantity AlgoWrapperOrder::quantity() const {
-   return Quantity(me->state().quantity());
+   const auto &s = me->state();
+   return s.has_fractional_quantity()
+              ? Quantity{s.quantity(), s.fractional_quantity().fraction()}
+              : Quantity{s.quantity()};
 }

 Quantity AlgoWrapperOrder::total_fills() const {
-   return Quantity(me->state().filled_quantity());
+   const auto &s = me->state();
+   return s.has_fractional_filled_quantity()
+              ? Quantity{s.filled_quantity(), s.fractional_filled_quantity().fraction()}
+              : Quantity{s.filled_quantity()};
 }

 bool AlgoWrapperOrder::is_shutting_down() const {
@@ -46,7 +46,7 @@ OrderOpResult IcebergAlgo::do_before_launch(IBeforeLaunchAlgoServices &serv) {
          "Show quantity must be greater than 0"};

   me->pause_between_orders_ms = iceberg.pause_between_orders_ms();
-   me->show_quantity = Quantity((uint32_t)iceberg.show_quantity());
+   me->show_quantity = CQuantity::create_new(iceberg.show_quantity());

   // Create a child template so we can quickly launch our child when
   // the time comes.
@@ -91,7 +91,11 @@ void IcebergAlgo::do_update(api::orderman::Update const &update) {
      return;
   }

-   if (update.quantity() < algo().total_fills().value()) {
+   const double frac_qty = update.has_fractional_quantity()
+                               ? update.fractional_quantity().fraction()
+                               : 0;
+   const Quantity new_qty{update.quantity(), frac_qty};
+   if (new_qty < algo().total_fills()) {
      algo().complete();
      return;
   }
@@ -108,7 +112,7 @@ void IcebergAlgo::do_update(api::orderman::Update const &update) {
         return;
      }

-      me->show_quantity = Quantity((uint32_t)iceberg_update.show_quantity());
+      me->show_quantity = CQuantity::create_new(iceberg_update.show_quantity());
      bool had_timer = me->pause_between_orders_ms > 0;
      bool should_have_timer = iceberg_update.pause_between_orders_ms() > 0;

@@ -122,7 +126,10 @@ void IcebergAlgo::do_update(api::orderman::Update const &update) {
         init_child_template(algo(), iceberg_update.child());
      }
   }
-   child_update.set_quantity(remaining_quantity().value());
+   const auto rem_qty = remaining_quantity();
+   child_update.set_quantity(rem_qty.value());
+   child_update.mutable_fractional_quantity()->set_fraction(
+       rem_qty.fractional_part());
   if (me->status == Status::WORKING)
      algo().update_child(me->child_key, child_update);
 }
@@ -130,10 +137,9 @@ void IcebergAlgo::do_update(api::orderman::Update const &update) {
 void IcebergAlgo::on_child_change(OrderKey k, ChildState const &st) {
   if (k == me->child_key) {
      algo().update_fill_state(
-          Quantity(algo().total_fills().value() + st.last_fills.value()),
-          st.last_price);
+          algo().total_fills() + st.last_fills, st.last_price);

-      if (algo().total_fills().value() >= algo().quantity().value()) {
+      if (algo().total_fills() >= algo().quantity()) {
         algo().complete();
         return;
      }
@@ -164,7 +170,7 @@ void IcebergAlgo::on_timer() {

 void IcebergAlgo::send_next_order() {
   bts_assert(me->status == Status::WORKING);
-   bts_assert(algo().quantity().value() > algo().total_fills().value());
+   bts_assert(algo().quantity() > algo().total_fills());

   me->child_key = algo().launch_child(
       *me->child_template, algo().price(), remaining_quantity());
@@ -172,9 +178,8 @@ void IcebergAlgo::send_next_order() {
   LOG(OrderMan, Info,
       "Iceberg order {} creating new child order {} with size "
       "{} price {}. Filled {} so far out of {}",
-       algo().order_key().value(), me->child_key.value(),
-       remaining_quantity().value(), algo().price().value(),
-       algo().total_fills().value(), algo().quantity().value());
+       algo().order_key(), me->child_key, remaining_quantity(), algo().price(),
+       algo().total_fills(), algo().quantity());
 }

 void orderman::IcebergAlgo::init_child_template(
@@ -186,9 +191,9 @@ void orderman::IcebergAlgo::init_child_template(
 }

 Quantity IcebergAlgo::remaining_quantity() const {
-   bts_assert(algo().quantity().value() > algo().total_fills().value());
-   auto remaining = algo().quantity().value() - algo().total_fills().value();
-   return Quantity(std::min(remaining, me->show_quantity.value()));
+   bts_assert(algo().quantity() > algo().total_fills());
+   auto remaining = algo().quantity() - algo().total_fills();
+   return std::min(remaining, me->show_quantity);
 }

 } // namespace orderman
@@ -134,15 +134,14 @@ bool SpreaderUtils::resolve_synthetic_spread(
                         bts::api::orderman::TimeInForce::TIME_IN_FORCE_IOC);
                  else
                     quoter_order_def.set_time_in_force(
-                         bts::api::orderman::TimeInForce::TIME_IN_FORCE_DAY);
+                         order_def.time_in_force());
                  // hedger order def
                  api::orderman::OrderDefinition hedger_order_def =
                      quoter_order_def;
                  hedger_order_def.mutable_custom_def()
                      ->mutable_limit()
                      ->MergeFrom(api::orderman::LimitOrderDefinition());
-                  hedger_order_def.set_time_in_force(
-                      bts::api::orderman::TimeInForce::TIME_IN_FORCE_DAY);
+                  quoter_order_def.set_time_in_force(order_def.time_in_force());
                  hedger_order_def.set_sub_type(
                      bts::api::orderman::OrderSubType::
                          ORDER_SUB_TYPE_HEDGING_LEG);
@@ -638,7 +638,7 @@ int num_cores() {
 #endif
 }

-std::string expand_env(std::string name, bool assert_vars_exist = true) {
+std::string expand_env(std::string name, bool assert_vars_exist) {
   static const std::regex env_re{R"--(\$\{([^}]+)\})--"};
   std::smatch match;
   while (std::regex_search(name, match, env_re)) {
@@ -656,6 +656,26 @@ std::string expand_env(std::string name, bool assert_vars_exist = true) {
   return name;
 }

+bool expand_env_in_place(std::string &name, bool assert_vars_exist) {
+   static const std::regex env_re{R"--(\$\{([^}]+)\})--"};
+   std::smatch match;
+   bool changed = false;
+   while (std::regex_search(name, match, env_re)) {
+      const auto from = match[0];
+      const auto var_name = match[1].str();
+      const auto var = std::getenv(var_name.c_str());
+      if (!var) {
+         bts_assert_msgf(
+             !assert_vars_exist, "Environment variable '{}' is undefined",
+             name);
+         continue;
+      }
+      name.replace(from.first, from.second, var);
+      changed = true;
+   }
+   return changed;
+}
+
 std::optional<std::string> try_expand_env(const std::string &name) {
   static const std::regex env_re{R"--(\$\{([^}]+)\})--"};
   std::smatch match;
@@ -90,6 +90,9 @@ boost::filesystem::path app_data_path();
 boost::filesystem::path app_data_path(
    const boost::filesystem::path &relative_path, bool mkdirs = true);

+std::string expand_env(std::string name, bool assert_vars_exist = true);
+bool expand_env_in_place(std::string &name, bool assert_vars_exist = true);
+
 std::string username();
 std::string find_newest_file(
    const std::string &directory,
@@ -0,0 +1,41 @@
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <mutex>
+
+namespace bts::utils {
+
+template <typename T>
+class Lazy {
+ public:
+   template <typename... Args>
+   explicit Lazy(Args &&...args)
+       : create_([=]() mutable { return std::make_unique<T>(args...); }) {}
+
+   T &get() {
+      std::call_once(init_flag_, [this]() { instance_ = create_(); });
+      return *instance_;
+   }
+
+   static Lazy<T> from_delegate(std::function<T()> create);
+
+ private:
+   explicit Lazy(std::function<T()> create)
+       : create_([&, create]() { return std::make_unique<T>(create()); }) {}
+
+   std::once_flag init_flag_;
+   std::function<std::unique_ptr<T>()> create_;
+   std::unique_ptr<T> instance_;
+};
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+template <typename T>
+Lazy<T> Lazy<T>::from_delegate(std::function<T()> create) {
+   return Lazy<T>(create);
+}
+
+} // namespace bts::utils
@@ -2,6 +2,7 @@
 #include "lib/io/http/http_client.h"
 #include "lib/io/http/http_response.h"
 #include "lib/utils/crypto/hmac.h"
+#include "lib/utils/type_name.h"

 namespace binance_execution::cross_margin {

@@ -18,16 +19,24 @@ using std::string;
 #define ERROR_FN(fn, ...) _LOG_FN(bts::Error, fn, __VA_ARGS__)

 template <std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_string(std::string &str) {
+static void shrink_str(std::string &str, const char *name) {
   if (str.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_str",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, str.size(), DEFAULT_SIZE, MAX_SIZE);
      str.resize(DEFAULT_SIZE);
      str.shrink_to_fit();
   }
 }

 template <class T, std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_vec(std::vector<T> &vec) {
+static void shrink_vec(std::vector<T> &vec, const char *name) {
   if (vec.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_vec",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, vec.size(), DEFAULT_SIZE, MAX_SIZE);
      vec.resize(DEFAULT_SIZE);
      vec.shrink_to_fit();
   }
@@ -57,7 +66,8 @@ void RestApiClient::request_(
   // Use thread_local vars to reduce memory allocations. Shrink strings
   // to reduce memory footprint from previous requests.
   thread_local bts::http::HttpRequest http_req;
-   shrink_string<4096, 128>(http_req.body);
+   shrink_str<4096, 128>(
+       http_req.body, bts::utils::cached_type_name<RequestT>());
   http_req.reset();
   // Prepare request
   http_req.url = url_;
@@ -66,7 +76,9 @@ void RestApiClient::request_(
   http_req.method = method;
   http_req.headers["X-MBX-APIKEY"] = api_key_;
   thread_local std::string qps;
-   shrink_string<256, 16>(qps);
+   static const std::string query_params_name =
+       fmt::format("{}_params", bts::utils::cached_type_name<RequestT>());
+   shrink_str<256, 16>(qps, query_params_name.c_str());
   qps.clear();
   to_query_params(qps, req);
   if (!qps.empty()) {
@@ -78,7 +90,8 @@ void RestApiClient::request_(
   }
   // Send it
   thread_local HttpResponse http_res;
-   shrink_string<4096, 128>(http_res.body);
+   shrink_str<4096, 128>(
+       http_res.body, bts::utils::cached_type_name<ResponseT>());
   http_res.reset();
   HttpClient::send(http_req, &http_res);
   res.http_code = http_res.code;
@@ -1,6 +1,7 @@
 #pragma once

 #include "lib/io/JsonReader.h"
+#include "lib/os/os.h"

 #include <nlohmann/json.hpp>
 #include <string>
@@ -15,6 +16,15 @@ struct GatewayConfig {
   std::string ws_endpoint;
   std::string ws_stream_endpoint;
   bool use_cross_margin_api = false;
+
+   void expand_env() {
+      bts::os::expand_env_in_place(api_key);
+      bts::os::expand_env_in_place(secret_key);
+      bts::os::expand_env_in_place(account);
+      bts::os::expand_env_in_place(rest_endpoint);
+      bts::os::expand_env_in_place(ws_endpoint);
+      bts::os::expand_env_in_place(ws_stream_endpoint);
+   }
 };

 inline void to_json(nlohmann::json &j, const GatewayConfig &gw) {
@@ -1,19 +1,31 @@
 #include "listen_key_requestor.h"
 #include "lib/core/TimeStamp.h"
-#include "lib/io/json_formatter.h"

+#include <chrono>
 #include <string>
 #include <thread>

 namespace binance_execution {

+#define _LOG_FN(level, fn, ...)                                                \
+   LOG(OrderMan, level, "ListenKeyRequestor::" fn ". " __VA_ARGS__)
+#define DIAG_FN(fn, ...) _LOG_FN(bts::Diag, fn, __VA_ARGS__)
+#define INFO_FN(fn, ...) _LOG_FN(bts::Info, fn, __VA_ARGS__)
+#define WARN_FN(fn, ...) _LOG_FN(bts::Warn, fn, __VA_ARGS__)
+#define ERROR_FN(fn, ...) _LOG_FN(bts::Error, fn, __VA_ARGS__)
+
 using bts::core::Delegate;
 using bts::core::TimeStamp;
-using bts::io::json_formatter;
 using std::optional;
 using std::string;
 using std::thread;

+static bts::core::TimeSpan retry_interval(std::uint32_t n_attempts) {
+   std::int64_t secs = 5 * n_attempts;
+   secs = std::clamp(secs, 5L, 300L);
+   return bts::core::TimeSpan::from_seconds(secs);
+}
+
 struct ListenKeyRequestor::PImpl {
   std::function<RestApiResult<ListenKeyResponse>(const ListenKeyRequest &)>
       get_listen_key;
@@ -42,6 +54,9 @@ struct ListenKeyRequestor::PImpl {
   }

   void run() {
+      static constexpr const std::uint32_t MAX_RETRIES = 30;
+      // Counts consecutive failed attempts (resets to 0 on successful request)
+      std::uint32_t retry_count = 0;
      while (is_running) {
         static const auto refresh_period =
             bts::core::TimeSpan::from_seconds(60 * 30);
@@ -49,10 +64,19 @@ struct ListenKeyRequestor::PImpl {
         if (listen_key.empty()) {
            const auto &res = get_listen_key({});
            if (!res.ok()) {
+               bts_assert(retry_count < MAX_RETRIES);
+               retry_count++;
+               const auto sleep_t = retry_interval(retry_count);
+               WARN_FN(
+                   "run",
+                   "get_listen_key request failed ({}). Will retry in {:.2f} "
+                   "secs",
+                   retry_count, sleep_t.total_seconds());
+               std::this_thread::sleep_for(
+                   std::chrono::milliseconds(sleep_t.millis()));
               continue;
            }
-            bts_assert_msgf(
-                res.ok(), "Failed to get listen key. {}", json_formatter(res));
+            retry_count = 0;
            listen_key = res.response->listenKey;
            bts_assert(!listen_key.empty());
            listen_key_timestamp = now;
@@ -60,9 +84,20 @@ struct ListenKeyRequestor::PImpl {
         }
         if ((now - listen_key_timestamp) > refresh_period) {
            const auto &res = refresh_listen_key({listen_key});
-            bts_assert_msgf(
-                res.ok(), "Failed to refresh listen key. {}",
-                json_formatter(res));
+            if (!res.ok()) {
+               bts_assert(retry_count < MAX_RETRIES);
+               retry_count++;
+               const auto sleep_t = retry_interval(retry_count);
+               WARN_FN(
+                   "run",
+                   "refresh_listen_key request failed ({}). Will retry in "
+                   "{:.2f} secs",
+                   retry_count, sleep_t.total_seconds());
+               std::this_thread::sleep_for(
+                   std::chrono::milliseconds(sleep_t.millis()));
+               continue;
+            }
+            retry_count = 0;
            listen_key_timestamp = now;
         }
         std::this_thread::sleep_for(std::chrono::seconds(2));
@@ -4,6 +4,7 @@
 #include "lib/io/http/http_response.h"
 #include "lib/io/json_formatter.h"
 #include "lib/utils/crypto/hmac.h"
+#include "lib/utils/type_name.h"

 namespace binance_execution::spots {

@@ -20,16 +21,24 @@ using std::string;
 #define ERROR_FN(fn, ...) _LOG_FN(bts::Error, fn, __VA_ARGS__)

 template <std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_string(std::string &str) {
+static void shrink_str(std::string &str, const char *name) {
   if (str.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_str",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, str.size(), DEFAULT_SIZE, MAX_SIZE);
      str.resize(DEFAULT_SIZE);
      str.shrink_to_fit();
   }
 }

 template <class T, std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_vec(std::vector<T> &vec) {
+static void shrink_vec(std::vector<T> &vec, const char *name) {
   if (vec.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_vec",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, vec.size(), DEFAULT_SIZE, MAX_SIZE);
      vec.resize(DEFAULT_SIZE);
      vec.shrink_to_fit();
   }
@@ -59,7 +68,8 @@ void RestApiClient::request_(
   // Use thread_local vars to reduce memory allocations. Shrink strings
   // to reduce memory footprint from previous requests.
   thread_local bts::http::HttpRequest http_req;
-   shrink_string<4096, 128>(http_req.body);
+   shrink_str<4096, 128>(
+       http_req.body, bts::utils::cached_type_name<RequestT>());
   http_req.reset();
   // Prepare request
   http_req.url = url_;
@@ -68,7 +78,9 @@ void RestApiClient::request_(
   http_req.method = method;
   http_req.headers["X-MBX-APIKEY"] = api_key_;
   thread_local std::string qps;
-   shrink_string<256, 16>(qps);
+   static const std::string query_params_name =
+       fmt::format("{}_params", bts::utils::cached_type_name<RequestT>());
+   shrink_str<256, 16>(qps, query_params_name.c_str());
   qps.clear();
   to_query_params(qps, req);
   if (!qps.empty()) {
@@ -80,7 +92,8 @@ void RestApiClient::request_(
   }
   // Send it
   thread_local HttpResponse http_res;
-   shrink_string<4096, 128>(http_res.body);
+   shrink_str<4096, 128>(
+       http_res.body, bts::utils::cached_type_name<ResponseT>());
   http_res.reset();
   HttpClient::send(http_req, &http_res);
   res.http_code = http_res.code;
@@ -2,6 +2,7 @@
 #include "lib/io/http/http_client.h"
 #include "lib/io/http/http_response.h"
 #include "lib/utils/crypto/hmac.h"
+#include "lib/utils/type_name.h"

 namespace binance_execution::usds_futures {

@@ -18,16 +19,24 @@ using std::string;
 #define ERROR_FN(fn, ...) _LOG_FN(bts::Error, fn, __VA_ARGS__)

 template <std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_string(std::string &str) {
+static void shrink_str(std::string &str, const char *name) {
   if (str.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_str",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, str.size(), DEFAULT_SIZE, MAX_SIZE);
      str.resize(DEFAULT_SIZE);
      str.shrink_to_fit();
   }
 }

 template <class T, std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_vec(std::vector<T> &vec) {
+static void shrink_vec(std::vector<T> &vec, const char *name) {
   if (vec.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_vec",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, vec.size(), DEFAULT_SIZE, MAX_SIZE);
      vec.resize(DEFAULT_SIZE);
      vec.shrink_to_fit();
   }
@@ -57,7 +66,8 @@ void RestApiClient::request_(
   // Use thread_local vars to reduce memory allocations. Shrink strings
   // to reduce memory footprint from previous requests.
   thread_local bts::http::HttpRequest http_req;
-   shrink_string<4096, 128>(http_req.body);
+   shrink_str<4096, 128>(
+       http_req.body, bts::utils::cached_type_name<RequestT>());
   http_req.reset();
   // Prepare request
   http_req.url = url_;
@@ -66,7 +76,9 @@ void RestApiClient::request_(
   http_req.method = method;
   http_req.headers["X-MBX-APIKEY"] = api_key_;
   thread_local std::string qps;
-   shrink_string<256, 16>(qps);
+   static const std::string query_params_name =
+       fmt::format("{}_params", bts::utils::cached_type_name<RequestT>());
+   shrink_str<256, 16>(qps, query_params_name.c_str());
   qps.clear();
   to_query_params(qps, req);
   if (!qps.empty()) {
@@ -78,7 +90,8 @@ void RestApiClient::request_(
   }
   // Send it
   thread_local HttpResponse http_res;
-   shrink_string<4096, 128>(http_res.body);
+   shrink_str<4096, 128>(
+       http_res.body, bts::utils::cached_type_name<ResponseT>());
   http_res.reset();
   HttpClient::send(http_req, &http_res);
   res.http_code = http_res.code;
@@ -0,0 +1,25 @@
+ARG IMAGE="sparkd"
+ARG CONFIG_DIR="src/server/sparkd/configurations/dev"
+ARG SOURCE_DIR="install/sparkd-release-gcc"
+
+FROM bluetradingsystems/linux_dotnet7 AS sparkd_base
+ARG CONFIG_DIR
+ARG SOURCE_DIR
+
+USER "root"
+RUN apt-get update && apt-get install -y liblttng-ust-dev
+RUN apt-get update && apt-get install -y libpcap0.8-dev
+RUN apt-get update && apt-get install -y python3.10-dev
+
+USER "bts"
+COPY --chown=bts:bts "${SOURCE_DIR}" "."
+COPY --chown=bts:bts "${CONFIG_DIR}" "/home/bts/sparkdata/conf"
+WORKDIR "/home/bts/bin"
+
+FROM sparkd_base AS sparkd
+
+FROM sparkd_base AS sparkd_md_replay
+ARG MD_RECORDING_DIR="src/server/sparkd/.md_recording"
+COPY --chown=bts:bts "${MD_RECORDING_DIR}" "/home/bts/sparkdata/data/md_recording"
+
+FROM ${IMAGE}
@@ -1,32 +1,48 @@
 #include "DumpSecurity.h"
-#include "lib/config/protos/market_data.pb.h"
 #include "lib/core/TableFormatter.h"
 #include "lib/domain_model/Security.h"
 #include "lib/domain_model/Security.pb.h"
 #include "lib/domain_model/SecurityType.h"
-#include "lib/md/Diag/ShmDumper.h"
 #include "lib/md/SecurityDB.h"
 #include "lib/md/SymbolLoader.h"
-#include "lib/md/UserSymbol.h"
+#include "lib/utils/lazy.h"
+
 #include <unordered_set>
 #include <vector>

 namespace bts::cmds::md::cmds {

 using bts::api::Security;
+using bts::api::security_type_parse_smart;
+using bts::api::SecurityType;
 using bts::api::SecurityType_Name;
+using bts::api::Venue;
+using bts::api::venue_parse_smart;
+using bts::core::TableFormatter;
+using bts::md::securitytype_of;
 using bts::md::SymbolLoader;
+using bts::md::db::load_metadata;
+using bts::md::db::load_securities;
 using bts::securities::security_type;
+using bts::utils::Lazy;
+using std::function;
 using std::make_tuple;
+using std::string;
+using std::unordered_set;
+using std::vector;

 template <class EnumT>
 using EnumNameMap = std::map<std::string, EnumT>;

-void print_security(const api::Security &security) {
+void print_security(
+    const Security &security,
+    Lazy<vector<api::synthetic::Metadata>> &metadata) {
+   static const char *SEPARATOR =
+       "\n**************************************************\n";
+   std::cout << SEPARATOR;
   security.PrintDebugString();
-   if (bts::md::securitytype_of(security) == api::SECURITYTYPE_SYNTHETIC) {
-      auto metadata = bts::md::db::load_metadata();
-      for (const auto &m : metadata) {
+   if (securitytype_of(security) == api::SECURITYTYPE_SYNTHETIC) {
+      for (const auto &m : metadata.get()) {
         if (m.security_key() == security.id()) {
            m.PrintDebugString();
            break;
@@ -35,17 +51,20 @@ void print_security(const api::Security &security) {
   }
 }

-void print_securities(std::function<bool(const api::Security &)> predicate) {
+void print_securities(function<bool(const Security &)> predicate) {
+   auto metadata =
+       Lazy<vector<api::synthetic::Metadata>>::from_delegate(load_metadata);
   const auto symbols = SymbolLoader::load();
-   bts::md::db::load_securities(symbols, [&](const api::Security &security) {
-      if (predicate(security))
-         print_security(security);
+   load_securities(symbols, [&](const Security &security) {
+      if (predicate(security)) {
+         print_security(security, metadata);
+      }
   });
 }

-void print_db_securities(std::function<bool(const api::Security &)> predicate) {
-   std::vector<bts::api::Security> secs;
-   const bts::core::TableFormatter::Schema<bts::api::Security> schema{
+void print_db_securities(function<bool(const Security &)> predicate) {
+   vector<Security> secs;
+   const TableFormatter::Schema<Security> schema{
       make_tuple(
           "id", 12, [](const Security &s) { return std::to_string(s.id()); }),
       make_tuple(
@@ -62,22 +81,67 @@ void print_db_securities(std::function<bool(const api::Security &)> predicate) {
           "name", 32,
           [](const Security &s) { return bts::securities::descriptor(s); }),
   };
-   auto on_sec = [&](const bts::api::Security &sec) {
+   auto on_sec = [&](const Security &sec) {
      if (!predicate(sec)) {
         return;
      }
      secs.push_back(sec);
   };
-   bts::md::db::load_securities([](auto) { return true; }, on_sec);
-   bts::core::TableFormatter::format(std::cout, secs, schema);
+   load_securities([](auto) { return true; }, on_sec);
+   TableFormatter::format(std::cout, secs, schema);
+}
+
+template <class T, class ParseT = function<bool(const string &, T *)>>
+static vector<T> parse_enums(const vector<string> &names, ParseT parse) {
+   vector<T> rv;
+   for (const auto &t : names) {
+      if (!parse(t, &rv.emplace_back())) {
+         rv.pop_back();
+      }
+   }
+   return rv;
 }

 struct DumpSecurity::Args {
   bool all;
   bool db = false;
-   std::vector<std::string> types;
-   std::vector<std::string> venues;
-   std::vector<uint32_t> security_ids;
+   vector<string> types;
+   vector<string> venues;
+   vector<uint32_t> security_ids;
+
+   function<bool(const Security &)> get_filter() {
+      if (all) {
+         return [](const auto &) { return true; };
+      }
+      auto types =
+          parse_enums<SecurityType>(this->types, security_type_parse_smart);
+      auto venues = parse_enums<Venue>(this->venues, venue_parse_smart);
+      unordered_set<uint32_t> ids;
+      for (auto id : security_ids) {
+         ids.insert(id);
+      }
+      return [&, types = std::move(types), venues = std::move(venues),
+              ids = std::move(ids)](const auto &s) {
+         if (!ids.empty()) {
+            return ids.find(s.id()) != ids.end();
+         }
+         if (!types.empty()) {
+            auto st = security_type(s);
+            if (!std::any_of(
+                    types.begin(), types.end(),
+                    [st](SecurityType _st) { return _st == st; })) {
+               return false;
+            }
+         }
+         if (!venues.empty() &&
+             !std::any_of(
+                 venues.begin(), venues.end(),
+                 [v = s.exchange()](Venue _v) { return _v == v; })) {
+            return false;
+         }
+         return true;
+      };
+   }
 };

 DumpSecurity::DumpSecurity()
@@ -94,44 +158,11 @@ DumpSecurity::DumpSecurity()
       "Print information about specific securities");

   callback([&] {
-      if (_args->all) {
-         print_securities([](const auto &) { return true; });
-      } else if (_args->db) {
-         std::vector<api::SecurityType> types;
-         for (const auto &t : _args->types) {
-            if (!api::security_type_parse_smart(t, &types.emplace_back())) {
-               types.pop_back();
-            }
-         }
-         std::vector<api::Venue> venues;
-         for (const auto &t : _args->venues) {
-            if (!api::venue_parse_smart(t, &venues.emplace_back())) {
-               venues.pop_back();
-            }
-         }
-         print_db_securities([&](const auto &s) {
-            if (!types.empty()) {
-               auto st = bts::securities::security_type(s);
-               if (!std::any_of(
-                       types.begin(), types.end(),
-                       [st](api::SecurityType _st) { return _st == st; })) {
-                  return false;
-               }
-            }
-            if (!venues.empty() &&
-                !std::any_of(
-                    venues.begin(), venues.end(),
-                    [v = s.exchange()](api::Venue _v) { return _v == v; })) {
-               return false;
-            }
-            return true;
-         });
+      auto filter = _args->get_filter();
+      if (_args->db) {
+         print_db_securities(filter);
      } else {
-         std::unordered_set<uint32_t> ids;
-         for (auto id : _args->security_ids)
-            ids.insert(id);
-         print_securities(
-             [&](const auto &s) { return ids.find(s.id()) != ids.end(); });
+         print_securities(filter);
      }
   });
 }
@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/alphaworx_prod
+SPARKD__SHM_SIZE=8gb
@@ -0,0 +1,6 @@
+* Connects to Binance and Okx PRODUCTION sites for market data and execution
+* Api keys NOT included
+* Configured for Alphaworx
+
+To retrieve api keys, run "python init_env.py". It should dump them to a file
+called  ".env.local".
@@ -0,0 +1,9 @@
+{
+  "api_key": "${BINANCE_API_KEY}",
+  "secret_key": "${BINANCE_SECRET_KEY}",
+  "account": "BINANCE_PROD",
+  "rest_endpoint": "https://api.binance.com",
+  "ws_endpoint": "wss://ws-api.binance.com:443/ws-api/v3",
+  "ws_stream_endpoint": "wss://stream.binance.com:9443",
+  "use_cross_margin_api": true
+}
@@ -0,0 +1,9 @@
+{
+  "api_key": "${BINANCE_API_KEY}",
+  "secret_key": "${BINANCE_SECRET_KEY}",
+  "account": "BINANCE_PROD",
+  "rest_endpoint": "https://fapi.binance.com",
+  "ws_endpoint": "wss://ws-fapi.binance.com/ws-fapi/v1",
+  "ws_stream_endpoint": "wss://fstream.binance.com",
+  "use_cross_margin_api": false
+}
@@ -0,0 +1,88 @@
+import botocore
+import json
+import os
+
+from boto3.session import Session
+from botocore.exceptions import ClientError
+from fileinput import FileInput
+
+
+def update_env_file(path, env_vars: dict):
+    """Update or add vars in a .env file in place."""
+    if not os.path.exists(path):
+        with open(path, "w") as file:
+            for k, v in env_vars.items():
+                file.write(f"{k}={v}\n")
+        return
+
+    with FileInput(path, inplace=True) as file:
+        for line in file:
+            if not line.startswith("#"):
+                stripped: str = line.lstrip()
+                key_len = stripped.find("=")
+                if key_len >= 0:
+                    k = stripped[0:key_len]
+                    if k in env_vars:
+                        print(f"{k}={env_vars[k]}")
+                        del env_vars[k]
+                        continue
+            print(line, end="")
+
+    if len(env_vars) > 0:
+        with open(path, "a") as file:
+            for k, v in env_vars.items():
+                file.write(f"{k}={v}\n")
+
+
+def get_secret():
+    secret_name = "alphaworx-binance-api-keys"
+    region_name = "ap-northeast-1"
+    profile_name = "AdministratorAccess-849458505639"
+    session = Session(profile_name=profile_name)
+    client = session.client(
+        service_name="secretsmanager",
+        region_name=region_name,
+    )
+
+    def on_token_error(e: Exception):
+        print(f"""
+Failed to get secret value. {e}
+Verify you are logged in to aws, then try again.
+
+  aws sso login --profile {profile_name}
+
+If this error persists, verify you have access to these api keys.
+For more help or information, talk to John.
+""")
+
+    try:
+        get_secret_value_response = client.get_secret_value(
+            SecretId=secret_name)
+    except ClientError as e:
+        print(f"Failed to get secret value.\n{e}")
+        raise e
+    except botocore.exceptions.SSOTokenLoadError as tl_err:
+        on_token_error(tl_err)
+        return None
+    except botocore.exceptions.TokenRetrievalError as tr_err:
+        on_token_error(tr_err)
+        return None
+    return get_secret_value_response["SecretString"]
+
+
+def _run():
+    this_dir = os.path.dirname(__file__)
+    secret_str = get_secret()
+    if secret_str is None:
+        exit(1)
+    secret = json.loads(secret_str)
+    env_vars = {}
+    if "api-key" in secret:
+        env_vars["BINANCE_API_KEY"] = secret["api-key"]
+    if "secret-key" in secret:
+        env_vars["BINANCE_SECRET_KEY"] = secret["secret-key"]
+    update_env_file(f"{this_dir}/.env.local", env_vars)
+
+
+if __name__ == "__main__":
+    _run()
@@ -0,0 +1,121 @@
+QuickbusCfg {
+  value {
+    node {
+      reconnect_period_secs: 1
+      hb_period_msecs: 15000
+      max_missed_hbs: 5
+      parent_host: ""
+      parent_port: 0
+      my_port: 0
+    }
+    client {
+      reconnect_period_secs: 1
+      hb_period_msecs: 15000
+      max_missed_hbs: 5
+      parent_host: @SystemConfig.config_ip
+      parent_port: @Ports.qbus_port
+      my_port: 0
+    }
+  }
+}
+
+JetFIX {
+  value {
+    connection {
+      name: "binance_1"
+      venue: VENUE_BINANCE
+    }
+    connection {
+      name: "binance_usds_futures_1"
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    connection {
+      name: "okx_1"
+      venue: VENUE_OKX
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    binance {
+      num_depth_levels: 5
+      url_rest_base: "https://api.binance.com"
+      url_ws_base: "wss://stream.binance.com:9443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: @SymbolLists.list[Binance].symbol
+          name_regexps: "USDC|USDT$"
+        }
+      }
+    }
+    binance_usds_futures {
+      num_depth_levels: 5
+      url_rest_base: "https://fapi.binance.com"
+      url_ws_base: "wss://fstream.binance.com"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
+        }
+      }
+    }
+    okx {
+      num_depth_levels: 5
+      url_rest_base: "https://aws.okx.com"
+      url_ws_base: "wss://wsaws.okx.com:8443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[Okx].symbol
+        }
+      }
+      is_application_ping_pong_enabled: true
+      application_ping_interval_ms: 20000
+      application_pong_timeout_ms: 10000
+      is_protocol_ping_pong_enabled: true
+      protocol_ping_interval_ms: 60000
+      protocol_pong_timeout_ms: 30000
+    }
+  }
+}
+
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_BINANCE
+    }
+    entries {
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    entries {
+      venue: VENUE_GLBX
+    }
+    entries {
+      venue: VENUE_OKX
+    }
+    entries {
+      venue: VENUE_TAIFEX
+    }
+  }
+}
+
+OkxExeCfg {
+  value {
+    account: "prod"
+    ops_id: "op_id_bts"
+    secret_key: "240038D7A28D7F9461247CFCB8E43710"
+    api_key: "dc188532-c74c-4d40-affe-3c6bf4d0ca9a"
+    passphrase: ":,7i%tA%ivfj%SCbX?Ut"
+    ws_url: "wss://wsaws.okx.com:8443"
+    rest_url: "https://www.okx.com"
+    ws_ping_timeout_seconds: 10 
+    ws_ping_interval_seconds: 25
+    ws_reconnect_interval_seconds: 5
+    ws_reconnect_max_count: 5
+	  simulated_trading: false
+    tdmode: "cross"
+  }
+}
@@ -0,0 +1,70 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "AAVE" 
+      symbol: "ACT"
+      symbol: "ADA"
+      symbol: "ALGO"
+      symbol: "AR"
+      symbol: "ATOM"
+      symbol: "AVAX"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "HIVE"
+      symbol: "IO"
+      symbol: "PAXG"
+      symbol: "SPELL"
+      symbol: "WBETH"
+      symbol: "WIF"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "ADA"
+      symbol: "HIVE"
+      symbol: "SPELL"
+      symbol: "IO"
+      symbol: "UNI"
+    }
+    list {
+      key: "Okx"
+      symbol: "AAVE"
+      symbol: "ACT"
+      symbol: "ADA"
+      symbol: "ALGO"
+      symbol: "AR"
+      symbol: "ATOM"
+      symbol: "AVAX"
+      symbol: "BETH"
+      symbol: "BTC"
+      symbol: "CRV"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "ETHFI"
+      symbol: "UNI"
+      symbol: "USDC"
+      symbol: "USDT"
+      symbol: "WIF"
+      symbol: "XAUT"
+      symbol: "XRP"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE"
+      spot: @SymbolLists.list[Binance].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    }
+    venues {
+      name: "VENUE_OKX"
+      spot: @SymbolLists.list[Okx].symbol
+      future: @SymbolLists.list[Okx].symbol
+    }
+  }
+}
@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/binance_dev
+SPARKD__SHM_SIZE=8gb
@@ -0,0 +1 @@
+* Connects to Binance test nets for market data and execution
@@ -1,7 +1,7 @@
 {
  "api_key": "QGwn9OVcpRTU76gOLV0gmQyVAJUmXW6w7H6y9tH8ctanzVnpZN1TP27liObE1Wsd",
  "secret_key": "7ohKg6X3DcsZMGaJmngC84EglsCFse9flMl8aKisa5FUY18LjGCEEdm0KEZljk9T",
-  "account": "binance_dev",
+  "account": "BINANCE_DEV",
  "rest_endpoint": "https://testnet.binance.vision",
  "ws_endpoint": "wss://testnet.binance.vision/ws-api/v3",
  "ws_stream_endpoint": "wss://stream.testnet.binance.vision:9443"
@@ -1,7 +1,7 @@
 {
  "api_key": "b838c108cd2648e34a4f088a7fc635a9cbccf3c2faa6bcaabcb532ffacc087b0",
  "secret_key": "a68f0ab27403c42fa0306b6341b2ddd51d99ac42916f849404155449457d0492",
-  "account": "binance_dev",
+  "account": "BINANCE_DEV",
  "rest_endpoint": "https://testnet.binancefuture.com",
  "ws_endpoint": "wss://testnet.binancefuture.com/ws-fapi/v1",
  "ws_stream_endpoint": "wss://stream.binancefuture.com"
@@ -1,26 +1,11 @@
-ExchangeSubAccounts {
-  value {
-    accounts{
-      exchange_subaccount: "ZmIKL6fb"
-      bts_account_name: "MAIN"     
-    }
-    accounts{
-      exchange_subaccount: "v1iSzZrY"
-      bts_account_name: "SMALL"
-    }
-  }
-}
-
 JetFIX {
  value {
    connection {
      name: "binance_1"
-      max_order_msgs_per_window: 300
      venue: VENUE_BINANCE
    }
    connection {
      name: "binance_usds_futures_1"
-      max_order_msgs_per_window: 300
      venue: VENUE_BINANCE_USDS_FUTURES
    }
  }
@@ -46,15 +31,11 @@ Logging {
    initial_queue_size: 120001
    max_queue_growth: 4
    console {
-      default_level: "Warn"
+      default_level: "Info"
      subsystem {
        name: "OrderMan"
        level: "Diag"
      }
-      subsystem {
-        name: "Http"
-        level: "Diag"
-      }
    }
    file {
      default_level: "Info"
@@ -90,17 +71,12 @@ MarketDataConfig {
      security_criteria {
        clauses {
          security_types: SECURITY_TYPE_SPOT
-          symbols: "AAVE"
-          symbols: "AVAX"
-          symbols: "BNB"
-          symbols: "BTC"
-          symbols: "DOGE"
-          symbols: "EIGEN"
-          symbols: "ETH"
+          symbols: @SymbolLists.list[Binance].symbol
          name_regexps: "USDC|USDT$"
        }
      }
    }
+
    binance_usds_futures {
      num_depth_levels: 5
      # url_rest_base: "https://fapi.binance.com"
@@ -111,44 +87,13 @@ MarketDataConfig {
      security_criteria {
        clauses {
          security_types: SECURITY_TYPE_FUTURE
-          symbols: "AAVE"
-          symbols: "AVAX"
-          symbols: "BNB"
-          symbols: "BTC"
-          symbols: "DOGE"
-          symbols: "EIGEN"
-          symbols: "ETH"
+          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
        }
      }
    }
  }
 }

-Symbols {
-  value {
-    venues {
-      name: "VENUE_BINANCE"
-      spot: "AAVE"
-      spot: "AVAX"
-      spot: "BNB"
-      spot: "BTC"
-      spot: "DOGE"
-      spot: "EIGEN"
-      spot: "ETH"
-    }
-    venues {
-      name: "VENUE_BINANCE_USDS_FUTURES"
-      future: "AAVE"
-      future: "AVAX"
-      future: "BNB"
-      future: "BTC"
-      future: "DOGE"
-      future: "EIGEN"
-      future: "ETH"
-    }
-  }
-}
-
 NotSimExecution {
  value {
    entries {
@@ -0,0 +1,37 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE"
+      spot: @SymbolLists.list[Binance].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    }
+  }
+}
@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
+SPARKD__SHM_SIZE=8gb
@@ -0,0 +1 @@
+* Connects to Binance and Okx test nets for market data and execution
@@ -0,0 +1,8 @@
+{
+  "api_key": "QGwn9OVcpRTU76gOLV0gmQyVAJUmXW6w7H6y9tH8ctanzVnpZN1TP27liObE1Wsd",
+  "secret_key": "7ohKg6X3DcsZMGaJmngC84EglsCFse9flMl8aKisa5FUY18LjGCEEdm0KEZljk9T",
+  "account": "BINANCE_DEV",
+  "rest_endpoint": "https://testnet.binance.vision",
+  "ws_endpoint": "wss://testnet.binance.vision/ws-api/v3",
+  "ws_stream_endpoint": "wss://stream.testnet.binance.vision:9443"
+}
@@ -0,0 +1,8 @@
+{
+  "api_key": "b838c108cd2648e34a4f088a7fc635a9cbccf3c2faa6bcaabcb532ffacc087b0",
+  "secret_key": "a68f0ab27403c42fa0306b6341b2ddd51d99ac42916f849404155449457d0492",
+  "account": "BINANCE_DEV",
+  "rest_endpoint": "https://testnet.binancefuture.com",
+  "ws_endpoint": "wss://testnet.binancefuture.com/ws-fapi/v1",
+  "ws_stream_endpoint": "wss://stream.binancefuture.com"
+}
@@ -0,0 +1,135 @@
+JetFIX {
+  value {
+    connection {
+      name: "binance_1"
+      venue: VENUE_BINANCE
+    }
+    connection {
+      name: "binance_usds_futures_1"
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    connection {
+      name: "okx_1"
+      venue: VENUE_OKX
+    }
+  }
+}
+
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "OrderMan"
+        level: "Diag"
+      }
+    }
+    file {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Diag"
+      }
+      subsystem {
+        name: "OrderMan"
+        level: "Diag"
+      }
+      subsystem {
+        name: "Http"
+        level: "Diag"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    binance {
+      num_depth_levels: 5
+      # url_rest_base: "wss://ws-api.binance.com:9443"
+      # According to Binance, the "apiN" urls offer performance at the cost of stability.
+      # https://developers.binance.com/docs/binance-spot-api-docs/rest-api/general-api-information
+      # url_rest_base: "https://api1.binance.com:9443"
+      # url_rest_base: "https://api.binance.com"
+      url_rest_base: "https://testnet.binance.vision"
+      # url_ws_base: "wss://ws-api.binance.com"
+      url_ws_base: "wss://testnet.binance.vision"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: @SymbolLists.list[Binance].symbol
+          name_regexps: "USDC|USDT$"
+        }
+      }
+    }
+    binance_usds_futures {
+      num_depth_levels: 5
+      # url_rest_base: "https://fapi.binance.com"
+      # url_ws_base: "wss://fstream.binance.com"
+      url_rest_base: "https://testnet.binancefuture.com"
+      url_ws_base: "wss://stream.binancefuture.com"
+
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
+        }
+      }
+    }
+    okx {
+      num_depth_levels: 5
+      url_rest_base: "https://aws.okx.com"
+      url_ws_base: "wss://wsaws.okx.com:8443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[Okx].symbol
+        }
+      }
+      is_application_ping_pong_enabled: true                                                                                                                                            
+      application_ping_interval_ms: 20000
+      application_pong_timeout_ms: 10000
+      is_protocol_ping_pong_enabled: true
+      protocol_ping_interval_ms: 60000
+      protocol_pong_timeout_ms: 30000
+    }
+  }
+}
+
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_BINANCE
+    }
+    entries {
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    entries {
+      venue: VENUE_GLBX
+    }
+    entries {
+      venue: VENUE_OKX
+    }
+    entries {
+      venue: VENUE_TAIFEX
+    }
+  }
+}
@@ -0,0 +1,63 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+    list {
+      key: "Okx"
+      symbol: "AAVE"
+      symbol: "ACT"
+      symbol: "ADA"
+      symbol: "ALGO"
+      symbol: "AR"
+      symbol: "ATOM"
+      symbol: "AVAX"
+      symbol: "BETH"
+      symbol: "BTC"
+      symbol: "CRV"
+      symbol: "EIGEN"
+      symbol: "ETHFI"
+      symbol: "UNI"
+      symbol: "WIF"
+      symbol: "XAUT"
+      symbol: "XRP"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE"
+      spot: @SymbolLists.list[Binance].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    }
+    venues {
+      name: "VENUE_OKX"
+      spot: @SymbolLists.list[Okx].symbol
+      future: @SymbolLists.list[Okx].symbol
+    }
+  }
+}
@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/sim_dev
+SPARKD__SHM_SIZE=8gb
@@ -0,0 +1,2 @@
+* Connects to Binance and Okx test nets for market data
+* Execution is SIM
@@ -0,0 +1,77 @@
+JetFIX {
+  value {
+    connection {
+      name: "binance_1"
+      venue: VENUE_BINANCE
+    }
+    connection {
+      name: "binance_usds_futures_1"
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    connection {
+      name: "okx_1"
+      venue: VENUE_OKX
+    }
+  }
+}
+
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+MarketDataConfig {
+  value {
+    binance {
+      num_depth_levels: 5
+      url_rest_base: "https://testnet.binance.vision"
+      url_ws_base: "wss://testnet.binance.vision"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: @SymbolLists.list[Binance].symbol
+          name_regexps: "USDC|USDT$"
+        }
+      }
+    }
+    binance_usds_futures {
+      num_depth_levels: 5
+      url_rest_base: "https://testnet.binancefuture.com"
+      url_ws_base: "wss://stream.binancefuture.com"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
+        }
+      }
+    }
+    okx {
+      num_depth_levels: 5
+      url_rest_base: "https://aws.okx.com"
+      url_ws_base: "wss://wsaws.okx.com:8443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[Okx].symbol
+        }
+      }
+      is_application_ping_pong_enabled: true                                                                                                                                            
+      application_ping_interval_ms: 20000
+      application_pong_timeout_ms: 10000
+      is_protocol_ping_pong_enabled: true
+      protocol_ping_interval_ms: 60000
+      protocol_pong_timeout_ms: 30000
+    }
+  }
+}
@@ -0,0 +1,63 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+    list {
+      key: "Okx"
+      symbol: "AAVE"
+      symbol: "ACT"
+      symbol: "ADA"
+      symbol: "ALGO"
+      symbol: "AR"
+      symbol: "ATOM"
+      symbol: "AVAX"
+      symbol: "BETH"
+      symbol: "BTC"
+      symbol: "CRV"
+      symbol: "EIGEN"
+      symbol: "ETHFI"
+      symbol: "UNI"
+      symbol: "WIF"
+      symbol: "XAUT"
+      symbol: "XRP"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE"
+      spot: @SymbolLists.list[Binance].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    }
+    venues {
+      name: "VENUE_OKX"
+      spot: @SymbolLists.list[Okx].symbol
+      future: @SymbolLists.list[Okx].symbol
+    }
+  }
+}
@@ -88,8 +88,9 @@ struct ExecutionGatewayProvider::PImpl {
         return std::make_unique<OkxExecutionGateway>(o);
      }
      case api::VENUE_BINANCE: {
-         const auto cfg =
+         auto cfg =
             parse_config_json<binance_execution::GatewayConfig>(conn.name());
+         cfg.expand_env();
         if (cfg.use_cross_margin_api) {
            return std::make_unique<
                binance_execution::cross_margin::CrossMarginGateway>(
@@ -101,6 +102,7 @@ struct ExecutionGatewayProvider::PImpl {
      case api::VENUE_BINANCE_USDS_FUTURES: {
         auto cfg =
             parse_config_json<binance_execution::GatewayConfig>(conn.name());
+         cfg.expand_env();
         if (cfg.use_cross_margin_api) {
            WARN_FN(
                "create_gateway(BinanceUsdsFutures)",
@@ -143,6 +143,7 @@ struct MDRecorder::PImpl {
   uintptr_t mdc_id = 0;
   bts::md::MDClient &md_client;
   std::map<SecurityKey, MDRecordingSecEntry> handles;
+   std::optional<std::thread> run_thread;
   MDBatchWriter batch_writer{settings.batch_size,std::vector<bts::md::recording::MDBatchWriter::DBWriteFunction>{DbWriters::writeToSQL}};
   uint64_t start_ns;
   PImpl(
@@ -160,6 +161,14 @@ struct MDRecorder::PImpl {
                 1000 * 1000 * 1000,
             100000, 1000000000,recording_config.batch_size()}),
         md_client(md_client),start_ns(bts::core::TimeStamp::now_ns()) {}
+
+   ~PImpl() {
+      active = false;
+      if (run_thread && run_thread->joinable()) {
+         run_thread->join();
+      }
+      run_thread.reset();
+   }
 };

 MDRecorder::MDRecorder(
@@ -171,7 +180,8 @@ MDRecorder::MDRecorder(
   me->file.open(file_name, std::ios::out | std::ios::binary | std::ios::trunc);
   LOG(md, Info, "MDRecorder::MDRecorder venue: {} file: {}",
       api::Venue_Name(venue), file_name);
-   bts::threading::make_thread("MDRecorder", [&]() { _poll_md(); }).detach();
+   me->run_thread.emplace(
+       bts::threading::make_thread("MDRecorder", [&]() { _poll_md(); }));
 }

 MDRecorder::~MDRecorder() {}
@@ -0,0 +1,42 @@
+MarketDataConfig {
+  value {
+    binance_us {
+      num_depth_levels: 5
+      url_rest_base: "https://api.binance.us"
+      url_ws_base: "wss://stream.binance.us:9443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: "BTC"
+          symbols: "ETH"
+        }
+      }
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE_US"
+      spot: "BTC"
+      spot: "ETH"
+    }
+  }
+}
+
+Logging {
+  value {
+    console {
+      default_level: "Info"
+    }
+    syslog {
+      default_level: "Info"
+    }
+    file {
+      default_level: "Info"
+    }
+    initial_queue_size: 120001
+    max_queue_growth: 4
+  }
+}
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+this_dir="$(dirname "${BASH_SOURCE[0]}")"
+work_dir="$this_dir/.tmp"
+# Clean previous install
+rm -rf "$work_dir"
+mkdir -p "$work_dir"
+cd "$work_dir"
+# Download installer and run it
+curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+  unzip awscliv2.zip && \
+  sudo ./aws/install
+cd "$this_dir"
+# Clean up
+rm -rf "$work_dir"
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+tools_path="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+eval "$tools_path/ensure_venv.sh" 1> /dev/stdout 2> /dev/stderr
+eval "$tools_path/venv_exec.sh python $tools_path/py/deploy.py $@" 1> /dev/stdout 2> /dev/stderr
@@ -0,0 +1,131 @@
+import argparse
+import dotenv
+import subprocess
+
+import publish_sparkd
+
+from core.argparse_utils import add_toggle_opt
+from core.file_utils import SparkPaths
+
+
+def _get_arg_parser():
+    parser = argparse.ArgumentParser(
+        description=
+        "Deploy sparkd, pubsub, md-publisher, webapp, and api server")
+    parser.add_argument(
+        "--cmake-jobs",
+        type=int,
+        help="The number of cmake parallel jobs",
+    )
+    parser.add_argument(
+        "--profile",
+        help="The configuration profile dirs to use, relative to each service")
+    parser.add_argument("--compose-file",
+                        help="The docker compose (yaml) file to use")
+    parser.add_argument(
+        "--clean",
+        action="store_true",
+        help="Perform a clean build (remove artifacts from previous build)",
+    )
+    parser.add_argument("--env-files",
+                        action="append",
+                        help=".env files to pass to docker compose")
+    parser.add_argument(
+        "--verbosity",
+        metavar="{0-5}",
+        choices=range(1, 6),
+        type=int,
+        help="The output verbosity level",
+    )
+    add_toggle_opt(parser,
+                   "build",
+                   help="Whether or not to build the services first")
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Preview the build commands without running them",
+    )
+    # TODO: add more services (api-server, pubsub, dsqa web server, etc.)
+    all_services = ["sparkd"]
+    parser.add_argument("services", nargs="*")
+    parser.set_defaults(
+        configuration="release",
+        services=all_services,
+        build=True,
+        clean=False,
+        profile="dev",
+        verbosity=2,
+    )
+    return parser
+
+
+def _get_env_files(services, profile):
+    root = SparkPaths.root()
+    files = []
+
+    def add(file):
+        if file.exists():
+            files.append(file)
+
+    add(root.joinpath(".env"))
+    add(root.joinpath(".env.local"))
+    if profile is not None:
+        if "sparkd" in services:
+            profile_dir=root.joinpath("src/server/sparkd/configurations", profile)
+            add(profile_dir.joinpath(".env"))
+            add(profile_dir.joinpath(".env.local"))
+    return files
+
+
+def deploy(args):
+    def run(cmd):
+        print(cmd)
+        if args.dry_run:
+            return
+        subprocess.run(cmd, shell=True).check_returncode()
+
+    services: list[str] = args.services or []
+    if args.build:
+        build_all = len(services) == 0
+        if build_all or "sparkd" in services:
+            publish_sparkd.publish(
+                publish_sparkd.PublishArgs(
+                    configuration="release",
+                    cmake_jobs=args.cmake_jobs,
+                    clean=args.clean,
+                    verbosity=args.verbosity,
+                    dry_run=args.dry_run,
+                ))
+    service_args = " ".join(services)
+    compose_cmd = "sudo docker compose"
+    if args.compose_file:
+        compose_cmd += f" -f {args.compose_file}"
+    env_files = _get_env_files(args.services, args.profile)
+    if len(env_files) > 0:
+        compose_cmd += " " + " ".join([f"--env-file {f}" for f in env_files])
+        for ef in env_files:
+            if args.verbosity > 2:
+                print(f"Loading env file {ef}")
+                if args.verbosity > 3:
+                    env_vars = dotenv.dotenv_values(ef)
+                    for k, v in env_vars.items():
+                        print(f"    {k}={v}")
+            dotenv.load_dotenv(ef)
+    run(f"{compose_cmd} down {service_args}")
+    run(f"{compose_cmd} build {service_args}")
+    run(f"{compose_cmd} up {service_args} -d")
+    run(f"{compose_cmd} logs {service_args} --follow")
+
+
+def _main():
+    parser = _get_arg_parser()
+    args = parser.parse_args()
+    deploy(args)
+
+
+if __name__ == "__main__":
+    try:
+        _main()
+    except Exception as e:
+        print(f"deploy.py failed: {e}")
+        exit(1)
@@ -0,0 +1,217 @@
+#!/usr/bin/env bash
+
+help_text="\
+Manage files in sparkd data volumes
+
+Usage: sparkd_data.sh [OPTIONS] [COMMAND] [...]
+
+Commands:
+  get               Copy files FROM data volume
+  set               Copy files TO data volume
+  shell             Start shell in data volume
+
+Options:
+  -h,--help         Print this message and exit
+
+"
+
+cp_help_text="\
+Copy files to/from sparkd data volume
+
+Usage: sparkd_data.sh <get|set> [OPTIONS] [POSITIONALS]
+
+Positionals:
+  SOURCE            Source file
+  DEST              Dest file
+
+Options:
+  --clean           Recreate the container that mounts to the volume (even if it already exists)
+  --source          Source file
+  --dest            Dest file
+  --config-port     The config port for the volume to manage
+  -h,--help         Print this message and exit
+"
+
+shell_help_text="\
+Start shell in sparkd data volume
+
+Usage: sparkd_data.sh shell [OPTIONS]
+
+Options:
+  --clean           Recreate the container that mounts to the volume (even if it already exists)
+  --config-port     The config port for the volume to manage
+  -h,--help         Print this message and exit
+"
+
+print_help_and_exit() {
+  echo "$1"
+  exit "${2:-0}"
+}
+
+bounce_container() {
+  if [ "$#" -ne 3 ]; then
+    echo "bounce_container invalid args '$@'."
+    echo "Usage: bounce_container <clean:0|1> <container_name> <volume_name>"
+    return 1
+  fi
+  clean="$1"
+  cont_name="$2"
+  vol_name="$3"
+  image_name="bluetradingsystems/linux_dotnet7"
+  if [ -n "$(sudo docker container ls -a -q --filter "name=$cont_name")" ]; then
+    if [[ $clean -eq 1 ]]; then
+      sudo docker stop $cont_name
+      sudo docker rm $cont_name > /dev/null 2>&1
+      sudo docker run --name $cont_name --rm -it -d -v $vol_name:/home/bts/data $image_name sh
+    fi
+  else
+    sudo docker run --name $cont_name --rm -it -d -v $vol_name:/home/bts/data $image_name sh
+  fi
+}
+
+get_container_name() {
+  if [ "$#" -ne 1 ]; then
+    echo "get_container_name invalid args '$@'."
+    echo "Usage: container_name <config_port>"
+    return 1
+  fi
+  echo "sparkd_data_man_${1}"
+}
+
+get_volume_name() {
+  if [ "$#" -ne 1 ]; then
+    echo "get_volume_name invalid args '$@'."
+    echo "Usage: get_volume_name <config_port>"
+    return 1
+  fi
+  echo "sparkd_${1}"
+}
+
+cp_file() {
+  config_port=${SPARKD__CONFIG_PORT:-8000}
+  is_get=${1:-1}
+  shift 1
+  source=""
+  dest=""
+  clean=0
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --clean)
+        clean=1
+        shift 1
+        ;;
+      --source)
+        if [[ -z "$2" ]]; then
+          echo "Source (--source) must be non-empty"
+          print_help_and_exit "$cp_help_text" 1
+        fi
+        source="$2"
+        shift 2
+        ;;
+      --dest)
+        if [[ -z "$2" ]]; then
+          echo "Dest (--Dest) must be non-empty"
+          print_help_and_exit "$cp_help_text" 1
+        fi
+        dest="$2"
+        shift 2
+        ;;
+      --config-port)
+        if [[ -z "$2" ]]; then
+          echo "Config port (--config-port) must be non-empty"
+          print_help_and_exit "$cp_help_text" 1
+        fi
+        config_port="$2"
+        shift 2
+        ;;
+      -h)
+        ;&
+      --help)
+        print_help_and_exit "$cp_help_text" 0
+        ;;
+      *)
+        if [[ -z "$source" ]]; then
+          source="$1"
+        elif [[ -z "$dest" ]]; then
+          dest="$1"
+        else
+          echo "Invalid positional argument '$1'"
+          print_help_and_exit "$cp_help_text" 1
+        fi
+        shift 1
+        ;;
+    esac
+  done
+
+  if [ -z "$source" ]; then
+    echo -e "Error: No source specified to copy\n"
+    print_help_and_exit "$cp_help_text" 1
+  fi
+
+  if [ -z "$dest" ]; then
+    dest="."
+  fi
+
+  cont_name="$(get_container_name "$config_port")"
+  vol_name="$(get_volume_name "$config_port")"
+  bounce_container "$clean" "$cont_name" "$vol_name"
+  if [[ $is_get -eq 1 ]]; then
+    sudo docker cp $cont_name:/home/bts/data/$source $dest
+  else
+    sudo docker cp $source $cont_name:/home/bts/data/$dest
+  fi
+}
+
+shell() {
+  clean=0
+  config_port=${SPARKD__CONFIG_PORT:-8000}
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --clean)
+        clean=1
+        shift 1
+        ;;
+      --config-port)
+        if [[ -z "$2" ]]; then
+          echo "Config port (--config-port) must be non-empty"
+          print_help_and_exit "$shell_help_text" 1
+        fi
+        config_port="$2"
+        shift 2
+        ;;
+      -h)
+        ;&
+      --help)
+        print_help_and_exit "$shell_help_text" 0
+        ;;
+      *)
+        echo "Invalid positional argument '$1'"
+        print_help_and_exit "$shell_help_text" 1
+        ;;
+    esac
+  done
+
+  cont_name="$(get_container_name "$config_port")"
+  vol_name="$(get_volume_name "$config_port")"
+  bounce_container "$clean" "$cont_name" "$vol_name"
+  sudo docker exec -it -w /home/bts/data $cont_name bash
+}
+
+case "$1" in
+  get)
+    shift 1
+    cp_file 1 $@
+    ;;
+  set)
+    shift 1
+    cp_file 0 $@
+    ;;
+  shell)
+    shift 1
+    shell $@
+    ;;
+  *)
+    echo "Invalid argument '$1'"
+    print_help_and_exit "$help_text" 1
+    ;;
+esac
Author	SHA1	Message	Date
John Colvin	066a0ba693	ui: Use dark theme foreground in mini orderbook, fills popups It was black in a couple places before, making it hard to read.	2025-03-10 14:47:58 -04:00
John Colvin	c2b67b4813	fix: Check for null in orderbook root algo type filter	2025-03-10 14:47:58 -04:00
John Colvin	ecad456a2d	fix: Apply exp filter to popup grids in pnl window	2025-03-10 14:47:58 -04:00
Taha Afzal	2cb3cfdcdd	Spreader's orders were hardcoded to be DAY, now it gets it from the parent	2025-03-10 17:22:08 +00:00
Taha Afzal	0ebc8e5b4b	Orders in DB are deleted on server bounce	2025-03-10 15:42:12 +00:00
John Colvin	2072105c0f	cleanup: Add using aliases to make code more concise	2025-03-10 14:53:07 +00:00
John Colvin	d663c86aa0	improvement: Only load synth metadata from disk once in md dumper Instead of every time a synth is dumped (i.e., do it lazily, outside the loop body)	2025-03-10 14:53:07 +00:00
John Colvin	4a2dfa46a1	improvement: Print delimiter line when dumping secs to console	2025-03-10 14:53:07 +00:00
John Colvin	6e35db6a24	improvement: Apply filter to shared mem secs in md dumper Previously, filter only applied to db securities (i.e., when --db flag was used)	2025-03-10 14:52:39 +00:00
John Colvin	b45dc3bc2d	fix: Preserve create time when order is updated Bugs (fixed): * Order create time mirrors last update time (it should never change).	2025-03-10 05:24:49 +00:00
John Colvin	a1912907b8	Add brief readmes for sparkd config profiles	2025-03-10 05:24:49 +00:00
John Colvin	b1b3dcdeb8	Add sim_dev config profile for sparkd	2025-03-10 05:24:49 +00:00
John Colvin	03dc8ef767	Add boto3 to py venv, so we can retrieve secrets from aws	2025-03-10 05:24:49 +00:00
John Colvin	975bf56c2a	fix: Don't spam binance w/ listen key requests Sleep for a few seconds before trying again on failure.	2025-03-10 05:24:49 +00:00
John Colvin	3f58c2ce84	Add script to install aws cli The aws cli can be used to configure credentials in the shell in order to retrieve secrets (e.g., production api keys for alphaworx, etc.).	2025-03-10 05:24:49 +00:00
John Colvin	96f28acf91	Add dev config profile for sparkd	2025-03-10 05:24:49 +00:00
John Colvin	c2c2658e21	Add alphaworx_prod config profile for sparkd	2025-03-10 05:24:49 +00:00
John Colvin	d6b8e359cd	Expand environment vars in binance gw config	2025-03-10 03:05:59 +00:00
John Colvin	84a0850d95	Tweak binance_dev config profile * Factor out repeated symbol list into SymbolLists entry * Change "binance_dev" account to "BINANCE_DEV" in exe gw json * Adjust log level * Remove unused config: - ExchangeSubAccounts - JetFIX_Connection.max_order_msgs_per_window	2025-03-10 03:05:59 +00:00
John Colvin	4effdc4160	diag: Log when http buffers are resized in binance exe gws	2025-03-10 03:05:59 +00:00
John Colvin	b9fe39f6e3	refactor: Make MDRecorder join the run thread in destructor This wasn't broken, but detached threads bother me.	2025-03-10 03:05:59 +00:00
John Colvin	b555850015	build: Package & deploy sparkd with docker compose	2025-03-10 03:05:59 +00:00
John Colvin	23794fa30d	devcontainer: Add docker in docker Also remove default port bindings in devcontainer (so the "docker in docker" sparkd can use them). It's easy enough to manually bind ports in vs code.	2025-03-08 05:20:08 +00:00
John Colvin	90b5a309a3	cleanup: Make a couple fns const Probably a typo, since these fns don't mutate anything.	2025-03-08 05:20:08 +00:00
John Colvin	07beda3fc6	fix: Support fractional show qty in iceberg orders	2025-03-07 20:42:24 +00:00
				`@@ -0,0 +1 @@`
				`* Connects to Binance test nets for market data and execution`
				`@@ -0,0 +1 @@`
				`* Connects to Binance and Okx test nets for market data and execution`