ui: Use dark theme foreground in mini orderbook, fills popups

It was black in a couple places before, making it hard to read.

.devcontainer/Dockerfile

@@ -90,6 +90,12 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 # Databento (deps)
 RUN apt-get update && apt-get -y install libzstd-dev
 
+# Docker in Docker
+RUN apt-get update && apt-get install -y \
+    docker.io \
+    docker-compose \
+    docker-compose-v2
+
 # Set up default executables (update-alternatives)
 RUN \
    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \

.devcontainer/devcontainer.json

@@ -12,17 +12,10 @@
   // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
   "runArgs": [
     "--shm-size",
-    "8gb",
-    "-p",
-    "8000:8000",
-    "-p",
-    "8001:8001",
-    "-p",
-    "8002:8002",
-    "-p",
-    "8003:8003",
-    "-p",
-    "12345:12345"
+    "8gb"
+  ],
+  "mounts": [
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
   ],
   "customizations": {
     "vscode": {

.env

@@ -0,0 +1,47 @@
+# This file defines the global default configuration for docker deployments.
+# It is tracked in source control. For temporary modifications, please create
+# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
+# If a variable is defined in both files, the value in .env.local will be used.
+#
+# About the deployment procedure:
+#   We use Docker Compose to build images and launch containers for various services
+#   that make up our software suite:
+#     - sparkd
+#     - TODO: pubsub
+#     - TODO: md-publisher
+#     - TODO: webapp (DSQA on the web)
+#     - TODO: api-server (websocket based)
+#   From inside a devcontainer, you can launch all services in separate containers
+#   on your machine by running:
+#
+#     tools/deploy.sh
+#
+#   This script builds release versions for each service, packaged (generally) in
+#   self-contained directories. Next, it runs a few "docker compose" commands to
+#   build docker images loaded with the self-contained binaries from the previous
+#   step, and start containers with those images to run each service. To view the
+#   commands that deploy.sh runs internally (without actually running them), run:
+#
+#     tools/deploy.sh --dry-run
+#
+#   For additional information/help, run:
+#
+#     tools/deploy.sh --help
+#
+
+### Sparkd ###
+# The internal image for sparkd to use. For more info, view:
+#   src/server/sparkd/Dockerfile
+SPARKD__IMAGE=sparkd
+# The config dir for this service to use
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
+# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
+# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
+# SPARKD__DATA_DIR=/home/bts/sparkdata/data
+SPARKD__SOURCE_DIR=install/sparkd-release-gcc
+SPARKD__CONFIG_PORT=8000
+SPARKD__QUICKBUS_PORT=8001
+SPARKD__SIM_FIX_PORT=8002
+SPARKD__SHM_SIZE=4gb
+
+# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)

.gitignore

@@ -59,3 +59,4 @@ trace_data/
 .devcontainer/
 *.Generated.cs
 *.lst
+.env.local

compose.yaml

@@ -0,0 +1,63 @@
+services:
+  sparkd:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
+    build:
+      dockerfile: src/server/sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: >
+      sh -c "
+        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
+        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
+          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
+        fi
+        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
+          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
+          set -a
+          . /home/bts/sparkdata/conf/.env.local
+          set +a
+        fi
+        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+
+  sparkd-tail:
+    image: bluetradingsystems/linux_dotnet7
+    volumes:
+      - type: volume
+        read_only: true
+        source: sparkd_data
+        target: /home/bts/sparkdata/data/logging.sparkd
+        volume:
+          subpath: logging.sparkd
+    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
+
+  sparkd-ping-test:
+    depends_on:
+      - sparkd
+    profiles:
+      - ping-test
+    build:
+      context: .
+      dockerfile_inline: |
+        FROM bluetradingsystems/linux_dotnet7
+        USER "root"
+        RUN apt-get update && apt-get -y install iputils-ping
+    command: [ "ping", "sparkd" ]
+
+volumes:
+  sparkd_data:
+    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"

docker/images/bluetradingsystems/linux_dotnet7/Dockerfile

@@ -0,0 +1,38 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y pkg-config
+RUN apt-get update && apt-get install -y locales
+RUN apt-get update && apt-get install -y ca-certificates
+RUN apt-get update && apt-get install -y libssh2-1-dev
+RUN apt-get update && apt-get install -y software-properties-common
+RUN apt-get update && apt-get install -y sudo
+
+# GCC
+ARG gcc_major=13
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+  && apt-get update \
+  && apt-get install -y g++-$gcc_major
+
+RUN apt-get update \
+  && apt-get install -y dotnet7
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+  groupadd -g "$docker_uid" "$docker_user" \
+  && useradd \
+  -m \
+  -u "$docker_uid" \
+  -g "$docker_uid" \
+  -G sudo \
+  --password "$(openssl passwd -1 "$docker_pass")" \
+  "$docker_user"
+
+USER bts
+WORKDIR /home/bts

requirements.txt

@@ -1,10 +1,13 @@
 astroid==2.7.3
 autopep8==1.5.7
+boto3==1.37.9
+botocore==1.37.9
 certifi==2023.5.7
 charset-normalizer==3.1.0
 colorama==0.4.4
 idna==3.4
 isort==5.9.3
+jmespath==1.0.1
 lazy-object-proxy==1.6.0
 mccabe==0.6.1
 platformdirs==2.3.0
@@ -14,7 +17,11 @@ pycodestyle==2.7.0
 pycparser==2.20
 pylint==2.10.2
 PyMeta3==0.5.1
+python-dotenv==1.0.1
+python-dateutil==2.9.0.post0
 requests==2.31.0
+s3transfer==0.11.4
+six==1.17.0
 termcolor==1.1.0
 toml==0.10.2
 urllib3==2.0.3

src/client/spark/Modules/Market/MarketView/MarketViewVM.cs

@@ -629,16 +629,16 @@ namespace mbts.spark {
 
       static (string desc, Color fg) _fills_title(ColKey key) =>
          key switch {
-            ColKey.Position => ("Day Fills", Colors.Black),
-            ColKey.RawDayPosition => ("Raw Day Fills", Colors.Black),
-            _ => ("Fills", Colors.Black),
+            ColKey.Position => ("Day Fills", BTSColors.DayFills.foreground),
+            ColKey.RawDayPosition => ("Raw Day Fills", BTSColors.DayFills.foreground),
+            _ => ("Fills", BTSColors.UI.default_text),
          };
 
       static (string desc, Color fg) _orderbook_title(ColKey key) =>
          key switch {
             ColKey.WorkingBids => ((string desc, Color fg))("Working Buy Orders", BTSColors.MarketData.bid),
             ColKey.WorkingAsks => ((string desc, Color fg))("Working Sell Orders", BTSColors.MarketData.ask),
-            _ => ("Working Orders", Colors.Black),
+            _ => ("Working Orders", BTSColors.UI.default_text),
          };
 
       static GridSettings _hide_cols(GridSettings grid, IEnumerable<OrderBookGridView.ColumnKeys> columns) =>

src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookFilterStates.cs

@@ -150,7 +150,11 @@ namespace bts.shell.orderbook {
          pb.add_discrete_values(fs.exchanges, o => o.order.security?.Exchange ?? Venue.Unset);
          pb.add_discrete_values(fs.tifs, o => o.order.order.time_in_force());
          pb.add_discrete_values(fs.order_types, o => AlgoID.from_custom_def(o.order.order.Def.CustomDef));
-         pb.add_discrete_values(fs.root_algo_types, o => AlgoID.from_algo_key(o.root.algo?.key ?? default));
+         pb.add_discrete_values(
+            Array.ConvertAll(fs.root_algo_types, a => (AlgoID?)a),
+            o => o.root.algo.HasValue
+               ? AlgoID.from_algo_key(o.root.algo.Value.key)
+               : null);
          pb.add_discrete_values(fs.order_sub_types, o => o.order.order.sub_type());
          pb.add_discrete_values(fs.sides, o => o.order.order.side());
          pb.add_discrete_values(fs.statuses, o => o.order.order.status());

src/client/spark/Modules/Trading/PNL/VM/PNLVM.cs

@@ -774,6 +774,17 @@ namespace bts.shell.pnl {
             case ColumnKeys.Account: filter.accounts = new[] { g.Name.ToString() }; break;
             case ColumnKeys.Symbol: filter.symbols = new[] { g.Name.ToString() }; break;
             case ColumnKeys.Exchange: filter.exchanges = new[] { (Venue)g.Name }; break;
+            case ColumnKeys.Expiration: {
+                  if (g.Name is DateTime exp) {
+                     if (filter.security_criteria is null) {
+                        filter.security_criteria = SecurityCriteria.from_expirations(exp);
+                     } else {
+                        filter.security_criteria.clauses = ArrayHelper.add(
+                           filter.security_criteria.clauses, SecurityCriteriaClause.from_expirations(exp));
+                     }
+                  }
+                  break;
+               }
             case ColumnKeys.Security:
                if (g.Name is SecurityImmutable sec)
                   filter.security_criteria = SecurityCriteria.from_securities(sec);

src/client/spark/core/FontsAndColors/BTSColors.cs

@@ -1040,6 +1040,7 @@ namespace mbts.ui {
       }
       public static class DayFills {
          public static BTSColor background => UI.common_background;
+         public static BTSColor foreground => UI.default_foreground;
       }
       public static class ComboReport {
          public static BTSColor foreground => UI.default_text;

src/client/spark/view/Controls/TickUpDown/PriceEntryCommitControl.xaml

@@ -8,7 +8,8 @@
              xmlns:cv="clr-namespace:mbts.ui.converters"
              xmlns:c="clr-namespace:mbts.ui.controls"
              xmlns:usp="clr-namespace:mbts.ui.spark"
-             d:DesignHeight="100" d:DesignWidth="800" x:Name="_me_price">
+             d:DesignHeight="100" d:DesignWidth="800" x:Name="_me_price"
+             Foreground="{u:BTSColor UI.input_text}">
   <UserControl.Resources>
     <cv:NullableDoubleConverter x:Key="TickDefaultPriceConverter" DefaultValue="0"/>
     <cv:TickDisplayConverter x:Key="TickDisplayConverter" x:Shared="False"/>

src/client/spark_api/data/SecurityCriteria.cs

@@ -27,6 +27,12 @@ namespace mbts.spark {
       internal static SecurityCriteria from_keys(IEnumerable<uint> keys) =>
          new(new[] { new SecurityCriteriaClause(security_keys: keys.ToArray()) });
 
+      internal static SecurityCriteria from_expirations(params DateTime[] exp_dates) =>
+         from_expirations(exp_dates.AsEnumerable());
+
+      internal static SecurityCriteria from_expirations(IEnumerable<DateTime> expirations) =>
+         new(new[] { SecurityCriteriaClause.from_expirations(expirations) });
+
       internal static SecurityCriteria from_securities(params SecurityImmutable[] securities) =>
          from_securities(securities.AsEnumerable());
 

src/client/spark_api/data/SecurityCriteriaClause.cs

@@ -58,6 +58,14 @@ namespace mbts.spark {
          return new SecurityCriteriaClause(security_keys: keys, pretty_name: pretty_name);
       }
 
+      public static SecurityCriteriaClause from_expirations(IEnumerable<DateTime> expirations) =>
+         expirations is null
+            ? new()
+            : from_expirations(expirations as DateTime[] ?? expirations.ToArray());
+
+      public static SecurityCriteriaClause from_expirations(params DateTime[] expirations) =>
+         new(expirations: expirations);
+
       [ProtoMember(1)]
       public string? pretty_name { get; }
 

src/lib/api/md/DatabentoMarketDataConfig.proto

@@ -15,4 +15,7 @@ message DatabentoMarketDataConfig {
   string dataset = 70;
   uint32 filter_securities_upto_n_months= 80;
   bool load_reference_data = 90;
+  uint32 debug_sec = 100;
+  bool only_securities =  110;
+  string bbo_start_time = 120;
 }

src/lib/domain_model/common.h

@@ -33,8 +33,8 @@ struct CQuantity {
    uint32_t value() const { return static_cast<uint32_t>(qty); }
    double full_value() const { return qty; }
    float full_value_float() const { return static_cast<float>(full_value()); }
-   uint32_t int_part() { return static_cast<uint32_t>(qty); }
-   double fractional_part() { return full_value() - int_part(); }
+   uint32_t int_part() const { return static_cast<uint32_t>(qty); }
+   double fractional_part() const { return full_value() - int_part(); }
    static bool equal_d(double d1, double d2) {
       return std::abs(d1 - d2) <= err;
    }

src/lib/io/websocket/websocket_client.cpp

@@ -3,6 +3,7 @@
 #include "lib/core/TimeSpan.h"
 #include "lib/core/TimeStamp.h"
 #include "lib/io/websocket/websocket_client_config.h"
+#include "lib/io/websocket/websocket_client_logger.h"
 #include "lib/io/websocket/websocket_client_state.h"
 #include "lib/io/websocket/websocket_close_status.h"
 #include "lib/io/websocket/websocket_error_code.h"
@@ -33,6 +34,45 @@
 #include <websocketpp/config/core_client.hpp>
 #include <websocketpp/roles/client_endpoint.hpp>
 
+namespace websocketpp::config {
+
+/// Client config with asio transport and TLS enabled
+struct bts_asio_tls_client : public core_client {
+   typedef bts_asio_tls_client type;
+   typedef core_client base;
+
+   typedef base::concurrency_type concurrency_type;
+
+   typedef base::request_type request_type;
+   typedef base::response_type response_type;
+
+   typedef base::message_type message_type;
+   typedef base::con_msg_manager_type con_msg_manager_type;
+   typedef base::endpoint_msg_manager_type endpoint_msg_manager_type;
+
+   typedef websocketpp::log::WebsocketClientLogger<
+       websocketpp::log::ElevelAdapter>
+       elog_type;
+   typedef websocketpp::log::WebsocketClientLogger<
+       websocketpp::log::AlevelAdapter>
+       alog_type;
+
+   typedef base::rng_type rng_type;
+
+   struct transport_config : public base::transport_config {
+      typedef type::concurrency_type concurrency_type;
+      typedef type::alog_type alog_type;
+      typedef type::elog_type elog_type;
+      typedef type::request_type request_type;
+      typedef type::response_type response_type;
+      typedef transport::asio::tls_socket::endpoint socket_type;
+   };
+
+   typedef transport::asio::endpoint<transport_config> transport_type;
+};
+
+} // namespace websocketpp::config
+
 namespace bts::websocket {
 
 struct WebsocketClientSnapshot {
@@ -70,10 +110,9 @@ using std::thread;
 using std::uint32_t;
 using std::chrono::nanoseconds;
 using std::this_thread::sleep_for;
-using websocketpp::config::asio_tls_client;
 
 using WebsocketppClient =
-    websocketpp::client<websocketpp::config::asio_tls_client>;
+    websocketpp::client<websocketpp::config::bts_asio_tls_client>;
 using ConnectionHdl = websocketpp::connection_hdl;
 using ConnectionPtr = WebsocketppClient::connection_ptr;
 using MessageTypePtr = websocketpp::config::core_client::message_type::ptr;
@@ -117,7 +156,10 @@ struct WebsocketClient::PImpl {
    int version = 0;
    int n_reconnects = 0;
 
-   PImpl() {
+   PImpl(WebsocketClientConfig config, boost::asio::io_context *io_context)
+       : config(std::move(config)), io_context(io_context) {
+      client.get_alog().set_name(this->config.name);
+      client.get_elog().set_name(this->config.name);
       client.set_access_channels(websocketpp::log::alevel::all);
       // client.clear_access_channels(websocketpp::log::alevel::frame_payload);
       client.set_error_channels(websocketpp::log::elevel::all);
@@ -404,10 +446,7 @@ struct WebsocketClient::PImpl {
 
 WebsocketClient::WebsocketClient(
     WebsocketClientConfig config, boost::asio::io_context *io_context)
-    : me_(std::make_unique<PImpl>()) {
-   me_->config = std::move(config);
-   me_->io_context = io_context;
-}
+    : me_(std::make_unique<PImpl>(std::move(config), io_context)) {}
 
 WebsocketClient::~WebsocketClient() {
    me_->close(websocketpp::close::status::going_away, "");

src/lib/io/websocket/websocket_client_config.h

@@ -11,6 +11,8 @@ struct WebsocketClientConfig {
    bool reconnect_on_close = false;
    bool reconnect_on_fail = false;
    std::map<std::string, std::string> headers{};
+   // name to use in logs
+   std::string name{};
 };
 
 } // namespace bts::websocket

src/lib/io/websocket/websocket_client_logger.h

@@ -0,0 +1,88 @@
+#pragma once
+
+#include "lib/logging/Common.h"
+#include "lib/logging/logging.h"
+
+#include <websocketpp/concurrency/basic.hpp>
+#include <websocketpp/logger/basic.hpp>
+#include <websocketpp/logger/levels.hpp>
+
+namespace websocketpp::log {
+
+struct ElevelAdapter {
+   typedef elevel LevelT;
+
+   constexpr bts::log::Level operator()(level channel) {
+      switch (channel) {
+      case elevel::devel:
+         return bts::Diag;
+      case elevel::library:
+      case elevel::warn:
+         return bts::Warn;
+      case elevel::rerror:
+         return bts::Error;
+      case elevel::fatal:
+         return bts::Fatal;
+      default:
+         return bts::Info;
+      }
+   }
+};
+
+struct AlevelAdapter {
+   typedef alevel LevelT;
+
+   constexpr bts::log::Level operator()(level channel) {
+      switch (channel) {
+      case alevel::control:
+      case alevel::frame_header:
+      case alevel::frame_payload:
+      case alevel::message_header:
+      case alevel::message_payload:
+      case alevel::debug_handshake:
+      case alevel::debug_close:
+      case alevel::devel:
+         return bts::Diag;
+      case alevel::disconnect:
+         return bts::Warn;
+      case alevel::fail:
+         return bts::Error;
+      default:
+         return bts::Info;
+      }
+   }
+};
+
+template <typename LevelAdapter>
+class WebsocketClientLogger
+    : public basic<concurrency::basic, typename LevelAdapter::LevelT> {
+   typedef basic<concurrency::basic, typename LevelAdapter::LevelT> base;
+
+   std::string name;
+   LevelAdapter level_adapter;
+
+ public:
+   WebsocketClientLogger(
+       level c, channel_type_hint::value h = channel_type_hint::access)
+       : base(c, h) {}
+
+   void set_name(std::string name) {
+      concurrency::basic::scoped_lock_type lock{this->m_lock};
+      this->name = std::move(name);
+   }
+
+   void write(level channel, const std::string &msg) {
+      concurrency::basic::scoped_lock_type lock{this->m_lock};
+      if (!this->dynamic_test(channel)) {
+         return;
+      }
+      LOG(Http, level_adapter(channel), "WebsocketClient-{} {}. {}", name,
+          std::string{LevelAdapter::LevelT::channel_name(channel)}, msg);
+   }
+
+   void write(level channel, const char *msg) {
+      write(channel, std::string{msg});
+   }
+};
+
+} // namespace websocketpp::log

src/lib/logging/Sink.cpp

@@ -9,6 +9,7 @@
 #include <boost/date_time/posix_time/posix_time.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/path.hpp>
+#include <cstdint>
 #include <fmt/ostream.h>
 #include <fstream>
 #include <iostream>
@@ -94,7 +95,7 @@ std::ifstream::pos_type filesize(std::string const &filename) {
 Sink file_sink(LoggingConfig const &cfg) {
    static std::ofstream stream;
    static uint64_t bytes_written = 0;
-   const auto max_sz = 120 * 1024 * 1024;
+   const uintmax_t max_sz =  1024 * 1024 * 1024 ;
 
    auto f = formatter(cfg);
    auto path = log_path();

src/lib/orderman/actors/FillStore.cpp

@@ -2,7 +2,10 @@
 #include "lib/api/services/OrderService.pb.h"
 #include "lib/config/config.h"
 #include "lib/config/protos/toggles.pb.h"
+#include "lib/core/TimeStamp.h"
 #include "lib/domain_model/Security.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/domain_model/SecurityKey.h"
 #include "lib/logging/logging.h"
 #include "lib/mercury/events/Update.h"
 #include "lib/orderman/actors/SODPosition.h"
@@ -14,10 +17,13 @@
 #include "lib/os/os.h"
 #include "lib/persistence/SqliteStore.h"
 #include "lib/utils/PBUtils.h"
+#include "lib/utils/TimeStamp.h"
 #include "lib/utils/bts_assert.h"
+#include "lib/utils/bts_chrono.h"
 #include "lib/utils/overloaded.h"
 
 #include <boost/date_time/posix_time/posix_time.hpp>
+#include <google/protobuf/timestamp.pb.h>
 #include <string>
 #include <sys/time.h>
 
@@ -32,6 +38,67 @@ namespace orderman {
 
 auto const fills_ns = "fills";
 
+struct FillAggregator {
+   struct SecPosition {
+    private:
+      Fill agg_fill;
+
+    public:
+      double agg_price = 0, agg_qty = 0;
+      SecPosition(const Fill &fill) : agg_fill(fill) {
+         const auto &fill_key = fill.fill_key();
+         std::string new_fill_key = "aggpos_" + fill_key;
+         agg_fill.set_fill_key(new_fill_key);
+         auto now_ns = bts::core::TimeStamp::now_ns();
+         agg_fill.mutable_timestamp()->set_nanos(now_ns);
+         add_fill(fill);
+      }
+      void add_fill(const Fill &fill) {
+         bts_assert(fill.security_key() == agg_fill.security_key());
+         double mult = fill.side() == api::SIDE_BUY ? 1.0 : -1.0;
+         double fp = fill.avg_price();
+         double fq = mult * fill.last_shares();
+         double new_agg_price =
+             ((agg_price * agg_qty) + (fp * fq)) / (agg_qty + fq);
+         double new_agg_qty = agg_qty + fq;
+         agg_price = new_agg_price;
+         agg_qty = new_agg_qty;
+      }
+
+    public:
+      Fill fill() const {
+         Fill rv = agg_fill;
+         rv.set_avg_price(agg_price);
+         rv.set_last_shares(std::abs(agg_qty));
+         rv.set_side(::bts::api::Side::SIDE_BUY);
+         if (agg_qty < 0) {
+            rv.set_side(::bts::api::Side::SIDE_SELL);
+         }
+         return rv;
+      }
+   };
+   std::map<SecurityKey, SecPosition> pos_map = {};
+
+   void add_fill(const Fill &fill) {
+      SecurityKey k = SecurityKey(fill.security_key());
+      auto it = pos_map.find(k);
+      if (it != pos_map.end()) {
+         SecPosition &sec_pos = it->second;
+         sec_pos.add_fill(fill);
+      } else {
+         pos_map.insert({k, {fill}});
+      }
+   }
+
+   std::vector<Fill> get_aggregated_fills() const {
+      std::vector<Fill> rv = {};
+      for (const auto &kv : pos_map) {
+         rv.push_back(kv.second.fill());
+      }
+      return rv;
+   }
+};
+
 struct FillStore::UnresolvedFills {
    std::vector<api::orderman::Fill> fills;
    api::Security const *security;
@@ -48,6 +115,8 @@ struct FillStore::PImpl {
    std::unordered_map<OrderKey, UnresolvedFills> fills_with_unresolved_accounts;
    mercury::Stream<StoredFillEvent> fill_stream;
    md::SecurityOracle oracle;
+   FillAggregator fill_aggregator = {};
+   bool aggregate_pos = false;
 };
 
 FillStore::FillStore(md::SecurityOracle o) : me(new PImpl(o)) {
@@ -100,40 +169,83 @@ void FillStore::init() {
        os::get_thread_id(), os::get_thread_name());
    LOG(OrderMan, Info, "FillStore::init oracle n_secs: {}",
        me->oracle.get_securities().size());
-   // Subscribe to NewExecutionReportEvent for new fills
-   subscribe<NewExecutionReportEvent>(
-       [this](auto const &ev) { this->on_execution_report(ev); });
-
    // Load stored fills, resolve their securities and publish the lot
-   api::orderman::Fill fill;
    std::vector<StoredFillEvent> stored_batch;
    LOG(persist, Info, "Loading load all fills");
    auto all = me->store->retrieve_all()[fills_ns];
    LOG(persist, Info, "Finished loading all {} fills", all.size());
-   SODPosition sod_helper;
-   for (auto const &kvp : all)
-      if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
-         if (sod_helper.is_sod(fill.timestamp())) {
-            auto fill_kind = sod_fill_kind(fill);
-            fill.set_fill_kind(fill_kind);
+   auto aggregate_clean_and_publish = [&]() {
+      api::orderman::Fill fill;
+      for (auto const &kvp : all) {
+         if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
+            me->fill_aggregator.add_fill(fill);
+         } else {
+            LOG(persist, Error,
+                "Could not recover fill {} from storage. "
+                "This should be investigated",
+                kvp.first);
          }
-         auto sec = me->oracle.get_security(SecurityKey(fill.security_key()));
+      }
+      // 1. go through all fills in aggregator and make StoredFillEvent
+      auto aggregated_fills = me->fill_aggregator.get_aggregated_fills();
+      for (auto &f : aggregated_fills) {
+         auto sec = me->oracle.get_security(SecurityKey(f.security_key()));
          if (sec) {
-            auto fk = kvp.first;
-            me->known_fills.insert(fk);
-            if (!config::get<config::toggles::FilterExpiredFills>().enabled() ||
-                !securities::is_expired(*sec)) {
-               stored_batch.emplace_back(fill, *sec);
+            stored_batch.emplace_back(f, *sec);
+         } else {
+            LOG(persist, Error,
+                "Stored fill {} contains unresolved security {}. ",
+                utils::pb::to_string(f), fill.security_key());
+         }
+      }
+      // 2 delete all fills in the DB
+      me->store->remove_all(fills_ns);
+      // 3. add the aggregated fills back
+      for (const auto &e : stored_batch) {
+         const Fill &f = e.fill;
+         me->store->insert(fills_ns, f.fill_key(), utils::pb::to_string(f));
+      }
+   };
+   auto publish_with_sod_marking = [&]() {
+      api::orderman::Fill fill;
+      SODPosition sod_helper;
+      for (auto const &kvp : all) {
+         if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill)) {
+            if (sod_helper.is_sod(fill.timestamp())) {
+               auto fill_kind = sod_fill_kind(fill);
+               fill.set_fill_kind(fill_kind);
             }
+            auto sec =
+                me->oracle.get_security(SecurityKey(fill.security_key()));
+            if (sec) {
+               auto fk = kvp.first;
+               me->known_fills.insert(fk);
+               if (!config::get<config::toggles::FilterExpiredFills>()
+                        .enabled() ||
+                   !securities::is_expired(*sec)) {
+                  stored_batch.emplace_back(fill, *sec);
+               }
+            } else
+               LOG(persist, Error,
+                   "Stored fill {} contains unresolved security {}. ",
+                   kvp.first, fill.security_key());
          } else
             LOG(persist, Error,
-                "Stored fill {} contains unresolved security {}. ", kvp.first,
-                fill.security_key());
-      } else
-         LOG(persist, Error,
-             "Could not recover fill {} from storage. "
-             "This should be investigated",
-             kvp.first);
+                "Could not recover fill {} from storage. "
+                "This should be investigated",
+                kvp.first);
+      }
+   };
+
+   // Subscribe to NewExecutionReportEvent for new fills
+   subscribe<NewExecutionReportEvent>(
+       [this](auto const &ev) { this->on_execution_report(ev); });
+   if (me->aggregate_pos) {
+      aggregate_clean_and_publish();
+   } else {
+      publish_with_sod_marking();
+   }
+
    me->fill_stream = make_stream<StoredFillEvent>();
    LOG(OrderMan, Info,
        "Fill Store published {} fills recoved from disk out of {} "

src/lib/orderman/actors/OrderStore.cpp

@@ -5,6 +5,7 @@
 #include "lib/config/protos/orders_fills_cleanup.pb.h"
 #include "lib/config/protos/toggles.pb.h"
 #include "lib/logging/logging.h"
+#include "lib/md/ExchangeSecurityKey.h"
 #include "lib/orderman/actors/SODPosition.h"
 #include "lib/orderman/events/OrderDefinitionUpdateEvent.h"
 #include "lib/orderman/events/OrderStateUpdateEvent.h"
@@ -12,6 +13,7 @@
 #include "lib/persistence/SqliteStore.h"
 #include "lib/utils/PBUtils.h"
 
+#include <google/protobuf/timestamp.pb.h>
 #include <unordered_set>
 
 namespace bts {
@@ -38,6 +40,7 @@ struct OrderStore::PImpl {
    bool soft_cleanup_enabled;
    md::SecurityOracle oracle;
    timeval cutoff_time;
+   bool clear_all_orders = true;
 };
 
 OrderStore::OrderStore(md::SecurityOracle o) : me(new PImpl(o)) {}
@@ -63,34 +66,46 @@ void OrderStore::init() {
 
    me->cutoff_time = SODPosition::get_order_cutoff_time(all);
 
-   for (auto const &kvp : all)
-      if (utils::pb::from_string<api::orderman::OrderRecord>(
-              kvp.second, &rec)) {
-         auto ok = OrderKey(std::stoi(kvp.first));
-         auto sk = SecurityKey(rec.def().security_key());
-         auto sec = me->oracle.get_security(sk);
+   auto load_orders = [&]() {
+      for (auto const &kvp : all) {
+         if (utils::pb::from_string<api::orderman::OrderRecord>(
+                 kvp.second, &rec)) {
+            auto ok = OrderKey(std::stoi(kvp.first));
+            auto sk = SecurityKey(rec.def().security_key());
+            auto sec = me->oracle.get_security(sk);
 
-         if (config::get<config::toggles::FilterOrdersWithDanglingSecurities>()
-                 .enabled() &&
-             sec == nullptr) {
-            LOG(OrderMan, Diag, "Order {} refers to unresolved security {}",
-                ok.value(), sk.value());
-         } else {
-            if (!_order_cutoff(rec))
-               stored_batch.emplace_back(rec);
-            me->known_orders[ok] = PImpl::ORInfo{rec, true};
-         }
-      } else
-         LOG(OrderMan, Error,
-             "Could not recover order {} from storage. "
-             "This should be investigated",
-             kvp.first);
+            if (config::get<
+                    config::toggles::FilterOrdersWithDanglingSecurities>()
+                    .enabled() &&
+                sec == nullptr) {
+               LOG(OrderMan, Diag, "Order {} refers to unresolved security {}",
+                   ok.value(), sk.value());
+            } else {
+               if (!_order_cutoff(rec))
+                  stored_batch.emplace_back(rec);
+               me->known_orders[ok] = PImpl::ORInfo{rec, true};
+            }
+         } else
+            LOG(OrderMan, Error,
+                "Could not recover order {} from storage. "
+                "This should be investigated",
+                kvp.first);
+      }
+   };
 
    me->order_stream = make_stream<StoredOrderEvent>();
    LOG(OrderMan, Info,
        "Order Store published {} orders recoved from disk out of {} "
        "total orders",
        stored_batch.size(), all.size());
+
+   if (me->clear_all_orders) {
+      // clear orders
+      me->store->remove_all(orders_ns);
+   } else {
+      load_orders();
+   }
+
    publish_batch(me->order_stream, stored_batch);
 }
 
@@ -114,17 +129,22 @@ void OrderStore::on_order_state_update(
          rec.set_key(ev.key.value());
          rec.set_parent_key(ev.parent_key.value());
          modified_orders.insert(ev.key);
+         const auto now = utils::pb::pb_now();
          if (rec.has_state()) {
+            google::protobuf::Timestamp created;
+            if (rec.state().has_created()) {
+               created = rec.state().created();
+            }
             rec.mutable_state()->CopyFrom(ev.rebuild_state());
-            *rec.mutable_state()->mutable_created() = utils::pb::pb_now();
-            *rec.mutable_state()->mutable_last_change() = rec.state().created();
+            *rec.mutable_state()->mutable_created() =
+                created.seconds() > 0 || created.nanos() > 0 ? created : now;
             // if filled qty went from 0.99 -> 1, it was copying the int part and not clearing the fraction part,
             // so changing MergeFrom to CopyFrom
          } else {
             *rec.mutable_state() = ev.rebuild_state();
-            *rec.mutable_state()->mutable_created() = utils::pb::pb_now();
+            *rec.mutable_state()->mutable_created() = now;
          }
-         *rec.mutable_state()->mutable_last_change() = utils::pb::pb_now();
+         *rec.mutable_state()->mutable_last_change() = now;
          LOG(OrderMan, Info, "OrderStore::on_order_state_update k: {} rec: {}",
              ev.key, utils::pb::to_string(rec));
       }

src/lib/orderman/gateways/OkxExecutionGateway.cpp

@@ -6,6 +6,7 @@
 #include "lib/io/http/http_client.h"
 #include "lib/io/http/http_response.h"
 #include "lib/io/websocket/websocket_client.h"
+#include "lib/io/websocket/websocket_client_config.h"
 #include "lib/logging/logging.h"
 #include "lib/md/SecurityOracle.h"
 #include "lib/orderman/CancelOrderMsg.h"
@@ -77,6 +78,16 @@ void OkxExecutionGateway::send_msgs(
           msg);
    }
 }
+
+static websocket::WebsocketClientConfig get_ws_client_config(
+    const std::string &url,
+    core::TimeSpan connect_timeout,
+    core::TimeSpan pong_timeout) {
+   websocket::WebsocketClientConfig rv{url, connect_timeout, pong_timeout};
+   rv.name = "OkxExecutionGateway";
+   return rv;
+}
+
 struct OkxExecutionGateway::PImpl {
    const std::string api_key = config::get<config::OkxExeCfg>().api_key();
    const std::string secret_key = config::get<config::OkxExeCfg>().secret_key();
@@ -99,8 +110,8 @@ struct OkxExecutionGateway::PImpl {
    md::SecurityOracle o;
    std::atomic<OkxExecutionState> state = OkxExecutionState::OFF;
    PImpl(md::SecurityOracle o)
-       : ws_client(
-             websocket::WebsocketClient({url, connect_timeout, pong_timeout})),
+       : ws_client(websocket::WebsocketClient(
+             get_ws_client_config(url, connect_timeout, pong_timeout))),
          cb(dummy), o(o) {
       ws_client.message_handler().bind<&PImpl::on_message>(*this);
       ws_client.close_handler().bind<&PImpl::on_close>(*this);
@@ -284,25 +295,38 @@ struct OkxExecutionGateway::PImpl {
 
    void start_impl() {
       size_t attempt = 0;
-      for (size_t version = 0; state != OkxExecutionState::OFF; version++) {
+      while (state != OkxExecutionState::OFF) {
          try {
             LOG(OrderMan, Warn,
                 "OkxExecutionAdapter::start Attempting login (Attempt: {})",
                 attempt + 1);
             if (ws_run_thread) {
+               // Join this thread to block until the websocket disconnects (then reconnect)
                if (ws_run_thread->joinable()) {
                   ws_run_thread->join();
                }
                ws_run_thread.reset();
             }
+            // Connect
             ws_client.connect();
             ws_run_thread = threading::make_thread(
-                "okx_execution_ws", [&, orig_version = version]() {
-                   while (version == orig_version) {
-                      ws_client.run_one();
-                   }
-                });
-            sleep(2);
+                "okx_execution_ws", [&]() { ws_client.run(); });
+            // Block until the websocket connects (or times out)
+            {
+               const auto start_time = bts::core::TimeStamp::now();
+               for (auto elapsed = core::TimeSpan::zero();
+                    !ws_client.is_connected() &&
+                    elapsed < core::TimeSpan::from_seconds(20);
+                    elapsed = core::TimeStamp::now() - start_time) {
+                  std::this_thread::sleep_for(std::chrono::milliseconds(500));
+               }
+               if (!ws_client.is_connected()) {
+                  LOG(OrderMan, Warn,
+                      "OkxExecutionAdapter::start. Timed out waiting for "
+                      "websocket to connect");
+                  continue;
+               }
+            }
             if (ws_ping_thread.has_value()) {
                if (ws_ping_thread->joinable()) {
                   ws_ping_thread
@@ -312,7 +336,7 @@ struct OkxExecutionGateway::PImpl {
                    .reset(); // Reset the optional before creating a new thread
             }
 
-            ws_ping_thread.emplace([&, orig_version = version]() {
+            ws_ping_thread.emplace([&]() {
                LOG(OrderMan, Warn,
                    "OkxExecutionAdapter::ws_ping_thread thread starting. url: "
                    "{}",
@@ -323,7 +347,7 @@ struct OkxExecutionGateway::PImpl {
                std::this_thread::sleep_for(
                    std::chrono::milliseconds(ping_interval.millis()));
 
-               while (ws_client.is_connected() && version == orig_version) {
+               while (ws_client.is_connected()) {
                   const auto now = bts::core::TimeStamp::now();
                   if ((now - last_ping_time) > ping_interval) {
                      LOG(OrderMan, Warn,

src/lib/orderman/orders/algo/AlgoWrapperOrder.cpp

@@ -248,7 +248,7 @@ OrderOpResult AlgoWrapperOrder::on_activate(
       me->set_price_quantity(p, q);
       for (auto &ct : me->child_templates)
          ct->builder.activate(*me->services);
-      if (q.value() == 0) {
+      if (q.full_value() == 0) {
          ReportStateGuard _(*this);
          terminate(ORDER_STATUS_COMPLETED);
       } else if (is_attaching())
@@ -292,8 +292,11 @@ OrderOpResult AlgoWrapperOrder::on_update(Update const &update) {
    }
    case Status::WORKING: {
       ReportStateGuard _(*this);
+      const double frac_qty = update.has_fractional_quantity()
+                                  ? update.fractional_quantity().fraction()
+                                  : 0;
       me->set_price_quantity(
-          Price(update.price()), Quantity(update.quantity()));
+          Price(update.price()), Quantity(update.quantity(), frac_qty));
       if (update.has_custom()) {
          *me->definition.mutable_custom_def() = update.custom();
          me->services->report_definition(me->definition);
@@ -313,6 +316,7 @@ OrderOpResult AlgoWrapperOrder::on_update(Price p, Quantity q) {
    Update up;
    up.set_price(p.value());
    up.set_quantity(q.value());
+   up.mutable_fractional_quantity()->set_fraction(q.fractional_part());
    return on_update(up);
 }
 
@@ -396,11 +400,17 @@ std::vector<api::Security const *> AlgoWrapperOrder::known_securities() const {
 Price AlgoWrapperOrder::price() const { return Price(me->state().price()); }
 
 Quantity AlgoWrapperOrder::quantity() const {
-   return Quantity(me->state().quantity());
+   const auto &s = me->state();
+   return s.has_fractional_quantity()
+              ? Quantity{s.quantity(), s.fractional_quantity().fraction()}
+              : Quantity{s.quantity()};
 }
 
 Quantity AlgoWrapperOrder::total_fills() const {
-   return Quantity(me->state().filled_quantity());
+   const auto &s = me->state();
+   return s.has_fractional_filled_quantity()
+              ? Quantity{s.filled_quantity(), s.fractional_filled_quantity().fraction()}
+              : Quantity{s.filled_quantity()};
 }
 
 bool AlgoWrapperOrder::is_shutting_down() const {

src/lib/orderman/orders/algo/algo_orders/IcebergAlgo.cpp

@@ -46,7 +46,7 @@ OrderOpResult IcebergAlgo::do_before_launch(IBeforeLaunchAlgoServices &serv) {
           "Show quantity must be greater than 0"};
 
    me->pause_between_orders_ms = iceberg.pause_between_orders_ms();
-   me->show_quantity = Quantity((uint32_t)iceberg.show_quantity());
+   me->show_quantity = CQuantity::create_new(iceberg.show_quantity());
 
    // Create a child template so we can quickly launch our child when
    // the time comes.
@@ -91,7 +91,11 @@ void IcebergAlgo::do_update(api::orderman::Update const &update) {
       return;
    }
 
-   if (update.quantity() < algo().total_fills().value()) {
+   const double frac_qty = update.has_fractional_quantity()
+                               ? update.fractional_quantity().fraction()
+                               : 0;
+   const Quantity new_qty{update.quantity(), frac_qty};
+   if (new_qty < algo().total_fills()) {
       algo().complete();
       return;
    }
@@ -108,7 +112,7 @@ void IcebergAlgo::do_update(api::orderman::Update const &update) {
          return;
       }
 
-      me->show_quantity = Quantity((uint32_t)iceberg_update.show_quantity());
+      me->show_quantity = CQuantity::create_new(iceberg_update.show_quantity());
       bool had_timer = me->pause_between_orders_ms > 0;
       bool should_have_timer = iceberg_update.pause_between_orders_ms() > 0;
 
@@ -122,7 +126,10 @@ void IcebergAlgo::do_update(api::orderman::Update const &update) {
          init_child_template(algo(), iceberg_update.child());
       }
    }
-   child_update.set_quantity(remaining_quantity().value());
+   const auto rem_qty = remaining_quantity();
+   child_update.set_quantity(rem_qty.value());
+   child_update.mutable_fractional_quantity()->set_fraction(
+       rem_qty.fractional_part());
    if (me->status == Status::WORKING)
       algo().update_child(me->child_key, child_update);
 }
@@ -130,10 +137,9 @@ void IcebergAlgo::do_update(api::orderman::Update const &update) {
 void IcebergAlgo::on_child_change(OrderKey k, ChildState const &st) {
    if (k == me->child_key) {
       algo().update_fill_state(
-          Quantity(algo().total_fills().value() + st.last_fills.value()),
-          st.last_price);
+          algo().total_fills() + st.last_fills, st.last_price);
 
-      if (algo().total_fills().value() >= algo().quantity().value()) {
+      if (algo().total_fills() >= algo().quantity()) {
          algo().complete();
          return;
       }
@@ -164,7 +170,7 @@ void IcebergAlgo::on_timer() {
 
 void IcebergAlgo::send_next_order() {
    bts_assert(me->status == Status::WORKING);
-   bts_assert(algo().quantity().value() > algo().total_fills().value());
+   bts_assert(algo().quantity() > algo().total_fills());
 
    me->child_key = algo().launch_child(
        *me->child_template, algo().price(), remaining_quantity());
@@ -172,9 +178,8 @@ void IcebergAlgo::send_next_order() {
    LOG(OrderMan, Info,
        "Iceberg order {} creating new child order {} with size "
        "{} price {}. Filled {} so far out of {}",
-       algo().order_key().value(), me->child_key.value(),
-       remaining_quantity().value(), algo().price().value(),
-       algo().total_fills().value(), algo().quantity().value());
+       algo().order_key(), me->child_key, remaining_quantity(), algo().price(),
+       algo().total_fills(), algo().quantity());
 }
 
 void orderman::IcebergAlgo::init_child_template(
@@ -186,9 +191,9 @@ void orderman::IcebergAlgo::init_child_template(
 }
 
 Quantity IcebergAlgo::remaining_quantity() const {
-   bts_assert(algo().quantity().value() > algo().total_fills().value());
-   auto remaining = algo().quantity().value() - algo().total_fills().value();
-   return Quantity(std::min(remaining, me->show_quantity.value()));
+   bts_assert(algo().quantity() > algo().total_fills());
+   auto remaining = algo().quantity() - algo().total_fills();
+   return std::min(remaining, me->show_quantity);
 }
 
 } // namespace orderman

src/lib/orderman/orders/spreader_orders/SpreaderOrder.cpp

@@ -68,6 +68,8 @@ OrderOpResult SpreaderOrder::on_activate(
       return OrderOpResult{api::orderman::ORDER_ERROR_OP_NOT_SUPPORTED};
    quantity =
        _bookkeeping_data().fqty_helper().in_transform_spreader_qty(quantity);
+   LOG(OrderMan, Info, "SpreaderOrder::on_activate price: {} quantity: {}",
+       price, quantity);
    SpreaderOrderServices spreader_order_services(&services);
    _on_activate(price, quantity, spreader_order_services);
    // the reason why this is immediatly reported (vs in the timer
@@ -82,8 +84,9 @@ OrderOpResult SpreaderOrder::on_update(Price price, Quantity quantity) {
       return OrderOpResult{api::orderman::ORDER_ERROR_OP_NOT_SUPPORTED};
    quantity =
        _bookkeeping_data().fqty_helper().in_transform_spreader_qty(quantity);
-   LOG(OrderMan, Info, "SpreaderOrder.on_update key:{} price:{} quantity:{}",
-       key().value(), price.value(), quantity.value());
+
+   LOG(OrderMan, Info, "SpreaderOrder::on_update price: {} quantity: {}", price,
+       quantity);
    _state_worker->on_update(price, quantity);
    _report_state();
    return OrderOpResult{};
@@ -329,8 +332,11 @@ void SpreaderOrder::_on_activate(
     Price price, Quantity quantity, SpreaderOrderServices &order_services) {
    bts_assert(_inited());
    bts_assert(!_activated());
-   quantity =
-       _bookkeeping_data().fqty_helper().in_transform_spreader_qty(quantity);
+   LOG(OrderMan, Info, "SpreaderOrder::_on_activate price: {} quantity: {}",
+       price, quantity);
+   // quantity has already been fractional_normalized
+   // quantity =
+   //     _bookkeeping_data().fqty_helper().in_transform_spreader_qty(quantity);
    LOG(OrderMan, Info, "SpreaderOrder.on_activate key:{} price:{} quantity:{}",
        key().value(), price.value(), quantity.value());
    _order_services = new SpreaderOrderServices(order_services);

src/lib/orderman/orders/spreader_orders/SpreaderOrder.h

@@ -285,6 +285,9 @@ class SpreaderOrder : public Order, public ISpreaderManager {
    }
    // just sets the price and qty in the bookkeeping data
    void _set_price_quantity(Price input_price, Quantity quantity) {
+      LOG(OrderMan, Info,
+          "SpreaderOrder::_set_price_quantity input_price: {} quantity: {}",
+          input_price, quantity);
       Price effective_price = _compute_effective_price(input_price);
       _bookkeeping_data().update_order(input_price, effective_price, quantity);
    }

src/lib/orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.h

@@ -1,4 +1,5 @@
 #pragma once
+#include "lib/logging/logging.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"
 #include <boost/container/flat_map.hpp>
 #include <memory>
@@ -76,7 +77,11 @@ class SpreaderBookkeepingData {
    }
    Price input_price() const { return _input_price; }
    Price effective_price() const { return _effective_price; }
-   Quantity quantity() const { return _quantity; }
+   Quantity quantity() const {
+      LOG(OrderMan, Info, "SpreaderBookkeepingData::quantity _quantity: {}",
+          _quantity);
+      return _quantity;
+   }
    NLegsHedgeRecompData &hedge_recomp_data() { return _hedge_recomp_data; }
    // gets leg data from leg's sec key
    SpreaderLegData &get_leg_data(SecurityKey security_key) {
@@ -146,6 +151,10 @@ class SpreaderBookkeepingData {
    }
    void update_order(
        Price input_price, Price effective_price, Quantity quantity) {
+      LOG(OrderMan, Info,
+          "SpreaderBookkeepingData::update_order input_price: {} "
+          "effective_price: {} quantity: {}",
+          input_price, effective_price, quantity.full_value());
       _input_price = input_price;
       _effective_price = effective_price;
       _quantity = quantity;
@@ -157,9 +166,6 @@ class SpreaderBookkeepingData {
    }
    size_t n_quoting_legs() const { return _n_quoting_legs; }
    SpreaderFractionalQtyHelper &fqty_helper() { return _fqty_helper; }
-   // SpreaderFractionalQtyHelper const *spreader_qty_helper() {
-   //    return &_fqty_helper;
-   // }
 };
 } // namespace orderman
 } // namespace bts

src/lib/orderman/orders/spreader_orders/spreader/SpreaderFractionalQtyHelper.cpp

@@ -9,6 +9,7 @@
 #include "lib/orderman/events/BookUpdateEvent.h"
 #include "lib/orderman/events/OrderStateUpdateEvent.h"
 //#include "lib/orderman/orders/spreader_orders/SpreaderOrder.h"
+#include "lib/orderman/orderman_utils/orderman_common.h"
 #include "lib/orderman/orders/spreader_orders/spreader/LegPropertyData.h"
 #include "lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h"
 #include "lib/os/os.h"
@@ -63,7 +64,9 @@ struct SpreaderFractionalQtyHelper ::PImpl {
    Book op_book = {};
    OrderStateUpdateEvent op_ord = {};
    const bool passthrough = false;
-   //const double in_qty_mult;
+   // we can keep the mult per leg, based on their tick info, but its easier
+   // (and correct) if we do it
+   const double in_qty_mult = 100000;
    const bool is_notional;
    const QtyTickInfo def_qty_tick_info = QtyTickInfo(10000.0, 1.0);
    std::map<bts::SecurityKey, QtyTickInfo> qty_info_map = {};
@@ -92,25 +95,43 @@ struct SpreaderFractionalQtyHelper ::PImpl {
    }
 };
 
+double SpreaderFractionalQtyHelper::_get_mult(size_t) const {
+   return me->in_qty_mult;
+}
+
+Quantity SpreaderFractionalQtyHelper::in_transform_spreader_qty(
+    Quantity qty) const {
+   // nothing needs to be done with spreader qty
+   if (me->passthrough)
+      return qty;
+   double qm = _get_mult(0);
+   double rv = qty.full_value() * qm;
+   auto rv_qty = Quantity::create_new(rv);
+   LOG(OrderMan, Info,
+       "SpreaderFractionalQtyHelper::in_transform_spreader_qty {} => {}", qty,
+       rv_qty);
+   return rv_qty;
+}
+
 std::vector<LegPropertyData> SpreaderFractionalQtyHelper::get_transformed_legs(
     const std::vector<LegPropertyData> &legs) const {
-   std::vector<LegPropertyData> rv = {};
-   for (const auto &l : legs) {
-      bts::api::ContractLeg leg_adj = l.leg_contract();
-      const auto &q_tinfo = get_qty_tick_info(
-          me->qty_info_map, bts::SecurityKey(l.leg_security_key()),
-          me->def_qty_tick_info);
-      double qm = q_tinfo.tick_mult;
-      leg_adj.set_quantity(l.size_multiplier() * qm);
-      double qty_tick_size = q_tinfo.tick_sz;
-      LegPropertyData lpd_adj(
-          l.leg_index(), leg_adj, l.leg_params(), l.trading_side(),
-          bts::api::SyntheticContract::PriceCalc::
-              SyntheticContract_PriceCalc_PRICECALC_STANDARD,
-          qty_tick_size);
-      rv.push_back(lpd_adj);
-   }
-   return rv;
+   // std::vector<LegPropertyData> rv = {};
+   // for (const auto &l : legs) {
+   //    bts::api::ContractLeg leg_adj = l.leg_contract();
+   //    const auto &q_tinfo = get_qty_tick_info(
+   //        me->qty_info_map, bts::SecurityKey(l.leg_security_key()),
+   //    double qm = q_tinfo.tick_mult;
+   //    leg_adj.set_quantity(l.size_multiplier() * qm);
+   //    double qty_tick_size = q_tinfo.tick_sz;
+   //    LegPropertyData lpd_adj(
+   //        l.leg_index(), leg_adj, l.leg_params(), l.trading_side(),
+   //        bts::api::SyntheticContract::PriceCalc::
+   //            SyntheticContract_PriceCalc_PRICECALC_STANDARD,
+   //        qty_tick_size);
+   //    rv.push_back(lpd_adj);
+   // }
+   // return rv;
+   return legs;
 }
 
 SpreaderFractionalQtyHelper::SpreaderFractionalQtyHelper(
@@ -140,9 +161,7 @@ const Book &SpreaderFractionalQtyHelper::in_transform_book(
    } else {
       me->op_book = bk;
    }
-   const auto &q_tinfo = get_qty_tick_info(
-       me->qty_info_map, bk.security_key(), me->def_qty_tick_info);
-   double qm = q_tinfo.tick_mult;
+   double qm = _get_mult(0);
    for (auto &ps : me->op_book.bid()) {
       ps = bts::md::PriceSize(ps.int_price(), ps.size() * qm);
    }
@@ -183,9 +202,7 @@ SpreaderFractionalQtyHelper::in_transform_order_event(
       me->op_ord.filled_at_attach_time_quantity = os::nan();
    }
    // factor mult
-   const auto &q_tinfo =
-       get_qty_tick_info(me->qty_info_map, sec_key, me->def_qty_tick_info);
-   double qmult = q_tinfo.tick_mult;
+   double qmult = _get_mult(sec_key.value());
    me->op_ord.quantity *= qmult;
    me->op_ord.filled_quantity *= qmult;
    me->op_ord.filled_at_attach_time_quantity *= qmult;
@@ -193,14 +210,6 @@ SpreaderFractionalQtyHelper::in_transform_order_event(
    return me->op_ord;
 }
 
-Quantity SpreaderFractionalQtyHelper::in_transform_spreader_qty(
-    Quantity qty) const {
-   // nothing needs to be done with spreader qty
-   if (me->passthrough)
-      return qty;
-   return qty;
-}
-
 Quantity SpreaderFractionalQtyHelper::out_transform_leg_order_qty(
     SecurityKey skey, Price price, Quantity qty) const {
    auto qty_init = qty;
@@ -214,13 +223,10 @@ Quantity SpreaderFractionalQtyHelper::out_transform_leg_order_qty(
              "notional -> physical {} -> {}",
              qty_init, qty);
       }
-      const auto &q_tinfo =
-          get_qty_tick_info(me->qty_info_map, skey, me->def_qty_tick_info);
-      double tick_sz = q_tinfo.tick_sz;
-      double qm = q_tinfo.tick_mult;
+      double qm = _get_mult(skey.value());
       double qty_out = qty.full_value() / qm;
       // rouning it to tick size
-      qty_out = std::floor(qty_out / tick_sz) * tick_sz;
+      //qty_out = std::floor(qty_out / tick_sz) * tick_sz;
       return Quantity::create_new(qty_out);
    };
    auto rv_qty = tr();
@@ -241,9 +247,15 @@ void SpreaderFractionalQtyHelper::out_transform_order_state(
        s_txt, s.filled_quantity(), s.avg_price());
    // todo:
    // this needs to divide the filled qty of the spreader
-   // if (me->passthrough)
-   //    return ord_st;
-   // return ord_st;
+   if (me->passthrough)
+      return;
+   double qm = _get_mult(0);
+   set_quantity(
+       s, StateQuantityType::OrderQuantity,
+       Quantity::create_new(static_cast<double>(s.quantity()) / qm));
+   set_quantity(
+       s, StateQuantityType::FilledQuantity,
+       static_cast<double>(s.filled_quantity()) / qm);
 }
 
 } // namespace bts::orderman

src/lib/orderman/orders/spreader_orders/spreader/SpreaderFractionalQtyHelper.h

@@ -15,31 +15,52 @@ class OrderState;
 namespace bts::orderman {
 class Book;
 class OrderStateUpdateEvent;
-// forinstruments with fractional qtys this sits on the outside layer and does conversions,
-//both for market data and orders
-// this resdies inside SpeaderBookkeepingData
+// for instruments with fractional qtys this sits on the outside layer and does conversions,
+// both for market data and orders
+// it resides inside SpeaderBookkeepingData
+// these are the trasnfo
+// spreader_qty in *mult
+// books coming in *mult
+// order out /mult
+// order cb in * mult
+// report out /mult
 class SpreaderFractionalQtyHelper {
  public:
    SpreaderFractionalQtyHelper(
        OrderKey spreader_ord_key,
        const api::orderman::SpreaderOrderDefinition &spreader_order_def,
        const std::vector<LegPropertyData> &legs);
-   // the leg qtys need to be translated
+   double _get_mult(size_t leg_idx) const;
+   ////////////////////////////////////
+   // 1. qty (in)
+   ////////////////////////////////////
+   Quantity in_transform_spreader_qty(Quantity qty) const;
+   ////////////////////////////////////
+   // 2. leg sizes (in)
+   ////////////////////////////////////
    std::vector<LegPropertyData> get_transformed_legs(
        const std::vector<LegPropertyData> &legs) const;
-   // in_ and out_ signify market data / orders coming in or out.
+   ////////////////////////////////////
+   // 3. books (in)
+   ////////////////////////////////////
    const Book &in_transform_book(const Book &book) const;
    // this is called when the order feedback fomr the outside in to the spreader
+   ////////////////////////////////////
+   // 4. orders event (in)
+   ////////////////////////////////////
    const OrderStateUpdateEvent &in_transform_order_event(
        SecurityKey sec_key, const OrderStateUpdateEvent &) const;
-   Quantity in_transform_spreader_qty(Quantity qty) const;
    // TODO add price
+   ////////////////////////////////////
+   // 5. orders (out)
+   ////////////////////////////////////
    Quantity out_transform_leg_order_qty(
        SecurityKey key, Price price, Quantity qty) const;
    // the state reported ot he rest of the system also has to be converted
+   ////////////////////////////////////
+   // 6. order report (out)
+   ////////////////////////////////////
    void out_transform_order_state(bts::api::orderman::OrderState &) const;
-   // struct QtyTickInfo;
-   // const QtyTickInfo &_get_tickinfo(SecurityKey skey);
 
  private:
    struct PImpl;

src/lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp

@@ -134,15 +134,14 @@ bool SpreaderUtils::resolve_synthetic_spread(
                          bts::api::orderman::TimeInForce::TIME_IN_FORCE_IOC);
                   else
                      quoter_order_def.set_time_in_force(
-                         bts::api::orderman::TimeInForce::TIME_IN_FORCE_DAY);
+                         order_def.time_in_force());
                   // hedger order def
                   api::orderman::OrderDefinition hedger_order_def =
                       quoter_order_def;
                   hedger_order_def.mutable_custom_def()
                       ->mutable_limit()
                       ->MergeFrom(api::orderman::LimitOrderDefinition());
-                  hedger_order_def.set_time_in_force(
-                      bts::api::orderman::TimeInForce::TIME_IN_FORCE_DAY);
+                  quoter_order_def.set_time_in_force(order_def.time_in_force());
                   hedger_order_def.set_sub_type(
                       bts::api::orderman::OrderSubType::
                           ORDER_SUB_TYPE_HEDGING_LEG);

src/lib/orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp

@@ -1,4 +1,5 @@
 #include "SpreaderStartStateWorker.h"
+#include "lib/logging/logging.h"
 #include "lib/orderman/OrderBuilder.h"
 #include "lib/orderman/orders/PreActivationOrderServices.h"
 #include "lib/orderman/orders/spreader_orders/SpreaderOrder.h"
@@ -47,6 +48,8 @@ void SpreaderStartStateWorker::on_init_new(
       std::string msg = "Could not resolve securities";
       _spreader->_transform_to_END(&msg);
    }
+   LOG(OrderMan, Info, "SpreaderStartStateWorker::on_init_new qty: {}",
+       _spreader->_bookkeeping_data().quantity().full_value());
 }
 void SpreaderStartStateWorker::on_init_recovered(
     const bts::api::orderman::OrderRecord &order_record,

src/lib/os/os.cpp

@@ -638,7 +638,7 @@ int num_cores() {
 #endif
 }
 
-std::string expand_env(std::string name, bool assert_vars_exist = true) {
+std::string expand_env(std::string name, bool assert_vars_exist) {
    static const std::regex env_re{R"--(\$\{([^}]+)\})--"};
    std::smatch match;
    while (std::regex_search(name, match, env_re)) {
@@ -656,6 +656,26 @@ std::string expand_env(std::string name, bool assert_vars_exist = true) {
    return name;
 }
 
+bool expand_env_in_place(std::string &name, bool assert_vars_exist) {
+   static const std::regex env_re{R"--(\$\{([^}]+)\})--"};
+   std::smatch match;
+   bool changed = false;
+   while (std::regex_search(name, match, env_re)) {
+      const auto from = match[0];
+      const auto var_name = match[1].str();
+      const auto var = std::getenv(var_name.c_str());
+      if (!var) {
+         bts_assert_msgf(
+             !assert_vars_exist, "Environment variable '{}' is undefined",
+             name);
+         continue;
+      }
+      name.replace(from.first, from.second, var);
+      changed = true;
+   }
+   return changed;
+}
+
 std::optional<std::string> try_expand_env(const std::string &name) {
    static const std::regex env_re{R"--(\$\{([^}]+)\})--"};
    std::smatch match;

src/lib/os/os.h

@@ -90,6 +90,9 @@ boost::filesystem::path app_data_path();
 boost::filesystem::path app_data_path(
     const boost::filesystem::path &relative_path, bool mkdirs = true);
 
+std::string expand_env(std::string name, bool assert_vars_exist = true);
+bool expand_env_in_place(std::string &name, bool assert_vars_exist = true);
+
 std::string username();
 std::string find_newest_file(
     const std::string &directory,

src/lib/persistence/IDGenerator.cpp

@@ -73,7 +73,7 @@ std::optional<uint64_t> IDGenerator::get_id(
       return std::optional<uint64_t>(rowid);
    } else {
       if (add_if_new) {
-         auto all = me->kv_store->retrieve_all();
+         // auto all = me->kv_store->retrieve_all();
          auto new_id = me->kv_store->insert(ns, key, "");
          LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key,
              new_id);

src/lib/utils/lazy.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <mutex>
+
+namespace bts::utils {
+
+template <typename T>
+class Lazy {
+ public:
+   template <typename... Args>
+   explicit Lazy(Args &&...args)
+       : create_([=]() mutable { return std::make_unique<T>(args...); }) {}
+
+   T &get() {
+      std::call_once(init_flag_, [this]() { instance_ = create_(); });
+      return *instance_;
+   }
+
+   static Lazy<T> from_delegate(std::function<T()> create);
+
+ private:
+   explicit Lazy(std::function<T()> create)
+       : create_([&, create]() { return std::make_unique<T>(create()); }) {}
+
+   std::once_flag init_flag_;
+   std::function<std::unique_ptr<T>()> create_;
+   std::unique_ptr<T> instance_;
+};
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+template <typename T>
+Lazy<T> Lazy<T>::from_delegate(std::function<T()> create) {
+   return Lazy<T>(create);
+}
+
+} // namespace bts::utils

src/lib_binance_execution/binance_execution/cross_margin/cross_margin_gateway.cpp

@@ -111,8 +111,6 @@ struct CrossMarginGateway::PImpl {
       pos.exchange = VENUE;
    }
 
-   static constexpr const char *CANCEL_REPLACE_CANCEL_SUFFIX = "crc";
-
    bts::md::SecurityOracle sec_oracle;
    GatewayConfig config;
    shared_ptr<bts::config::JetFIX_Connection> jetfix_connection;
@@ -152,16 +150,6 @@ struct CrossMarginGateway::PImpl {
 
    void on_order_update(trade_data_stream::OrderUpdate u) {
       INFO_FN("on_order_update", "\n{:2}", json_formatter(u));
-      if (u.orig_client_order_id.ends_with(CANCEL_REPLACE_CANCEL_SUFFIX)) {
-         // This is the "canceled" order part of a cancel/replace op. This op
-         // generates another update without such a suffix. That's the new
-         // order; that's the one we want.
-         DIAG_FN(
-             "on_order_update",
-             "Ignoring cancel/replace cancel order update. id: {}",
-             u.client_order_id);
-         return;
-      }
       incoming_order_updates.push(std::move(u));
    }
 
@@ -268,7 +256,7 @@ struct CrossMarginGateway::PImpl {
       trade_data_stream::OrderUpdate curr;
       auto &km = *key_map;
       while (incoming_order_updates.pop(curr)) {
-         auto bts_key = km.get_bts_key(curr.client_order_id);
+         const auto bts_key = km.get_bts_key(curr.client_order_id);
          if (!bts_key) {
             ERROR_FN(
                 "dispatch_order_updates",
@@ -277,7 +265,7 @@ struct CrossMarginGateway::PImpl {
                 curr.client_order_id);
             continue;
          }
-         auto exch_key = km.get_exch_key(*bts_key);
+         const auto exch_key = km.get_exch_key(*bts_key);
          if (exch_key != curr.client_order_id) {
             INFO_FN(
                 "dispatch_order_updates",

src/lib_binance_execution/binance_execution/cross_margin/cross_margin_gateway_config.h

@@ -1,26 +0,0 @@
-#pragma once
-
-#include <nlohmann/json.hpp>
-#include <string>
-
-namespace binance_execution::cross_margin {
-
-struct CrossMarginGatewayConfig {
-   std::string api_key;
-   std::string secret_key;
-   std::string account;
-   std::string rest_endpoint;
-   std::string ws_endpoint;
-   std::string ws_stream_endpoint;
-};
-
-NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(
-    CrossMarginGatewayConfig,
-    api_key,
-    secret_key,
-    account,
-    rest_endpoint,
-    ws_endpoint,
-    ws_stream_endpoint);
-
-} // namespace binance_execution::cross_margin

src/lib_binance_execution/binance_execution/cross_margin/messages/create_oco_request.h

@@ -1,49 +0,0 @@
-// #pragma once
-
-// #include "binance_execution/enums/new_order_response_type.h"
-// #include "binance_execution/enums/order_type.h"
-// #include "binance_execution/enums/side.h"
-// #include "binance_execution/enums/time_in_force.h"
-
-// #include <nlohmann/json.hpp>
-// #include <string>
-
-// namespace bts::orderman {
-// struct CreateOrderMsg;
-// }
-
-// namespace binance_execution::cross_margin {
-
-// /**
-//  * @a https://developers.binance.com/docs/binance-spot-api-docs/rest-api/trading-endpoints#new-order-trade
-//  */
-// struct CreateOcoRequest {
-//    std::string symbol;
-//    std::string listClientOrderId;
-//    Side side = Side::NONE;
-//    double quantity = 0;
-//    std::string limitClientOrderId;
-//    double price = std::numeric_limits<double>::quiet_NaN();
-//    double limitIcebergQty = 0;
-//    std::string stopClientOrderId;
-//    double stopPrice;
-//    double stopLimitPrice;
-//    double stopIcebergQty;
-//    TimeInForce stopLimitTimeInForce;
-//    NewOrderResponseType newOrderRespType;
-//    OrderType type = OrderType::NONE;
-//    std::int64_t timestamp = 0;
-//    std::string newClientOrderId;
-//    TimeInForce timeInForce = TimeInForce::NONE;
-//    std::int64_t recvWindow = 0;
-//    NewOrderResponseType newOrderRespType = NewOrderResponseType::NONE;
-
-//    bool init(const bts::orderman::CreateOrderMsg &m, std::string &error);
-// };
-
-// void to_json(nlohmann::json &j, const CreateOcoRequest &req);
-// void from_json(const nlohmann::json &j, CreateOcoRequest &req);
-
-// void to_query_params(std::string &query_params, const CreateOcoRequest &req);
-
-// } // namespace binance_execution::spots

src/lib_binance_execution/binance_execution/cross_margin/rest_api_client.cpp

@@ -2,6 +2,7 @@
 #include "lib/io/http/http_client.h"
 #include "lib/io/http/http_response.h"
 #include "lib/utils/crypto/hmac.h"
+#include "lib/utils/type_name.h"
 
 namespace binance_execution::cross_margin {
 
@@ -11,23 +12,31 @@ using bts::system::crypto::hmac_digest_hex;
 using std::string;
 
 #define _LOG_FN(level, fn, ...)                                                \
-   LOG(OrderMan, level, "Binance::" fn ". " __VA_ARGS__)
+   LOG(OrderMan, level, "BinanceCrossMargin::" fn ". " __VA_ARGS__)
 #define DIAG_FN(fn, ...) _LOG_FN(bts::Diag, fn, __VA_ARGS__)
 #define INFO_FN(fn, ...) _LOG_FN(bts::Info, fn, __VA_ARGS__)
 #define WARN_FN(fn, ...) _LOG_FN(bts::Warn, fn, __VA_ARGS__)
 #define ERROR_FN(fn, ...) _LOG_FN(bts::Error, fn, __VA_ARGS__)
 
 template <std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_string(std::string &str) {
+static void shrink_str(std::string &str, const char *name) {
    if (str.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_str",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, str.size(), DEFAULT_SIZE, MAX_SIZE);
       str.resize(DEFAULT_SIZE);
       str.shrink_to_fit();
    }
 }
 
 template <class T, std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_vec(std::vector<T> &vec) {
+static void shrink_vec(std::vector<T> &vec, const char *name) {
    if (vec.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_vec",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, vec.size(), DEFAULT_SIZE, MAX_SIZE);
       vec.resize(DEFAULT_SIZE);
       vec.shrink_to_fit();
    }
@@ -52,11 +61,13 @@ void RestApiClient::request_(
     const std::string &path,
     bts::io::http::HttpMethod method,
     const std::string &body,
+    bool sign,
     RestApiResult<ResponseT> &res) {
    // Use thread_local vars to reduce memory allocations. Shrink strings
    // to reduce memory footprint from previous requests.
    thread_local bts::http::HttpRequest http_req;
-   shrink_string<4096, 128>(http_req.body);
+   shrink_str<4096, 128>(
+       http_req.body, bts::utils::cached_type_name<RequestT>());
    http_req.reset();
    // Prepare request
    http_req.url = url_;
@@ -65,17 +76,22 @@ void RestApiClient::request_(
    http_req.method = method;
    http_req.headers["X-MBX-APIKEY"] = api_key_;
    thread_local std::string qps;
-   shrink_string<256, 16>(qps);
+   static const std::string query_params_name =
+       fmt::format("{}_params", bts::utils::cached_type_name<RequestT>());
+   shrink_str<256, 16>(qps, query_params_name.c_str());
    qps.clear();
    to_query_params(qps, req);
    if (!qps.empty()) {
-      sign_query_params_(qps);
+      if (sign) {
+         sign_query_params_(qps);
+      }
       http_req.url.append("?");
       http_req.url.append(qps);
    }
    // Send it
    thread_local HttpResponse http_res;
-   shrink_string<4096, 128>(http_res.body);
+   shrink_str<4096, 128>(
+       http_res.body, bts::utils::cached_type_name<ResponseT>());
    http_res.reset();
    HttpClient::send(http_req, &http_res);
    res.http_code = http_res.code;
@@ -102,11 +118,12 @@ void RestApiClient::request_(
 #define REQUEST_DEF(ResponseT, RequestT)                                       \
    template void RestApiClient::request_<ResponseT, RequestT>(                 \
        const RequestT &req, const std::string &path,                           \
-       bts::io::http::HttpMethod method, const std::string &body,              \
+       bts::io::http::HttpMethod method, const std::string &body, bool sign,   \
        RestApiResult<ResponseT> &res)
 
 REQUEST_DEF(CreateOrderResponse, CreateOrderRequest);
 REQUEST_DEF(CancelOrderResponse, CancelOrderRequest);
 REQUEST_DEF(ListenKeyResponse, ListenKeyRequest);
+REQUEST_DEF(RefreshListenKeyResponse, RefreshListenKeyRequest);
 
 } // namespace binance_execution::cross_margin

src/lib_binance_execution/binance_execution/cross_margin/rest_api_client.h

@@ -6,12 +6,13 @@
 #include "binance_execution/cross_margin/messages/create_order_response.h"
 #include "binance_execution/messages/listen_key_request.h"
 #include "binance_execution/messages/listen_key_response.h"
+#include "binance_execution/messages/refresh_listen_key_request.h"
+#include "binance_execution/messages/refresh_listen_key_response.h"
 #include "binance_execution/rest_api_result.h"
 #include "lib/io/http/HttpMethod.h"
 
 #include <nlohmann/json.hpp>
 #include <string>
-#include <vector>
 
 namespace binance_execution::cross_margin {
 
@@ -26,8 +27,8 @@ class RestApiClient {
        const CancelOrderRequest &req);
    const RestApiResult<ListenKeyResponse> &get_listen_key(
        const ListenKeyRequest &req = {});
-   const RestApiResult<ListenKeyResponse> &refresh_listen_key(
-       const ListenKeyRequest &req = {});
+   const RestApiResult<RefreshListenKeyResponse> &refresh_listen_key(
+       const RefreshListenKeyRequest &req = {});
 
  private:
    std::string url_;
@@ -40,6 +41,7 @@ class RestApiClient {
        const std::string &path,
        bts::io::http::HttpMethod method,
        const std::string &body,
+       bool sign,
        RestApiResult<ResponseT> &op_res);
 
    bool sign_query_params_(std::string &qp);
@@ -60,7 +62,7 @@ inline const RestApiResult<CreateOrderResponse> &RestApiClient::create_order(
    thread_local RestApiResult<CreateOrderResponse> res;
    res.reset();
    request_<CreateOrderResponse>(
-       req, path, bts::io::http::HttpMethod::POST, "", res);
+       req, path, bts::io::http::HttpMethod::POST, "", true, res);
    return res;
 }
 
@@ -70,7 +72,7 @@ inline const RestApiResult<CancelOrderResponse> &RestApiClient::cancel_order(
    thread_local RestApiResult<CancelOrderResponse> res;
    res.reset();
    request_<CancelOrderResponse>(
-       req, path, bts::io::http::HttpMethod::DELETE, "", res);
+       req, path, bts::io::http::HttpMethod::DELETE, "", true, res);
    return res;
 }
 
@@ -79,16 +81,16 @@ inline const RestApiResult<ListenKeyResponse> &RestApiClient::get_listen_key(
    static const std::string path{"/sapi/v1/userDataStream"};
    thread_local RestApiResult<ListenKeyResponse> res;
    res.reset();
-   request_(req, path, bts::io::http::HttpMethod::POST, "", res);
+   request_(req, path, bts::io::http::HttpMethod::POST, "", false, res);
    return res;
 }
 
-inline const RestApiResult<ListenKeyResponse> &
-RestApiClient::refresh_listen_key(const ListenKeyRequest &req) {
+inline const RestApiResult<RefreshListenKeyResponse> &
+RestApiClient::refresh_listen_key(const RefreshListenKeyRequest &req) {
    static const std::string path{"/sapi/v1/userDataStream"};
-   thread_local RestApiResult<ListenKeyResponse> res;
+   thread_local RestApiResult<RefreshListenKeyResponse> res;
    res.reset();
-   request_(req, path, bts::io::http::HttpMethod::PUT, "", res);
+   request_(req, path, bts::io::http::HttpMethod::PUT, "", false, res);
    return res;
 }
 

src/lib_binance_execution/binance_execution/cross_margin/trade_data_stream/trade_data_subscription.cpp

@@ -16,7 +16,7 @@
 namespace binance_execution::cross_margin::trade_data_stream {
 
 #define _LOG_FN(level, fn, ...)                                                \
-   LOG(OrderMan, level, "Binance::" fn ". " __VA_ARGS__)
+   LOG(OrderMan, level, "BinanceCrossMargin::" fn ". " __VA_ARGS__)
 #define DIAG_FN(fn, ...) _LOG_FN(bts::Diag, fn, __VA_ARGS__)
 #define INFO_FN(fn, ...) _LOG_FN(bts::Info, fn, __VA_ARGS__)
 #define WARN_FN(fn, ...) _LOG_FN(bts::Warn, fn, __VA_ARGS__)
@@ -58,6 +58,7 @@ struct TradeDataSubscription::PImpl {
       LOG(OrderMan, bts::Info, "Creating websocket client. url: {}", url);
       WebsocketClientConfig ws_stream_client_cfg;
       ws_stream_client_cfg.url = url;
+      ws_stream_client_cfg.name = "BinanceCrossMargin_TradeData";
       auto &ws_cli = ws_client.emplace(ws_stream_client_cfg);
       ws_cli.message_handler().bind<&PImpl::on_message>(*this);
       ws_cli.close_handler().bind<&PImpl::on_close>(*this);

src/lib_binance_execution/binance_execution/gateway_config.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "lib/io/JsonReader.h"
+#include "lib/os/os.h"
 
 #include <nlohmann/json.hpp>
 #include <string>
@@ -15,6 +16,15 @@ struct GatewayConfig {
    std::string ws_endpoint;
    std::string ws_stream_endpoint;
    bool use_cross_margin_api = false;
+
+   void expand_env() {
+      bts::os::expand_env_in_place(api_key);
+      bts::os::expand_env_in_place(secret_key);
+      bts::os::expand_env_in_place(account);
+      bts::os::expand_env_in_place(rest_endpoint);
+      bts::os::expand_env_in_place(ws_endpoint);
+      bts::os::expand_env_in_place(ws_stream_endpoint);
+   }
 };
 
 inline void to_json(nlohmann::json &j, const GatewayConfig &gw) {

src/lib_binance_execution/binance_execution/listen_key_requestor.cpp

@@ -1,23 +1,36 @@
 #include "listen_key_requestor.h"
 #include "lib/core/TimeStamp.h"
-#include "lib/io/json_formatter.h"
 
+#include <chrono>
 #include <string>
 #include <thread>
 
 namespace binance_execution {
 
+#define _LOG_FN(level, fn, ...)                                                \
+   LOG(OrderMan, level, "ListenKeyRequestor::" fn ". " __VA_ARGS__)
+#define DIAG_FN(fn, ...) _LOG_FN(bts::Diag, fn, __VA_ARGS__)
+#define INFO_FN(fn, ...) _LOG_FN(bts::Info, fn, __VA_ARGS__)
+#define WARN_FN(fn, ...) _LOG_FN(bts::Warn, fn, __VA_ARGS__)
+#define ERROR_FN(fn, ...) _LOG_FN(bts::Error, fn, __VA_ARGS__)
+
 using bts::core::Delegate;
 using bts::core::TimeStamp;
-using bts::io::json_formatter;
 using std::optional;
 using std::string;
 using std::thread;
 
+static bts::core::TimeSpan retry_interval(std::uint32_t n_attempts) {
+   std::int64_t secs = 5 * n_attempts;
+   secs = std::clamp(secs, 5L, 300L);
+   return bts::core::TimeSpan::from_seconds(secs);
+}
+
 struct ListenKeyRequestor::PImpl {
    std::function<RestApiResult<ListenKeyResponse>(const ListenKeyRequest &)>
        get_listen_key;
-   std::function<RestApiResult<ListenKeyResponse>(const ListenKeyRequest &)>
+   std::function<RestApiResult<RefreshListenKeyResponse>(
+       const RefreshListenKeyRequest &)>
        refresh_listen_key;
    string listen_key;
    TimeStamp listen_key_timestamp;
@@ -41,6 +54,9 @@ struct ListenKeyRequestor::PImpl {
    }
 
    void run() {
+      static constexpr const std::uint32_t MAX_RETRIES = 30;
+      // Counts consecutive failed attempts (resets to 0 on successful request)
+      std::uint32_t retry_count = 0;
       while (is_running) {
          static const auto refresh_period =
              bts::core::TimeSpan::from_seconds(60 * 30);
@@ -48,20 +64,40 @@ struct ListenKeyRequestor::PImpl {
          if (listen_key.empty()) {
             const auto &res = get_listen_key({});
             if (!res.ok()) {
+               bts_assert(retry_count < MAX_RETRIES);
+               retry_count++;
+               const auto sleep_t = retry_interval(retry_count);
+               WARN_FN(
+                   "run",
+                   "get_listen_key request failed ({}). Will retry in {:.2f} "
+                   "secs",
+                   retry_count, sleep_t.total_seconds());
+               std::this_thread::sleep_for(
+                   std::chrono::milliseconds(sleep_t.millis()));
                continue;
             }
-            bts_assert_msgf(
-                res.ok(), "Failed to get listen key. {}", json_formatter(res));
+            retry_count = 0;
             listen_key = res.response->listenKey;
             bts_assert(!listen_key.empty());
             listen_key_timestamp = now;
             callback(listen_key);
          }
          if ((now - listen_key_timestamp) > refresh_period) {
-            const auto &res = refresh_listen_key({});
-            bts_assert_msgf(
-                res.ok(), "Failed to refresh listen key. {}",
-                json_formatter(res));
+            const auto &res = refresh_listen_key({listen_key});
+            if (!res.ok()) {
+               bts_assert(retry_count < MAX_RETRIES);
+               retry_count++;
+               const auto sleep_t = retry_interval(retry_count);
+               WARN_FN(
+                   "run",
+                   "refresh_listen_key request failed ({}). Will retry in "
+                   "{:.2f} secs",
+                   retry_count, sleep_t.total_seconds());
+               std::this_thread::sleep_for(
+                   std::chrono::milliseconds(sleep_t.millis()));
+               continue;
+            }
+            retry_count = 0;
             listen_key_timestamp = now;
          }
          std::this_thread::sleep_for(std::chrono::seconds(2));
@@ -72,8 +108,8 @@ struct ListenKeyRequestor::PImpl {
 ListenKeyRequestor::ListenKeyRequestor(
     std::function<RestApiResult<ListenKeyResponse>(const ListenKeyRequest &)>
         get_listen_key,
-    std::function<RestApiResult<ListenKeyResponse>(const ListenKeyRequest &)>
-        refresh_listen_key)
+    std::function<RestApiResult<RefreshListenKeyResponse>(
+        const RefreshListenKeyRequest &)> refresh_listen_key)
     : me_(std::make_unique<PImpl>(get_listen_key, refresh_listen_key)) {}
 
 ListenKeyRequestor::~ListenKeyRequestor() = default;

src/lib_binance_execution/binance_execution/listen_key_requestor.h

@@ -2,6 +2,8 @@
 
 #include "binance_execution/messages/listen_key_request.h"
 #include "binance_execution/messages/listen_key_response.h"
+#include "binance_execution/messages/refresh_listen_key_request.h"
+#include "binance_execution/messages/refresh_listen_key_response.h"
 #include "binance_execution/rest_api_result.h"
 #include "lib/utils/Delegate.h"
 
@@ -19,8 +21,8 @@ class ListenKeyRequestor {
    explicit ListenKeyRequestor(
        std::function<RestApiResult<ListenKeyResponse>(const ListenKeyRequest &)>
            get_listen_key,
-       std::function<RestApiResult<ListenKeyResponse>(const ListenKeyRequest &)>
-           refresh_listen_key);
+       std::function<RestApiResult<RefreshListenKeyResponse>(
+           const RefreshListenKeyRequest &)> refresh_listen_key);
    ~ListenKeyRequestor();
 
    void start();

src/lib_binance_execution/binance_execution/messages/refresh_listen_key_request.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "lib/io/JsonReader.h"
+
+#include <nlohmann/json.hpp>
+#include <string>
+
+namespace binance_execution {
+
+struct RefreshListenKeyRequest {
+   std::string listenKey;
+};
+
+inline void to_json(nlohmann::json &j, const RefreshListenKeyRequest &req) {
+   j["listenKey"] = req.listenKey;
+}
+
+inline void from_json(const nlohmann::json &j, RefreshListenKeyRequest &req) {
+   const bts::io::JsonReader jr{&j};
+   jr.read_or("listenKey", req.listenKey);
+}
+
+inline void to_query_params(
+    std::string &qps, const RefreshListenKeyRequest &req) {
+   if (!req.listenKey.empty()) {
+      qps.append("listenKey=");
+      qps.append(req.listenKey);
+   }
+}
+
+} // namespace binance_execution

src/lib_binance_execution/binance_execution/messages/refresh_listen_key_response.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "lib/io/JsonReader.h"
+
+#include <nlohmann/json.hpp>
+#include <string>
+
+namespace binance_execution {
+
+struct RefreshListenKeyResponse {
+   std::string listenKey;
+};
+
+inline void to_json(nlohmann::json &j, const RefreshListenKeyResponse &res) {
+   if (!res.listenKey.empty()) {
+      j["listenKey"] = res.listenKey;
+   }
+}
+
+inline void from_json(const nlohmann::json &j, RefreshListenKeyResponse &res) {
+   const bts::io::JsonReader jr{&j};
+   jr.read_or("listenKey", res.listenKey);
+}
+
+} // namespace binance_execution

src/lib_binance_execution/binance_execution/spots/rest_api_client.cpp

@@ -4,6 +4,7 @@
 #include "lib/io/http/http_response.h"
 #include "lib/io/json_formatter.h"
 #include "lib/utils/crypto/hmac.h"
+#include "lib/utils/type_name.h"
 
 namespace binance_execution::spots {
 
@@ -20,16 +21,24 @@ using std::string;
 #define ERROR_FN(fn, ...) _LOG_FN(bts::Error, fn, __VA_ARGS__)
 
 template <std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_string(std::string &str) {
+static void shrink_str(std::string &str, const char *name) {
    if (str.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_str",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, str.size(), DEFAULT_SIZE, MAX_SIZE);
       str.resize(DEFAULT_SIZE);
       str.shrink_to_fit();
    }
 }
 
 template <class T, std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_vec(std::vector<T> &vec) {
+static void shrink_vec(std::vector<T> &vec, const char *name) {
    if (vec.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_vec",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, vec.size(), DEFAULT_SIZE, MAX_SIZE);
       vec.resize(DEFAULT_SIZE);
       vec.shrink_to_fit();
    }
@@ -54,11 +63,13 @@ void RestApiClient::request_(
     const std::string &path,
     bts::io::http::HttpMethod method,
     const std::string &body,
+    bool sign,
     RestApiResult<ResponseT> &res) {
    // Use thread_local vars to reduce memory allocations. Shrink strings
    // to reduce memory footprint from previous requests.
    thread_local bts::http::HttpRequest http_req;
-   shrink_string<4096, 128>(http_req.body);
+   shrink_str<4096, 128>(
+       http_req.body, bts::utils::cached_type_name<RequestT>());
    http_req.reset();
    // Prepare request
    http_req.url = url_;
@@ -67,17 +78,22 @@ void RestApiClient::request_(
    http_req.method = method;
    http_req.headers["X-MBX-APIKEY"] = api_key_;
    thread_local std::string qps;
-   shrink_string<256, 16>(qps);
+   static const std::string query_params_name =
+       fmt::format("{}_params", bts::utils::cached_type_name<RequestT>());
+   shrink_str<256, 16>(qps, query_params_name.c_str());
    qps.clear();
    to_query_params(qps, req);
    if (!qps.empty()) {
-      sign_query_params_(qps);
+      if (sign) {
+         sign_query_params_(qps);
+      }
       http_req.url.append("?");
       http_req.url.append(qps);
    }
    // Send it
    thread_local HttpResponse http_res;
-   shrink_string<4096, 128>(http_res.body);
+   shrink_str<4096, 128>(
+       http_res.body, bts::utils::cached_type_name<ResponseT>());
    http_res.reset();
    HttpClient::send(http_req, &http_res);
    res.http_code = http_res.code;
@@ -104,7 +120,7 @@ void RestApiClient::request_(
 #define REQUEST_DEF(ResponseT, RequestT)                                       \
    template void RestApiClient::request_<ResponseT, RequestT>(                 \
        const RequestT &req, const std::string &path,                           \
-       bts::io::http::HttpMethod method, const std::string &body,              \
+       bts::io::http::HttpMethod method, const std::string &body, bool sign,   \
        RestApiResult<ResponseT> &res)
 
 REQUEST_DEF(OrderRecord, CreateOrderRequest);
@@ -112,5 +128,6 @@ REQUEST_DEF(ModifyOrderResponse, ModifyOrderRequest);
 REQUEST_DEF(OrderRecord, CancelOrderRequest);
 REQUEST_DEF(std::vector<OrderRecord>, AllOrdersRequest);
 REQUEST_DEF(ListenKeyResponse, ListenKeyRequest);
+REQUEST_DEF(RefreshListenKeyResponse, RefreshListenKeyRequest);
 
 } // namespace binance_execution::spots

src/lib_binance_execution/binance_execution/spots/rest_api_client.h

@@ -2,6 +2,8 @@
 
 #include "binance_execution/messages/listen_key_request.h"
 #include "binance_execution/messages/listen_key_response.h"
+#include "binance_execution/messages/refresh_listen_key_request.h"
+#include "binance_execution/messages/refresh_listen_key_response.h"
 #include "binance_execution/rest_api_result.h"
 #include "binance_execution/spots/messages/all_orders_request.h"
 #include "binance_execution/spots/messages/cancel_order_request.h"
@@ -34,8 +36,8 @@ class RestApiClient {
        const AllOrdersRequest &req);
    const RestApiResult<ListenKeyResponse> &get_listen_key(
        const ListenKeyRequest &req = {});
-   const RestApiResult<ListenKeyResponse> &refresh_listen_key(
-       const ListenKeyRequest &req = {});
+   const RestApiResult<RefreshListenKeyResponse> &refresh_listen_key(
+       const RefreshListenKeyRequest &req = {});
 
  private:
    std::string url_;
@@ -48,6 +50,7 @@ class RestApiClient {
        const std::string &path,
        bts::io::http::HttpMethod method,
        const std::string &body,
+       bool sign,
        RestApiResult<ResponseT> &op_res);
 
    bool sign_query_params_(std::string &qp);
@@ -67,7 +70,8 @@ inline const RestApiResult<OrderRecord> &RestApiClient::create_order(
    static const std::string path{"/api/v3/order"};
    thread_local RestApiResult<OrderRecord> res;
    res.reset();
-   request_<OrderRecord>(req, path, bts::io::http::HttpMethod::POST, "", res);
+   request_<OrderRecord>(
+       req, path, bts::io::http::HttpMethod::POST, "", true, res);
    return res;
 }
 
@@ -76,7 +80,8 @@ inline const RestApiResult<OrderRecord> &RestApiClient::create_test_order(
    static const std::string path{"/api/v3/order/test"};
    thread_local RestApiResult<OrderRecord> res;
    res.reset();
-   request_<OrderRecord>(req, path, bts::io::http::HttpMethod::POST, "", res);
+   request_<OrderRecord>(
+       req, path, bts::io::http::HttpMethod::POST, "", true, res);
    return res;
 }
 
@@ -86,7 +91,7 @@ inline const RestApiResult<ModifyOrderResponse> &RestApiClient::modify_order(
    thread_local RestApiResult<ModifyOrderResponse> res;
    res.reset();
    request_<ModifyOrderResponse>(
-       req, path, bts::io::http::HttpMethod::POST, "", res);
+       req, path, bts::io::http::HttpMethod::POST, "", true, res);
    return res;
 }
 
@@ -95,7 +100,8 @@ inline const RestApiResult<OrderRecord> &RestApiClient::cancel_order(
    static const std::string path{"/api/v3/order"};
    thread_local RestApiResult<OrderRecord> res;
    res.reset();
-   request_<OrderRecord>(req, path, bts::io::http::HttpMethod::DELETE, "", res);
+   request_<OrderRecord>(
+       req, path, bts::io::http::HttpMethod::DELETE, "", true, res);
    return res;
 }
 
@@ -105,7 +111,7 @@ RestApiClient::get_all_orders(const AllOrdersRequest &req) {
    thread_local RestApiResult<std::vector<OrderRecord>> res;
    res.reset();
    request_<std::vector<OrderRecord>>(
-       req, path, bts::io::http::HttpMethod::GET, "", res);
+       req, path, bts::io::http::HttpMethod::GET, "", true, res);
    return res;
 }
 
@@ -114,16 +120,16 @@ inline const RestApiResult<ListenKeyResponse> &RestApiClient::get_listen_key(
    static const std::string path{"/api/v3/userDataStream"};
    thread_local RestApiResult<ListenKeyResponse> res;
    res.reset();
-   request_(req, path, bts::io::http::HttpMethod::POST, "", res);
+   request_(req, path, bts::io::http::HttpMethod::POST, "", false, res);
    return res;
 }
 
-inline const RestApiResult<ListenKeyResponse> &
-RestApiClient::refresh_listen_key(const ListenKeyRequest &req) {
+inline const RestApiResult<RefreshListenKeyResponse> &
+RestApiClient::refresh_listen_key(const RefreshListenKeyRequest &req) {
    static const std::string path{"/api/v3/userDataStream"};
-   thread_local RestApiResult<ListenKeyResponse> res;
+   thread_local RestApiResult<RefreshListenKeyResponse> res;
    res.reset();
-   request_(req, path, bts::io::http::HttpMethod::PUT, "", res);
+   request_(req, path, bts::io::http::HttpMethod::PUT, "", false, res);
    return res;
 }
 

src/lib_binance_execution/binance_execution/spots/spots_gateway.cpp

@@ -280,7 +280,7 @@ struct SpotsGateway::PImpl {
       user_data_stream::OrderUpdate curr;
       auto &km = *key_map;
       while (incoming_order_updates.pop(curr)) {
-         auto bts_key = km.get_bts_key(curr.client_order_id);
+         const auto bts_key = km.get_bts_key(curr.client_order_id);
          if (!bts_key) {
             ERROR_FN(
                 "dispatch_order_updates",
@@ -289,7 +289,7 @@ struct SpotsGateway::PImpl {
                 curr.client_order_id);
             continue;
          }
-         auto exch_key = km.get_exch_key(*bts_key);
+         const auto exch_key = km.get_exch_key(*bts_key);
          if (exch_key != curr.client_order_id) {
             INFO_FN(
                 "dispatch_order_updates",

src/lib_binance_execution/binance_execution/spots/user_data_stream/user_data_subscription.cpp

@@ -58,6 +58,7 @@ struct UserDataSubscription::PImpl {
       LOG(OrderMan, bts::Info, "Creating websocket client. url: {}", url);
       WebsocketClientConfig ws_stream_client_cfg;
       ws_stream_client_cfg.url = url;
+      ws_stream_client_cfg.name = "Binance_UserData";
       auto &ws_cli = ws_client.emplace(ws_stream_client_cfg);
       ws_cli.message_handler().bind<&PImpl::on_message>(*this);
       ws_cli.close_handler().bind<&PImpl::on_close>(*this);

src/lib_binance_execution/binance_execution/usds_futures/rest_api_client.cpp

@@ -2,6 +2,7 @@
 #include "lib/io/http/http_client.h"
 #include "lib/io/http/http_response.h"
 #include "lib/utils/crypto/hmac.h"
+#include "lib/utils/type_name.h"
 
 namespace binance_execution::usds_futures {
 
@@ -18,16 +19,24 @@ using std::string;
 #define ERROR_FN(fn, ...) _LOG_FN(bts::Error, fn, __VA_ARGS__)
 
 template <std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_string(std::string &str) {
+static void shrink_str(std::string &str, const char *name) {
    if (str.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_str",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, str.size(), DEFAULT_SIZE, MAX_SIZE);
       str.resize(DEFAULT_SIZE);
       str.shrink_to_fit();
    }
 }
 
 template <class T, std::size_t MAX_SIZE, std::size_t DEFAULT_SIZE>
-static void shrink_vec(std::vector<T> &vec) {
+static void shrink_vec(std::vector<T> &vec, const char *name) {
    if (vec.size() > MAX_SIZE) {
+      WARN_FN(
+          "shrink_vec",
+          "Shrinking {} buffer size from {} to {} (max resting size is {})",
+          std::string{name}, vec.size(), DEFAULT_SIZE, MAX_SIZE);
       vec.resize(DEFAULT_SIZE);
       vec.shrink_to_fit();
    }
@@ -52,11 +61,13 @@ void RestApiClient::request_(
     const std::string &path,
     bts::io::http::HttpMethod method,
     const std::string &body,
+    bool sign,
     RestApiResult<ResponseT> &res) {
    // Use thread_local vars to reduce memory allocations. Shrink strings
    // to reduce memory footprint from previous requests.
    thread_local bts::http::HttpRequest http_req;
-   shrink_string<4096, 128>(http_req.body);
+   shrink_str<4096, 128>(
+       http_req.body, bts::utils::cached_type_name<RequestT>());
    http_req.reset();
    // Prepare request
    http_req.url = url_;
@@ -65,17 +76,22 @@ void RestApiClient::request_(
    http_req.method = method;
    http_req.headers["X-MBX-APIKEY"] = api_key_;
    thread_local std::string qps;
-   shrink_string<256, 16>(qps);
+   static const std::string query_params_name =
+       fmt::format("{}_params", bts::utils::cached_type_name<RequestT>());
+   shrink_str<256, 16>(qps, query_params_name.c_str());
    qps.clear();
    to_query_params(qps, req);
    if (!qps.empty()) {
-      sign_query_params_(qps);
+      if (sign) {
+         sign_query_params_(qps);
+      }
       http_req.url.append("?");
       http_req.url.append(qps);
    }
    // Send it
    thread_local HttpResponse http_res;
-   shrink_string<4096, 128>(http_res.body);
+   shrink_str<4096, 128>(
+       http_res.body, bts::utils::cached_type_name<ResponseT>());
    http_res.reset();
    HttpClient::send(http_req, &http_res);
    res.http_code = http_res.code;
@@ -102,7 +118,7 @@ void RestApiClient::request_(
 #define REQUEST_DEF(ResponseT, RequestT)                                       \
    template void RestApiClient::request_<ResponseT, RequestT>(                 \
        const RequestT &req, const std::string &path,                           \
-       bts::io::http::HttpMethod method, const std::string &body,              \
+       bts::io::http::HttpMethod method, const std::string &body, bool sign,   \
        RestApiResult<ResponseT> &res)
 
 REQUEST_DEF(CreateOrderResponse, CreateOrderRequest);
@@ -111,5 +127,6 @@ REQUEST_DEF(CancelOrderResponse, CancelOrderRequest);
 REQUEST_DEF(std::vector<OrderRecord>, AllOrdersRequest);
 REQUEST_DEF(std::vector<PositionInfo>, PositionInfoRequest);
 REQUEST_DEF(ListenKeyResponse, ListenKeyRequest);
+REQUEST_DEF(RefreshListenKeyResponse, RefreshListenKeyRequest);
 
 } // namespace binance_execution::usds_futures

src/lib_binance_execution/binance_execution/usds_futures/rest_api_client.h

@@ -2,6 +2,8 @@
 
 #include "binance_execution/messages/listen_key_request.h"
 #include "binance_execution/messages/listen_key_response.h"
+#include "binance_execution/messages/refresh_listen_key_request.h"
+#include "binance_execution/messages/refresh_listen_key_response.h"
 #include "binance_execution/rest_api_result.h"
 #include "binance_execution/usds_futures/messages/all_orders_request.h"
 #include "binance_execution/usds_futures/messages/cancel_order_request.h"
@@ -38,8 +40,8 @@ class RestApiClient {
        const PositionInfoRequest &req);
    const RestApiResult<ListenKeyResponse> &get_listen_key(
        const ListenKeyRequest &req = {});
-   const RestApiResult<ListenKeyResponse> &refresh_listen_key(
-       const ListenKeyRequest &req = {});
+   const RestApiResult<RefreshListenKeyResponse> &refresh_listen_key(
+       const RefreshListenKeyRequest &req = {});
 
  private:
    std::string url_;
@@ -52,6 +54,7 @@ class RestApiClient {
        const std::string &path,
        bts::io::http::HttpMethod method,
        const std::string &body,
+       bool sign,
        RestApiResult<ResponseT> &op_res);
 
    bool sign_query_params_(std::string &qp);
@@ -72,7 +75,7 @@ inline const RestApiResult<CreateOrderResponse> &RestApiClient::create_order(
    thread_local RestApiResult<CreateOrderResponse> res;
    res.reset();
    request_<CreateOrderResponse>(
-       req, path, bts::io::http::HttpMethod::POST, "", res);
+       req, path, bts::io::http::HttpMethod::POST, "", true, res);
    return res;
 }
 
@@ -82,7 +85,7 @@ inline const RestApiResult<ModifyOrderResponse> &RestApiClient::modify_order(
    thread_local RestApiResult<ModifyOrderResponse> res;
    res.reset();
    request_<ModifyOrderResponse>(
-       req, path, bts::io::http::HttpMethod::PUT, "", res);
+       req, path, bts::io::http::HttpMethod::PUT, "", true, res);
    return res;
 }
 
@@ -92,7 +95,7 @@ inline const RestApiResult<CancelOrderResponse> &RestApiClient::cancel_order(
    thread_local RestApiResult<CancelOrderResponse> res;
    res.reset();
    request_<CancelOrderResponse>(
-       req, path, bts::io::http::HttpMethod::DELETE, "", res);
+       req, path, bts::io::http::HttpMethod::DELETE, "", true, res);
    return res;
 }
 
@@ -102,7 +105,7 @@ RestApiClient::get_all_orders(const AllOrdersRequest &req) {
    thread_local RestApiResult<std::vector<OrderRecord>> res;
    res.reset();
    request_<std::vector<OrderRecord>>(
-       req, path, bts::io::http::HttpMethod::GET, "", res);
+       req, path, bts::io::http::HttpMethod::GET, "", true, res);
    return res;
 }
 
@@ -111,7 +114,7 @@ RestApiClient::get_position_info(const PositionInfoRequest &req) {
    static const std::string path{"/fapi/v3/positionRisk"};
    thread_local RestApiResult<std::vector<PositionInfo>> res;
    res.reset();
-   request_(req, path, bts::io::http::HttpMethod::GET, "", res);
+   request_(req, path, bts::io::http::HttpMethod::GET, "", true, res);
    return res;
 }
 
@@ -120,16 +123,16 @@ inline const RestApiResult<ListenKeyResponse> &RestApiClient::get_listen_key(
    static const std::string path{"/fapi/v1/listenKey"};
    thread_local RestApiResult<ListenKeyResponse> res;
    res.reset();
-   request_(req, path, bts::io::http::HttpMethod::POST, "", res);
+   request_(req, path, bts::io::http::HttpMethod::POST, "", false, res);
    return res;
 }
 
-inline const RestApiResult<ListenKeyResponse> &
-RestApiClient::refresh_listen_key(const ListenKeyRequest &req) {
+inline const RestApiResult<RefreshListenKeyResponse> &
+RestApiClient::refresh_listen_key(const RefreshListenKeyRequest &req) {
    static const std::string path{"/fapi/v1/listenKey"};
-   thread_local RestApiResult<ListenKeyResponse> res;
+   thread_local RestApiResult<RefreshListenKeyResponse> res;
    res.reset();
-   request_(req, path, bts::io::http::HttpMethod::PUT, "", res);
+   request_(req, path, bts::io::http::HttpMethod::PUT, "", false, res);
    return res;
 }
 

src/lib_binance_execution/binance_execution/usds_futures/user_data_stream/user_data_subscription.cpp

@@ -54,6 +54,7 @@ struct UserDataSubscription::PImpl {
       LOG(OrderMan, bts::Info, "Creating websocket client. url: {}", url);
       WebsocketClientConfig ws_stream_client_cfg;
       ws_stream_client_cfg.url = url;
+      ws_stream_client_cfg.name = "BinanceUsdsFutures_UserData";
       auto &ws_cli = ws_client.emplace(ws_stream_client_cfg);
       ws_cli.message_handler().bind<&PImpl::on_message>(*this);
       ws_cli.close_handler().bind<&PImpl::on_close>(*this);

src/server/sparkd/Dockerfile

@@ -0,0 +1,25 @@
+ARG IMAGE="sparkd"
+ARG CONFIG_DIR="src/server/sparkd/configurations/dev"
+ARG SOURCE_DIR="install/sparkd-release-gcc"
+
+FROM bluetradingsystems/linux_dotnet7 AS sparkd_base
+ARG CONFIG_DIR
+ARG SOURCE_DIR
+
+USER "root"
+RUN apt-get update && apt-get install -y liblttng-ust-dev
+RUN apt-get update && apt-get install -y libpcap0.8-dev
+RUN apt-get update && apt-get install -y python3.10-dev
+
+USER "bts"
+COPY --chown=bts:bts "${SOURCE_DIR}" "."
+COPY --chown=bts:bts "${CONFIG_DIR}" "/home/bts/sparkdata/conf"
+WORKDIR "/home/bts/bin"
+
+FROM sparkd_base AS sparkd
+
+FROM sparkd_base AS sparkd_md_replay
+ARG MD_RECORDING_DIR="src/server/sparkd/.md_recording"
+COPY --chown=bts:bts "${MD_RECORDING_DIR}" "/home/bts/sparkdata/data/md_recording"
+
+FROM ${IMAGE}

src/server/sparkd/cmds/md/cmds/DumpSecurity.cpp

@@ -1,32 +1,48 @@
 #include "DumpSecurity.h"
-#include "lib/config/protos/market_data.pb.h"
 #include "lib/core/TableFormatter.h"
 #include "lib/domain_model/Security.h"
 #include "lib/domain_model/Security.pb.h"
 #include "lib/domain_model/SecurityType.h"
-#include "lib/md/Diag/ShmDumper.h"
 #include "lib/md/SecurityDB.h"
 #include "lib/md/SymbolLoader.h"
-#include "lib/md/UserSymbol.h"
+#include "lib/utils/lazy.h"
+
 #include <unordered_set>
 #include <vector>
 
 namespace bts::cmds::md::cmds {
 
 using bts::api::Security;
+using bts::api::security_type_parse_smart;
+using bts::api::SecurityType;
 using bts::api::SecurityType_Name;
+using bts::api::Venue;
+using bts::api::venue_parse_smart;
+using bts::core::TableFormatter;
+using bts::md::securitytype_of;
 using bts::md::SymbolLoader;
+using bts::md::db::load_metadata;
+using bts::md::db::load_securities;
 using bts::securities::security_type;
+using bts::utils::Lazy;
+using std::function;
 using std::make_tuple;
+using std::string;
+using std::unordered_set;
+using std::vector;
 
 template <class EnumT>
 using EnumNameMap = std::map<std::string, EnumT>;
 
-void print_security(const api::Security &security) {
+void print_security(
+    const Security &security,
+    Lazy<vector<api::synthetic::Metadata>> &metadata) {
+   static const char *SEPARATOR =
+       "\n**************************************************\n";
+   std::cout << SEPARATOR;
    security.PrintDebugString();
-   if (bts::md::securitytype_of(security) == api::SECURITYTYPE_SYNTHETIC) {
-      auto metadata = bts::md::db::load_metadata();
-      for (const auto &m : metadata) {
+   if (securitytype_of(security) == api::SECURITYTYPE_SYNTHETIC) {
+      for (const auto &m : metadata.get()) {
          if (m.security_key() == security.id()) {
             m.PrintDebugString();
             break;
@@ -35,17 +51,20 @@ void print_security(const api::Security &security) {
    }
 }
 
-void print_securities(std::function<bool(const api::Security &)> predicate) {
+void print_securities(function<bool(const Security &)> predicate) {
+   auto metadata =
+       Lazy<vector<api::synthetic::Metadata>>::from_delegate(load_metadata);
    const auto symbols = SymbolLoader::load();
-   bts::md::db::load_securities(symbols, [&](const api::Security &security) {
-      if (predicate(security))
-         print_security(security);
+   load_securities(symbols, [&](const Security &security) {
+      if (predicate(security)) {
+         print_security(security, metadata);
+      }
    });
 }
 
-void print_db_securities(std::function<bool(const api::Security &)> predicate) {
-   std::vector<bts::api::Security> secs;
-   const bts::core::TableFormatter::Schema<bts::api::Security> schema{
+void print_db_securities(function<bool(const Security &)> predicate) {
+   vector<Security> secs;
+   const TableFormatter::Schema<Security> schema{
        make_tuple(
            "id", 12, [](const Security &s) { return std::to_string(s.id()); }),
        make_tuple(
@@ -62,22 +81,67 @@ void print_db_securities(std::function<bool(const api::Security &)> predicate) {
            "name", 32,
            [](const Security &s) { return bts::securities::descriptor(s); }),
    };
-   auto on_sec = [&](const bts::api::Security &sec) {
+   auto on_sec = [&](const Security &sec) {
       if (!predicate(sec)) {
          return;
       }
       secs.push_back(sec);
    };
-   bts::md::db::load_securities([](auto) { return true; }, on_sec);
-   bts::core::TableFormatter::format(std::cout, secs, schema);
+   load_securities([](auto) { return true; }, on_sec);
+   TableFormatter::format(std::cout, secs, schema);
+}
+
+template <class T, class ParseT = function<bool(const string &, T *)>>
+static vector<T> parse_enums(const vector<string> &names, ParseT parse) {
+   vector<T> rv;
+   for (const auto &t : names) {
+      if (!parse(t, &rv.emplace_back())) {
+         rv.pop_back();
+      }
+   }
+   return rv;
 }
 
 struct DumpSecurity::Args {
    bool all;
    bool db = false;
-   std::vector<std::string> types;
-   std::vector<std::string> venues;
-   std::vector<uint32_t> security_ids;
+   vector<string> types;
+   vector<string> venues;
+   vector<uint32_t> security_ids;
+
+   function<bool(const Security &)> get_filter() {
+      if (all) {
+         return [](const auto &) { return true; };
+      }
+      auto types =
+          parse_enums<SecurityType>(this->types, security_type_parse_smart);
+      auto venues = parse_enums<Venue>(this->venues, venue_parse_smart);
+      unordered_set<uint32_t> ids;
+      for (auto id : security_ids) {
+         ids.insert(id);
+      }
+      return [&, types = std::move(types), venues = std::move(venues),
+              ids = std::move(ids)](const auto &s) {
+         if (!ids.empty()) {
+            return ids.find(s.id()) != ids.end();
+         }
+         if (!types.empty()) {
+            auto st = security_type(s);
+            if (!std::any_of(
+                    types.begin(), types.end(),
+                    [st](SecurityType _st) { return _st == st; })) {
+               return false;
+            }
+         }
+         if (!venues.empty() &&
+             !std::any_of(
+                 venues.begin(), venues.end(),
+                 [v = s.exchange()](Venue _v) { return _v == v; })) {
+            return false;
+         }
+         return true;
+      };
+   }
 };
 
 DumpSecurity::DumpSecurity()
@@ -94,44 +158,11 @@ DumpSecurity::DumpSecurity()
        "Print information about specific securities");
 
    callback([&] {
-      if (_args->all) {
-         print_securities([](const auto &) { return true; });
-      } else if (_args->db) {
-         std::vector<api::SecurityType> types;
-         for (const auto &t : _args->types) {
-            if (!api::security_type_parse_smart(t, &types.emplace_back())) {
-               types.pop_back();
-            }
-         }
-         std::vector<api::Venue> venues;
-         for (const auto &t : _args->venues) {
-            if (!api::venue_parse_smart(t, &venues.emplace_back())) {
-               venues.pop_back();
-            }
-         }
-         print_db_securities([&](const auto &s) {
-            if (!types.empty()) {
-               auto st = bts::securities::security_type(s);
-               if (!std::any_of(
-                       types.begin(), types.end(),
-                       [st](api::SecurityType _st) { return _st == st; })) {
-                  return false;
-               }
-            }
-            if (!venues.empty() &&
-                !std::any_of(
-                    venues.begin(), venues.end(),
-                    [v = s.exchange()](api::Venue _v) { return _v == v; })) {
-               return false;
-            }
-            return true;
-         });
+      auto filter = _args->get_filter();
+      if (_args->db) {
+         print_db_securities(filter);
       } else {
-         std::unordered_set<uint32_t> ids;
-         for (auto id : _args->security_ids)
-            ids.insert(id);
-         print_securities(
-             [&](const auto &s) { return ids.find(s.id()) != ids.end(); });
+         print_securities(filter);
       }
    });
 }

src/server/sparkd/configurations/alphaworx_prod/.env

@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/alphaworx_prod
+SPARKD__SHM_SIZE=8gb

src/server/sparkd/configurations/alphaworx_prod/README.txt

@@ -0,0 +1,6 @@
+* Connects to Binance and Okx PRODUCTION sites for market data and execution
+* Api keys NOT included
+* Configured for Alphaworx
+
+To retrieve api keys, run "python init_env.py". It should dump them to a file
+called  ".env.local".

src/server/sparkd/configurations/alphaworx_prod/binance_1.json

@@ -0,0 +1,9 @@
+{
+  "api_key": "${BINANCE_API_KEY}",
+  "secret_key": "${BINANCE_SECRET_KEY}",
+  "account": "BINANCE_PROD",
+  "rest_endpoint": "https://api.binance.com",
+  "ws_endpoint": "wss://ws-api.binance.com:443/ws-api/v3",
+  "ws_stream_endpoint": "wss://stream.binance.com:9443",
+  "use_cross_margin_api": true
+}

src/server/sparkd/configurations/alphaworx_prod/binance_usds_futures_1.json

@@ -0,0 +1,9 @@
+{
+  "api_key": "${BINANCE_API_KEY}",
+  "secret_key": "${BINANCE_SECRET_KEY}",
+  "account": "BINANCE_PROD",
+  "rest_endpoint": "https://fapi.binance.com",
+  "ws_endpoint": "wss://ws-fapi.binance.com/ws-fapi/v1",
+  "ws_stream_endpoint": "wss://fstream.binance.com",
+  "use_cross_margin_api": false
+}

src/server/sparkd/configurations/alphaworx_prod/init_env.py

@@ -0,0 +1,88 @@
+import botocore
+import json
+import os
+
+from boto3.session import Session
+from botocore.exceptions import ClientError
+from fileinput import FileInput
+
+
+def update_env_file(path, env_vars: dict):
+    """Update or add vars in a .env file in place."""
+    if not os.path.exists(path):
+        with open(path, "w") as file:
+            for k, v in env_vars.items():
+                file.write(f"{k}={v}\n")
+        return
+
+    with FileInput(path, inplace=True) as file:
+        for line in file:
+            if not line.startswith("#"):
+                stripped: str = line.lstrip()
+                key_len = stripped.find("=")
+                if key_len >= 0:
+                    k = stripped[0:key_len]
+                    if k in env_vars:
+                        print(f"{k}={env_vars[k]}")
+                        del env_vars[k]
+                        continue
+            print(line, end="")
+
+    if len(env_vars) > 0:
+        with open(path, "a") as file:
+            for k, v in env_vars.items():
+                file.write(f"{k}={v}\n")
+
+
+def get_secret():
+    secret_name = "alphaworx-binance-api-keys"
+    region_name = "ap-northeast-1"
+    profile_name = "AdministratorAccess-849458505639"
+    session = Session(profile_name=profile_name)
+    client = session.client(
+        service_name="secretsmanager",
+        region_name=region_name,
+    )
+
+    def on_token_error(e: Exception):
+        print(f"""
+Failed to get secret value. {e}
+Verify you are logged in to aws, then try again.
+
+  aws sso login --profile {profile_name}
+
+If this error persists, verify you have access to these api keys.
+For more help or information, talk to John.
+""")
+
+    try:
+        get_secret_value_response = client.get_secret_value(
+            SecretId=secret_name)
+    except ClientError as e:
+        print(f"Failed to get secret value.\n{e}")
+        raise e
+    except botocore.exceptions.SSOTokenLoadError as tl_err:
+        on_token_error(tl_err)
+        return None
+    except botocore.exceptions.TokenRetrievalError as tr_err:
+        on_token_error(tr_err)
+        return None
+    return get_secret_value_response["SecretString"]
+
+
+def _run():
+    this_dir = os.path.dirname(__file__)
+    secret_str = get_secret()
+    if secret_str is None:
+        exit(1)
+    secret = json.loads(secret_str)
+    env_vars = {}
+    if "api-key" in secret:
+        env_vars["BINANCE_API_KEY"] = secret["api-key"]
+    if "secret-key" in secret:
+        env_vars["BINANCE_SECRET_KEY"] = secret["secret-key"]
+    update_env_file(f"{this_dir}/.env.local", env_vars)
+
+
+if __name__ == "__main__":
+    _run()

src/server/sparkd/configurations/alphaworx_prod/spark.conf

@@ -0,0 +1,121 @@
+QuickbusCfg {
+  value {
+    node {
+      reconnect_period_secs: 1
+      hb_period_msecs: 15000
+      max_missed_hbs: 5
+      parent_host: ""
+      parent_port: 0
+      my_port: 0
+    }
+    client {
+      reconnect_period_secs: 1
+      hb_period_msecs: 15000
+      max_missed_hbs: 5
+      parent_host: @SystemConfig.config_ip
+      parent_port: @Ports.qbus_port
+      my_port: 0
+    }
+  }
+}
+
+JetFIX {
+  value {
+    connection {
+      name: "binance_1"
+      venue: VENUE_BINANCE
+    }
+    connection {
+      name: "binance_usds_futures_1"
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    connection {
+      name: "okx_1"
+      venue: VENUE_OKX
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    binance {
+      num_depth_levels: 5
+      url_rest_base: "https://api.binance.com"
+      url_ws_base: "wss://stream.binance.com:9443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: @SymbolLists.list[Binance].symbol
+          name_regexps: "USDC|USDT$"
+        }
+      }
+    }
+    binance_usds_futures {
+      num_depth_levels: 5
+      url_rest_base: "https://fapi.binance.com"
+      url_ws_base: "wss://fstream.binance.com"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
+        }
+      }
+    }
+    okx {
+      num_depth_levels: 5
+      url_rest_base: "https://aws.okx.com"
+      url_ws_base: "wss://wsaws.okx.com:8443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[Okx].symbol
+        }
+      }
+      is_application_ping_pong_enabled: true
+      application_ping_interval_ms: 20000
+      application_pong_timeout_ms: 10000
+      is_protocol_ping_pong_enabled: true
+      protocol_ping_interval_ms: 60000
+      protocol_pong_timeout_ms: 30000
+    }
+  }
+}
+
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_BINANCE
+    }
+    entries {
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    entries {
+      venue: VENUE_GLBX
+    }
+    entries {
+      venue: VENUE_OKX
+    }
+    entries {
+      venue: VENUE_TAIFEX
+    }
+  }
+}
+
+OkxExeCfg {
+  value {
+    account: "prod"
+    ops_id: "op_id_bts"
+    secret_key: "240038D7A28D7F9461247CFCB8E43710"
+    api_key: "dc188532-c74c-4d40-affe-3c6bf4d0ca9a"
+    passphrase: ":,7i%tA%ivfj%SCbX?Ut"
+    ws_url: "wss://wsaws.okx.com:8443"
+    rest_url: "https://www.okx.com"
+    ws_ping_timeout_seconds: 10 
+    ws_ping_interval_seconds: 25
+    ws_reconnect_interval_seconds: 5
+    ws_reconnect_max_count: 5
+	  simulated_trading: false
+    tdmode: "cross"
+  }
+}

src/server/sparkd/configurations/alphaworx_prod/symbols.conf

@@ -0,0 +1,70 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "AAVE" 
+      symbol: "ACT"
+      symbol: "ADA"
+      symbol: "ALGO"
+      symbol: "AR"
+      symbol: "ATOM"
+      symbol: "AVAX"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "HIVE"
+      symbol: "IO"
+      symbol: "PAXG"
+      symbol: "SPELL"
+      symbol: "WBETH"
+      symbol: "WIF"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "ADA"
+      symbol: "HIVE"
+      symbol: "SPELL"
+      symbol: "IO"
+      symbol: "UNI"
+    }
+    list {
+      key: "Okx"
+      symbol: "AAVE"
+      symbol: "ACT"
+      symbol: "ADA"
+      symbol: "ALGO"
+      symbol: "AR"
+      symbol: "ATOM"
+      symbol: "AVAX"
+      symbol: "BETH"
+      symbol: "BTC"
+      symbol: "CRV"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "ETHFI"
+      symbol: "UNI"
+      symbol: "USDC"
+      symbol: "USDT"
+      symbol: "WIF"
+      symbol: "XAUT"
+      symbol: "XRP"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE"
+      spot: @SymbolLists.list[Binance].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    }
+    venues {
+      name: "VENUE_OKX"
+      spot: @SymbolLists.list[Okx].symbol
+      future: @SymbolLists.list[Okx].symbol
+    }
+  }
+}

src/server/sparkd/configurations/binance_dev/.env

@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/binance_dev
+SPARKD__SHM_SIZE=8gb

src/server/sparkd/configurations/binance_dev/README.txt

@@ -0,0 +1 @@
+* Connects to Binance test nets for market data and execution

src/server/sparkd/configurations/binance_dev/binance_1.json

@@ -1,7 +1,7 @@
 {
   "api_key": "QGwn9OVcpRTU76gOLV0gmQyVAJUmXW6w7H6y9tH8ctanzVnpZN1TP27liObE1Wsd",
   "secret_key": "7ohKg6X3DcsZMGaJmngC84EglsCFse9flMl8aKisa5FUY18LjGCEEdm0KEZljk9T",
-  "account": "binance_dev",
+  "account": "BINANCE_DEV",
   "rest_endpoint": "https://testnet.binance.vision",
   "ws_endpoint": "wss://testnet.binance.vision/ws-api/v3",
   "ws_stream_endpoint": "wss://stream.testnet.binance.vision:9443"

src/server/sparkd/configurations/binance_dev/binance_usds_futures_1.json

@@ -1,7 +1,7 @@
 {
   "api_key": "b838c108cd2648e34a4f088a7fc635a9cbccf3c2faa6bcaabcb532ffacc087b0",
   "secret_key": "a68f0ab27403c42fa0306b6341b2ddd51d99ac42916f849404155449457d0492",
-  "account": "binance_dev",
+  "account": "BINANCE_DEV",
   "rest_endpoint": "https://testnet.binancefuture.com",
   "ws_endpoint": "wss://testnet.binancefuture.com/ws-fapi/v1",
   "ws_stream_endpoint": "wss://stream.binancefuture.com"

src/server/sparkd/configurations/binance_dev/spark.conf

@@ -1,30 +1,11 @@
-ExchangeSubAccounts {
-  value {
-    accounts {
-      exchange_subaccount: "binance_dev"
-      bts_account_name: "BINANCE_DEV"
-    }
-    accounts{
-      exchange_subaccount: "ZmIKL6fb"
-      bts_account_name: "MAIN"     
-    }
-    accounts{
-      exchange_subaccount: "v1iSzZrY"
-      bts_account_name: "SMALL"
-    }
-  }
-}
-
 JetFIX {
   value {
     connection {
       name: "binance_1"
-      max_order_msgs_per_window: 300
       venue: VENUE_BINANCE
     }
     connection {
       name: "binance_usds_futures_1"
-      max_order_msgs_per_window: 300
       venue: VENUE_BINANCE_USDS_FUTURES
     }
   }
@@ -51,14 +32,10 @@ Logging {
     max_queue_growth: 4
     console {
       default_level: "Info"
-      # subsystem {
-      #   name: "OrderMan"
-      #   level: "Diag"
-      # }
-      # subsystem {
-      #   name: "Http"
-      #   level: "Diag"
-      # }
+      subsystem {
+        name: "OrderMan"
+        level: "Diag"
+      }
     }
     file {
       default_level: "Info"
@@ -94,16 +71,12 @@ MarketDataConfig {
       security_criteria {
         clauses {
           security_types: SECURITY_TYPE_SPOT
-          symbols: "AAVE"
-          symbols: "AVAX"
-          symbols: "BNB"
-          symbols: "BTC"
-          symbols: "DOGE"
-          symbols: "EIGEN"
-          symbols: "ETH"
+          symbols: @SymbolLists.list[Binance].symbol
+          name_regexps: "USDC|USDT$"
         }
       }
     }
+
     binance_usds_futures {
       num_depth_levels: 5
       # url_rest_base: "https://fapi.binance.com"
@@ -114,44 +87,13 @@ MarketDataConfig {
       security_criteria {
         clauses {
           security_types: SECURITY_TYPE_FUTURE
-          symbols: "AAVE"
-          symbols: "AVAX"
-          symbols: "BNB"
-          symbols: "BTC"
-          symbols: "DOGE"
-          symbols: "EIGEN"
-          symbols: "ETH"
+          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
         }
       }
     }
   }
 }
 
-Symbols {
-  value {
-    venues {
-      name: "VENUE_BINANCE"
-      spot: "AAVE"
-      spot: "AVAX"
-      spot: "BNB"
-      spot: "BTC"
-      spot: "DOGE"
-      spot: "EIGEN"
-      spot: "ETH"
-    }
-    venues {
-      name: "VENUE_BINANCE_USDS_FUTURES"
-      future: "AAVE"
-      future: "AVAX"
-      future: "BNB"
-      future: "BTC"
-      future: "DOGE"
-      future: "EIGEN"
-      future: "ETH"
-    }
-  }
-}
-
 NotSimExecution {
   value {
     entries {

src/server/sparkd/configurations/binance_dev/symbols.conf

@@ -0,0 +1,37 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE"
+      spot: @SymbolLists.list[Binance].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    }
+  }
+}

src/server/sparkd/configurations/dev/.env

@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
+SPARKD__SHM_SIZE=8gb

src/server/sparkd/configurations/dev/README.txt

@@ -0,0 +1 @@
+* Connects to Binance and Okx test nets for market data and execution

src/server/sparkd/configurations/dev/binance_1.json

@@ -0,0 +1,8 @@
+{
+  "api_key": "QGwn9OVcpRTU76gOLV0gmQyVAJUmXW6w7H6y9tH8ctanzVnpZN1TP27liObE1Wsd",
+  "secret_key": "7ohKg6X3DcsZMGaJmngC84EglsCFse9flMl8aKisa5FUY18LjGCEEdm0KEZljk9T",
+  "account": "BINANCE_DEV",
+  "rest_endpoint": "https://testnet.binance.vision",
+  "ws_endpoint": "wss://testnet.binance.vision/ws-api/v3",
+  "ws_stream_endpoint": "wss://stream.testnet.binance.vision:9443"
+}

src/server/sparkd/configurations/dev/binance_usds_futures_1.json

@@ -0,0 +1,8 @@
+{
+  "api_key": "b838c108cd2648e34a4f088a7fc635a9cbccf3c2faa6bcaabcb532ffacc087b0",
+  "secret_key": "a68f0ab27403c42fa0306b6341b2ddd51d99ac42916f849404155449457d0492",
+  "account": "BINANCE_DEV",
+  "rest_endpoint": "https://testnet.binancefuture.com",
+  "ws_endpoint": "wss://testnet.binancefuture.com/ws-fapi/v1",
+  "ws_stream_endpoint": "wss://stream.binancefuture.com"
+}

src/server/sparkd/configurations/dev/spark.conf

@@ -0,0 +1,135 @@
+JetFIX {
+  value {
+    connection {
+      name: "binance_1"
+      venue: VENUE_BINANCE
+    }
+    connection {
+      name: "binance_usds_futures_1"
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    connection {
+      name: "okx_1"
+      venue: VENUE_OKX
+    }
+  }
+}
+
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "OrderMan"
+        level: "Diag"
+      }
+    }
+    file {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Diag"
+      }
+      subsystem {
+        name: "OrderMan"
+        level: "Diag"
+      }
+      subsystem {
+        name: "Http"
+        level: "Diag"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    binance {
+      num_depth_levels: 5
+      # url_rest_base: "wss://ws-api.binance.com:9443"
+      # According to Binance, the "apiN" urls offer performance at the cost of stability.
+      # https://developers.binance.com/docs/binance-spot-api-docs/rest-api/general-api-information
+      # url_rest_base: "https://api1.binance.com:9443"
+      # url_rest_base: "https://api.binance.com"
+      url_rest_base: "https://testnet.binance.vision"
+      # url_ws_base: "wss://ws-api.binance.com"
+      url_ws_base: "wss://testnet.binance.vision"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: @SymbolLists.list[Binance].symbol
+          name_regexps: "USDC|USDT$"
+        }
+      }
+    }
+    binance_usds_futures {
+      num_depth_levels: 5
+      # url_rest_base: "https://fapi.binance.com"
+      # url_ws_base: "wss://fstream.binance.com"
+      url_rest_base: "https://testnet.binancefuture.com"
+      url_ws_base: "wss://stream.binancefuture.com"
+
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
+        }
+      }
+    }
+    okx {
+      num_depth_levels: 5
+      url_rest_base: "https://aws.okx.com"
+      url_ws_base: "wss://wsaws.okx.com:8443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[Okx].symbol
+        }
+      }
+      is_application_ping_pong_enabled: true                                                                                                                                            
+      application_ping_interval_ms: 20000
+      application_pong_timeout_ms: 10000
+      is_protocol_ping_pong_enabled: true
+      protocol_ping_interval_ms: 60000
+      protocol_pong_timeout_ms: 30000
+    }
+  }
+}
+
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_BINANCE
+    }
+    entries {
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    entries {
+      venue: VENUE_GLBX
+    }
+    entries {
+      venue: VENUE_OKX
+    }
+    entries {
+      venue: VENUE_TAIFEX
+    }
+  }
+}

src/server/sparkd/configurations/dev/symbols.conf

@@ -0,0 +1,63 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+    list {
+      key: "Okx"
+      symbol: "AAVE"
+      symbol: "ACT"
+      symbol: "ADA"
+      symbol: "ALGO"
+      symbol: "AR"
+      symbol: "ATOM"
+      symbol: "AVAX"
+      symbol: "BETH"
+      symbol: "BTC"
+      symbol: "CRV"
+      symbol: "EIGEN"
+      symbol: "ETHFI"
+      symbol: "UNI"
+      symbol: "WIF"
+      symbol: "XAUT"
+      symbol: "XRP"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE"
+      spot: @SymbolLists.list[Binance].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    }
+    venues {
+      name: "VENUE_OKX"
+      spot: @SymbolLists.list[Okx].symbol
+      future: @SymbolLists.list[Okx].symbol
+    }
+  }
+}

src/server/sparkd/configurations/sim_dev/.env

@@ -0,0 +1,2 @@
+SPARKD__CONFIG_DIR=src/server/sparkd/configurations/sim_dev
+SPARKD__SHM_SIZE=8gb

src/server/sparkd/configurations/sim_dev/README.txt

@@ -0,0 +1,2 @@
+* Connects to Binance and Okx test nets for market data
+* Execution is SIM

src/server/sparkd/configurations/sim_dev/spark.conf

@@ -0,0 +1,77 @@
+JetFIX {
+  value {
+    connection {
+      name: "binance_1"
+      venue: VENUE_BINANCE
+    }
+    connection {
+      name: "binance_usds_futures_1"
+      venue: VENUE_BINANCE_USDS_FUTURES
+    }
+    connection {
+      name: "okx_1"
+      venue: VENUE_OKX
+    }
+  }
+}
+
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+MarketDataConfig {
+  value {
+    binance {
+      num_depth_levels: 5
+      url_rest_base: "https://testnet.binance.vision"
+      url_ws_base: "wss://testnet.binance.vision"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: @SymbolLists.list[Binance].symbol
+          name_regexps: "USDC|USDT$"
+        }
+      }
+    }
+    binance_usds_futures {
+      num_depth_levels: 5
+      url_rest_base: "https://testnet.binancefuture.com"
+      url_ws_base: "wss://stream.binancefuture.com"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[BinanceUsdsFutures].symbol
+        }
+      }
+    }
+    okx {
+      num_depth_levels: 5
+      url_rest_base: "https://aws.okx.com"
+      url_ws_base: "wss://wsaws.okx.com:8443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[Okx].symbol
+        }
+      }
+      is_application_ping_pong_enabled: true                                                                                                                                            
+      application_ping_interval_ms: 20000
+      application_pong_timeout_ms: 10000
+      is_protocol_ping_pong_enabled: true
+      protocol_ping_interval_ms: 60000
+      protocol_pong_timeout_ms: 30000
+    }
+  }
+}

src/server/sparkd/configurations/sim_dev/symbols.conf

@@ -0,0 +1,63 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "AAVE"
+      symbol: "AVAX"
+      symbol: "BNB"
+      symbol: "BTC"
+      symbol: "DOGE"
+      symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+    list {
+      key: "Okx"
+      symbol: "AAVE"
+      symbol: "ACT"
+      symbol: "ADA"
+      symbol: "ALGO"
+      symbol: "AR"
+      symbol: "ATOM"
+      symbol: "AVAX"
+      symbol: "BETH"
+      symbol: "BTC"
+      symbol: "CRV"
+      symbol: "EIGEN"
+      symbol: "ETHFI"
+      symbol: "UNI"
+      symbol: "WIF"
+      symbol: "XAUT"
+      symbol: "XRP"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE"
+      spot: @SymbolLists.list[Binance].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    }
+    venues {
+      name: "VENUE_OKX"
+      spot: @SymbolLists.list[Okx].symbol
+      future: @SymbolLists.list[Okx].symbol
+    }
+  }
+}

src/server/sparkd/execution/execution_gateway_provider.cpp

@@ -88,8 +88,9 @@ struct ExecutionGatewayProvider::PImpl {
          return std::make_unique<OkxExecutionGateway>(o);
       }
       case api::VENUE_BINANCE: {
-         const auto cfg =
+         auto cfg =
              parse_config_json<binance_execution::GatewayConfig>(conn.name());
+         cfg.expand_env();
          if (cfg.use_cross_margin_api) {
             return std::make_unique<
                 binance_execution::cross_margin::CrossMarginGateway>(
@@ -101,6 +102,7 @@ struct ExecutionGatewayProvider::PImpl {
       case api::VENUE_BINANCE_USDS_FUTURES: {
          auto cfg =
              parse_config_json<binance_execution::GatewayConfig>(conn.name());
+         cfg.expand_env();
          if (cfg.use_cross_margin_api) {
             WARN_FN(
                 "create_gateway(BinanceUsdsFutures)",

src/server/sparkd/md/databento/DatabentoDataHandler.cpp

@@ -1,5 +1,6 @@
 #include "DatabentoDataHandler.h"
 #include "databento/datetime.hpp"
+#include "databento/dbn_encoder.hpp"
 #include "databento/enums.hpp"
 #include "databento/exceptions.hpp"
 #include "databento/live_blocking.hpp"
@@ -7,6 +8,7 @@
 #include "databento/record.hpp"
 #include "databento/timeseries.hpp"
 #include "lib/api/domain/Venue.pb.h"
+#include "lib/api/md/MarketDataConfig.pb.h"
 #include "lib/config/config.h"
 #include "lib/config/protos/system_config.pb.h"
 #include "lib/core/TimeStamp.h"
@@ -22,14 +24,21 @@
 #include "lib/utils/TimeStamp.h"
 #include "lib/utils/bts_assert.h"
 #include "lib/utils/bts_chrono.h"
+#include "lib/utils/datetime.h"
 #include "lib/utils/helper_macros.h"
+#include <boost/filesystem/operations.hpp>
 #include <cassert>
 #include <chrono>
+#include <cstddef>
 #include <databento/constants.hpp>
 #include <databento/historical.hpp>
 #include <databento/live.hpp>
 #include <optional>
+#include <ratio>
 #include <string>
+#include <string_view>
+#include <tuple>
+#include <unordered_map>
 #include <vector>
 
 namespace bts::md::DatabentoHandler {
@@ -61,7 +70,87 @@ static SecurityType parse_sec_type(std::string_view str) {
    }
    return SEC_TYPE(UNSPECIFIED);
 }
+void update_file(
+    const std::string &key,
+    const std::string &dataset,
+    const std::vector<std::string> &parent_symbols,
+    const uint time_to_run,
+    std::string file_path) {
 
+   auto client =
+       databento::LiveBuilder{}
+           .SetKey(key)
+           .SetDataset(dataset)
+           .SetUpgradePolicy(databento::VersionUpgradePolicy::UpgradeToV2)
+           .BuildBlocking();
+
+   client.Subscribe(
+       parent_symbols, databento::Schema::Definition, databento::SType::Parent,
+       databento::UnixNanos{});
+   // We start streaming
+   databento::Metadata metadata = client.Start();
+   {
+      // We open a file for writing and encode the metadata
+      databento::OutFileStream out_file{file_path};
+      databento::DbnEncoder encoder{metadata, &out_file};
+      // Wait for at least one record and 5 seconds before closing the connection
+      const std::chrono::seconds timeout{time_to_run};
+      const auto start = std::chrono::steady_clock::now();
+      while (auto *record = client.NextRecord(timeout)) {
+         encoder.EncodeRecord(*record);
+         if (std::chrono::steady_clock::now() - start >= timeout) {
+            break;
+         }
+      }
+   }
+   return;
+}
+void log_time_statistics(
+    const std::vector<std::vector<std::tuple<uint64_t, std::string>>>
+        &all_times) {
+   std::unordered_map<std::string, std::vector<uint64_t>> subpart_times;
+
+   // Collect all timing values per subpart
+   for (const auto &entry : all_times) {
+      for (const auto &[time, label] : entry) {
+         subpart_times[label].push_back(time);
+      }
+   }
+
+   // Function to compute statistics
+   auto compute_statistics = [](std::vector<uint64_t> &values)
+       -> std::tuple<double, double, double, double, double, double, double> {
+      if (values.empty())
+         return std::make_tuple(0.0, 0.0, 0, 0, 0, 0, 0);
+
+      std::sort(values.begin(), values.end());
+
+      double sum = std::accumulate(values.begin(), values.end(), 0.0);
+      double avg = sum / values.size();
+
+      double sq_sum =
+          std::inner_product(values.begin(), values.end(), values.begin(), 0.0);
+      double stddev = std::sqrt(sq_sum / values.size() - avg * avg);
+
+      auto percentile = [&](double p) -> uint64_t {
+         size_t idx = static_cast<size_t>(p * (values.size() - 1));
+         return values[idx];
+      };
+
+      return std::make_tuple(
+          avg, stddev, percentile(0.00), percentile(0.25), percentile(0.50),
+          percentile(0.75), percentile(1.00));
+   };
+
+   // Compute and log statistics
+   for (auto &[label, times] : subpart_times) {
+      auto [avg, stddev, p0, p25, p50, p75, p100] = compute_statistics(times);
+      LOG(md, Sys,
+          "Subpart: {}, Avg: {:.2f} ns, Std: {:.2f} ns, P0: {} ns, P25: {} ns, "
+          "P50: {} ns, P75: {} ns, P100: {} ns",
+          label, avg, stddev, p0, p25, p50, p75, p100);
+   }
+}
 void DatabentoDatahandler::run(
     bool use_file,
     std::shared_ptr<IMarketDataListener> listener,
@@ -74,59 +163,71 @@ void DatabentoDatahandler::run(
     const uint filter_securities_upto_n_months,
     const bool load_reference_data,
     const std::chrono::duration<long double> sleep_duration) {
-
+   uint DEBUG_SEC_ID = 0;
+   if (venue == api::domain::VENUE_CBOE)
+      DEBUG_SEC_ID = config::get<bts::api::md::MarketDataConfig>()
+                         .cboe()
+                         .at(0)
+                         .databento_settings()
+                         .debug_sec();
    auto sleep_duration_updated = sleep_duration;
-
+   std::vector<std::vector<std::tuple<uint64_t, std::string>>> all_times;
+   std::unordered_map<uint32_t, bts::domain::SecurityID> sec_id_map;
    if (!use_file) {
       sleep_duration_updated = std::chrono::duration<
           long double>::zero(); // Set to zero for live data
    }
-   auto process_security = [&](const databento::Record &record) {
-      const auto &msg = record.Get<databento::InstrumentDefMsg>();
-      std::string id = remove_spaces(msg.RawSymbol());
-      std::string symbol;
-
+   auto sec_filter = [&](const core::TimeStamp expiration_time,
+                         const std::string &symbol,
+                         const api::domain::Venue &venue, uint strike) {
+      uint months = filter_securities_upto_n_months;
       switch (venue) {
       case api::domain::Venue::VENUE_CBOE: {
-         symbol = remove_spaces(
-             std::string(msg.underlying.data(), msg.underlying.size()));
+         if (strike < 3800 || strike > 8000) {
+            return false;
+         }
+         if (symbol == "SPX")
+            months = 6;
+         else if (symbol == "SPXW")
+            months = 1;
          break;
       }
       case api::domain::Venue::VENUE_GLBX:
-         if (id.length() > 5)
-            return databento::KeepGoing::Continue;
-         symbol = id.substr(0, 2);
-         //  sec.security_common.cpp = msg.unit_of_measure_qty;
-
-         break;
+         return true;
       default:
          bts_assert_msg(false, "Exchange not supported on databento");
-         return databento::KeepGoing::Stop;
+      }
+
+      auto duration = months * 32 * 24 * 60 * 60;
+      return expiration_time.seconds() <
+             bts::core::TimeStamp::now().seconds() + duration;
+   };
+   auto process_security = [&](const databento::Record &record) -> bool {
+      const auto &msg = record.Get<databento::InstrumentDefMsg>();
+      const auto exp_nanos = msg.expiration;
+      std::string exp_string = databento::ToIso8601(exp_nanos);
+      const auto expiration_time =
+          bts::core::TimeStamp::from_iso_string(exp_string);
+      if (!sec_filter(
+              expiration_time, msg.Underlying(), venue,
+              msg.strike_price / 1000000000)) {
+         return false;
       }
       bts::domain::Security sec;
       sec.security_common = domain::SecurityCommon();
       sec.security_common.venue = venue;
       sec.security_common.exchange_sec_id = msg.hd.instrument_id;
       sec.security_common.desc = msg.RawSymbol();
-      sec.security_common.symbol = msg.Asset();
+      sec.security_common.symbol = msg.Underlying();
       sec.security_common.asset = msg.Asset();
       sec.security_common.group = msg.Group();
       sec.security_common.min_price_increment = 0.01;
       sec.security_common.min_quantity = 0.01;
       sec.security_common.min_quantity_increment = 0.01;
       sec.security_common.currency = {msg.currency.data(), msg.currency.size()};
-      // sec.security_common.cpp = msg.hd.
 
       const SecurityType sec_type =
           parse_sec_type({msg.security_type.data(), msg.security_type.size()});
-      const auto exp_nanos = msg.expiration;
-      std::string exp_string = databento::ToIso8601(exp_nanos);
-      const auto expiration_time =
-          bts::core::TimeStamp::from_iso_string(exp_string);
-      auto duration = filter_securities_upto_n_months*30*24*60*60;
-      if( expiration_time.seconds() >= bts::core::TimeStamp::now().seconds() + duration){
-         return databento::KeepGoing::Continue;
-      }
 
       switch (sec_type) {
       case SEC_TYPE(FUTURE):
@@ -134,11 +235,10 @@ void DatabentoDatahandler::run(
          break;
       case SEC_TYPE(OPTION): {
          auto call_put = CallPut::PUT;
-         if (msg.instrument_class == 'C') {
+         if (msg.instrument_class == 'C')
             call_put = CallPut::CALL;
-         } else {
+         else
             bts_assert(msg.instrument_class == 'P');
-         }
          sec.security.emplace<bts::domain::OptionSecurity>(
              expiration_time, bts::domain::Price(msg.strike_price, -9),
              call_put);
@@ -147,36 +247,62 @@ void DatabentoDatahandler::run(
       default:
          bts_assert_msg(
              false, "Only futures and options supported on databento");
-         return databento::KeepGoing::Stop;
       }
-
       *sec.security_common.id = sec_id_store->get_or_add(
           venue, sec_type, std::to_string(msg.hd.instrument_id));
       listener->on_securities({&sec});
-      return databento::KeepGoing::Continue;
+      sec_id_map.insert({msg.hd.instrument_id, sec.security_common.id});
+      return true;
+   };
+   auto debug_log = [&](const databento::Mbp1Msg &msg) {
+      if (DEBUG_SEC_ID != msg.hd.instrument_id)
+         return;
+      const double denom = 1000000000;
+      // auto delta = std::to_string(std::chrono::duration<double, std::micro>(msg.ts_in_delta).count());
+      auto seq_num = msg.sequence;
+      auto ap = msg.levels[0].ask_px / denom;
+      auto bp = msg.levels[0].bid_px / denom;
+      auto as = msg.levels[0].ask_sz;
+      auto bs = msg.levels[0].bid_sz;
+      LOG(md, Sys, "ID: {}, seq_num: {}, B_P: {}, B_S: {}, A_P: {}, A_S: {}",
+          msg.hd.instrument_id, seq_num, bp, bs, ap, as);
    };
 
    auto process_bbo = [&](const databento::Record &record) {
       const auto &msg = record.Get<databento::Mbp1Msg>();
+      debug_log(msg);
+
+      // std::vector<std::tuple<uint64_t,std::string>>& times = all_times.emplace_back();
+      // times.reserve(8);
+      // return databento::KeepGoing::Continue; //only added so the same functions work with the threaded builder, not really need in the blocking version
       // The security type to is needed to get the id. Since the sec type isn't known,
       // check for an id using both types (OPTION and FUTURE). This is safe to do since
       // options and futures never have common names (instrument ids).
-      const auto name = std::to_string(msg.hd.instrument_id);
-      auto sec_id_opt =
-          sec_id_store->get_id(venue, SEC_TYPE(OPTION), name, false);
-      if (!sec_id_opt.has_value()) {
-         sec_id_opt =
-             sec_id_store->get_id(venue, SEC_TYPE(FUTURE), name, false);
-         if (!sec_id_opt.has_value()) {
-            return databento::KeepGoing::Continue;
-         }
-      }
+      // auto current = bts::datetime::unix_nano_now();
+      // auto prev = current;
+      // times.push_back({current-prev,"Start Process bbo"});
+
+      // prev = current;
+      // current = bts::datetime::unix_nano_now();
+      // times.push_back({current-prev,"MISC to check log time"});
+      if (!sec_id_map.contains(msg.hd.instrument_id))
+         return databento::KeepGoing::Continue;
+      bts::domain::SecurityID sec_id = sec_id_map.at(msg.hd.instrument_id);
+
+      // prev = current;
+      // current = bts::datetime::unix_nano_now();
+      // times.push_back({current-prev,"Sec_ID done"});
+
+      // sleep_for(sleep_duration_updated);
+      // prev = current;
+      // current = bts::datetime::unix_nano_now();
+      // times.push_back({current-prev,"Sleep done"});
 
-      sleep_for(sleep_duration_updated);
-      bts::domain::SecurityID sec_id =
-          bts::domain::SecurityID(sec_id_opt.value());
       domain::Book book;
       book.set_security_id(sec_id);
+      // prev = current;
+      // current = bts::datetime::unix_nano_now();
+      // times.push_back({current-prev,"Empty Book Made"});
 
       auto &bids = book.bids();
       auto &asks = book.asks();
@@ -186,130 +312,215 @@ void DatabentoDatahandler::run(
       asks.begin()->price = bts::domain::Price(msg.levels[0].ask_px, -9);
       asks.begin()->total_quantity =
           bts::domain::Quantity(msg.levels[0].ask_sz);
+      //     prev = current;
+      //     current = bts::datetime::unix_nano_now();
+      // times.push_back({current-prev,"Complete book done"});
 
       auto recv_time = bts::core::TimeStamp::from_iso_string(
           databento::ToIso8601(msg.ts_recv));
+      // prev = current;
+      // current = bts::datetime::unix_nano_now();
+      // times.push_back({current-prev,"Time converted"});
+
       listener->on_book(
           book,
           utils::TimeStamp{datetime::ptime_to_timestamp(recv_time.to_ptime())});
-      return databento::KeepGoing::Continue;
+      // prev = current;
+      // current = bts::datetime::unix_nano_now();
+      // times.push_back({current-prev,"On book done"});
+
+      return databento::KeepGoing::
+          Continue; //only added so the same functions work with the threaded builder, not really need in the blocking version
    };
+
    auto process_trades = [&](const databento::Record &record) {
       const auto &msg = record.Get<databento::TradeMsg>();
       // The security type to is needed to get the id. Since the sec type isn't known,
       // check for an id using both types (OPTION and FUTURE). This is safe to do since
       // options and futures never have common names (instrument ids).
-      const auto name = std::to_string(msg.hd.instrument_id);
-      auto sec_id_opt =
-          sec_id_store->get_id(venue, SEC_TYPE(OPTION), name, false);
-      if (!sec_id_opt.has_value()) {
-         sec_id_opt =
-             sec_id_store->get_id(venue, SEC_TYPE(FUTURE), name, false);
-         if (!sec_id_opt.has_value()) {
-            return databento::KeepGoing::Continue;
-         }
-      }
+      if (!sec_id_map.contains(msg.hd.instrument_id))
+         return databento::KeepGoing::Continue;
+      bts::domain::SecurityID sec_id = sec_id_map.at(msg.hd.instrument_id);
 
-      sleep_for(sleep_duration_updated);
-      bts::domain::SecurityID sec_id =
-          bts::domain::SecurityID(sec_id_opt.value());
+      // sleep_for(sleep_duration_updated);
       domain::TradeRecord trade;
       trade.trade_price = bts::domain::Price(msg.price, -9);
       trade.trade_quantity = bts::domain::Quantity(msg.size);
-      trade.sec_id =  sec_id;
-      switch(msg.side){
-         case databento::Side::Ask:
-            trade.agg_side = domain::AggressorSide::Sell;
-            break;
-         case databento::Side::Bid:
-            trade.agg_side = domain::AggressorSide::Buy;
-            break;
-         case databento::Side::None:
-            trade.agg_side = domain::AggressorSide::None;
-            break;
+      trade.sec_id = sec_id;
+      switch (msg.side) {
+      case databento::Side::Ask:
+         trade.agg_side = domain::AggressorSide::Sell;
+         break;
+      case databento::Side::Bid:
+         trade.agg_side = domain::AggressorSide::Buy;
+         break;
+      case databento::Side::None:
+         trade.agg_side = domain::AggressorSide::None;
+         break;
       }
       trade.time_stamp = bts::core::TimeStamp::from_iso_string(
           databento::ToIso8601(msg.ts_recv));
       std::vector<const domain::TradeRecord *> trade_ptrs;
       trade_ptrs.push_back(&trade);
       listener->on_trades(trade_ptrs);
-     
-      return databento::KeepGoing::Continue;
+
+      return databento::KeepGoing::
+          Continue; //only added so the same functions work with the threaded builder, not really need in the blocking version
    };
    if (use_file) {
       auto client = databento::HistoricalBuilder{}.SetKey(key).Build();
       databento::DbnFileStore dbn_file_store(securities_path);
-      dbn_file_store.Replay(process_security);
+      // dbn_file_store.Replay(process_security); // fix this
       databento::DbnFileStore dbn_file_store2(books_path);
       dbn_file_store2.Replay(process_bbo);
-   }else {
-    bts_assert(dataset!="");
-    // auto client = databento::LiveBuilder{}
-    //               .SetKey(key)
-    //               .SetDataset(dataset)
-    //               .BuildThreaded();
-    auto client = databento::LiveBlocking{
-        databento::ILogReceiver::Default(),
-        key,
-        dataset,
-        true,
-        databento::VersionUpgradePolicy::UpgradeToV2,
-        {}
-    };
-    //hardcoding for now will update later
-    std::vector<std::string> symbols;
-    switch (venue) {
-        case api::domain::VENUE_CBOE: symbols= {"SPX.OPT","SPXW.OPT"}; break;
-        case api::domain::VENUE_GLBX: symbols= {"ES.FUT"}; break;
-        default:bts_assert_msg(false, "ONLY GLBX AND CBOE SUPPORTED FOR NOW");
-    }
-    if(load_reference_data) 
-         client.Subscribe(symbols, databento::Schema::Definition, databento::SType::Parent,databento::UnixNanos{});
-    else client.Subscribe(symbols, databento::Schema::Definition, databento::SType::Parent);
-    client.Subscribe(symbols, databento::Schema::Mbp1, databento::SType::Parent);
-    client.Subscribe(symbols, databento::Schema::Trades, databento::SType::Parent);
-    client.Start();
-    while (true) {
-    try {
-    auto* record = client.NextRecord(std::chrono::seconds{500});
-    if (record == nullptr) {
-        LOG(md, Error, "Timed out waiting for data");
-        break;
-    }
-    if (record->Holds<databento::Mbp1Msg>()) {
-        process_bbo(*record);
-    }
-    if (record->Holds<databento::InstrumentDefMsg>()) {
-        process_security(*record);
-    }
-    if (record->Holds<databento::TradeMsg>()) {
-        process_trades(*record);
-    }
-  } catch (const databento::TcpError& error) {
-    std::string err = error.what();
-    LOG(md, Error, "Caught TCP error: {} \nAttempting reconnect\n",err);
-    client.Reconnect();
-  } catch(const databento::DbnResponseError& err){
-    std::string msg = err.Message();
-    std::string what = err.what();
-    LOG(md, Error, "Caught DbnResponseError what: {} ,msg:{} \nShutting down\n",msg, what);
-    client.Stop();
-    // bts_assert(false);
-  }
-  catch(const databento::Exception& excp){
-    std::string err = excp.Message();
-    std::string err2 = excp.what();
-    LOG(md, Error, "Caught unknown dadtabento::exception what: {}, msg: {} \nShutting down\n",err, err2);
-    client.Stop();
-    // bts_assert(false);
-  }catch (const std::exception& exc) {
-    std::string err = exc.what();
-    LOG(md, Error, "Caught unknown exception: {} \nShutting down\n",err);
-    client.Stop();
-    // bts_assert(false);
-  }
-    
-    }
+   } else {
+      bts_assert(dataset != "");
+
+      //hardcoding for now will update later
+      std::vector<std::string> symbols;
+      switch (venue) {
+      case api::domain::VENUE_CBOE:
+         symbols = {"SPX.OPT", "SPXW.OPT"};
+         break;
+      case api::domain::VENUE_GLBX:
+         symbols = {"ES.FUT"};
+         break;
+      default:
+         bts_assert_msg(false, "ONLY GLBX AND CBOE SUPPORTED FOR NOW");
+      }
+      //ocassionaly run security writer to update security dbn
+      if (!boost::filesystem::exists(securities_path) || load_reference_data) {
+         update_file(key, dataset, symbols, 15 * 60, securities_path);
+      }
+
+      //regular flow
+      //read from security dbn, filter contracts that we need and subscribe to those in batches of two thousand
+      LOG(md, Info, "starting loading symbols");
+      databento::DbnFileStore store{securities_path};
+      std::vector<std::vector<std::string>> all_sec;
+
+      all_sec.emplace_back();
+      all_sec.back().reserve(2000);
+      size_t total_count = 0;
+
+      while (auto *record = store.NextRecord()) {
+         if (auto *def = record->GetIf<databento::InstrumentDefMsg>()) {
+            std::string s = def->RawSymbol();
+            if (process_security(*record)) {
+               ++total_count;
+               // Check if the last vector is full
+               if (all_sec.back().size() == 2000) {
+                  all_sec.emplace_back();
+                  all_sec.back().reserve(2000);
+                  LOG(md, Info, "{} symbols added", total_count);
+               }
+               all_sec.back().push_back(std::move(s)); // Move to avoid copying
+            }
+         }
+      }
+      // std::ostringstream output;
+
+      // for (const auto& sec_group : all_sec) {
+      //    for (size_t i = 0; i < sec_group.size(); ++i) {
+      //       if (i > 0) output << ", ";
+      //       output << sec_group[i];
+      //    }
+      //    output << '\n';  // Newline after each group
+      // }
+
+      // // Store the final output in a stringls
+
+      // std::string final_output = output.str();
+      // size_t size = all_sec.size();
+      LOG(md, Info, "total {} secs loaded", total_count);
+
+
+      bool first = true;
+      auto client = databento::LiveBlocking{
+          databento::ILogReceiver::Default(),           key, dataset, true,
+          databento::VersionUpgradePolicy::UpgradeToV2, {}};
+      std::string start = config::get<bts::api::md::MarketDataConfig>()
+          .cboe()
+          .at(0)
+          .databento_settings()
+          .bbo_start_time();
+      if(start != ""){
+         for (std::vector vec : all_sec) {
+            client.Subscribe(
+                vec, databento::Schema::Mbp1, databento::SType::RawSymbol, start);
+            client.Subscribe(
+                vec, databento::Schema::Trades, databento::SType::RawSymbol,start);
+         }   
+      }else{
+         for (std::vector vec : all_sec) {
+            client.Subscribe(
+                vec, databento::Schema::Mbp1, databento::SType::RawSymbol);
+            client.Subscribe(
+                vec, databento::Schema::Trades, databento::SType::RawSymbol);
+         } 
+      }
+      bool only_securities = config::get<bts::api::md::MarketDataConfig>()
+                         .cboe()
+                         .at(0)
+                         .databento_settings()
+                         .only_securities();
+      client.Start();
+      while (true) {
+         try {
+            auto *record = client.NextRecord(std::chrono::seconds{500});
+            if (first) {
+               LOG(md, Info, "first record");
+               first = false;
+            }
+            if (record == nullptr) {
+               LOG(md, Error, "Timed out waiting for data");
+               break;
+            }
+            if(only_securities)
+               continue;
+            if (record->Holds<databento::Mbp1Msg>()) {
+               process_bbo(*record);
+            }
+            if (all_times.size() == 1001) {
+               log_time_statistics(all_times);
+               client.Stop();
+            }
+            //  if (record->Holds<databento::InstrumentDefMsg>()) {
+            //      process_security(*record);
+            //  }
+            if (record->Holds<databento::TradeMsg>()) {
+               process_trades(*record);
+            }
+         } catch (const databento::TcpError &error) {
+            std::string err = error.what();
+            LOG(md, Error, "Caught TCP error: {} \nAttempting reconnect\n",
+                err);
+            client.Reconnect();
+         } catch (const databento::DbnResponseError &err) {
+            std::string msg = err.Message();
+            std::string what = err.what();
+            LOG(md, Error,
+                "Caught DbnResponseError what: {} ,msg:{} \nShutting down\n",
+                msg, what);
+            client.Stop();
+            // bts_assert(false);
+         } catch (const databento::Exception &excp) {
+            std::string err = excp.Message();
+            std::string err2 = excp.what();
+            LOG(md, Error,
+                "Caught unknown dadtabento::exception what: {}, msg: {} "
+                "\nShutting down\n",
+                err, err2);
+            client.Stop();
+            // bts_assert(false);
+         } catch (const std::exception &exc) {
+            std::string err = exc.what();
+            LOG(md, Error, "Caught unknown exception: {} \nShutting down\n",
+                err);
+            client.Stop();
+            // bts_assert(false);
+         }
+      }
    }
 }
 

src/server/sparkd/md/recording/MDRecorder.cpp

@@ -143,6 +143,7 @@ struct MDRecorder::PImpl {
    uintptr_t mdc_id = 0;
    bts::md::MDClient &md_client;
    std::map<SecurityKey, MDRecordingSecEntry> handles;
+   std::optional<std::thread> run_thread;
    MDBatchWriter batch_writer{settings.batch_size,std::vector<bts::md::recording::MDBatchWriter::DBWriteFunction>{DbWriters::writeToSQL}};
    uint64_t start_ns;
    PImpl(
@@ -160,6 +161,14 @@ struct MDRecorder::PImpl {
                  1000 * 1000 * 1000,
              100000, 1000000000,recording_config.batch_size()}),
          md_client(md_client),start_ns(bts::core::TimeStamp::now_ns()) {}
+
+   ~PImpl() {
+      active = false;
+      if (run_thread && run_thread->joinable()) {
+         run_thread->join();
+      }
+      run_thread.reset();
+   }
 };
 
 MDRecorder::MDRecorder(
@@ -171,7 +180,8 @@ MDRecorder::MDRecorder(
    me->file.open(file_name, std::ios::out | std::ios::binary | std::ios::trunc);
    LOG(md, Info, "MDRecorder::MDRecorder venue: {} file: {}",
        api::Venue_Name(venue), file_name);
-   bts::threading::make_thread("MDRecorder", [&]() { _poll_md(); }).detach();
+   me->run_thread.emplace(
+       bts::threading::make_thread("MDRecorder", [&]() { _poll_md(); }));
 }
 
 MDRecorder::~MDRecorder() {}

src/server/sparkd/spark.conf

@@ -0,0 +1,42 @@
+MarketDataConfig {
+  value {
+    binance_us {
+      num_depth_levels: 5
+      url_rest_base: "https://api.binance.us"
+      url_ws_base: "wss://stream.binance.us:9443"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: "BTC"
+          symbols: "ETH"
+        }
+      }
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_BINANCE_US"
+      spot: "BTC"
+      spot: "ETH"
+    }
+  }
+}
+
+Logging {
+  value {
+    console {
+      default_level: "Info"
+    }
+    syslog {
+      default_level: "Info"
+    }
+    file {
+      default_level: "Info"
+    }
+    initial_queue_size: 120001
+    max_queue_growth: 4
+  }
+}

tools/aws/install_aws_cli.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+this_dir="$(dirname "${BASH_SOURCE[0]}")"
+work_dir="$this_dir/.tmp"
+# Clean previous install
+rm -rf "$work_dir"
+mkdir -p "$work_dir"
+cd "$work_dir"
+# Download installer and run it
+curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+  unzip awscliv2.zip && \
+  sudo ./aws/install
+cd "$this_dir"
+# Clean up
+rm -rf "$work_dir"

tools/binance/binance.env

@@ -74,6 +74,16 @@ BINANCE_USDS_FUTURES_POSITION_INFO_PARAMS='symbol=${SYMBOL:-OMIT}&recvWindow=${R
 BINANCE_USDS_FUTURES_POSITION_INFO_METHOD="GET"
 BINANCE_USDS_FUTURES_POSITION_INFO_IS_SIGNED=1
 
+BINANCE_USDS_FUTURES_LISTEN_KEY_PATH="/fapi/v1/listenKey"
+BINANCE_USDS_FUTURES_LISTEN_KEY_PARAMS=''
+BINANCE_USDS_FUTURES_LISTEN_KEY_METHOD="POST"
+BINANCE_USDS_FUTURES_LISTEN_KEY_IS_KEYED=1
+
+BINANCE_USDS_FUTURES_REFRESH_LISTEN_KEY_PATH="/fapi/v1/listenKey"
+BINANCE_USDS_FUTURES_REFRESH_LISTEN_KEY_PARAMS='listenKey=${LISTEN_KEY:-OMIT}'
+BINANCE_USDS_FUTURES_REFRESH_LISTEN_KEY_METHOD="PUT"
+BINANCE_USDS_FUTURES_REFRESH_LISTEN_KEY_IS_KEYED=1
+
 BINANCE_COIN_FUTURES_PROD_URL="https://dapi.binance.com"
 BINANCE_COIN_FUTURES_DEV_URL="https://testnet.binancefuture.com"
 BINANCE_COIN_FUTURES_EXCHANGE_INFO_PATH="/dapi/v1/exchangeInfo"
@@ -116,3 +126,57 @@ BINANCE_CANCEL_REPLACE_PATH="/api/v3/order/cancelReplace"
 BINANCE_CANCEL_REPLACE_PARAMS='symbol=${SYMBOL}&side=${SIDE:-BUY}&type=${TYPE:-LIMIT}&timeInForce=${TIME_IN_FORCE:-GTC}&quantity=${QUANTITY:-OMIT}&price=${PRICE:-OMIT}&cancelReplaceMode=${CANCEL_REPLACE_MODE}&cancelNewClientOrderId=${CANCEL_NEW_CLIENT_ORDER_ID:-OMIT}&cancelOrigClientOrderId=${CANCEL_ORIG_CLIENT_ORDER_ID:-OMIT}&cancelOrderId=${CANCEL_ORDER_ID:-OMIT}&newClientOrderId=${NEW_CLIENT_ORDER_ID:-OMIT}&recvWindow=${RECV_WINDOW:-6000}'
 BINANCE_CANCEL_REPLACE_METHOD="POST"
 BINANCE_CANCEL_REPLACE_IS_SIGNED=1
+
+BINANCE_LISTEN_KEY_PATH="/api/v3/userDataStream"
+BINANCE_LISTEN_KEY_PARAMS=''
+BINANCE_LISTEN_KEY_METHOD="POST"
+BINANCE_LISTEN_KEY_IS_KEYED=1
+
+BINANCE_REFRESH_LISTEN_KEY_PATH="/api/v3/userDataStream"
+BINANCE_REFRESH_LISTEN_KEY_PARAMS='listenKey=${LISTEN_KEY}'
+BINANCE_REFRESH_LISTEN_KEY_METHOD="PUT"
+BINANCE_REFRESH_LISTEN_KEY_IS_KEYED=1
+
+BINANCE_CROSS_MARGIN_COLLATERAL_RATIO_PATH="/sapi/v1/margin/crossMarginCollateralRatio"
+BINANCE_CROSS_MARGIN_COLLATERAL_RATIO_METHOD="GET"
+BINANCE_CROSS_MARGIN_COLLATERAL_RATIO_IS_SIGNED=1
+
+BINANCE_CROSS_MARGIN_PAIRS_PATH="/sapi/v1/margin/allPairs"
+BINANCE_CROSS_MARGIN_PAIRS_PARAMS='symbol=${SYMBOL:-OMIT}'
+BINANCE_CROSS_MARGIN_PAIRS_METHOD="GET"
+BINANCE_CROSS_MARGIN_PAIRS_IS_SIGNED=1
+
+BINANCE_CROSS_MARGIN_ORDER_PATH="/sapi/v1/margin/order"
+BINANCE_CROSS_MARGIN_ORDER_PARAMS='symbol=${SYMBOL}&side=${SIDE}&type=${TYPE}&quantity=${QUANTITY}&price=${PRICE}&timeInForce=GTC&autoRepayAtCancel=true&sideEffectType=AUTO_BORROW_REPAY&recvWindow=${RECV_WINDOW:-6000}'
+BINANCE_CROSS_MARGIN_ORDER_METHOD="POST"
+BINANCE_CROSS_MARGIN_ORDER_IS_SIGNED=1
+
+BINANCE_CROSS_MARGIN_OPEN_ORDERS_PATH="/sapi/v1/margin/openOrders"
+BINANCE_CROSS_MARGIN_OPEN_ORDERS_PARAMS='symbol=${SYMBOL}&isIsolated=${IS_ISOLATED:-OMIT}&recvWindow=${RECV_WINDOW:-6000}'
+BINANCE_CROSS_MARGIN_OPEN_ORDERS_METHOD="GET"
+BINANCE_CROSS_MARGIN_OPEN_ORDERS_IS_SIGNED=1
+
+BINANCE_CROSS_MARGIN_CANCEL_PATH="/sapi/v1/margin/order"
+BINANCE_CROSS_MARGIN_CANCEL_PARAMS='symbol=${SYMBOL}&isIsolated=${IS_ISOLATED:-OMIT}&orderId=${ORDER_ID:-OMIT}&origClientOrderId=${ORIG_CLIENT_ORDER_ID:-OMIT}&newClientOrderId=${NEW_CLIENT_ORDER_ID:-OMIT}&recvWindow=${RECV_WINDOW:-6000}'
+BINANCE_CROSS_MARGIN_CANCEL_METHOD="DELETE"
+BINANCE_CROSS_MARGIN_CANCEL_IS_SIGNED=1
+
+BINANCE_CROSS_MARGIN_LISTEN_KEY_PATH="/sapi/v1/userDataStream"
+BINANCE_CROSS_MARGIN_LISTEN_KEY_PARAMS=''
+BINANCE_CROSS_MARGIN_LISTEN_KEY_METHOD="POST"
+BINANCE_CROSS_MARGIN_LISTEN_KEY_IS_KEYED=1
+
+BINANCE_CROSS_MARGIN_REFRESH_LISTEN_KEY_PATH="/sapi/v1/userDataStream"
+BINANCE_CROSS_MARGIN_REFRESH_LISTEN_KEY_PARAMS='listenKey=${LISTEN_KEY}'
+BINANCE_CROSS_MARGIN_REFRESH_LISTEN_KEY_METHOD="PUT"
+BINANCE_CROSS_MARGIN_REFRESH_LISTEN_KEY_IS_KEYED=1
+
+BINANCE_ISOLATED_MARGIN_SYMBOL_PATH="/sapi/v1/margin/order"
+BINANCE_ISOLATED_MARGIN_SYMBOL_PARAMS='symbol=${SYMBOL}&side=${SIDE}&type=${TYPE}&quantity=${QUANTITY}&price=${PRICE}&autoRepayAtCancel=true&recvWindow=${RECV_WINDOW:-6000}'
+BINANCE_ISOLATED_MARGIN_SYMBOL_METHOD="GET"
+BINANCE_ISOLATED_MARGIN_SYMBOL_IS_SIGNED=1
+
+BINANCE_ALL_MARGIN_ASSETS_PATH="/sapi/v1/margin/allAssets"
+BINANCE_ALL_MARGIN_ASSETS_PARAMS='asset=${ASSET:-OMIT}'
+BINANCE_ALL_MARGIN_ASSETS_METHOD="GET"
+BINANCE_ALL_MARGIN_ASSETS_IS_SIGNED=1

tools/binance/binance_orders_client.sh

@@ -133,6 +133,8 @@ method_name="${venue}_${op}_METHOD"
 method="${!method_name:-GET}"
 is_signed_name="${venue}_${op}_IS_SIGNED"
 is_signed="${!is_signed_name:-0}"
+is_keyed_name="${venue}_${op}_IS_KEYED"
+is_keyed="${!is_keyed_name:-0}"
 
 # Notes on query params
 # =============================================================================
@@ -168,13 +170,21 @@ params=$(echo "$params" | sed -E 's/(&)?[^&=]+=OMIT//g' | sed 's/^&//;s/&$//')
 # Sign the request (if necessary)
 declare -A headers
 if [ "$is_signed" -eq 1 ]; then
+  if [ -n "$params" ]; then
+    params="$params&"
+  fi
   # Signed requests require a timestamp, and api key header
-  params="$params&timestamp=$(date +%s%3N)"
+  params="${params}timestamp=$(date +%s%3N)"
   signature="$(echo -n "$params" | openssl dgst -sha256 -hmac "$secret_key" | awk '{print $NF}')"
   params="$params&signature=$signature"
   headers["X-MBX-APIKEY"]="$api_key"
+elif [ "$is_keyed" -eq 1 ]; then
+  headers["X-MBX-APIKEY"]="$api_key"
+fi
+url="${!base_url_name}${!path_name}"
+if [ -n "$params" ]; then
+  url="$url?$params"
 fi
-url="${!base_url_name}${!path_name}?${params}"
 
 headers_str=""
 for key in "${!headers[@]}"; do

tools/deploy.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+tools_path="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+eval "$tools_path/ensure_venv.sh" 1> /dev/stdout 2> /dev/stderr
+eval "$tools_path/venv_exec.sh python $tools_path/py/deploy.py $@" 1> /dev/stdout 2> /dev/stderr

tools/py/deploy.py

@@ -0,0 +1,131 @@
+import argparse
+import dotenv
+import subprocess
+
+import publish_sparkd
+
+from core.argparse_utils import add_toggle_opt
+from core.file_utils import SparkPaths
+
+
+def _get_arg_parser():
+    parser = argparse.ArgumentParser(
+        description=
+        "Deploy sparkd, pubsub, md-publisher, webapp, and api server")
+    parser.add_argument(
+        "--cmake-jobs",
+        type=int,
+        help="The number of cmake parallel jobs",
+    )
+    parser.add_argument(
+        "--profile",
+        help="The configuration profile dirs to use, relative to each service")
+    parser.add_argument("--compose-file",
+                        help="The docker compose (yaml) file to use")
+    parser.add_argument(
+        "--clean",
+        action="store_true",
+        help="Perform a clean build (remove artifacts from previous build)",
+    )
+    parser.add_argument("--env-files",
+                        action="append",
+                        help=".env files to pass to docker compose")
+    parser.add_argument(
+        "--verbosity",
+        metavar="{0-5}",
+        choices=range(1, 6),
+        type=int,
+        help="The output verbosity level",
+    )
+    add_toggle_opt(parser,
+                   "build",
+                   help="Whether or not to build the services first")
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Preview the build commands without running them",
+    )
+    # TODO: add more services (api-server, pubsub, dsqa web server, etc.)
+    all_services = ["sparkd"]
+    parser.add_argument("services", nargs="*")
+    parser.set_defaults(
+        configuration="release",
+        services=all_services,
+        build=True,
+        clean=False,
+        profile="dev",
+        verbosity=2,
+    )
+    return parser
+
+
+def _get_env_files(services, profile):
+    root = SparkPaths.root()
+    files = []
+
+    def add(file):
+        if file.exists():
+            files.append(file)
+
+    add(root.joinpath(".env"))
+    add(root.joinpath(".env.local"))
+    if profile is not None:
+        if "sparkd" in services:
+            profile_dir=root.joinpath("src/server/sparkd/configurations", profile)
+            add(profile_dir.joinpath(".env"))
+            add(profile_dir.joinpath(".env.local"))
+    return files
+
+
+def deploy(args):
+    def run(cmd):
+        print(cmd)
+        if args.dry_run:
+            return
+        subprocess.run(cmd, shell=True).check_returncode()
+
+    services: list[str] = args.services or []
+    if args.build:
+        build_all = len(services) == 0
+        if build_all or "sparkd" in services:
+            publish_sparkd.publish(
+                publish_sparkd.PublishArgs(
+                    configuration="release",
+                    cmake_jobs=args.cmake_jobs,
+                    clean=args.clean,
+                    verbosity=args.verbosity,
+                    dry_run=args.dry_run,
+                ))
+    service_args = " ".join(services)
+    compose_cmd = "sudo docker compose"
+    if args.compose_file:
+        compose_cmd += f" -f {args.compose_file}"
+    env_files = _get_env_files(args.services, args.profile)
+    if len(env_files) > 0:
+        compose_cmd += " " + " ".join([f"--env-file {f}" for f in env_files])
+        for ef in env_files:
+            if args.verbosity > 2:
+                print(f"Loading env file {ef}")
+                if args.verbosity > 3:
+                    env_vars = dotenv.dotenv_values(ef)
+                    for k, v in env_vars.items():
+                        print(f"    {k}={v}")
+            dotenv.load_dotenv(ef)
+    run(f"{compose_cmd} down {service_args}")
+    run(f"{compose_cmd} build {service_args}")
+    run(f"{compose_cmd} up {service_args} -d")
+    run(f"{compose_cmd} logs {service_args} --follow")
+
+
+def _main():
+    parser = _get_arg_parser()
+    args = parser.parse_args()
+    deploy(args)
+
+
+if __name__ == "__main__":
+    try:
+        _main()
+    except Exception as e:
+        print(f"deploy.py failed: {e}")
+        exit(1)

tools/sparkd_data.sh

@@ -0,0 +1,217 @@
+#!/usr/bin/env bash
+
+help_text="\
+Manage files in sparkd data volumes
+
+Usage: sparkd_data.sh [OPTIONS] [COMMAND] [...]
+
+Commands:
+  get               Copy files FROM data volume
+  set               Copy files TO data volume
+  shell             Start shell in data volume
+
+Options:
+  -h,--help         Print this message and exit
+
+"
+
+cp_help_text="\
+Copy files to/from sparkd data volume
+
+Usage: sparkd_data.sh <get|set> [OPTIONS] [POSITIONALS]
+
+Positionals:
+  SOURCE            Source file
+  DEST              Dest file
+
+Options:
+  --clean           Recreate the container that mounts to the volume (even if it already exists)
+  --source          Source file
+  --dest            Dest file
+  --config-port     The config port for the volume to manage
+  -h,--help         Print this message and exit
+"
+
+shell_help_text="\
+Start shell in sparkd data volume
+
+Usage: sparkd_data.sh shell [OPTIONS]
+
+Options:
+  --clean           Recreate the container that mounts to the volume (even if it already exists)
+  --config-port     The config port for the volume to manage
+  -h,--help         Print this message and exit
+"
+
+print_help_and_exit() {
+  echo "$1"
+  exit "${2:-0}"
+}
+
+bounce_container() {
+  if [ "$#" -ne 3 ]; then
+    echo "bounce_container invalid args '$@'."
+    echo "Usage: bounce_container <clean:0|1> <container_name> <volume_name>"
+    return 1
+  fi
+  clean="$1"
+  cont_name="$2"
+  vol_name="$3"
+  image_name="bluetradingsystems/linux_dotnet7"
+  if [ -n "$(sudo docker container ls -a -q --filter "name=$cont_name")" ]; then
+    if [[ $clean -eq 1 ]]; then
+      sudo docker stop $cont_name
+      sudo docker rm $cont_name > /dev/null 2>&1
+      sudo docker run --name $cont_name --rm -it -d -v $vol_name:/home/bts/data $image_name sh
+    fi
+  else
+    sudo docker run --name $cont_name --rm -it -d -v $vol_name:/home/bts/data $image_name sh
+  fi
+}
+
+get_container_name() {
+  if [ "$#" -ne 1 ]; then
+    echo "get_container_name invalid args '$@'."
+    echo "Usage: container_name <config_port>"
+    return 1
+  fi
+  echo "sparkd_data_man_${1}"
+}
+
+get_volume_name() {
+  if [ "$#" -ne 1 ]; then
+    echo "get_volume_name invalid args '$@'."
+    echo "Usage: get_volume_name <config_port>"
+    return 1
+  fi
+  echo "sparkd_${1}"
+}
+
+cp_file() {
+  config_port=${SPARKD__CONFIG_PORT:-8000}
+  is_get=${1:-1}
+  shift 1
+  source=""
+  dest=""
+  clean=0
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --clean)
+        clean=1
+        shift 1
+        ;;
+      --source)
+        if [[ -z "$2" ]]; then
+          echo "Source (--source) must be non-empty"
+          print_help_and_exit "$cp_help_text" 1
+        fi
+        source="$2"
+        shift 2
+        ;;
+      --dest)
+        if [[ -z "$2" ]]; then
+          echo "Dest (--Dest) must be non-empty"
+          print_help_and_exit "$cp_help_text" 1
+        fi
+        dest="$2"
+        shift 2
+        ;;
+      --config-port)
+        if [[ -z "$2" ]]; then
+          echo "Config port (--config-port) must be non-empty"
+          print_help_and_exit "$cp_help_text" 1
+        fi
+        config_port="$2"
+        shift 2
+        ;;
+      -h)
+        ;&
+      --help)
+        print_help_and_exit "$cp_help_text" 0
+        ;;
+      *)
+        if [[ -z "$source" ]]; then
+          source="$1"
+        elif [[ -z "$dest" ]]; then
+          dest="$1"
+        else
+          echo "Invalid positional argument '$1'"
+          print_help_and_exit "$cp_help_text" 1
+        fi
+        shift 1
+        ;;
+    esac
+  done
+
+  if [ -z "$source" ]; then
+    echo -e "Error: No source specified to copy\n"
+    print_help_and_exit "$cp_help_text" 1
+  fi
+
+  if [ -z "$dest" ]; then
+    dest="."
+  fi
+
+  cont_name="$(get_container_name "$config_port")"
+  vol_name="$(get_volume_name "$config_port")"
+  bounce_container "$clean" "$cont_name" "$vol_name"
+  if [[ $is_get -eq 1 ]]; then
+    sudo docker cp $cont_name:/home/bts/data/$source $dest
+  else
+    sudo docker cp $source $cont_name:/home/bts/data/$dest
+  fi
+}
+
+shell() {
+  clean=0
+  config_port=${SPARKD__CONFIG_PORT:-8000}
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --clean)
+        clean=1
+        shift 1
+        ;;
+      --config-port)
+        if [[ -z "$2" ]]; then
+          echo "Config port (--config-port) must be non-empty"
+          print_help_and_exit "$shell_help_text" 1
+        fi
+        config_port="$2"
+        shift 2
+        ;;
+      -h)
+        ;&
+      --help)
+        print_help_and_exit "$shell_help_text" 0
+        ;;
+      *)
+        echo "Invalid positional argument '$1'"
+        print_help_and_exit "$shell_help_text" 1
+        ;;
+    esac
+  done
+
+  cont_name="$(get_container_name "$config_port")"
+  vol_name="$(get_volume_name "$config_port")"
+  bounce_container "$clean" "$cont_name" "$vol_name"
+  sudo docker exec -it -w /home/bts/data $cont_name bash
+}
+
+case "$1" in
+  get)
+    shift 1
+    cp_file 1 $@
+    ;;
+  set)
+    shift 1
+    cp_file 0 $@
+    ;;
+  shell)
+    shift 1
+    shell $@
+    ;;
+  *)
+    echo "Invalid argument '$1'"
+    print_help_and_exit "$help_text" 1
+    ;;
+esac