check pt: Combine lib_py_market_data and provider py scripts into one project

fix build: Ensure py scripts use unix line endings
I'm not sure why, but my sparkd build (linux) suddenly began failing to generate cpp headers for protos. The error message indicated it was due to an unexpected carriage return (\r) in plugin.py: /usr/bin/env: ‘python\r’: No such file or directory
2024-08-12 14:03:33 +00:00 · 2024-08-10 15:17:01 +00:00 · 2024-08-10 03:21:06 +00:00 · 2024-08-09 16:50:49 +00:00 · 2024-08-09 16:50:49 +00:00 · 2024-08-09 16:50:49 +00:00
1168 changed files with 18278 additions and 27391 deletions
@@ -87,10 +87,6 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev

-# Databento (deps)
-RUN apt-get update && apt-get -y install libzstd-dev
-
-# Docker in Docker
 RUN apt-get update && apt-get install -y \
    docker.io \
    docker-compose \
@@ -21,16 +21,6 @@
        ],
        "cwd": "build/Debug/bin/net7.0",
      },
-      "linux": {
-        "preLaunchTask": "Build Api Server (Debug)",
-        "program": "spark_api_server",
-        "args": [
-          "run",
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -48,15 +38,6 @@
        ],
        "cwd": "build/Debug/bin/net7.0",
      },
-      "linux": {
-        "program": "spark_api_server",
-        "args": [
-          "run",
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
@@ -124,18 +105,6 @@
      "stopAtEntry": false,
      "console": "integratedTerminal"
    },
-    {
-      "name": "(gdb) Launch",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
-      "args": ["run"],
-      "stopAtEntry": false,
-      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
-      "environment": [],
-      "externalConsole": false,
-      "MIMode": "gdb",
-    },
    {
      "name": "DataStreamQueryAnalyzer",
      "type": "coreclr",
@@ -234,12 +203,12 @@
      "preLaunchTask": "Build Sparkd (Debug | Clang)",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
      },
      "args": [
        "run"
      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
      "stopAtEntry": false,
      "externalConsole": false,
      "MIMode": "gdb",
@@ -262,12 +231,12 @@
      "request": "launch",
      "program": "",
      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
+        "program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
      },
      "args": [
        "run"
      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
+      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
      "stopAtEntry": false,
      "externalConsole": false,
      "MIMode": "gdb",
@@ -283,35 +252,7 @@
          "ignoreFailures": true
        }
      ]
-    },
-    {
-      "name": "Sparkd Web (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "web"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
+    }
  ],
  "inputs": [
    {
@@ -18,7 +18,7 @@ set(
 # SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
 # can run simultaneously without fighting over the same shm files (e.g., sparkd,
 # api server, etc.).
-set(SHARED_MEMORY_PREFIX "\${BTS_SHARED_MEMORY_PREFIX}" CACHE STRING "Prefix to use when creating shared memory files")
+set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
 set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
 set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
 set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
@@ -33,85 +33,95 @@ else()
  set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()

+include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${EXTERNALS_LIB})
+
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 if(DEFINED APP_DATA_DIR)
  add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
 endif()
 add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
+if(WIN32)
+  # subdirs
+  add_subdirectory(src/lib_spark)
+else()
+  # make sparkd look for libspark.so in its directory
+  # set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+  # set(CMAKE_INSTALL_RPATH ".")
+  # set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install/$<CONFIG>")

-if(BUILD_SPARKD)
-  # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
-  # read by their code). We override some below.
-  add_compile_definitions(SPARK_ENABLE_CCAPI)
+  if(BUILD_SPARKD)
+    # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
+    # read by their code). We override some below.
+    add_compile_definitions(SPARK_ENABLE_CCAPI)

-  # This allows us to override hardcoded exchange-urls with ones read from
-  # config (i.e., resolved at runtime rather than compile-time)
-  add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
+    # This allows us to override hardcoded exchange-urls with ones read from
+    # config (i.e., resolved at runtime rather than compile-time)
+    add_compile_definitions(CCAPI_EXPOSE_INTERNAL)

-  # Enable ccapi services
-  add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
-  add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
-  # Enable ccapi exchanges
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
-  add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
+    # Enable ccapi services
+    add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
+    add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
+    # Enable ccapi exchanges
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
+    add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)

-  set(CCAPI_LOG_LEVEL
-    "Warn"
-    CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
-  string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
-  add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
+    set(CCAPI_LOG_LEVEL
+      "Warn"
+      CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
+    string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
+    add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")

-  # ** Binance **
-  add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
-  add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-  add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+    # ** Binance **
+    add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
+    add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+    add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")

-  # ** Binance US **
-  add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
-  add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
+    # ** Binance US **
+    add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
+    add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")

-  # ** Binance USDS Futures **
-  add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
-  add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
-  add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
+    # ** Binance USDS Futures **
+    add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
+    add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
+    add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")

-  # ** Deribit **
-  add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
-  add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
-  add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
-  add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
-  add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
-  add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
-  add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
-  add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
-  add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
+    # ** Deribit **
+    add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
+    add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
+    add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
+    add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
+    add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
+    add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
+    add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
+    add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
+    add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")

-  # ** Okx **
-  add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
-  add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
-endif()
+    # ** Okx **
+    add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
+    add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
+  endif()

-# subdirs
-add_subdirectory(src/lib_system)
-add_subdirectory(src/lib_domain)
-add_subdirectory(src/lib_http)
-if (BUILD_SPARKD)
-  add_subdirectory(src/lib_python_md)
-  add_subdirectory(src/app_sparkd)
-  add_subdirectory(src/data_files)
-endif()
-add_subdirectory(src/lib_spark)
-if (BUILD_ADMIN_WEB_APP)
-  add_subdirectory(src/Spark.Admin.WebApp)
-endif()
-if(BUILD_DOCS)
-  add_subdirectory(doc/user)
-endif()
-if(BUILD_SDK)
-  add_subdirectory(src/sdk)
+  # subdirs
+  if (BUILD_SPARKD)
+    add_subdirectory(src/app_sparkd)
+    add_subdirectory(src/data_files)
+    add_subdirectory(src/lib_py_market_data)
+    # add_subdirectory(src/py_market_data_providers)
+  endif()
+  add_subdirectory(src/lib_spark)
+  if (BUILD_ADMIN_WEB_APP)
+    add_subdirectory(src/Spark.Admin.WebApp)
+  endif()
+  if(BUILD_DOCS)
+    add_subdirectory(doc/user)
+  endif()
+  if(BUILD_SDK)
+    add_subdirectory(src/sdk)
+  endif()
 endif()
@@ -28,7 +28,7 @@
        "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -53,7 +53,6 @@
        "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
        "CMAKE_PREFIX_PATH": "${sourceDir}/cmake/;${sourceDir}/.venv/lib/python3.10/site-packages/pybind11/share/cmake/pybind11",
        "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
        "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
        "BUILD_SDK": {
          "type": "BOOL",
@@ -77,7 +76,7 @@
          "value": "ON"
        },
        "CCAPI_LOG_LEVEL": "Info",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_dev/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -88,7 +87,7 @@
      "cacheVariables": {
        "CMAKE_BUILD_TYPE": "Release",
        "CCAPI_LOG_LEVEL": "Warn",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/md_prod/*.conf;spark/config/profiles/default/*.conf"
+        "CONFIG_DEFAULTS_PATH": "config/profiles/md_prod/*.conf;config/profiles/default/*.conf"
      }
    },

@@ -168,8 +167,8 @@
          "type": "BOOL",
          "value": "ON"
        },
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net7.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
          "value": "ON"
@@ -207,7 +206,7 @@
      "description": "Data Stream utils ported over from Edge",
      "binaryDir": "${sourceDir}/build/${presetName}",
      "cacheVariables": {
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
      }
    },
@@ -240,7 +239,8 @@
        "CMAKE_CXX_COMPILER": "clang++",
        "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
-        "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
+        "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
+        "CONFIG_PROFILES": "default;data_stream",
        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
        "EXPORT_MD_CONSUMER": {
          "type": "BOOL",
@@ -294,14 +294,6 @@
      "inherits": "sparkd"
    },

-    {
-      "name": "sparkd-debug-clang-j6",
-      "configurePreset": "sparkd-debug-clang",
-      "displayName": "Sparkd (Debug | Clang | jobs=6)",
-      "inherits": "sparkd",
-      "jobs": 6
-    },
-
    {
      "name": "sparkd-debug-gcc",
      "configurePreset": "sparkd-debug-gcc",
@@ -14,16 +14,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\DataStreamSuite\Bts.Net\Bts.Net.csproj", "{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{05995ED5-650A-4955-9DFC-3242B737E77A}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{EFA9F78D-083E-457A-BABB-9386C88C32C0}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -80,46 +70,6 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.Build.0 = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.ActiveCfg = Release|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.Build.0 = Release|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.Build.0 = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.ActiveCfg = Release|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.Build.0 = Release|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.Build.0 = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.ActiveCfg = Release|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.Build.0 = Release|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.Build.0 = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.ActiveCfg = Release|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.Build.0 = Release|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.Build.0 = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.ActiveCfg = Release|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.Build.0 = Release|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1,7 +0,0 @@
-include_guard(GLOBAL)
-
-macro(export_to_parent)
-  foreach(var IN ITEMS ${ARGN})
-    set(${var} ${${var}} PARENT_SCOPE)
-  endforeach()
-endmacro()
@@ -0,0 +1,66 @@
+function(init_executable)
+  set(source_dir)
+  if (${ARGC} EQUAL 0)
+    set(source_dir ${PROJECT_NAME})
+  elseif(${ARGC} GREATER 1)
+    message(FATAL_ERROR "init_shared_library received invalid arguments. Usage: init_shared_library(<OPTIONAL:source_dir>)")
+  else()
+    set(source_dir ${ARGV})
+  endif()
+
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  if (${_ARGS_PROTO_SOURCES} EQUAL "GLOB_RECURSIVE")
+    file(
+      GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
+      RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+      *.proto
+    )
+  endif()
+
+  set(${PROJECT_NAME}_SOURCES PARENT_SCOPE)
+  file(
+    GLOB_RECURSE ${PROJECT_NAME}_SOURCES
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${source_dir}/*.cpp
+  )
+  set(${PROJECT_NAME}_SOURCES ${${PROJECT_NAME}_SOURCES} PARENT_SCOPE)
+  # set(${PROJECT_NAME}_Sources ${${PROJECT_NAME}_Sources} PARENT_SCOPE)
+  # Note: cpp files inside 'tests' dir(s) are considered tests.
+  # Remove tests from main sources, and add them to test sources
+  set(${PROJECT_NAME}_TEST_SOURCES)
+  foreach(item ${${PROJECT_NAME}_SOURCES})
+    if(${item} MATCHES "tests/*.cpp")
+      list(REMOVE_ITEM ${PROJECT_NAME}_SOURCES ${item})
+      list(APPEND ${PROJECT_NAME}_TEST_SOURCES ${item})
+    endif()
+  set(${PROJECT_NAME}_TEST_SOURCES ${${PROJECT_NAME}_TEST_SOURCES} PARENT_SCOPE)
+  endforeach()
+
+  # Add executable
+  add_executable(${PROJECT_NAME} ${${PROJECT_NAME}_SOURCES})
+  target_include_directories(
+    ${PROJECT_NAME}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+
+  target_link_directories(
+    ${PROJECT_NAME}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+  )
+
+  # Add tests executable (if there are any)
+  if (${PROJECT_NAME}_TEST_SOURCES)
+    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_TEST_SOURCEs})
+    target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
+    # Catch2
+    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
+    target_link_libraries(${PROJECT_NAME}-tests PRIVATE Catch2::Catch2WithMain)
+  endif()
+endfunction()
@@ -1,142 +0,0 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-
-function(init_library)
-  set(_options SHARED STATIC)
-  set(_one_value_args
-    NAME
-    SOURCE_DIR
-    INCLUDE_DIR
-  )
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  if (_ARGS_SHARED AND _ARGS_STATIC)
-    message(FATAL_ERROR "Invalid args passed to init_library: cannot specify STATIC and SHARED")
-  endif()
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_source_dir _ARGS_SOURCE_DIR _name)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _source_dir)
-
-  # Cpp sources
-  file(
-    GLOB_RECURSE ${_name}_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/*.cpp
-  )
-  # Note: cpp files inside 'tests' dir(s) are considered tests.
-  # Remove tests from main sources, and add them to test sources
-  foreach(item ${${_name}_SOURCES})
-    if(${item} MATCHES "tests/.*\.cpp$")
-      list(APPEND ${_name}_TEST_SOURCES ${item})
-    endif()
-  endforeach()
-  foreach(item ${${_name}_TEST_SOURCES})
-    list(REMOVE_ITEM ${_name}_SOURCES ${item})
-  endforeach()
-
-  # Proto sources 
-  file(
-    GLOB_RECURSE ${_name}_PROTO_SOURCES
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_source_dir}/protobufs/*.proto
-  )
-  update_deps_file("${${_name}_PROTO_SOURCES};")
-  protobuf_generate_cpp(
-    CXX_SOURCES ${_name}_CXX_PROTO_SOURCES
-    CXX_HEADERS ${_name}_CXX_PROTO_HEADERS
-    CXX_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}
-    PROTO_INCLUDE_DIRS
-      ${EXTERNALS_INCLUDE}
-      ${CMAKE_CURRENT_SOURCE_DIR}
-    PROTO_SOURCES
-      ${${_name}_PROTO_SOURCES}
-  )
-
-  export_to_parent(
-    ${_name}_SOURCES
-    ${_name}_TEST_SOURCES
-    ${_name}_PROTO_SOURCES
-    ${_name}_CXX_PROTO_SOURCES
-    ${_name}_CXX_PROTO_HEADERS
-  )
-
-  # Add main lib
-  if (_ARGS_STATIC)
-    add_library(${_name} STATIC ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  elseif(_ARGS_SHARED)
-    add_library(${_name} SHARED ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  else()
-    add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
-  endif()
-  target_include_directories(
-    ${_name}
-    SYSTEM
-    BEFORE
-    PUBLIC
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
-  )
-  target_include_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-  )
-  # Include protos (if any)
-  if (${_name}_PROTO_SOURCES)
-    target_include_directories(
-      ${_name}
-      PUBLIC
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    )
-  endif()
-  target_link_directories(
-    ${_name}
-    PUBLIC
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
-      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-  )
-  # Install main lib
-  foreach(_hdr ${${_name}_CXX_PROTO_HEADERS})
-    file(RELATIVE_PATH _relative_path ${CMAKE_CURRENT_BINARY_DIR} ${_hdr})
-    install(FILES ${_hdr} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${_relative_path})
-  endforeach()
-  install_basic(
-    NAME ${_name}
-    INCLUDE_DIR ${_include_dir}
-  )
-
-  # Add tests executable (if there are any)
-  if (${_name}_TEST_SOURCES)
-    add_executable(${_name}-tests ${${_name}_TEST_SOURCES})
-
-    get_target_property(_lib_type ${_name} TYPE)
-    if (_lib_type STREQUAL "SHARED_LIBRARY")
-      # Set RPATH so the tests exe knows where to find the shared lib.
-      set_target_properties(
-        ${_name}-tests
-        PROPERTIES
-          BUILD_RPATH "$ORIGIN"
-          INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
-      )
-    endif()
-
-    # Link main lib
-    target_link_directories(
-      ${_name}-tests
-      PRIVATE
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-        $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
-    )
-    target_link_libraries(${_name}-tests PRIVATE ${_name})
-
-    # Catch2
-    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
-    target_link_libraries(${_name}-tests PRIVATE Catch2::Catch2WithMain)
-  endif()
-endfunction()
@@ -0,0 +1,52 @@
+function(init_shared_library)
+  set(source_dir)
+  if (${ARGC} EQUAL 0)
+    set(source_dir ${PROJECT_NAME})
+  elseif(${ARGC} GREATER 1)
+    message(FATAL_ERROR "init_shared_library received invalid arguments. Usage: init_shared_library(<OPTIONAL:source_dir>)")
+  else()
+    set(source_dir ${ARGV})
+  endif()
+  set(${PROJECT_NAME}_Sources PARENT_SCOPE)
+  file(
+    GLOB_RECURSE ${PROJECT_NAME}_Sources
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    ${source_dir}/*.cpp)
+  set(${PROJECT_NAME}_Sources ${${PROJECT_NAME}_Sources} PARENT_SCOPE)
+  # set(${PROJECT_NAME}_Sources ${${PROJECT_NAME}_Sources} PARENT_SCOPE)
+  # Note: cpp files inside 'tests' dir(s) are considered tests.
+  # Remove tests from main sources, and add them to test sources
+  set(${PROJECT_NAME}_Test_Sources)
+  foreach(item ${${PROJECT_NAME}_Sources})
+    if(${item} MATCHES "tests/*.cpp")
+      list(REMOVE_ITEM ${PROJECT_NAME}_Sources ${item})
+      list(APPEND ${PROJECT_NAME}_Test_Sources ${item})
+    endif()
+  set(${PROJECT_NAME}_Test_Sources ${${PROJECT_NAME}_Test_Sources} PARENT_SCOPE)
+  endforeach()
+
+  # Add main library
+  add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_Sources})
+  target_include_directories(
+    ${PROJECT_NAME}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+  )
+
+  target_link_directories(
+    ${PROJECT_NAME}
+    PUBLIC
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
+  )
+
+  # Add tests executable (if there are any)
+  if (${PROJECT_NAME}_Test_Sources)
+    add_executable(${PROJECT_NAME}-tests ${${PROJECT_NAME}_Test_Sources})
+    target_link_libraries(${PROJECT_NAME}-tests PRIVATE ${PROJECT_NAME})
+    # Catch2
+    find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2WithMain)
+    target_link_libraries(${PROJECT_NAME}-tests PRIVATE Catch2::Catch2WithMain)
+  endif()
+endfunction(init_shared_library)
@@ -1,39 +0,0 @@
-include_guard(DIRECTORY)
-include(GNUInstallDirs)
-include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
-
-function(install_basic)
-  set(_options)
-  set(_one_value_args NAME INCLUDE_DIR)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-  set_first_defined(_include_dir _ARGS_INCLUDE_DIR _name)
-
-  file(
-    GLOB_RECURSE ${_name}_HEADERS
-    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${_include_dir}/*.h)
-
-  target_sources(${_name}
-    PUBLIC
-      FILE_SET HEADERS FILES ${${_name}_HEADERS})
-
-  install(
-    TARGETS ${_name}
-    EXPORT ${_name}Targets
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  )
-
-  install(
-    EXPORT ${_name}Targets
-    FILE ${_name}Targets.cmake
-    NAMESPACE ${_name}::
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
-
-  pack_basic()
-endfunction()
@@ -0,0 +1,29 @@
+include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
+
+function(install_basic_config)
+  file(
+    GLOB_RECURSE ${PROJECT_NAME}_HEADERS
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+    *.h)
+
+  target_sources(${PROJECT_NAME}
+    PUBLIC
+      FILE_SET HEADERS FILES ${${PROJECT_NAME}_HEADERS})
+
+  install(
+    TARGETS ${PROJECT_NAME}
+    EXPORT ${PROJECT_NAME}Targets
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    FILE_SET HEADERS DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  )
+
+  install(
+    EXPORT ${PROJECT_NAME}Targets
+    FILE ${PROJECT_NAME}Targets.cmake
+    NAMESPACE ${PROJECT_NAME}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
+
+  pack_basic()
+endfunction()
@@ -1,44 +1,21 @@
-include_guard(DIRECTORY)
 include(CMakePackageConfigHelpers)
-include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)

 function(pack_basic)
-  set(_options)
-  set(_one_value_args NAME)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  set_first_defined(_name _ARGS_NAME PROJECT_NAME)
-
-  find_package(Git QUIET)
-
-  if(GIT_FOUND)
-    execute_process(
-      COMMAND ${GIT_EXECUTABLE} describe --tags --always --dirty
-      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-      OUTPUT_VARIABLE PROJECT_VERSION
-      OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-  endif()
-
-  if(NOT GIT_FOUND OR PROJECT_VERSION STREQUAL "")
-    set(PROJECT_VERSION "0.0.0")
-  endif()
-
  write_basic_package_version_file(
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
+    "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
    VERSION ${PROJECT_VERSION}
    COMPATIBILITY AnyNewerVersion
  )

  configure_package_config_file(
    "${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
+    "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
  )

  install(
    FILES
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}Config.cmake"
-      "${CMAKE_CURRENT_BINARY_DIR}/${_name}ConfigVersion.cmake"
-    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name})
+      "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
+      "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
 endfunction()
@@ -0,0 +1,167 @@
+set(Protobuf_PROTOC_EXECUTABLE "${EXTERNALS_BIN}/protoc")
+set(PROTOBUF_INCLUDE_DIRS "${EXTERNALS_INCLUDE}")
+set(PROTOBUF_STATIC_LIB "libprotobuf")
+set(_protobuf_src_dir ${CMAKE_SOURCE_DIR}/src/lib_spark)
+set(_protobuf_bin_dir ${CMAKE_BINARY_DIR}/src/lib_spark)
+
+function(Protobuf_Generate_Cpp)
+set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    add_custom_command(
+      OUTPUT
+        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+        "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+      COMMAND
+        ${Protobuf_PROTOC_EXECUTABLE}
+      ARGS
+        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM)
+  endforeach()
+
+  set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
+      PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
+      PARENT_SCOPE)
+endfunction()
+
+function(Service_Compiler_Generate_Cpp)
+  set(_options)
+  set(_one_value_args CXX_SOURCES CXX_HEADERS CXX_OUT_DIR)
+  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
+  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+  set(${_ARGS_CXX_SOURCES})
+  set(${_ARGS_CXX_HEADERS})
+
+  set(_proto_include_cli_args)
+  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
+    list(APPEND _proto_include_cli_args -I ${_dir})
+  endforeach()
+
+  foreach(FIL ${_ARGS_PROTO_SOURCES})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${FIL} DIRECTORY)
+    if(FIL_DIR)
+      set(FIL_WE "${FIL_DIR}/${FIL_WE}")
+    endif()
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h")
+
+    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc")
+    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
+
+    set(TEMPLATES
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
+        ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
+    )
+
+    set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
+    if(WIN32)
+      set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
+      add_custom_command(
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND SET PYTHONPATH=${SPARK_PYTHON}
+        COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
+        COMMAND
+          ${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_cmd}
+          --service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
+          ${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM)
+    else()
+      add_custom_command(
+        OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
+               "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
+        COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
+        ARGS
+          ${Protobuf_PROTOC_EXECUTABLE}
+          --cpp_out=${_ARGS_CXX_OUT_DIR}
+          --plugin=protoc-gen-service=${_plugin_py}
+          --service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
+          ${_proto_include_cli_args} ${ABS_FIL}
+        DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
+        COMMENT "Running C++ service compiler on ${FIL}"
+        VERBATIM)
+    endif()
+
+  endforeach()
+
+  # remove some warnings from protobuf generated files
+  if(WIN32)
+    set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
+                                                      "/wd4309 /wd4838")
+  else()
+    set_source_files_properties(
+      ${${_ARGS_CXX_SOURCES}}
+      PROPERTIES
+        COMPILE_FLAGS
+        "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
+    )
+  endif()
+
+  set(${_ARGS_CXX_SOURCES}
+      ${${_ARGS_CXX_SOURCES}}
+      PARENT_SCOPE)
+  set(${_ARGS_CXX_HEADERS}
+      ${${_ARGS_CXX_HEADERS}}
+      PARENT_SCOPE)
+
+endfunction()
+
@@ -1,76 +0,0 @@
-include_guard(GLOBAL)
-
-function(protobuf_generate_cpp)
-  set(_options)
-  set(_one_value_args
-    CXX_SOURCES
-    CXX_HEADERS
-    CXX_OUT_DIR
-    PROTOC
-  )
-  set(_multi_value_args
-    PROTO_INCLUDE_DIRS
-    PROTO_SOURCES
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-  endif()
-  set(${_ARGS_CXX_SOURCES})
-  set(${_ARGS_CXX_HEADERS})
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  foreach(_fil ${_ARGS_PROTO_SOURCES})
-    get_filename_component(_abs_fil ${_fil} ABSOLUTE)
-    get_filename_component(_fil_we ${_fil} NAME_WE)
-    get_filename_component(_fil_dir ${_fil} DIRECTORY)
-    if(_fil_dir)
-      set(_fil_we "${_fil_dir}/${_fil_we}")
-    endif()
-    list(APPEND ${_ARGS_CXX_SOURCES} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc")
-    list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    message(STATUS "add_custom_command for ${_fil} -> ${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h")
-    add_custom_command(
-      OUTPUT
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.cc"
-        "${_ARGS_CXX_OUT_DIR}/${_fil_we}.pb.h"
-      COMMAND
-        ${_ARGS_PROTOC}
-      ARGS
-        --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${_fil}
-      DEPENDS ${_abs_fil}
-      COMMENT "Running C++ protocol buffer compiler on ${_fil}"
-      VERBATIM
-    )
-  endforeach()
-
-  set_source_files_properties(
-    ${${_ARGS_CXX_SOURCES}}
-    ${${_ARGS_CXX_HEADERS}}
-    PROPERTIES
-      GENERATED TRUE
-  )
-
-  # remove some warnings from protobuf generated files
-  if(WIN32)
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "/wd4309 /wd4838"
-    )
-  else()
-    set_source_files_properties(
-      ${${_ARGS_CXX_SOURCES}}
-      PROPERTIES
-        COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
-    )
-  endif()
-
-  set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
-  set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
-endfunction()
@@ -1,58 +0,0 @@
-function(protobuf_generate_cpp_config)
-  set(_options)
-  set(_one_value_args
-    PROTO_SOURCES_HEADER
-    PROTO_DEFAULTS_HEADER
-  )
-  set(_multi_value_args
-    PROTO_SOURCES
-    PROTO_DEFAULTS
-  )
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  # Configure the 'config_proto_files' header. It provides an array of strings populated with the
-  # files in PROTO_SOURCES. At runtime, we use this array to generate a special 'config' proto
-  # message with fields for each message defined in said files. A global instance of this 'config'
-  # proto is then constructed, and populated with [compile-time] defaults, and [runtime] overrides
-  # (i.e., in spark.conf). Calls to `bts::config::get<T>()` retrieve the value from the field
-  # associated with type `T` in this global instance.
-  message(STATUS "PROTO_SOURCES: ${_ARGS_PROTO_SOURCES}")
-  message(STATUS "PROTO_SOURCES_HEADER: ${_ARGS_PROTO_SOURCES_HEADER}")
-  message(STATUS "PROTO_DEFAULTS_HEADER: ${_ARGS_PROTO_DEFAULTS_HEADER}")
-  if (_ARGS_PROTO_SOURCES_HEADER)
-    set(PROTO_SOURCES)
-    set(PROTO_SOURCES_CXX_ARRAY_ITEMS "")
-    foreach(_proto_path ${_ARGS_PROTO_SOURCES})
-      list(APPEND PROTO_SOURCES ${_proto_path})
-      string(APPEND PROTO_SOURCES_CXX_ARRAY_ITEMS "\"${_proto_path}\",\n")
-    endforeach()
-    list(LENGTH PROTO_SOURCES PROTO_SOURCES_LENGTH)
-    message(STATUS "CONFIGURING PROTO_SOURCES_HEADER): ${_ARGS_PROTO_SOURCES_HEADER}")
-    configure_file(${CMAKE_SOURCE_DIR}/cmake/config_proto_files.h.in ${_ARGS_PROTO_SOURCES_HEADER})
-  endif()
-
-  # Generate static defaults. Ideally, this would be a single string, but we use an array because
-  # the compiler complains if the string is longer than 65k.
-  set(_cfg_defaults_str "inline const char * config_static_defaults[] = { ")
-
-  foreach(_cfg_file_path ${_ARGS_PROTO_DEFAULTS})
-    file(READ ${_cfg_file_path} _cfg_file_content)
-    string(REPLACE "\"" "\\\"" _cfg_file_content_escaped ${_cfg_file_content})
-    string(REPLACE "\n" ";" _cfg_lines ${_cfg_file_content_escaped})
-
-    foreach(_line ${_cfg_lines})
-      string(APPEND _cfg_defaults_str "\n  \"${_line}\",")
-    endforeach(_line)
-  endforeach(_cfg_file_path)
-
-  string(APPEND _cfg_defaults_str "};")
-  set(_old_cfg_defaults_str "")
-
-  if(EXISTS ${_ARGS_PROTO_DEFAULTS_HEADER})
-    file(READ ${_ARGS_PROTO_DEFAULTS_HEADER} _old_cfg_defaults_str)
-  endif()
-
-  if(NOT _cfg_defaults_str STREQUAL _old_cfg_defaults_str)
-    file(WRITE "${_ARGS_PROTO_DEFAULTS_HEADER}" "${_cfg_defaults_str}")
-  endif()
-endfunction()
@@ -1,49 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_fs)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
-  endif()
-  if (NOT DEFINED _ARGS_PROTOC)
-    set(_ARGS_PROTOC "${EXTERNALS_BIN}/protoc")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -I ${_dir})
-  endforeach()
-
-  set(_outputs)
-  foreach(_proto_source ${_ARGS_PROTO_SOURCES})
-    # get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
-    get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
-    # get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
-    list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
-  endforeach()
-
-  set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
-  set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
-  set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs)
-  add_custom_command(
-    OUTPUT ${_outputs}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND ${_venv_exec}
-      ${_ARGS_PROTOC}
-      --plugin=protoc-gen-service=${_plugin_py}
-      --service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${_venv_exec}
-      ${_plugin_py}
-      ${_template_fs}
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
-endfunction()
@@ -1,33 +0,0 @@
-include_guard(DIRECTORY)
-
-function(protobuf_generate_js)
-  set(_options)
-  set(_one_value_args TARGET_NAME OUTPUT)
-  set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-
-  if (NOT DEFINED _ARGS_TARGET_NAME)
-    set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
-  endif()
-
-  set(_proto_include_cli_args)
-  foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
-    list(APPEND _proto_include_cli_args -p ${_dir})
-  endforeach()
-
-  add_custom_command(
-    OUTPUT ${_ARGS_OUTPUT}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND npx
-      --no-install pbjs
-      -t json-module
-      -w commonjs
-      -o ${_ARGS_OUTPUT}
-      ${_proto_include_cli_args}
-      ${_ARGS_PROTO_SOURCES}
-    DEPENDS
-      ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-      ${_ARGS_PROTO_SOURCES}
-  )
-  add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
-endfunction()
@@ -1,9 +1,7 @@
-include_guard(GLOBAL)
-
-macro(set_first_defined RESULT)
-  foreach(_arg ${ARGN})
-    if (DEFINED ${_arg})
-      set(${RESULT} ${${_arg}})
+macro(set_first_defined RESULT CANDIDATE_VARS)
+  foreach(_candidate_var ${CANDIDATE_VARS})
+    if (DEFINED _candidate_var)
+      set(${RESULT} ${_candidate_var})
      break()
    endif()
  endforeach()
@@ -1,33 +1,26 @@
-include_guard(DIRECTORY)
-include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
-
-function(sym_link_dir)
-  set(_options)
-  set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
-  set(_multi_value_args)
-  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
+function(sym_link_dir TARGET_NAME SOURCE DEST)
  add_custom_command(
-    OUTPUT ${_ARGS_DEST}
-    COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
+    OUTPUT ${DEST}
+    COMMAND ln ARGS -sf ${SOURCE} ${DEST}
  )
-  set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
  add_custom_target(
-    ${_ARGS_TARGET_NAME} ALL
-    DEPENDS ${_ARGS_DEST}
-    COMMENT "${_comment}"
+    ${TARGET_NAME} ALL
+    DEPENDS ${DEST}
  )
 endfunction()

 function(sym_link_file)
  set(_options)
-  set(_one_value_args SOURCE DEST INSTALL_DEST)
+  set(_one_value_args TARGET_NAME SOURCE DEST INSTALL_DEST)
  set(_multi_value_args)
  cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  file(
-    CREATE_LINK
-      ${_ARGS_SOURCE}
-      ${_ARGS_DEST}
-    SYMBOLIC
+  add_custom_command(
+    OUTPUT ${_ARGS_DEST}
+    COMMAND ln ARGS -s ${_ARGS_SOURCE} ${_ARGS_DEST}
+  )
+  add_custom_target(
+    ${_ARGS_TARGET_NAME} ALL
+    DEPENDS ${_ARGS_DEST}
  )
  if (DEFINED _ARGS_INSTALL_DEST)
    install(
@@ -1,5 +1,3 @@
-include_guard(GLOBAL)
-
 macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
  if(${CONDITION})
    set(${RESULT} ${TRUE_VALUE})
@@ -1,11 +0,0 @@
-#pragma once
-
-#include <array>
-
-namespace bts::@PROJECT_NAME@::protobufs {
-
-inline std::array<const char *, @PROTO_SOURCES_LENGTH@> config_proto_files = {
-@PROTO_SOURCES_CXX_ARRAY_ITEMS@
-};
-
-} // namespace bts::@PROJECT_NAME@::protobufs
@@ -1,15 +1,9 @@
 services:
  sparkd:
    build:
-      dockerfile: src/app_sparkd/sparkd/Dockerfile
-      args:
-        - SOURCE_DIR=install/sparkd-release-gcc
-        - CONF_FILE=src/app_sparkd/spark.current.conf
+      dockerfile: src/server/sparkd/Dockerfile
    command: [ "./sparkd", "run" ]
-    ports:
-      - 8000:8000
-      - 8001:8001
-      - 8002:8002
+    ports: [ "8000:8000", "8001:8001", "8002:8002" ]

  pubsub:
    environment:
@@ -53,7 +53,7 @@ The status callback is called when the market for this security opens, closes, o
 // Subscribe to security trading status
 void IAlgoServices::subscribe_status(SecurityKey);
 // You implement
-virtual void on_status(SecurityKey, domain::protobufs::TradingStatus) override;
+virtual void on_status(SecurityKey, co2::TradingStatus) override;
 ```

 # Child Orders
@@ -24,6 +24,7 @@ RUN locale-gen en_US.UTF-8
 ARG docker_user="bts"
 ARG docker_uid="1000"
 ARG docker_pass="bts"
+# ARG entrypoint

 RUN \
  groupadd -g "$docker_uid" "$docker_user" \
@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443
@@ -0,0 +1,32 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.3.1' } },
+  plugins: ['react-refresh'],
+  rules: {
+    "import/order": [
+      "error",
+      {
+        "groups": [
+          "builtin",
+          "external",
+          "internal",
+          ["parent", "sibling", "index"]
+        ],
+        "newlines-between": "always"
+      }
+    ],
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}
@@ -1,36 +0,0 @@
-import importPlugin from "eslint-plugin-import";
-import reactPkg from "eslint-plugin-react";
-const { configs: reactConfigs } = reactPkg;
-import js from "@eslint/js";
-import defineConfig from "./vite.config.js";
-
-export default [
-  js.configs.recommended,
-  importPlugin.flatConfigs.recommended,
-  reactConfigs.flat.recommended,
-  reactConfigs.flat["jsx-runtime"],
-  {
-    files: ["**/*.{js,jsx}"],
-    languageOptions: {
-      ecmaVersion: "latest",
-      sourceType: "module",
-      globals: {
-        console: "readonly"
-      }
-    },
-    settings: {
-      "import/resolver": {
-        vite: {
-          viteConfig: defineConfig({ mode: "development" })
-        }
-      }
-    },
-    rules: {
-      semi: "error",
-      "prefer-const": "error",
-      "no-unused-vars": "warn",
-      "import/no-dynamic-require": "warn",
-      "import/no-nodejs-modules": "warn",
-    }
-  }
-];
@@ -10,37 +10,35 @@
    "preview": "vite preview"
  },
  "dependencies": {
-    "@angular/material": "^18.2.8",
-    "@emotion/react": "^11.13.3",
-    "@emotion/styled": "^11.13.0",
-    "@fontsource/noto-sans": "^5.1.0",
-    "@material/data-table": "^15.0.0-canary.423edc3dc.0",
-    "@mui/icons-material": "^6.1.4",
-    "@mui/material": "^6.1.4",
-    "@mui/x-data-grid": "^7.20.0",
-    "bulma": "^1.0.2",
+    "@angular/material": "^18.0.4",
+    "@emotion/react": "^11.11.4",
+    "@emotion/styled": "^11.11.5",
+    "@fontsource/noto-sans": "^5.0.22",
+    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
+    "@mui/icons-material": "^5.15.20",
+    "@mui/material": "^5.15.20",
+    "@mui/x-data-grid": "^7.7.1",
+    "bulma": "^1.0.1",
    "classnames": "^2.5.1",
    "react": "^18.3.1",
-    "react-data-grid": "^7.0.0-beta.47",
+    "react-data-grid": "^7.0.0-beta.44",
    "react-data-table-component": "^7.6.2",
    "react-dom": "^18.3.1",
-    "react-json-view-lite": "^1.5.0",
+    "react-json-view-lite": "^1.4.0",
    "react-pro-sidebar": "^1.1.0",
-    "react-use-websocket": "^4.9.0",
-    "rpc-websockets": "^9.0.4"
+    "react-use-websocket": "^4.8.1",
+    "rpc-websockets": "^7.11.2"
  },
  "devDependencies": {
-    "@types/react": "^18.3.11",
-    "@types/react-dom": "^18.3.1",
-    "@vitejs/plugin-react": "^4.3.2",
-    "eslint": "^9.12.0",
-    "@eslint/js": "^9.12.0",
-    "eslint-plugin-import": "^2.31.0",
-    "eslint-plugin-react": "^7.37.1",
-    "eslint-plugin-react-hooks": "^5.0.0",
-    "eslint-plugin-react-refresh": "^0.4.12",
-    "eslint-import-resolver-vite": "^2.0.1",
-    "react-mui-sidebar": "^1.3.8",
-    "vite": "^5.4.9"
+    "@types/react": "^18.3.3",
+    "@types/react-dom": "^18.3.0",
+    "@vitejs/plugin-react": "^4.3.1",
+    "eslint": "^8.57.0",
+    "eslint-plugin-import": "^2.29.1",
+    "eslint-plugin-react": "^7.34.2",
+    "eslint-plugin-react-hooks": "^4.6.2",
+    "eslint-plugin-react-refresh": "^0.4.7",
+    "react-mui-sidebar": "^1.3.5",
+    "vite": "^5.2.13"
  }
 }
@@ -1,5 +1,5 @@
 import { fileURLToPath, URL } from 'node:url';
-import { defineConfig } from 'vite';
+import { defineConfig, loadEnv } from 'vite';
 import plugin from '@vitejs/plugin-react';
 import fs from 'fs';
 import path from 'path';
@@ -1,11 +1,7 @@
 FROM bluetradingsystems/linux_dotnet7

-ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
-ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
-ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
-
 RUN dotnet dev-certs https --trust
-COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
-COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
-COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
+COPY --chown=bts:bts "build/Release/bin/net7.0/linux-x64/publish" "bin"
+COPY --chown=bts:bts "build/Bts.DataStreamWebApp.Client/production/dist" "bin/wwwroot"
+COPY --chown=bts:bts "src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf" ".local/share/bts/Bts.DataStreamWebApp.Server/conf/spark.conf"
 WORKDIR "/home/bts/bin"
@@ -0,0 +1,6 @@
+FROM bluetradingsystems/linux_dotnet7
+
+RUN dotnet dev-certs https --trust
+COPY --chown=bts:bts "bin" "bin"
+COPY --chown=bts:bts "webapp.spark.conf" ".local/share/DataStream/conf/spark.conf"
+WORKDIR "/home/bts/bin"
@@ -0,0 +1,49 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Text;
+// using System.IO;
+
+// using System.Runtime.Serialization.Formatters.Binary;
+
+// namespace BCG
+// {
+//    public class DumpLoaderStream : ViewStream, ISideEffectStream
+//    {
+//       //wtf
+//       //public class Exception : System.Exception
+//       //{public Exception(string msg) : base(msg) { }}
+
+//       public DumpLoaderStream(string fileName, int secondsBetweenPolls)
+//       {
+//          mySetRealSource(
+//             new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
+//                ViewStreamDatabase.instance));
+//       }
+
+//       public DumpLoaderStream(string fileName)
+//       {
+//         if (fileName == null)
+//            throw new LoaderStream.Exception("File name cannot be null");
+
+//         for (int i = 0; i < 5; i++)
+//            try
+//            {
+//               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
+//               {
+//                  IClientDeserializer deser = new ClientUniversalDeserializer();
+//                  byte[] the_ops;
+//                  Array the_rows;
+//                  deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
+
+//                  mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
+//               }
+//               break;
+//            }
+//            catch (IOException)
+//            {
+//               System.Threading.Thread.Sleep(50);
+//            }
+
+//       }
+//    }
+// }
@@ -1,7 +1,6 @@
 using BCG;

 namespace bts {
-   // This is the other side of the SaverStream, its "loads" the stream and fires once
   [DataStreamUserRespresentation("loader")]
   public class LoaderStream : ViewStream {

@@ -5,11 +5,10 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using BCG;
+using bts;

-namespace bts {
-   // Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
-   // which just writes one update, as a create)
+namespace BCG {
+   // Writes all the udpates to a stream in a file.
   // File naming:
   // token_epoch_yyyyMMdd_HHmmss
   // File location: directory(passed)\token\
@@ -95,10 +94,10 @@ namespace bts {
      void _on_change(object new_value) {
         DataPacket dp = new_value as DataPacket;
         byte[] serialized_update = null;
-         bool first_write = _serializer == null || _n_updates_in_current_file == 0;
+         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
         _serializer = _serializer ?? new UniversalStreamIOSerializer();
-         if (first_write) {
-            // first file always starts with create
+         if (_write_create) {
+            // new files always starts with create
            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
         } else
            serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
@@ -106,8 +105,8 @@ namespace bts {
         long utc_epoch = epoch_now();

         //time, length of data, data
-         //bw.Write(utc_epoch);
-         //bw.Write(serialized_update.Length);
+         bw.Write(utc_epoch);
+         bw.Write(serialized_update.Length);
         bw.Write(serialized_update);
         _out_fs.Flush();
         myFireEvent(dp);
@@ -0,0 +1,190 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+// using System.Runtime.Serialization.Formatters.Binary;
+// using BCG.Utilities.General;
+// using bts.collections;
+// using System.Reflection.Metadata;
+
+
+// namespace BCG {
+
+//    public class PlaybackStream : ViewStream, IWorker {
+//       private readonly PlaybackType _type;
+//       private BlueTable _current_table;
+//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
+//       // these are the packets already downloaded from the current file
+//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
+//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
+//       private IEnumerable<string> _file_names;
+//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
+//       private DateTime _playback_t0, _playback_t1;
+//       private bool _all_files_read = false;
+//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
+
+//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
+//          _type = PlaybackType.Streaming;
+//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
+//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
+//          string token_directory_name = $"{directory_name}\\{token}";
+//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
+//          List<string> files_in_range = new List<string>();
+//          if (Directory.Exists(token_directory_name)) {
+//             _file_names =
+//          Directory.GetFiles(token_directory_name).Where(f => {
+//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
+//             return file_time > _playback_t0 && file_time < _playback_t1;
+//          });
+//          }
+//       }
+
+//       protected override void myOnFirstSubscriber() {
+//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
+//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
+//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
+//          if (snapshot_packet != null) {
+//             foreach (PlaybackPacket p in _snapshot_data.packets) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
+//                for (int i = 0; i < ops.Length; i++) {
+//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//                   byte op = ops[i];
+//                   if (op == StreamOpCode.create) {
+//                      IBlueRowCollection full_table = rows_args[0];
+//                      _current_table = new BlueTable(full_table);
+//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                   } else {
+//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
+//                   }
+//                   if (p.idx == snapshot_packet.idx) {
+//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
+//                      break;
+//                   }
+//                }
+//             }
+//          }
+//       }
+
+
+//       private void _load_packets_from_current_file() {
+//          int curr_idx = _current_fs.Item1;
+//          FileStream curr_fs = _current_fs.Item2;
+//          _packets = new List<PlaybackPacket>();
+//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
+//       }
+
+//       private bool current_file_all_replayed {
+//          get {
+//             bool rv = true;
+//             _packets.ForEach(p => rv = rv && p.replayed);
+//             return rv;
+//          }
+//       }
+
+//       protected override void myOnLastSubscriber() {
+//          var fs = _current_fs.Item2;
+//          if (fs != null) {
+//             fs.Close();
+//          }
+//          ViewStreamDatabase.instance.removeWorker(this);
+//       }
+
+//       // public event PlaybackStreamProgress onProgress;
+//       // public event Action onFileExhausted;
+//       private PlaybackPacket _locate_packet_to_replay() {
+//          foreach (var p in _packets) {
+//             // returns the first non replayed packet
+//             if (!p.replayed)
+//                return p;
+//          }
+//          //else
+//          {
+//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
+//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
+//             if (onProgress != null)
+//                onProgress(myFileReader.Position, myFileReader.Length);
+//          }
+//       }
+
+//       #region IWorker Members
+
+//       public void doWork() {
+//          if (_all_files_read)
+//             return;
+//          if (current_file_all_replayed) {
+//             _load_next_file();
+//          }
+//          PlaybackPacket curr_packet = _locate_packet_to_replay();
+//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
+//             byte[] ops;
+//             Array row_array;
+//             // what a stupid interface!
+//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+//             _fire_packet(ops, row_array);
+//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
+//          } else {
+//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
+//             // this is the time we need to fire (replay) this packet
+//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
+//             long now_epoch = PersistorStream.epoch_now();
+//             // fire when we are past the 
+//             if (now_epoch >= time_to_fire) {
+//                byte[] ops;
+//                Array row_array;
+//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
+//                _fire_packet(ops, row_array);
+//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
+//             }
+//          }
+//       }
+
+//       private void _fire_packet(byte[] ops, Array row_array) {
+//          for (int i = 0; i < ops.Length; i++) {
+//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
+//             byte op = ops[i];
+//             if (op == StreamOpCode.create) {
+//                IBlueRowCollection full_table = rows_args[0];
+//                _current_table = new BlueTable(full_table);
+//                //_current_table.addRows(full_table);
+//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
+//                                           (myLastValue as DataPacket).newRows));
+//             } else if (op == StreamOpCode.multipleRemoveAdd) {
+//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
+//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
+//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
+//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
+//                myFireEvent(new DataPacket(_current_table, adds, dels));
+//             } else {
+//                bts.Log.assert(false, $"Oprator: {op} is not expected");
+//             }
+
+//          }
+//       }
+
+//       private void _load_next_file() {
+//          var new_idx = _current_fs.Item1 + 1;
+//          if (new_idx >= _file_names.Count()) {
+//             _all_files_read = true;
+//             bts.Log.Gen.info("All files read");
+//             return;
+//          }
+//          var prev_fs = _current_fs.Item2;
+//          if (prev_fs != null)
+//             prev_fs.Close();
+//          var new_filename = _file_names.ElementAt(new_idx);
+//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
+//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
+//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
+//          _load_packets_from_current_file();
+//       }
+
+//       #endregion
+//    }
+// }
@@ -0,0 +1,68 @@
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
+// using System.IO;
+
+// namespace BCG
+// {
+//    public class RecorderStream : ViewStream
+//    {
+//       private ViewStream mySource;
+//       private DateTime myStartTime;
+//       private FileStream myOutFile;
+//       private string myFileName;
+//       private IBlueDataSerializer mySerializer;
+
+//       public RecorderStream(ViewStream src, string recordFile)
+//       {
+//          mySource = src;
+//          myFileName = recordFile;
+
+//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
+//       }
+
+//       protected override void myOnFirstSubscriber()
+//       {
+//          myStartTime = DateTime.Now;         
+//          mySource.onChange += new ChangeDelegate(mySource_onChange);
+//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
+//       }
+
+//       protected override void myOnLastSubscriber()
+//       {
+//          mySource.onChange -= mySource_onChange;
+//          myOutFile.Close();
+//       }
+
+//       void mySource_onChange(object newValue)
+//       {
+//          DataPacket dp = (DataPacket)newValue;         
+
+//          BlueTable toWrite = new BlueTable(dp.schema);
+
+//          if (mySerializer == null)
+//          {
+//             mySerializer = new UniversalSerializer(dp.schema);
+//          }
+//          foreach (IBlueRow r in dp.newRows)
+//             toWrite.addRow(r);
+
+//          byte[] result;
+
+//          if (myLastValue == null)
+//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
+//          else
+//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
+
+//          BinaryWriter bw = new BinaryWriter(myOutFile);
+//          //write down what time this happened
+//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
+//          bw.Write(result.Length);
+//          bw.Write(result);         
+//          myOutFile.Flush();
+
+//          myFireEvent(dp);
+//       }
+//    }
+// }
@@ -5,8 +5,7 @@ using System.IO;
 using BCG;

 namespace bts {
-   // this saves the updates as a Create on the same file, each update calls -save, which
-   // reqrites the whole data to the packet. The other side of this stream is the Loader
+   // this saves the total state in the filr provided in the cons
   [DataStreamUserRespresentation("saver")]
   public class SaverStream : ViewStream {
      private ViewStream _stream;
@@ -32,6 +31,7 @@ namespace bts {
         StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
         byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
         StreamSerializationUtils.write_to_file(str_bytes, _filename);
+
      }
   }
 }
@@ -3,7 +3,6 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
 using System.IO;
-using bts;


 namespace BCG {
@@ -17,8 +17,8 @@
  </ItemGroup>

  <ItemGroup>
-    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
    </Proto>
  </ItemGroup>

@@ -9,15 +9,7 @@
 Letter                   [a-zA-Z]
 BooleanConstant          'true'|'false'
 CSStringConstant         \"([^\"\\]*|\\.)*\"
-YearInt                  [0-9]{4}
-MonthInt                 [0-9]{1,2}
-DayInt                   [0-9]{1,2}
-/**
- * DateTimeConstant supports American (mm/dd/yyyy) and European (yyyy/mm/dd)
- * formats. Month and day can be 1 or 2 digits. It does not validate that the
- * numbers represent an actual date.
- */
-DateTimeConstant         ({MonthInt}\/{DayInt}\/{YearInt}|{YearInt}\/{MonthInt}\/{DayInt})
+DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
 IntegerConstant          -?[0-9]+
 NullConstant             'null'
 RealConstant             -?[0-9]+\.[0-9]+
@@ -1,8 +1,5 @@
 FROM bluetradingsystems/linux_dotnet7

-ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
-ARG CONF_FILE="src/DataStreamSuite/DataStreamPublisher/spark.conf"
-
 ENV DOCKERIZE_VERSION v0.7.0
 USER root
 RUN apt-get update \
@@ -11,6 +8,6 @@ RUN apt-get update \
    && apt-get autoremove -yqq --purge wget && rm -rf /var/lib/apt/lists/*

 USER bts
-COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
-COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/DataStreamPublisher/conf/spark.conf"
+COPY --chown=bts:bts "build/Release/bin/net7.0/linux-x64/publish" "bin"
+COPY --chown=bts:bts "src/DataStreamSuite/DataStreamPublisher/spark.conf" ".local/share/bts/DataStreamPublisher/conf/spark.conf"
 WORKDIR "/home/bts/bin"
@@ -37,17 +37,14 @@ using Bts.PubSub;
 using Spark.Core;
 using mbts.utils;

-namespace bts
-{
-   public class AsyncTCPConnection_APM : ITCPConnection, IWorker, IDisposable
-   {
+namespace bts {
+   public class AsyncTCPConnection_APM : ITCPConnection, IWorker, IDisposable {

      // this class is used as the argument of the completion function for BeginSend/Receive   
      // serves as the base class for both Send and ReeceiveMessage classes.
      // since both TCP sends and receives happen in multiple transfers, this
      // provides a wrapper around copying the partial transfers into the complete buffer
-      public class AsyncMessageBase
-      {
+      public class AsyncMessageBase {
         readonly string msg_id = Guid.NewGuid().ToString();
         // every BeginSend or BeginReceive has a unique key associated with it
         public int key;
@@ -59,45 +56,38 @@ namespace bts
         public int transfer_ptr;
         protected int _n_transfers = 0;
         public bool completed_sync = false;
-         public AsyncMessageBase(byte[] the_buffer, int the_offset)
-         {
+         public AsyncMessageBase(byte[] the_buffer, int the_offset) {
            buffer = the_buffer;
            transfer_ptr = the_offset;
         }
-         public int remaining_size
-         {
-            get
-            {
+         public int remaining_size {
+            get {
               return buffer.Length - transfer_ptr;
            }
         }

         public int original_size => buffer.Length;

-         public void advance_transfer_pointer(int advance_by)
-         {
+         public void advance_transfer_pointer(int advance_by) {
            transfer_ptr += advance_by;
            _n_transfers++;
-            if (transfer_ptr > original_size)
-            {
+            if (transfer_ptr > original_size) {
               bts.Log.Gen.error($"transfer went beyond initial size {ToString()}");
            }
            // TODO: log if it goes past the buffer
         }

-         public bool fully_transfered()
-         {
+         public bool fully_transfered() {
            return transfer_ptr >= original_size;
         }

-         public override string ToString()
-         {
+         public override string ToString() {
            return $"{GetType().ToString()} msg_id: {msg_id} key: {key} original_length: {original_size} transfer_ptr: {transfer_ptr} n_transfers: {_n_transfers} " +
                   $"fully_transfered: {fully_transfered()} completed_sync: {completed_sync}";
         }
      }
-      public class AsyncSendMessage : AsyncMessageBase
-      {
+
+      public class AsyncSendMessage : AsyncMessageBase {
         public string message_id;
         public SendConfirmationDelegate? on_send_del;
         //private int _n_transfers = 0;
@@ -115,31 +105,25 @@ namespace bts
         }
      }

-      public class AsyncReceiveMessage : AsyncMessageBase
-      {
-         public enum ReadState
-         {
+      public class AsyncReceiveMessage : AsyncMessageBase {
+         public enum ReadState {
            reading_size,
            reading_data
         };
         readonly ReadState read_state;

         public AsyncReceiveMessage(int size, ReadState the_read_state) :
-         this(new byte[size], 0, the_read_state)
-         { }
+         this(new byte[size], 0, the_read_state) { }
         public AsyncReceiveMessage(byte[] the_buffer, int the_transfer_ptr, ReadState the_read_state)
-         : base(the_buffer, the_transfer_ptr)
-         {
+         : base(the_buffer, the_transfer_ptr) {
            read_state = the_read_state;
         }

-         public int read_int32(int start_idx = 0)
-         {
+         public int read_int32(int start_idx = 0) {
            return BitConverter.ToInt32(buffer, start_idx);
         }

-         public byte[] read_bytes(int start_idx)
-         {
+         public byte[] read_bytes(int start_idx) {
            byte[] rv = new byte[buffer.Length - start_idx];
            Array.Copy(buffer, start_idx, rv, 0, rv.Length);
            return rv;
@@ -148,19 +132,16 @@ namespace bts
         public bool is_reading_size() => read_state == ReadState.reading_size;
      }

-      public class ErrorMessage
-      {
+      public class ErrorMessage {
         public readonly SocketError? socket_error = null;
         public readonly Exception? exception;
         public readonly bool is_exception = false;
         public readonly string context = String.Empty;
-         public ErrorMessage(SocketError socket_error_, string context_)
-         {
+         public ErrorMessage(SocketError socket_error_, string context_) {
            socket_error = socket_error_;
            context = context_;
         }
-         public ErrorMessage(Exception exception_, string context_)
-         {
+         public ErrorMessage(Exception exception_, string context_) {
            is_exception = true;
            exception = exception_;
            context = context_;
@@ -168,8 +149,7 @@ namespace bts

         public bool oom => exception != null && exception is OutOfMemoryException;

-         public override string ToString()
-         {
+         public override string ToString() {
            if (is_exception)
               return $"ErrorMessage exception: {exception} context: {context}";
            return $"socket_error: {socket_error} context: {context}";
@@ -195,15 +175,13 @@ namespace bts
      private bool _hb_received_yet = false, _data_received_yet = false, _hb_sent_once = false;
      private DateTime _last_hb_received_time = new DateTime(1900, 1, 1);

-      public AsyncTCPConnection_APM(Socket sock, TimeSpan hb_period, ITCPConnection.ConnectionSide connection_side)
-      {
+      public AsyncTCPConnection_APM(Socket sock, TimeSpan hb_period, ITCPConnection.ConnectionSide connection_side) {
         _sock = sock;
         _hb_period = hb_period;
         _connection_side = connection_side;
         _sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true);
         #region testing options
-         switch (connection_side)
-         {
+         switch (connection_side) {
            case ITCPConnection.ConnectionSide.Server:
               TCPSocketUtils.set_server_socket_options(ref _sock);
               break;
@@ -218,13 +196,10 @@ namespace bts
         _detect_connection_status_change();
      }

-      private void _detect_connection_status_change()
-      {
-         Action testing_func = () =>
-         {
+      private void _detect_connection_status_change() {
+         Action testing_func = () => {
            bool connected = _sock.Connected;
-            while (true)
-            {
+            while (true) {
               if (connected != _sock.Connected)
                  Log.Gen.error($"Connection status changed this: {ToString()} {connected} -> {_sock.Connected}");
               connected = _sock.Connected;
@@ -234,30 +209,26 @@ namespace bts
         ThreadPool.QueueUserWorkItem(_ => testing_func());
      }

-      public override string ToString()
-      {
+      public override string ToString() {
         return $"AsyncTCPConnection_APM side: {_connection_side} peer: {_peer_conn_info}, self: {_self_conn_info}";
      }

      #region ITCPConnection Members

-      public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del)
-      {
+      public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del) {
         conf_id = Guid.NewGuid().ToString();
         byte[] msg = _build_data_message(data);
         _begin_or_resume_sending(new AsyncSendMessage(msg, 0, conf_id, del));
      }

-      public void send(byte[] data)
-      {
+      public void send(byte[] data) {
         //Log.Gen.info($"AsyncTCPConnection_APM.send data: {AsyncTCPConnection_AsyncWrapper.to_hex(data, 0, data.Length)}");
         byte[] msg = _build_data_message(data);
         //Log.Gen.info($"AsyncTCPConnection_APM.send msg: {AsyncTCPConnection_AsyncWrapper.to_hex(msg, 0, msg.Length)}");
         _begin_or_resume_sending(new AsyncSendMessage(msg, 0));
      }

-      public void register(ITCPMessageHandler messageHandler)
-      {
+      public void register(ITCPMessageHandler messageHandler) {
         bts.Log.assert_d(_message_handler == null, "Message Handler is already assigned");
         //bts.Log.Gen.info($"AsyncTCPConnection_APM for peer = {_peer_conn_info} being registered");
         _message_handler = messageHandler;
@@ -266,18 +237,15 @@ namespace bts
         _read_next_msg();
      }

-      public RemoteConnectionInfo get_peer_address()
-      {
+      public RemoteConnectionInfo get_peer_address() {
         return TCPSocketUtils.get_peer_address(_sock);
      }

-      public RemoteConnectionInfo get_self_address()
-      {
+      public RemoteConnectionInfo get_self_address() {
         return TCPSocketUtils.get_peer_address(_sock);
      }

-      public bool isDisposed
-      {
+      public bool isDisposed {
         get { return !_is_alive; }
      }

@@ -290,14 +258,12 @@ namespace bts
         // TODO: check heartbeats
      }

-      private static int _tid()
-      {
+      private static int _tid() {
         var tid = System.Threading.Thread.CurrentThread.ManagedThreadId;
         return tid;
      }

-      private static string _cb_txt(IAsyncResult res)
-      {
+      private static string _cb_txt(IAsyncResult res) {
         return $"completed?: {res.IsCompleted} sync?: {res.CompletedSynchronously}";
      }

@@ -305,13 +271,10 @@ namespace bts

      #region sending data

-      private void _begin_or_resume_sending(AsyncSendMessage message)
-      {
+      private void _begin_or_resume_sending(AsyncSendMessage message) {
         //bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_sending this: {ToString()} tid: {_tid()} message: {message} _is_alive: {_is_alive}");
-         if (_is_alive)
-         {
-            try
-            {
+         if (_is_alive) {
+            try {
               SocketError begin_send_err;
               // since we get message on on_send we dont need to keep the state around
               message.key = _send_key++;
@@ -319,21 +282,16 @@ namespace bts
                                 out begin_send_err, new AsyncCallback(BeginSend_callback_function), message);
               if (begin_send_err != SocketError.Success)
                  _handle_error(new ErrorMessage(begin_send_err, "_begin_or_resume_sending"));
-            }
-            catch (Exception ex)
-            {
+            } catch (Exception ex) {
               _handle_error(new ErrorMessage(ex, "_begin_or_resume_sending"));
            }
         }
      }

-      private void BeginSend_callback_function(IAsyncResult res)
-      {
-         if (res.CompletedSynchronously)
-         {
+      private void BeginSend_callback_function(IAsyncResult res) {
+         if (res.CompletedSynchronously) {
            ThreadPool.QueueUserWorkItem(_ => _on_send_async(res));
-         }
-         else
+         } else
            _on_send_async(res);
      }

@@ -346,16 +304,13 @@ namespace bts
         _send_key_map.Add(message.key);
         // bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} " +
         //                         $"res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}");
-         if (_is_alive)
-         {
-            try
-            {
+         if (_is_alive) {
+            try {
               SocketError send_err;
               int bytes_sent = _sock.EndSend(res, out send_err);
               //message.offset += bytesSent;
               message.advance_transfer_pointer(bytes_sent);
-               if (bytes_sent == 0)
-               {
+               if (bytes_sent == 0) {
                  bts.Log.Gen.error($"AsyncTCPConnection_APM._on_send this: {this} sent 0 bytes, throwing exception");
                  _handle_error(new ErrorMessage(new ApplicationException("Sent 0 bytes, peer socket is likely shutdown"),
                        "_on_send"));
@@ -363,31 +318,23 @@ namespace bts
               }
               bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: post endSend this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} " +
               $"message: {res.AsyncState} bytes_sent: {bytes_sent} _is_alive: {_is_alive} send_err: {send_err}");
-               if (send_err == SocketError.Success)
-               {
+               if (send_err == SocketError.Success) {
                  // successful send of some bytes
                  //if (message.offset == message.buffer.Length) {
-                  if (message.fully_transfered())
-                  {
+                  if (message.fully_transfered()) {
                     // fully sent, if confirmation needs to be sent out                     
-                     if (!string.IsNullOrEmpty(message.message_id)) {
+                     if (!String.IsNullOrEmpty(message.message_id)) {
                        ArgumentNullException.ThrowIfNull(message.on_send_del);
                        message.on_send_del(message.message_id, true);
                     }
-                  }
-                  else
-                  {
+                  } else {
                     // there is more to send
                     _begin_or_resume_sending(message);
                  }
-               }
-               else
-               {
+               } else {
                  _handle_error(new ErrorMessage(send_err, "_on_send"));
               }
-            }
-            catch (Exception ex)
-            {
+            } catch (Exception ex) {
               _handle_error(new ErrorMessage(ex, "_on_send"));
            }
         }
@@ -398,10 +345,8 @@ namespace bts
      /// future references
      /// </summary>
      /// <returns></returns>
-      private byte[] _build_hb_message()
-      {
-         if (_cached_hb_message == null)
-         {
+      private byte[] _build_hb_message() {
+         if (_cached_hb_message == null) {
            BinaryWriter bw = new BinaryWriter(new MemoryStream());
            // the total msg =[msg_sz, msg_type] 
            int msg_sz = 4;
@@ -413,8 +358,7 @@ namespace bts
      }

      // writes size prefixed msg
-      private byte[] _build_data_message(byte[] data)
-      {
+      private byte[] _build_data_message(byte[] data) {
         // TODO, avoid unecessary memory creation (write directly into byte[] using BitConverter)
         MemoryStream str = new MemoryStream();
         BinaryWriter bw = new BinaryWriter(str);
@@ -433,34 +377,28 @@ namespace bts

      #region receiving data

-      private void _read_next_msg()
-      {
+      private void _read_next_msg() {
         _read_size();
      }

-      private void _read_size()
-      {
+      private void _read_size() {
         _begin_or_resume_reading(new AsyncReceiveMessage(4, AsyncReceiveMessage.ReadState.reading_size));
      }

-      private void _read_data(int data_size)
-      {
+      private void _read_data(int data_size) {
         // TODO, this could be a problem is the size is too big - maybe read it in chunks (later)
         _begin_or_resume_reading(new AsyncReceiveMessage(data_size, AsyncReceiveMessage.ReadState.reading_data));
      }

-      private static string IAsyncResult_txt(IAsyncResult res)
-      {
+      private static string IAsyncResult_txt(IAsyncResult res) {
         return $"IsCompleted: {res.IsCompleted} CompletedSynchronously: {res.CompletedSynchronously}";
      }

-      private void _begin_or_resume_reading(AsyncReceiveMessage rcv_msg)
-      {
+      private void _begin_or_resume_reading(AsyncReceiveMessage rcv_msg) {
         //bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_reading: this: {ToString()} tid: {_tid()} message: {rcv_msg} _is_alive: {_is_alive}");
         if (!_is_alive)
            return;
-         try
-         {
+         try {
            SocketError read_sock_err;
            rcv_msg.key = _receive_key++;
            var res = _sock.BeginReceive(rcv_msg.buffer, rcv_msg.transfer_ptr,
@@ -468,35 +406,28 @@ namespace bts
                                 SocketFlags.None, out read_sock_err,
                                 new AsyncCallback(BeginReceive_callback_function),
                                 rcv_msg);
-            if (read_sock_err != SocketError.Success)
-            {
+            if (read_sock_err != SocketError.Success) {
               bts.Log.Gen.error($"_begin_or_resume_reading this: {this} error: {read_sock_err} rcv_msg: {rcv_msg}");
               _handle_error(new ErrorMessage(read_sock_err, "_beging_or_resume_reading"));
            }
-         }
-         catch (Exception ex)
-         {
+         } catch (Exception ex) {
            _handle_error(new ErrorMessage(ex, "_beging_or_resume_reading"));
         }
      }

-      private void _log_incoming_msg(IAsyncResult res)
-      {
-         // AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage;
+      private void _log_incoming_msg(IAsyncResult res) {
+         // var rcv_message = ExceptionAssert.Cast(res.AsyncState).To<AsyncReceiveMessage>();
         // var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(rcv_message.buffer, 0, rcv_message.buffer.Length);
         // Log.Gen.info($"BeginReceive_callback_function _log_incoming_msg buff: {buff_txt}");

      }

      // this is the function passed to BeginReceive, we 
-      private void BeginReceive_callback_function(IAsyncResult res)
-      {
+      private void BeginReceive_callback_function(IAsyncResult res) {
         _log_incoming_msg(res);
-         if (res.CompletedSynchronously)
-         {
+         if (res.CompletedSynchronously) {
            ThreadPool.QueueUserWorkItem(_ => _on_read_async(res));
-         }
-         else
+         } else
            _on_read_async(res);
      }

@@ -512,15 +443,13 @@ namespace bts
         _read_key_map.Add(rcv_message.key);
         if (!_is_alive)
            return;
-         try
-         {
+         try {
            SocketError read_err;
            Log.Gen.info($"AsyncTCPConnection_APM._on_read pre-EndReceive rcv_message: {rcv_message} res: {IAsyncResult_txt(res)}");
            int bytes_read = _sock.EndReceive(res, out read_err);
            rcv_message.advance_transfer_pointer(bytes_read);
            Log.Gen.info($"AsyncTCPConnection_APM._on_read post-EndReceive rcv_message: {rcv_message} res: {IAsyncResult_txt(res)} bytes_read: {bytes_read}");
-            if (bytes_read == 0)
-            {
+            if (bytes_read == 0) {
               bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, throwing exception");
               //bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, sleeping and ignoring");
               //System.Threading.Thread.Sleep(50);
@@ -528,32 +457,23 @@ namespace bts
               return;
            }
            //return;
-            if (read_err == SocketError.Success)
-            {
-               if (rcv_message.fully_transfered())
-               {
-                  if (rcv_message.is_reading_size())
-                  {
+            if (read_err == SocketError.Success) {
+               if (rcv_message.fully_transfered()) {
+                  if (rcv_message.is_reading_size()) {
                     var msg_size = _on_read_size_completely(rcv_message);
-                     if (msg_size != null && msg_size.HasValue)
-                     {
+                     if (msg_size != null && msg_size.HasValue) {
                        _read_data(msg_size.Value);
                     }
                     return;
-                  }
-                  else
-                  {
+                  } else {
                     // data is read
                     int msg_type = rcv_message.read_int32();
                     bts.Log.assert(msg_type == HB_MSG_SIGNAL || msg_type == DATA_MSG_SIGNAL);
-                     if (msg_type == DATA_MSG_SIGNAL)
-                     {
+                     if (msg_type == DATA_MSG_SIGNAL) {
                        _on_read_data_completely(rcv_message);
                        _read_next_msg();
                        return;
-                     }
-                     else if (msg_type == HB_MSG_SIGNAL)
-                     {
+                     } else if (msg_type == HB_MSG_SIGNAL) {
                        _on_read_hb_completely(rcv_message);
                        _read_next_msg();
                        return;
@@ -561,28 +481,21 @@ namespace bts
                     // go back to reading size for a new message
                     //_read_size();
                  }
-               }
-               else
-               {
+               } else {
                  // msg has not been fully rd, we have already advanced the transfer ptr
                  _begin_or_resume_reading(rcv_msg: rcv_message);
               }
-            }
-            else
-            {
+            } else {
               _handle_error(new ErrorMessage(read_err, "_on_read_async"));
            }
-         }
-         catch (Exception ex)
-         {
+         } catch (Exception ex) {
            _handle_error(new ErrorMessage(ex, "_on_read_async"));
         }
      }

      #endregion

-      private int? _on_read_size_completely(AsyncReceiveMessage msg)
-      {
+      private int? _on_read_size_completely(AsyncReceiveMessage msg) {
         bts.Log.assert(msg.is_reading_size());
         bts.Log.assert(msg.fully_transfered());
         bts.Log.assert(msg.original_size == 4);
@@ -590,28 +503,23 @@ namespace bts
         //var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(msg.buffer, 0, msg.buffer.Length);
         //bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely buff: {buff_txt}");
         bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely msg: {msg} msg_sz: {msg_sz}");
-         if (msg_sz < 0 || msg_sz > _max_msg_size)
-         {
+         if (msg_sz < 0 || msg_sz > _max_msg_size) {
            string err_msg = $"Encountered malformed message, prefixed_size: {msg_sz}";
            _handle_error(new ErrorMessage(new ApplicationException(err_msg), "_on_read_size_completely"));
            return null;
-         }
-         else
-         {
+         } else {
            // sz is determined, now we read that many bytes
            //_read_data(msg_sz);
            return msg_sz;
         }
      }

-      private void _on_read_hb_completely(AsyncReceiveMessage msg)
-      {
+      private void _on_read_hb_completely(AsyncReceiveMessage msg) {
         bts.Log.assert(!msg.is_reading_size());
         bts.Log.assert(msg.fully_transfered());
         bts.Log.assert(msg.original_size == 4);
         bts.Log.assert(msg.read_int32(0) == HB_MSG_SIGNAL);
-         if (!_hb_received_yet)
-         {
+         if (!_hb_received_yet) {
            _hb_received_yet = true;
            //bts.Log.Gen.info($"AsyncTCPConnection_APM HB received for the first time from peer {_peer_conn_info}");
         }
@@ -620,8 +528,7 @@ namespace bts
         _message_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_conn_info));
      }

-      private void _on_read_data_completely(AsyncReceiveMessage msg)
-      {
+      private void _on_read_data_completely(AsyncReceiveMessage msg) {
         bts.Log.assert(!msg.is_reading_size());
         bts.Log.assert(msg.fully_transfered());
         var type = msg.read_int32(0);
@@ -630,8 +537,7 @@ namespace bts
         byte[] data = msg.read_bytes(4);
         ///callbacks can be made through thread pool to avoid this
         ///thread from being blocked, by this callback
-         if (!_data_received_yet)
-         {
+         if (!_data_received_yet) {
            _data_received_yet = true;
            //bts.Log.Gen.info($"AsyncTCPConnection_APM data received for the first time from peer {_peer_conn_info}");
         }
@@ -642,19 +548,15 @@ namespace bts

      #region IDisposable Members

-      public void Dispose()
-      {
+      public void Dispose() {
         // socket is closed and disposed here
-         lock (this)
-         {
+         lock (this) {
            bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} Dispose called _dipose_impetus: {_dipose_impetus}");
-            if (_is_alive)
-            {
+            if (_is_alive) {
               _master_worker.removeWorker(this);
               _is_alive = false;
               bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} being disposed, last HB was received at {_last_hb_received_time}");
-               try
-               {
+               try {
                  bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} shutting down socket now");
                  _sock.Shutdown(SocketShutdown.Both);
               } catch (OutOfMemoryException) {
@@ -672,10 +574,8 @@ namespace bts
      #endregion

      private string _dipose_impetus = String.Empty;
-      private void _handle_error(ErrorMessage err_msg)
-      {
-         lock (this)
-         {
+      private void _handle_error(ErrorMessage err_msg) {
+         lock (this) {
            Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
            if (err_msg.oom) {
               throw Log.notnull(err_msg.exception);
@@ -286,4 +286,4 @@ namespace BCG {



-}
+}
@@ -1,8 +1,5 @@
 FROM bluetradingsystems/linux_dotnet7

-ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
-ARG CONF_FILE="src/DataStreamSuite/PubSubServer/spark.conf"
-
-COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
-COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/PubSubServer/conf/spark.conf"
+COPY --chown=bts:bts "build/Release/bin/net7.0/linux-x64/publish" "bin"
+COPY --chown=bts:bts "src/DataStreamSuite/PubSubServer/spark.conf" ".local/share/bts/PubSubServer/conf/spark.conf"
 WORKDIR "/home/bts/bin"
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Define the bash array
+# values=("value1" "value2" "value3")
+values=$(dotnet sln DataStreamSuite.sln list | tail +3)
+
+# Function to get the key for a value
+get_key() {
+  local value=$1
+  # Replace this with your actual logic to get the key
+  echo "key_for_$value"
+}
+
+# Function to get the JSON array of child values for a value
+get_children() {
+  local value=$1
+  # Replace this with your actual logic to get the JSON array of child values
+  echo '["child1", "child2", "child3"]'
+}
+
+# Create an empty JSON object
+json_object="{}"
+
+# Iterate through the array and construct the JSON object
+for value in ${values[@]}; do
+  key=$(get_key "$value")
+  children=$(get_children "$value")
+  json_object=$(echo "$json_object" | jq --arg key "$key" --argjson children "$children" '. + {($key): $children}')
+done
+
+# Print the final JSON object
+echo "$json_object"
+
@@ -43,7 +43,7 @@
      Condition="Exists($(ComputerSpecificConfig))" />

  <PropertyGroup Label="ProtoBuf">
-    <ProtocPluginBase>$(SparkRoot)\src\lib_spark\spark\api\service_compiler</ProtocPluginBase>
+    <ProtocPluginBase>$(SparkRoot)\src\lib\api\service_compiler</ProtocPluginBase>
  </PropertyGroup>
  <Choose>
    <When Condition="'$(OS)' == 'Windows_NT'">
@@ -24,9 +24,7 @@
    <PropertyGroup>
      <GenProtosCmd>$(protoc) %(Proto.FullPath)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)\lib_system</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)\lib_domain</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)\lib_spark</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-spark-csharp=$(ProtocPlugin)&quot;</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --spark-csharp_out=lang=c#,templates=immutable.pb.cs,strip_out_path=1:%(Proto.OutDir)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(Proto.OutDir)</GenProtosCmd>
@@ -86,9 +84,7 @@
    <PropertyGroup>
      <GenProtosCmd>$(protoc) %(ProtoService.FullPath)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)\lib_system</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)\lib_domain</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)\lib_spark</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-service=$(ProtocPlugin)&quot;</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --service_out=lang=c#,strip_out_path=1:%(ProtoService.OutDir)</GenProtosCmd>
      <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(ProtoService.OutDir)</GenProtosCmd>
@@ -1,43 +1,55 @@
 cmake_minimum_required(VERSION 3.5)
 project(Spark.Admin.WebApp)

-include(GNUInstallDirs)
 include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateJs.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateFs.cmake)

-file(
-  GLOB_RECURSE ${PROJECT_NAME}_PROTO_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  ../lib_domain/domain/protobufs/*.proto
+# dependencies
+set(PROTO_GLOBS
  ../lib_spark/spark/api/*.proto
  ../lib_spark/spark/api/services/*.proto
  ../lib_spark/spark/domain_model/*.proto
  ../lib_spark/spark/sparkproto/*.proto
  ../lib_spark/spark/md/protos/*.proto
  ../../externals/linux/include/google/protobuf/descriptor.proto
-  ../../externals/linux/include/google/protobuf/timestamp.proto
-)
+  ../../externals/linux/include/google/protobuf/timestamp.proto)

-add_custom_command(
-  OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-  COMMAND npm install --include=dev
-  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/package.json
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-)
+file(
+  GLOB_RECURSE _proto_rel_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  ${PROTO_GLOBS})
+
+file(
+  GLOB_RECURSE _proto_abs_SOURCES
+  ${PROTO_GLOBS})
+
+file(
+  GLOB_RECURSE _ts_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  src/*.ts)
+
+file(
+  GLOB_RECURSE _fs_SOURCES
+  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+  src/*.fs)

 set(_tools ${CMAKE_SOURCE_DIR}/tools)
 set(_quiet_exec ${_tools}/quiet_exec.sh)
 set(_venv_exec ${_tools}/venv_exec.sh)
 set(_tools_py ${CMAKE_SOURCE_DIR}/tools/py)

+# The tsconfig file is generated as a function of the cmake build type, and
+# binary dir. Depending on the build type, it extends either
+# 'tsconfig.debug.json' or 'tsconfig.release.json'.
 set(_tsconfig_base "${CMAKE_CURRENT_SOURCE_DIR}/tsconfig.$<IF:$<CONFIG:Debug>,debug,release>.json")
+set(WEBPACK_MODE $<IF:$<CONFIG:Debug>,"development","production">)
 set(_tsconfig "${CMAKE_CURRENT_BINARY_DIR}/tsconfig.json")

 set(_venv_python ${CMAKE_SOURCE_DIR}/.venv/bin/python)
 set(_gen_tsconfig_py ${_tools_py}/gen_tsconfig.py)
 set(_build_admin_web_app_py ${_tools_py}/build_admin_web_app.py)
+
+set(_gen_tsconfig_deps ${_venv_exec} ${_venv_python} ${_gen_tsconfig_py} ${_tsconfig_base})
+
 set(_build_admin_web_app_deps
  ${_quiet_exec}
  ${_venv_exec}
@@ -47,143 +59,49 @@ set(_build_admin_web_app_deps
  ${CMAKE_CURRENT_SOURCE_DIR}/webpack.config.js
  ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/fs/fable.fs
  ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-  ${${PROJECT_NAME}_PROTO_SOURCES}
+  ${_proto_rel_SOURCES}
  ${_api_ts_SOURCES}
  ${_ui_fs_SOURCES}
  ${_tsconfig})

 update_deps_file("${_build_admin_web_app_deps}")

+function(npm_install path)
+  add_custom_command(
+    OUTPUT ${path}/package-lock.json
+    COMMAND "npm" ARGS "install" "--include=dev"
+    DEPENDS ${path}/package.json
+    WORKING_DIRECTORY ${path}
+    COMMENT "Run 'npm install' in ${path}")
+endfunction()
+
+npm_install(${CMAKE_CURRENT_SOURCE_DIR})
+
 add_custom_command(
  OUTPUT ${_tsconfig}
  COMMAND ${_venv_exec}
  ARGS
-    ${_venv_python}
-    ${_gen_tsconfig_py}
-    --config-base ${_tsconfig_base}
-    --config-out ${_tsconfig}
-    --build-out ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS
-    ${_venv_exec}
-    ${_venv_python}
-    ${_gen_tsconfig_py}
-    ${_tsconfig_base}
-    ${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
-)
-
-protobuf_generate_js(
-  TARGET_NAME generate_js_protos
-  PROTO_INCLUDE_DIRS
-    ../lib_domain
-    ../lib_spark
-    ../../externals/linux/include
-  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
-  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/proto.js
-)
-
-protobuf_generate_fs(
-  TARGET_NAME generate_fs_protos
-  PROTO_INCLUDE_DIRS
-    ../lib_domain
-    ../lib_spark
-    ../../externals/linux/include
-  PROTO_SOURCES ${${PROJECT_NAME}_PROTO_SOURCES}
-  OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/fable
-)
-
-# Each input file maps to multiple output files with different extensions
-# (js, js.map, d.ts, fs). Transpile ts -> d.ts (and js), then d.ts -> fs.
-# Who knew front-end development could be so complicated? `\_(=_=)_/`
-file(
-  GLOB_RECURSE ${PROJECT_NAME}_TS_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  src/*.ts
-)
-set(${PROJECT_NAME}_JS_PROTOS)
-set(${PROJECT_NAME}_TS_PROTOS)
-set(${PROJECT_NAME}_FS_PROTOS)
-foreach(_ts_source ${${PROJECT_NAME}_TS_SOURCES})
-  get_filename_component(_ts_source_we ${_ts_source} NAME_WE)
-  set(_prefix ${CMAKE_CURRENT_BINARY_DIR}/${_ts_source_we})
-  list(APPEND ${PROJECT_NAME}_JS_PROTOS ${_prefix}.js ${_prefix}.js.map)
-  list(APPEND ${PROJECT_NAME}_TS_PROTOS ${_prefix}.d.ts ${_prefix}.d.ts.map)
-  list(APPEND ${PROJECT_NAME}_FS_PROTOS ${_prefix}.fs)
-  add_custom_command(
-    OUTPUT ${_prefix}.fs
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    COMMAND npx --no-install ts2fable ${_prefix}.d.ts ${_prefix}.fs
-    DEPENDS ${_prefix}.d.ts
-  )
-endforeach()
+  ${_venv_python}
+  ${_gen_tsconfig_py}
+  --config-base ${_tsconfig_base}
+  --config-out ${_tsconfig}
+  --build-out ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDS ${_gen_tsconfig_deps})

 add_custom_command(
-  OUTPUT
-    ${${PROJECT_NAME}_JS_PROTOS}
-    ${${PROJECT_NAME}_TS_PROTOS}
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-  COMMAND npx
-    --no-install tsc
-    -p ${_tsconfig}
-  DEPENDS
-    ${_tsconfig}
-    ${CMAKE_CURRENT_BINARY_DIR}/proto.js
-)
-add_custom_target(
-  transpile_type_script
-  ALL
-  DEPENDS
-    ${${PROJECT_NAME}_JS_PROTOS}
-    ${${PROJECT_NAME}_TS_PROTOS}
-)
-add_custom_target(generate_fsts_stubs ALL DEPENDS ${${PROJECT_NAME}_FS_PROTOS})
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
+  COMMAND ${_quiet_exec}
+  ARGS
+  ${_venv_exec}
+  ${_venv_python}
+  ${_build_admin_web_app_py}
+  --protos ${_proto_abs_SOURCES}
+  --tsconfig ${_tsconfig}
+  --mode ${WEBPACK_MODE}
+  --output-dir ${CMAKE_CURRENT_BINARY_DIR}
+  COMMENT "Building Spark.Admin.WebApp"
+  DEPENDS ${_build_admin_web_app_deps})

-sym_link_dir(
-  TARGET_NAME link_webapp_public
-  SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/public
-  DEST ${CMAKE_CURRENT_BINARY_DIR}/public
-)
-sym_link_dir(
-  TARGET_NAME link_webapp_root
-  SOURCE ${CMAKE_CURRENT_BINARY_DIR}
-  DEST ${CMAKE_SOURCE_DIR}/build/Spark.Admin.WebApp
-)
+add_custom_target(admin_web_app_bundle ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/bundle.js)

-file(
-  GLOB_RECURSE ${PROJECT_NAME}_FS_SOURCES
-  RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-  src/*.fs
-)
-add_custom_command(
-  OUTPUT
-    ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
-    $<$<CONFIG:DEBUG>:${CMAKE_CURRENT_BINARY_DIR}/bundle.js.map>
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src
-  COMMAND dotnet restore
-  COMMAND dotnet tool restore
-  COMMAND export SPARK_ADMIN_WEB_APP_OUTPUT_DIR=${CMAKE_CURRENT_BINARY_DIR}
-  COMMAND dotnet fable -o ${CMAKE_CURRENT_BINARY_DIR}
-  COMMAND npm run build -- --mode $<IF:$<CONFIG:Debug>,"development","production">
-  DEPENDS
-    ${${PROJECT_NAME}_FS_SOURCES}
-    ${${PROJECT_NAME}_FS_PROTOS}
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/Spark.Admin.WebApp.fsproj
-)
-add_custom_target(
-  bundle_spark_admin_webapp ALL
-  DEPENDS
-    ${CMAKE_CURRENT_BINARY_DIR}/bundle.js
-    $<$<CONFIG:DEBUG>:${CMAKE_CURRENT_BINARY_DIR}/bundle.js.map>
-  COMMENT "Bundling Spark.Admin.WebApp"
-)
-
-# Exclude 'public' dir because it is a symlink
-install(
-  DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
-  DESTINATION "${CMAKE_INSTALL_BINDIR}"
-  PATTERN "public" EXCLUDE
-)
-# Install actual (non-symlinked) 'public' dir
-install(
-  DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/public"
-  DESTINATION "${CMAKE_INSTALL_BINDIR}/${PROJECT_NAME}"
-)
+install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" DESTINATION "bin")
@@ -42,7 +42,7 @@
    "ts-loader": "^9.4.2",
    "tsutils": "^3.21.0",
    "typescript": "^4.9.5",
-    "webpack-cli": "^5.1.4",
+    "webpack-cli": "^5.0.1",
    "webpack-dev-server": "^4.11.1",
    "ws": "^8.12.1"
  }
@@ -17,7 +17,7 @@ type Tab =
  | PositionManagementTab
  | MarginTab

-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.COINBASE; Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.SIM; |]

 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]

@@ -69,11 +69,6 @@ type ICEExchangeField =
  | AccountCode of string
  | MiscTagMapping of JSMap<string>

-type TaifexExchangeField =
-  | CTI of CTICode
-  | Origin of CustomerOrFirm
-  | ClearingAccount of string
-
 type SIMExchangeField =
  | CTI of CTICode
  | Origin of CustomerOrFirm
@@ -797,9 +797,9 @@ let venueToString (venue:bts.api.Venue) =
    | bts.api.Venue.BINANCE -> "BINANCE"
    | bts.api.Venue.BinanceUs -> "BinanceUs"
    | bts.api.Venue.BinanceUsdsFutures -> "BinanceUsdsFutures"
+    | bts.api.Venue.COINBASE -> "COINBASE"
    | bts.api.Venue.FTX -> "FTX"
    | bts.api.Venue.OKX -> "OKX"
-    | bts.api.Venue.TAIFEX -> "TAIFEX"

 let exchange_display expirationNameList =
  let s = sprintf "%O" expirationNameList.product.exchange
@@ -33,7 +33,6 @@
    <Compile Include="$(API)/fable/timestamp.fable.fs" />
    <Compile Include="$(API)/fable/apicore.fable.fs" />
    <Compile Include="$(API)/fable/apioptions.fable.fs" />
-    <Compile Include="$(API)/fable/aggressor_side.fable.fs" />
    <Compile Include="$(API)/fable/Security.fable.fs" />
    <Compile Include="$(API)/fable/SparkProtos.fable.fs" />
    <Compile Include="$(API)/fable/HelloService.fable.fs" />
@@ -1,10 +1,8 @@
-cmake_minimum_required(VERSION 3.14)
+cmake_minimum_required(VERSION 3.5)
 project(sparkd)

-include(FetchContent)
 include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+include(${CMAKE_SOURCE_DIR}/cmake/Protobuf.cmake)
 include(GNUInstallDirs)

 if (NOT DEFINED PROJECT_VERSION)
@@ -33,18 +31,15 @@ protobuf_generate_cpp(
    ${CMAKE_SOURCE_DIR}/src/lib_spark
    ${CMAKE_CURRENT_SOURCE_DIR}
  PROTO_SOURCES
-    ${${PROJECT_NAME}_PROTO_SOURCES}
-)
+    ${${PROJECT_NAME}_PROTO_SOURCES})
 add_executable(sparkd ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_PROTO_CXX_SOURCES})

 # include paths
 target_include_directories(
-  sparkd
-  PUBLIC
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
-    $<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>
-)
+  sparkd PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
+  $<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>)

 find_package(PkgConfig REQUIRED)
 pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-connector)
@@ -53,20 +48,11 @@ pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-conne
 list(APPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules)
 find_package(rapidjson REQUIRED)
 target_include_directories(
-  ${PROJECT_NAME} SYSTEM
-  PUBLIC
-    $<BUILD_INTERFACE:${rapidjson_INCLUDE_DIR}> 
+  ${PROJECT_NAME} SYSTEM PUBLIC
+  $<BUILD_INTERFACE:${rapidjson_INCLUDE_DIR}> 
 )
 add_dependencies(${PROJECT_NAME} rapidjson_project)

-# databento
-FetchContent_Declare(
-  databento
-  GIT_REPOSITORY https://github.com/databento/databento-cpp
-  GIT_TAG HEAD
-)
-FetchContent_MakeAvailable(databento)
-
 # deribit-demo log level
 if(NOT DEFINED DERIBIT_DEMO_LOG_LEVEL)
  add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
@@ -97,51 +83,61 @@ else()
  message("-- Building of DERIBIT_DEMO_LOG_LEVEL=TRACE")
 endif()

+# add_dependencies(${PROJECT_NAME} py_market_data)
+list(APPEND CMAKE_PREFIX_PATH "${CMAKE_SOURCE_DIR}/install/$<CONFIG>/lib/cmake/py-market-data")
+
 target_link_directories(
  ${PROJECT_NAME} PUBLIC
  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
  $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
 )

-# Set RPATH so sparkd knows where to find our shared libs (e.g., libspark).
+# Set RPATH so sparkd knows where to find our shared libs (e.g., libspark, lib_py_market_data).
 # The dirs are relative to sparkd's path, represented by "$ORIGIN".
 set_target_properties(
  ${PROJECT_NAME} PROPERTIES
-  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_python_md"
+  BUILD_RPATH "$ORIGIN/../lib_spark;$ORIGIN/../lib_py_market_data"
  INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
 )

-# Python interop (for python md providers)
-add_dependencies(${PROJECT_NAME}
-  python_md
-  python_md_module
-  python_md_providers
-  copy_python_md_providers
-)
 include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)

-sym_link_local_lib(python_md)
+sym_link_local_lib(py_market_data)
 sym_link_file(
-  SOURCE ${CMAKE_BINARY_DIR}/src/lib_python_md/python_md_module.cpython-310-x86_64-linux-gnu.so
-  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_module.cpython-310-x86_64-linux-gnu.so
+  TARGET_NAME link_py_market_data_module
+  SOURCE ${CMAKE_BINARY_DIR}/src/lib_py_market_data/py_market_data_module.cpython-310-x86_64-linux-gnu.so
+  DEST ${CMAKE_CURRENT_BINARY_DIR}/py_market_data_module.cpython-310-x86_64-linux-gnu.so
  INSTALL_DEST ${CMAKE_INSTALL_BINDIR}/
 )
-get_target_property(providers python_md_providers INTERFACE_PYTHON_SCRIPTS_DIR)
-sym_link_dir(
-  TARGET_NAME link_python_md_providers
-  SOURCE ${providers}
-  DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_providers
-)
+# add_custom_command(
+#   OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/py_market_data_module.cpython-310-x86_64-linux-gnu.so
+#   COMMAND ${CMAKE_COMMAND} -E create_symlink
+#     ${CMAKE_BINARY_DIR}/src/lib_py_market_data/py_market_data_module.cpython-310-x86_64-linux-gnu.so
+#     ${CMAKE_CURRENT_BINARY_DIR}/py_market_data_module.cpython-310-x86_64-linux-gnu.so
+#   DEPENDS 
+#     ${CMAKE_BINARY_DIR}/src/lib_py_market_data/py_market_data_module.cpython-310-x86_64-linux-gnu.so
+#   COMMENT "Creating symlink to py_market_data_module.so"
+# )
+# add_custom_target(create_symlinks ALL
+#   DEPENDS ${CMAKE_BINARY_DIR}/src/lib_py_market_data/py_market_data_module.cpython-310-x86_64-linux-gnu.so)
+# install(
+#   FILES
+#   ${CMAKE_CURRENT_BINARY_DIR}/py_market_data_module.cpython-310-x86_64-linux-gnu.so
+#   DESTINATION ${CMAKE_INSTALL_BINDIR}/
+# )

 target_link_libraries(
  sparkd
-  system
-  python_md
-  python_md_module
+  py_market_data
+  py_market_data_module
  spark
-  databento::databento
  ${emumba-deribit-connector-library_LIBRARIES}
+  ssl
+  cryptopp
  tbb
+  onixs-cmemdh-all-4.8.0
+  OnixS.IceImpactMulticastPriceFeedHandlerCpp
+  ACE
  quickfix
  benchmark
  Common++
@@ -153,33 +149,11 @@ target_link_libraries(
  -static-libstdc++
  -pthread)

-# Onixs
-target_link_libraries(
-  ${PROJECT_NAME}
-  onixs-cmemdh
-  OnixS.IceImpactMulticastPriceFeedHandlerCpp
-  # gcc doesn't resolve onixs-fix-engine and ACE in runtime path unless we specify
-  # --no-as-needed. These are onixs deps that aren't called directly by our code.
-  # Note: this isn't an issue for clang
-  -Wl,--no-as-needed
-  onixs-fix-engine
-  ACE
-)
-
-# Catch2
-find_package(Catch2 CONFIG REQUIRED)
-target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
-
-install_basic()
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasicConfig.cmake)
+install_basic_config()
 install(
  FILES
-    "${EXTERNALS_LIB}/libonixs-cmemdh.so"
-    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4"
-    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4.8.0"
    "${EXTERNALS_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-    "${EXTERNALS_LIB}/libonixs-fix-engine.so"
-    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4"
-    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4.10.1"
    "${EXTERNALS_LIB}/libACE-5.7.5.so"
  DESTINATION ${CMAKE_INSTALL_LIBDIR}
 )
@@ -1,19 +0,0 @@
-FROM bluetradingsystems/linux_dotnet7
-
-ARG SOURCE_DIR="install/sparkd-release-gcc"
-ARG CONF_FILE="src/app_sparkd/spark.current.conf"
-
-USER "root"
-RUN apt-get update \
-  && apt-get install -y liblttng-ust-dev
-
-RUN apt-get update \
-  && apt-get install -y libpcap0.8-dev
-
-RUN apt-get update \
-  && apt-get install -y python3.10-dev
-
-USER "bts"
-COPY --chown=bts:bts "${SOURCE_DIR}" "."
-COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/sparkd/conf/spark.conf"
-WORKDIR "/home/bts/bin"
@@ -0,0 +1,10 @@
+FROM bluetradingsystems/linux_dotnet7
+
+USER "root"
+RUN apt-get update \
+  && apt-get install -y liblttng-ust-dev
+
+USER "bts"
+COPY --chown=bts:bts "install/Release/bin" "bin"
+COPY --chown=bts:bts "src/server/sparkd/spark.current.conf" ".local/share/bts/sparkd/conf/spark.conf"
+WORKDIR "/home/bts/bin"
@@ -1,5 +1,4 @@
 #include "AlgoDefinitionServiceServer.h"
-#include "spark/spark_data_directory.h"

 namespace bts {
 namespace api {
@@ -20,11 +19,8 @@ struct AlgoDefinitionServiceServer::PImpl {
       const AlgoDefinitionServiceBase::Algo_definitionsStatefulPublisher
           &publisher)
       : table_name("algo_definitions"),
-         store(std::make_unique<SqliteStore>(
-             spark::SparkDataDirectory{}
-                 .database_file("algo_definitions")
-                 .string())),
-         context(context), publisher(publisher) {}
+         store(new SqliteStore("algo_definition")), context(context),
+         publisher(publisher) {}
 };

 AlgoDefinitionServiceServer::AlgoDefinitionServiceServer(
@@ -4,19 +4,19 @@
 #include "spark/api/services/AlgoDefinitionService.server.pb.h"
 #include "spark/api/services/OrderService.client.pb.h"
 #include "spark/api/services/OrderService.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/persistence/IStore.h"
 #include "spark/persistence/SqliteStore.h"
 #include "spark/utils/PBUtils.h"
 #include "spark/utils/TimeStamp.h"
 #include "spark/utils/bts_assert.h"
 #include "spark/utils/bts_no_copy.h"
-#include "system/logging/logging.h"
+#include <memory>

 #include <boost/tokenizer.hpp>
 #include <fmt/format.h>
 #include <google/protobuf/util/time_util.h>
 #include <map>
-#include <memory>
 #include <set>
 #include <string>
 #include <utility>
@@ -69,4 +69,4 @@ class AlgoDefinitionServiceServer : public AlgoDefinitionServiceBase {
 } // namespace algo
 } // namespace orderman
 } // namespace api
-} // namespace bts
+} // namespace bts
@@ -2,13 +2,13 @@
 #include "spark/api/apicore.pb.h"
 #include "spark/api/services/AnalyticsService.pb.h"
 #include "spark/api/services/AnalyticsService.server.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/persistence/IStore.h"
 #include "spark/persistence/SqliteStore.h"
 #include "spark/utils/PBUtils.h"
 #include "spark/utils/TimeStamp.h"
 #include "spark/utils/bts_assert.h"
 #include "sparkd/algo/AlgoDefinitionServiceServer.h"
-#include "system/logging/logging.h"

 #include <boost/algorithm/string.hpp>
 #include <boost/tokenizer.hpp>
@@ -50,7 +50,7 @@ void add_or_update_to_table(
    const std::string &ns,
    const std::string &key,
    const std::string &val) {
-   auto [exists, rowid, existing_val] = store->retrieve_one(ns, key);
+   auto [exists, existing_val] = store->retrieve_one(ns, key);
   LOG(Gen, Info,
       "add_or_update ns: {} key: {} val: {} exists: {} existing_val: {}", ns,
       key, val, exists, existing_val);
@@ -384,8 +384,7 @@ void AnalyticsServiceServer::save_report(
    const api::Response<::bts::api::ResultMessage> &resp) {
   Report report = report_arg;
   std::string key = std::to_string(report.id());
-   auto [present, rowid, val] =
-       me->reports_store->retrieve_one(me->reports_ns, key);
+   auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key);
   if (!present) {
      int new_id = get_max_id(me->reports_store.get(), me->reports_ns) + 1;
      report.set_id(new_id);
@@ -407,8 +406,7 @@ void AnalyticsServiceServer::delete_report(
    const bts::api::analytics::IntWrapper &id,
    const api::Response<::bts::api::ResultMessage> &resp) {
   std::string key = std::to_string(id.data());
-   auto [present, rowid, val] =
-       me->reports_store->retrieve_one(me->reports_ns, key);
+   auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key);
   api::ResultMessage res;
   res.set_success(false);
   if (!present) {
@@ -2,12 +2,12 @@

 #include "spark/api/public/ServiceContext.h"
 #include "spark/config/config.h"
+#include "spark/config/protos/logging.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/quickbus/quickbus.h"

 #include "spark/api/service_helpers/AdminServiceUtils.h"
 #include "spark/api/services/AdminService.client.pb.h"
-#include "spark/utils/init_spark_lib.h"
-#include "system/protobufs/logging.pb.h"

 #include <algorithm>
 #include <boost/algorithm/string.hpp>
@@ -17,9 +17,9 @@
 namespace bts::cmds {

 void md_report() {
-   auto cfg = config::get<logging::protobufs::Logging>();
+   auto cfg = config::get<config::Logging>();
   cfg.mutable_console()->set_default_level("Fatal");
-   utils::SparkLibInit spark_lib{cfg, false, true};
+   log::configure_logging(cfg);

   std::map<api::CMEMics, std::string> mic_map;
   mic_map[api::CMEMics::CME] = "XCME";
@@ -1,14 +1,13 @@
 #include "CMECert.h"
+#include "spark/logging/logging.h"
 #include "spark/orderman/ilink3/IL3Dispatcher.h"
 #include "spark/orderman/ilink3/IL3GW.h"
 #include "spark/orderman/ilink3/IL3Reactor.h"
 #include "spark/orderman/ilink3/ILink3Utils.h"
+#include "spark/os/os.h"
 #include "spark/utils/Dispatcher.h"
 #include "spark/utils/SecureLogon.h"
 #include "spark/utils/TimeStamp.h"
-#include "system/logging/logging.h"
-#include "system/os.h"
-
 #include <chrono>
 #include <fstream>
 #include <future>
@@ -2,6 +2,8 @@

 #include "spark/config/config.h"
 #include "spark/config/protos/keycode.pb.h"
+#include "spark/logging/logging.h"
+#include "spark/os/os.h"
 #include "spark/utils/AuthToken.h"
 #include "spark/utils/FileUtils.h"
 #include "spark/utils/bts_assert.h"
@@ -11,8 +13,6 @@
 #include "spark/utils/crypto/PublicKey.h"
 #include "spark/utils/datetime.h"
 #include "spark/utils/helper_macros.h"
-#include "system/logging/logging.h"
-#include "system/os.h"

 #include <boost/algorithm/string/replace.hpp>
 #include <boost/date_time/gregorian/gregorian.hpp>
@@ -1,5 +1,5 @@
 #include "QuickTest.h"
-#include "system/logging/logging.h"
+#include "spark/logging/logging.h"

 #include <thread>

@@ -1,33 +1,31 @@
 #include "Run.h"
-#include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/api/public/ServiceContext.h"
 #include "spark/config/config.h"
 #include "spark/config/protos/ports.pb.h"
 #include "spark/config/protos/quickbus_cfg.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/md/MBOProcessor.h"
 #include "spark/md/MBOServiceServer.h"
 #include "spark/md/MDClient.h"
 #include "spark/md/SyntheticSecurityService.h"
-#include "spark/md/relay/query_server.h"
+#include "spark/md/relay/QueryServer.h"
 #include "spark/md/service/MarketDataServiceServer.h"
 #include "spark/metrics/metrics.h"
 #include "spark/metrics/system_metrics.h"
 #include "spark/orderman/OrderServiceServer.h"
 #include "spark/orderman/events/TradeSummarySignalDispatcher.h"
 #include "spark/orderman/order_factory/DynamicOrderRegistry.h"
-#include "spark/persistence/IDGenerator.h"
 #include "spark/persistence/PersistenceServiceServer.h"
 #include "spark/plugin/SparkServices.h"
 #include "spark/plugin/plugin.h"
 #include "spark/quickbus/core/Quickbus.h"
 #include "spark/tracing/PerfLog.h"
 #include "spark/utils/Dispatcher.h"
+#include "md/cmds/Run.h"
 #include "sparkd/algo/AlgoDefinitionServiceServer.h"
 #include "sparkd/analytics/AnalyticsServiceServer.h"
-#include "sparkd/cmds/md/cmds/Cleanup.h"
 #include "sparkd/http/WebServer.h"
 #include "sparkd/md/MarketDataAuxiliaryServiceServer.h"
-#include "sparkd/md/MarketDataGateway.h"
 #include "sparkd/md/recording/MDRecorder.h"
 #include "sparkd/position/MarginServiceServer.h"
 #include "sparkd/position/PositionServiceServer.h"
@@ -38,7 +36,7 @@
 #include "sparkd/support/ShellLatencyServer.h"
 #include "sparkd/user/AdminServiceServer.h"
 #include "sparkd/user/AuthServiceServer.h"
-#include "system/logging/logging.h"
+

 namespace bts::cmds {

@@ -51,7 +49,8 @@ static void execute() {
   auto qb_cfg = config::get<config::QuickbusCfg>().node();
   auto &ports = config::get<config::Ports>();
   qb_cfg.set_my_port(ports.qbus_port());
-   qbus::Quickbus quickbus(qb_cfg, qbus::QuickbusThreadOptions::DEFERRED_START);
+   qbus::Quickbus quickbus(
+       qb_cfg, qbus::QuickbusThreadOptions::DEFERRED_START);
   api::ServiceContext sc(&quickbus);

   trace::PerfLog perf_log;
@@ -64,11 +63,7 @@ static void execute() {

   bts::md::MBOProcessor mbo_processor;
   bts::md::TradeSummarySignalDispatcher tss_dispatcher;
-   md::cmds::Cleanup::execute();
-   bts::md::MarketDataGateway md_gateway{
-       config::get<api::md::MarketDataConfig>(),
-       persistence::IDGenerator::instance_ptr()};
-   md_gateway.start();
+   md::cmds::Run::run();

   api::synthetic::SyntheticSecurityService synthetic_service(sc);
   bts::md::MDClient mdclient(
@@ -1,77 +0,0 @@
-#include "TestPythonMdProviderCmd.h"
-#include "python_md/md_sink_impl.h"
-#include "python_md/py_md_executor.h"
-#include "system/logging/logging.h"
-
-#include <pybind11/embed.h>
-#include <pybind11/pybind11.h>
-#include <string>
-#include <thread>
-
-using bts::md::py_interop::MdSinkImpl;
-using std::string;
-
-namespace bts::cmds {
-
-struct TestPythonMdProviderCmd::Args {
-   std::string py_md_provider;
-   std::vector<std::string> module_paths;
-};
-
-static void execute(
-    std::string py_md_provider, std::vector<std::string> module_paths) {
-   using namespace std::chrono_literals;
-   MdSinkImpl sink;
-   sink.set_securities_callback([](const auto &securities) {
-      LOG(md, Info, "Received securities: {}", securities.to_string());
-   });
-   sink.set_books_callback([](const auto &books) {
-      LOG(md, Info, "Received books: {}", books.to_string());
-   });
-   sink.set_trades_callback([](const auto &trades) {
-      LOG(md, Info, "Received trades: {}", trades.to_string());
-   });
-   try {
-      md::py_interop::PyMdExecutor py_md_exec{
-          std::move(py_md_provider), std::move(module_paths), &sink};
-      LOG(md, Info, "Starting py md executor.");
-      py_md_exec.start();
-      LOG(md, Info, "Stopping py md executor.");
-      py_md_exec.stop();
-      // The following brief sleep loop provides a means to test that 'ctrl+c' interrupts exit
-      // quickly (there was a bug during early development of the 'python market data' feature that
-      // caused such interrupts to hang for a few seconds).
-      std::cout << "Press ctrl+c to exit..." << std::endl;
-      static const int SLEEP_CYCLES = 3;
-      static const auto SLEEP_INTERVAL = 3s;
-      for (int i = 0; i < SLEEP_CYCLES; i++) {
-         std::this_thread::sleep_for(SLEEP_INTERVAL);
-      }
-   } catch (const std::exception &ex) {
-      LOG(md, Error, "Py md executor threw: {}", string{ex.what()});
-   }
-}
-
-TestPythonMdProviderCmd::TestPythonMdProviderCmd()
-    : CLI::App("Test python md provider plugin", "test-py-md-provider"),
-      args_(std::make_unique<Args>()) {
-   add_option(
-       "--provider", args_->py_md_provider,
-       "Python md provider plugin (py file)");
-   add_option(
-       "--module-path", args_->module_paths, "Module path for python plugin");
-   callback([this]() {
-      if (args_->py_md_provider.empty()) {
-         args_->py_md_provider = "dummy_md_provider.DummyMdProvider";
-      }
-      if (args_->module_paths.empty()) {
-         args_->module_paths.push_back(".");
-         args_->module_paths.push_back("./python_md_providers");
-      }
-      execute(std::move(args_->py_md_provider), std::move(args_->module_paths));
-   });
-}
-
-TestPythonMdProviderCmd::~TestPythonMdProviderCmd() = default;
-
-} // namespace bts::cmds
@@ -1,22 +0,0 @@
-#pragma once
-
-#include <CLI11/CLI11.hpp>
-#include <memory>
-
-namespace bts::cmds {
-
-/**
- * This command is designed to run a quick sanity check on Python md provider C++ interop.
- * It instantiates a python md provider (in C++) and calls <provider>.start() with a dummy
- * sink that prints updates to console, but does not write md to shared memory.
- */
-struct TestPythonMdProviderCmd : public CLI::App {
-   TestPythonMdProviderCmd();
-   ~TestPythonMdProviderCmd();
-
- private:
-   struct Args;
-   std::unique_ptr<Args> args_;
-};
-
-} // namespace bts::cmds
@@ -1,22 +1,30 @@
 #include "Web.h"
-#include "md/cmds/Run.h"
 #include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/MarketDataService.pb.h"
+#include "spark/api/services/SyntheticSecurityService.pb.h"
 #include "spark/config/config.h"
+#include "spark/config/protos/market_data.pb.h"
 #include "spark/config/protos/ports.pb.h"
 #include "spark/config/protos/quickbus_cfg.pb.h"
+#include "spark/md/MBOProcessor.h"
 #include "spark/md/MDClient.h"
 #include "spark/md/SyntheticSecurityService.h"
-#include "spark/md/relay/query_server.h"
+#include "spark/md/relay/QueryServer.h"
 #include "spark/metrics/metrics.h"
 #include "spark/metrics/system_metrics.h"
-#include "spark/quickbus/core/Quickbus.h"
+#include "spark/quickbus/quickbus.h"
 #include "spark/tracing/PerfLog.h"
 #include "spark/utils/Dispatcher.h"
+#include "spark/utils/thread_manager.h"
+#include "md/cmds/Cleanup.h"
+#include "md/cmds/Run.h"
+#include "sparkd/exchange_sim/ExchangeSim.h"
 #include "sparkd/http/WebServer.h"
 #include "sparkd/user/AdminServiceServer.h"
 #include "sparkd/user/AuthServiceServer.h"
-
+#include <optional>
 #include <string>
+#include <vector>

 namespace bts::cmds {

@@ -1,6 +1,6 @@
 #include "DeribitDemo.h"
+#include "spark/logging/logging.h"
 #include "sparkd/cmds/crypto/DeribitDemoListener.h"
-#include "system/logging/logging.h"

 #include <csignal>
 #include <emumba-deribit-connector/deribit_exception.h>
@@ -1,8 +1,7 @@
 #include "DeribitDemoCmd.h"
 #include "DeribitDemo.h"
 #include "spark/config/config.h"
-#include "system/os.h"
-
+#include "spark/os/os.h"
 #include <CLI11/CLI11.hpp>
 #include <functional>
 #include <optional>
@@ -1,8 +1,7 @@
 #include "DeribitDemoConfiguration.h"
 #include "RapidJsonReader.h"
+#include "spark/logging/logging.h"
 #include "rapidjson/istreamwrapper.h"
-#include "system/logging/logging.h"
-
 #include <emumba-deribit-connector/deribit_utils.h>
 #include <fstream>
 #include <list>
@@ -1,6 +1,6 @@
 #pragma once

-#include "system/FMT_IMPL.h"
+#include "spark/core/FMT_IMPL.h"

 #include <emumba-deribit-connector/deribit_utils.h>
 #include <iostream>
@@ -1,7 +1,5 @@
 #include "LiveLadderDemo.h"
 #include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/config/config.h"
-#include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/MarketDataGateway.h"
 #include "sparkd/md/listeners/LiveLadderPrinter.h"

@@ -9,16 +7,13 @@

 namespace bts::cmds {

-using bts::api::md::MarketDataConfig;
-using bts::persistence::IDGenerator;
-
-LiveLadderDemo::LiveLadderDemo(const MarketDataConfig &config) {
+LiveLadderDemo::LiveLadderDemo(const api::md::MarketDataConfig &) {
   char ssl_cert_file[] = "SSL_CERT_FILE=/etc/ssl/cert/ca-certificates.crt";
   putenv(ssl_cert_file);

-   md::MarketDataGateway gw{config, IDGenerator::instance_ptr()};
+   md::MarketDataGateway gw;
   md::listeners::LiveLadderPrinter ladder_printer;
-   gw.start();
+   gw.run();

   const std::size_t n_loops = 16;
   for (std::size_t n = 0; n < n_loops; n++) {
@@ -3,10 +3,9 @@
 #include "LiveLadderDemo.h"
 #include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/io/ProtoReader.h"
+#include "spark/logging/logging.h"
 #include "sparkd/cmds/common/Validators.h"
 #include "sparkd/cmds/crypto/LiveLadderDemo.h"
-#include "system/logging/logging.h"
-
 #include <CLI11/CLI11.hpp>
 #include <optional>

@@ -1,38 +1,11 @@
 #include "Channels.h"
-#include "domain/protobufs/venue.pb.h"
-#include "domain/security_criteria.h"
-#include "spark/api/md/MarketDataConfig.pb.h"
-#include "spark/config/config.h"
-#include "spark/md/SymbolLoader.h"
 #include "spark/md/cme/MDP3Utils.h"
-#include "spark/md/cme/mdp3_data_directory.h"

 namespace bts::cmds::md::cmds {

-using bts::domain::SecurityCriteria;
-using bts::md::await_cme_config;
-using bts::md::Mdp3DataDirectory;
-using bts::md::parse_channels;
-using bts::md::SymbolLoader;
-
 Channels::Channels() : CLI::App("List configured cme channels", "channels") {
   callback([] {
-      Mdp3DataDirectory data_dir;
-      auto md_cfg = bts::config::get<api::md::MarketDataConfig>();
-      if (md_cfg.glbx().empty()) {
-         std::cerr << "'MarketDataConfig.glbx' not defined (check spark.conf)."
-                   << std::endl;
-      }
-      const auto cfg_file = data_dir.config_file().string();
-      auto glbx_cfg = md_cfg.glbx()[0];
-      const auto templates_file = data_dir.templates_file().string();
-      await_cme_config(
-          glbx_cfg.cme_config_url(), cfg_file, glbx_cfg.cme_templates_url(),
-          templates_file);
-      const auto symbols = SymbolLoader::load(
-          SecurityCriteria::create({domain::protobufs::VENUE_GLBX}));
-      const auto channels =
-          parse_channels(cfg_file, {symbols.begin(), symbols.end()});
+      auto channels = bts::md::get_configured_cme_channels();
      for (const auto &ch : channels) {
         std::cout << ch.id << " - ";
         std::set<std::string> symbols;
@@ -1,10 +1,9 @@
 #include "Cleanup.h"
 #include "spark/config/config.h"
 #include "spark/config/protos/market_data.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/md/ShmSegmentName.h"
 #include "spark/md/protos/co2.pb.h"
-#include "system/logging/logging.h"
-
 #include <boost/filesystem.hpp>
 #include <string>

@@ -1,6 +1,5 @@
 #include "Record.h"
-#include "spark/md/relay/relay_recorder.h"
-
+#include "spark/md/relay/RelayRecorder.h"
 #include <boost/filesystem.hpp>

 namespace bts::cmds::md::cmds {
@@ -9,14 +8,13 @@ struct Record::Args {
   std::string path;
 };

-Record::Record()
-    : CLI::App("Record market data to a file", "record"),
-      _args(new Record::Args()) {
+Record::Record() :
+    CLI::App("Record market data to a file", "record"),
+    _args(new Record::Args()) {
   add_option("out,--out", _args->path, "Output file");
   callback([&] {
      boost::filesystem::path p(_args->path);
-      bts::md::record_everything(
-          p.parent_path().string(), p.filename().string());
+      bts::md::record_everything(p.parent_path().string(), p.filename().string());
   });
 }

@@ -1,5 +1,5 @@
 #include "Relay.h"
-#include "spark/md/relay/relay_control.h"
+#include "spark/md/relay/RelayClient.h"

 namespace bts::cmds::md::cmds {

@@ -1,11 +1,10 @@
 #include "Replay.h"
-#include "spark/config/config.h"
-#include "spark/config/protos/market_data.pb.h"
-#include "spark/md/relay/relay_recorder.h"
+#include "spark/md/relay/RelayRecorder.h"

 namespace bts::cmds::md::cmds {

-Replay::Replay() : CLI::App("Replay recorded market data", "replay") {
+Replay::Replay() :
+    CLI::App("Replay recorded market data", "replay") {
   callback([] {
      auto naggle_ms = config::get<bts::md::RelayOptions>().naggle_ms();
      const auto &options = config::get<bts::md::ReplayOptions>();
@@ -1,19 +1,25 @@
 #include "Run.h"
 #include "Cleanup.h"
-#include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/config/protos/market_data.pb.h"
-#include "spark/persistence/IDGenerator.h"
-#include "spark/utils/datetime.h"
+#include "spark/domain/SecurityCriteria.h"
+#include "spark/logging/logging.h"
+#include "spark/md/SymbolLoader.h"
+#include "spark/md/UserSymbol.h"
+#include "spark/md/relay/RelayRecorder.h"
+#include "spark/os/os.h"
+#include "spark/utils/bts_assert.h"
 #include "spark/utils/thread_manager.h"
 #include "sparkd/md/MarketDataGateway.h"
-#include "system/logging/logging.h"
-#include "system/os.h"
-
+#include "sparkd/md/platforms/IMDPlatform.h"
+#include "sparkd/md/platforms/ice_onixs/ICEMain.h"
+#include "sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
 #include <string>
-#include <thread>

 namespace bts::cmds::md::cmds {

+using bts::domain::SecurityCriteria;
+using bts::md::SymbolLoader;
+
 static void boot_sleep() {
   auto conf = config::get<bts::md::MDBootDelayWindow>();
   const auto *start = &conf.standard_start();
@@ -37,17 +43,57 @@ static void boot_sleep() {
   }
 }

+static void run_replay() {
+   LOG(md, Info, "Run::run_replay t_id:{} t_name", os::get_thread_id(),
+       os::get_thread_name());
+   const auto &replay_opts = config::get<bts::md::ReplayOptions>();
+   bts::md::replay_recording(
+       replay_opts.url(), config::get<bts::md::RelayOptions>().naggle_ms(),
+       replay_opts.fallback());
+}
+
+static void run_onixs(
+    bts::md::MBOWriter *mbo_writer,
+    bts::md::TradeSummarySignalDispatcher *tss_dispatcher) {
+   LOG(md, Info, "Run::run_onixs t_id:{} t_name", os::get_thread_id(),
+       os::get_thread_name());
+   std::vector<std::shared_ptr<bts::md::IMDPlatform>> platforms;
+   // get all glbx symbols
+   auto cme_symbols =
+       SymbolLoader::load(SecurityCriteria::create({api::domain::VENUE_GLBX}));
+   if (!cme_symbols.empty()) {
+      auto *main = new bts::md::MDP3Main();
+      main->start(
+          {cme_symbols.begin(), cme_symbols.end()}, mbo_writer, tss_dispatcher);
+      platforms.emplace_back(main);
+   }
+   // ice
+   const auto ice_symbols =
+       SymbolLoader::load(SecurityCriteria::create({api::domain::VENUE_ICE}));
+   if (!ice_symbols.empty()) {
+      auto *imain = new bts::md::ice::ICEMain();
+      imain->start({ice_symbols.begin(), ice_symbols.end()});
+      platforms.emplace_back(imain);
+   }
+   // deribit
+   // auto *d_platform = new bts::md::EmMDListener()
+   // Timers
+   auto &disp = utils::Dispatcher::current();
+   utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() {
+      for (auto &p : platforms)
+         p->send_heartbeats();
+   });
+   heartbeat_timer.start();
+
+   disp.run();
+}
+
 static void run_md_gwy() {
   LOG(md, Info, "Run::run_md_gwy t_id:{} t_name:{}", os::get_thread_id(),
       os::get_thread_name());
-   bts::md::MarketDataGateway gwy{
-       config::get<api::md::MarketDataConfig>(),
-       persistence::IDGenerator::instance_ptr()};
-   gwy.start();
-   while (true) {
-      using namespace std::chrono_literals;
-      std::this_thread::sleep_for(2s);
-   }
+   bts::md::MarketDataGateway gwy = {};
+   // this keeps going in a loop
+   gwy.run();
 }

 void Run::run() {
@@ -62,8 +108,13 @@ void Run::run() {
 Run::Run()
    : CLI::App(
          "Start market data feed, and store up-to-date snapshots of mbo, and "
-          "mbp in shared memory.",
+          "mbp in shared memory.  If no subcommand specified, uses the "
+          "configured default feed (replay, or onixs)",
          "run") {
+   add_subcommand("replay", "Run replay market data")->callback(run_replay);
+   add_subcommand("onixs", "Run onixs market data")->callback([] {
+      run_onixs(nullptr, nullptr);
+   });
   callback([] { run(); });
 }

@@ -9,6 +9,12 @@ namespace bts::cmds::md::cmds {
 * shared memory.
 */
 struct Run : public CLI::App {
+   enum class MDType {
+      Configured,
+      Replay,
+      Onixs
+   };
+
   Run();

   /**
@@ -1,9 +1,12 @@
 #include "Tail.h"
-#include "domain/protobufs/security_type.pb.h"
-#include "domain/security_criteria.h"
+#include "spark/api/domain/SecurityType.pb.h"
+#include "spark/api/domain/Venue.pb.h"
+#include "spark/config/protos/market_data.pb.h"
+#include "spark/domain/SecurityCriteria.h"
 #include "spark/md/Diag/MDPeriodicDumper.h"
+#include "spark/md/Diag/ShmDumper.h"
 #include "spark/md/SymbolLoader.h"
-
+#include "spark/md/UserSymbol.h"
 #include <map>

 namespace bts::cmds::md::cmds {
@@ -41,7 +44,7 @@ Tail::Tail()

   callback([&] {
      const auto products = SymbolLoader::load(SecurityCriteria::create(
-          {}, {domain::protobufs::SECURITY_TYPE_FUTURE}));
+          {}, {bts::api::domain::SECURITY_TYPE_FUTURE}));
      bts::md::run_periodic_dumper(
          products, _args->mod, _args->format, _args->all, _args->qualifier);
   });
@@ -1,5 +1,5 @@
 #include "Print.h"
-#include "spark/md/relay/relay_recorder.h"
+#include "spark/md/relay/RelayRecorder.h"

 namespace bts::cmds::md::cmds::rec_tool::cmds {

@@ -1,5 +1,5 @@
 #include "Trace.h"
-#include "spark/md/relay/relay_recorder.h"
+#include "spark/md/relay/RelayRecorder.h"

 namespace bts::cmds::md::cmds::rec_tool::cmds {

@@ -1,5 +1,5 @@
 #include "Unprint.h"
-#include "spark/md/relay/relay_recorder.h"
+#include "spark/md/relay/RelayRecorder.h"

 namespace bts::cmds::md::cmds::rec_tool::cmds {

@@ -18,9 +18,7 @@ EnumOptionNames<api::Venue> VenueOption::names() {
       NAME_PAIR(api::VENUE_BINANCE),
       NAME_PAIR(api::VENUE_BINANCE_US),
       NAME_PAIR(api::VENUE_COINBASE),
-       NAME_PAIR(api::VENUE_BINANCE_USDS_FUTURES),
-       NAME_PAIR(api::VENUE_CBOE),
-       NAME_PAIR(api::VENUE_TAIFEX)};
+       NAME_PAIR(api::VENUE_BINANCE_USDS_FUTURES)};
 }

 std::string VenueOption::name(api::Venue venue) {
@@ -47,10 +45,6 @@ std::string VenueOption::name(api::Venue venue) {
      return "coinbase";
   case api::VENUE_BINANCE_USDS_FUTURES:
      return "binance_usds_futures";
-   case api::VENUE_CBOE:
-      return "cboe";
-   case api::VENUE_TAIFEX:
-      return "taifex";
   default:
      return std::to_string(static_cast<int32_t>(venue));
   }
@@ -6,29 +6,31 @@
 #include <quickfix/SessionSettings.h>
 #include <quickfix/SocketAcceptor.h>

+#include <fstream>
 #include <iostream>
 #include <string>

 #include "spark/config/config.h"
 #include "spark/config/protos/exchange_sim_cfg.pb.h"
 #include "spark/config/protos/ports.pb.h"
+#include "spark/config/protos/system_config.pb.h"
 #include "spark/domain_model/Security.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/md/PriceSize.h"
-#include "system/logging/logging.h"
-#include "system/os.h"
-#include "system/protobufs/config/system_config.pb.h"
+#include "spark/os/os.h"

 #include "spark/domain_model/Security.pb.h"
 #include "spark/md/IntPrice.h"
 #include "spark/md/MBOProcessor.h"
-#include "spark/md/relay/relay_recorder.h"
+#include "spark/md/relay/RelayClient.h"
+#include "spark/md/relay/RelayRecorder.h"
+#include "spark/utils/bts_chrono.h"
 #include <quickfix/FileLog.h>

+#include <set>
+
 namespace bts {
 namespace sim {
-
-using bts::system::protobufs::SystemConfig;
-
 auto cfg = "[DEFAULT]\n"
           "ConnectionType=acceptor\n"
           "SocketAcceptPort=$EXCHANGE_SIM_FIX_PORT\n"
@@ -55,7 +57,7 @@ md::PriceSize coerce_precision(const co2::PriceSizeMessage &p) {
 struct MDPlayer : md::IRelayWriter {
   MDPlayer(
       IMDPlayerObserver &obs,
-       std::map<uint32_t, api::Security> &known_securities)
+       std::map<uint32_t, md::Security> &known_securities)
       : obs(obs), known_securities(known_securities),
         forward_implied(config::get<config::ExchangeSim>().forward_implied()) {
   }
@@ -128,7 +130,7 @@ struct MDPlayer : md::IRelayWriter {
         }
   }
   IMDPlayerObserver &obs;
-   std::map<uint32_t, api::Security> &known_securities;
+   std::map<uint32_t, md::Security> &known_securities;
   bool forward_implied;
 };

@@ -142,7 +144,7 @@ int start_exchange_sim(bts::api::Venue venue, md::MBOWriter *mbo) {
      auto venue_string = boost::to_lower_copy(api::Venue_Name(venue));
      boost::replace_all(
          str_settings, "$BTS_DATA_PATH",
-          config::get<SystemConfig>().data_path());
+          config::get<config::SystemConfig>().data_path());
      boost::replace_all(str_settings, "$SIM_VENUE", venue_string);
      std::string port_string;
      switch (venue) {
@@ -170,7 +172,7 @@ int start_exchange_sim(bts::api::Venue venue, md::MBOWriter *mbo) {

      acceptor.start();

-      std::map<uint32_t, api::Security> known_securities;
+      std::map<uint32_t, md::Security> known_securities;
      const auto &conf = config::get<config::ExchangeSim>();
      while (true) {
         MDPlayer player(application, known_securities);
@@ -5,9 +5,11 @@
 #include "spark/config/config.h"
 #include "spark/config/protos/exchange_sim_cfg.pb.h"
 #include "spark/domain_model/Security.h"
+#include "spark/logging/logging.h"
 #include "spark/md/MBOProcessor.h"
 #include "spark/md/MDProducer.h"
-#include "spark/md/shm/fixed_security.h"
+#include "spark/md/Shm/FixedSecurity.h"
+#include "spark/md/Shm/MDStructure.h"
 #include "spark/orderman/orderman_utils/fix_order_codec.h"
 #include "spark/persistence/SqliteStore.h"
 #include "spark/utils/Dispatcher.h"
@@ -15,7 +17,6 @@
 #include "sparkd/exchange_sim/book/Order.h"
 #include "sparkd/exchange_sim/book/OrderMatcher.h"
 #include "sparkd/exchange_sim/book/book_transform.h"
-#include "system/logging/logging.h"

 #include <boost/algorithm/string.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
@@ -33,8 +34,6 @@ namespace ex_sim {
 GCC_DIAG_OFF("-Wdeprecated")
 GCC_DIAG_OFF("-Wignored-qualifiers")

-#define AGGRESSOR_SIDE(x) bts::domain::protobufs::AGGRESSOR_SIDE_##x
-
 api::Side convert(const FIX::Side &side) {
   switch (side) {
   case FIX::Side_BUY:
@@ -540,9 +539,9 @@ void ExchangeSimApplication::update_md(std::string const &sec_desc) {
             md::PriceSize_from_double<ex_sim::precision>(
                 t.price.value() * price_factor, t.quantity.value()),
             datetime::ll_now(),
-             t.bid_aggressor ? AGGRESSOR_SIDE(BUY)
-                             : (t.ask_aggressor ? AGGRESSOR_SIDE(SELL)
-                                                : AGGRESSOR_SIDE(NONE)));
+             t.bid_aggressor ? co2::AGGRESSOR_SIDE_BUY
+                             : (t.ask_aggressor ? co2::AGGRESSOR_SIDE_SELL
+                                                : co2::AGGRESSOR_SIDE_NONE));
      }

      // publish mbo
@@ -582,11 +581,10 @@ void negate_id(api::Security *sec) {
 }

 void ExchangeSimApplication::persist_book(OrderBook const &b) {
-   auto [exists, rowid, oldval] =
-       me->store->retrieve_one("books", b.sec_desc());
+   auto existing = me->store->retrieve_one("books", b.sec_desc());
   auto val = utils::pb::to_string(b);

-   if (exists)
+   if (existing.first)
      me->store->update("books", b.sec_desc(), val);
   else
      me->store->insert("books", b.sec_desc(), val);
@@ -619,7 +617,7 @@ void ExchangeSimApplication::on_new_security(api::Security const &s) {
 }

 void ExchangeSimApplication::on_settlement(
-    api::Security const &sec, md::IntPrice price) {
+    md::Security const &sec, md::IntPrice price) {
   auto sec_desc = get_sim_sec_desc(sec);
   auto iter = me->md_producer_handles.find(sec_desc);
   if (iter != me->md_producer_handles.end()) {
@@ -629,8 +627,7 @@ void ExchangeSimApplication::on_settlement(
   }
 }

-void ExchangeSimApplication::on_high(
-    api::Security const &sec, md::Price price) {
+void ExchangeSimApplication::on_high(md::Security const &sec, md::Price price) {
   auto sec_desc = get_sim_sec_desc(sec);
   auto iter = me->md_producer_handles.find(sec_desc);
   if (iter != me->md_producer_handles.end()) {
@@ -639,7 +636,7 @@ void ExchangeSimApplication::on_high(
   }
 }

-void ExchangeSimApplication::on_low(api::Security const &sec, md::Price price) {
+void ExchangeSimApplication::on_low(md::Security const &sec, md::Price price) {
   auto sec_desc = get_sim_sec_desc(sec);
   auto iter = me->md_producer_handles.find(sec_desc);
   if (iter != me->md_producer_handles.end()) {
@@ -648,8 +645,7 @@ void ExchangeSimApplication::on_low(api::Security const &sec, md::Price price) {
   }
 }

-void ExchangeSimApplication::on_open(
-    api::Security const &sec, md::Price price) {
+void ExchangeSimApplication::on_open(md::Security const &sec, md::Price price) {
   auto sec_desc = get_sim_sec_desc(sec);
   auto iter = me->md_producer_handles.find(sec_desc);
   if (iter != me->md_producer_handles.end()) {
@@ -659,7 +655,7 @@ void ExchangeSimApplication::on_open(
 }

 void ExchangeSimApplication::on_close(
-    api::Security const &sec, md::Price price) {
+    md::Security const &sec, md::Price price) {
   auto sec_desc = get_sim_sec_desc(sec);
   auto iter = me->md_producer_handles.find(sec_desc);
   if (iter != me->md_producer_handles.end()) {
@@ -669,7 +665,7 @@ void ExchangeSimApplication::on_close(
 }

 void ExchangeSimApplication::on_depth(
-    api::Security const &sec,
+    md::Security const &sec,
    md::PriceSize bids[md::COCONUT_MAX_DEPTH],
    md::PriceSize asks[md::COCONUT_MAX_DEPTH],
    std::array<md::PriceSize, md::COCONUT_MAX_IMPLIED_DEPTH> *implied_bids,
@@ -69,8 +69,7 @@ class ExchangeSimApplication : public sim::IMDPlayerObserver,
   void onLogon(const FIX::SessionID &sessionID) override;
   void onLogout(const FIX::SessionID &sessionID) override;
   void toAdmin(FIX::Message &, const FIX::SessionID &) override;
-   void toApp(FIX::Message &, const FIX::SessionID &)
-       EXCEPT(FIX::DoNotSend) override;
+   void toApp(FIX::Message &, const FIX::SessionID &) EXCEPT(FIX::DoNotSend) override;
   void fromAdmin(const FIX::Message &, const FIX::SessionID &) EXCEPT(
       FIX::FieldNotFound,
       FIX::IncorrectDataFormat,
@@ -118,14 +117,14 @@ class ExchangeSimApplication : public sim::IMDPlayerObserver,
   void update_md(std::string const &sec_desc);
   void persist_book(OrderBook const &b);

-   void on_new_security(api::Security const &) override;
-   void on_high(api::Security const &sec, md::Price price) override;
-   void on_low(api::Security const &sec, md::Price price) override;
-   void on_open(api::Security const &sec, md::Price price) override;
-   void on_close(api::Security const &sec, md::Price price) override;
-   void on_settlement(api::Security const &, md::IntPrice) override;
+   void on_new_security(md::Security const &) override;
+   void on_high(md::Security const &sec, md::Price price) override;
+   void on_low(md::Security const &sec, md::Price price) override;
+   void on_open(md::Security const &sec, md::Price price) override;
+   void on_close(md::Security const &sec, md::Price price) override;
+   void on_settlement(md::Security const &, md::IntPrice) override;
   void on_depth(
-       api::Security const &s,
+       md::Security const &s,
       md::PriceSize bids[md::COCONUT_MAX_DEPTH],
       md::PriceSize asks[md::COCONUT_MAX_DEPTH],
       std::array<md::PriceSize, md::COCONUT_MAX_IMPLIED_DEPTH> *implied_bids,
@@ -1,7 +1,7 @@
 #pragma once

 #include "spark/md/PriceSize.h"
-#include "spark/md/shm/hot_depth_entry.h"
+#include "spark/md/Shm/MDStructure.h"

 namespace bts {
 namespace api {
@@ -10,14 +10,14 @@ class Security;
 namespace sim {

 struct IMDPlayerObserver {
-   virtual void on_new_security(api::Security const &) = 0;
-   virtual void on_high(const api::Security &, md::Price) = 0;
-   virtual void on_low(const api::Security &, md::Price) = 0;
-   virtual void on_open(const api::Security &, md::Price) = 0;
-   virtual void on_close(const api::Security &, md::Price) = 0;
-   virtual void on_settlement(const api::Security &, md::IntPrice) = 0;
+   virtual void on_new_security(md::Security const &) = 0;
+   virtual void on_high(const md::Security &, md::Price) = 0;
+   virtual void on_low(const md::Security &, md::Price) = 0;
+   virtual void on_open(const md::Security &, md::Price) = 0;
+   virtual void on_close(const md::Security &, md::Price) = 0;
+   virtual void on_settlement(const md::Security &, md::IntPrice) = 0;
   virtual void on_depth(
-       api::Security const &s,
+       md::Security const &s,
       md::PriceSize bids[md::COCONUT_MAX_DEPTH],
       md::PriceSize asks[md::COCONUT_MAX_DEPTH],
       std::array<md::PriceSize, md::COCONUT_MAX_IMPLIED_DEPTH> *implied_bids,
@@ -1,9 +1,9 @@
 #pragma once

+#include "spark/core/FMT_IMPL.h"
 #include "spark/domain_model/Security.pb.h"
 #include "spark/orderman/orderman_utils/orderman_common.h"
 #include "sparkd/exchange_sim/book/Order.h"
-#include "system/FMT_IMPL.h"

 #include <memory>
 #include <sstream>
@@ -1,6 +1,5 @@
 #include "sparkd/exchange_sim/book/book_transform.h"
-
-#include <catch2/catch_all.hpp>
+#include <catch/catch.hpp>
 #include <stdlib.h>

 namespace bts {
@@ -5,6 +5,7 @@
 #include "spark/domain_model/Security.pb.h"
 #include "spark/domain_model/SecurityKey.h"
 #include "spark/domain_model/common.h"
+#include "spark/logging/logging.h"
 #include "spark/md/SecurityOracle.h"
 #include "spark/orderman/CancelOrderMsg.h"
 #include "spark/orderman/CancelReplaceOrderMsg.h"
@@ -19,14 +20,13 @@
 #include "spark/orderman/orderman_utils/ExchangePosition.h"
 #include "spark/orderman/orderman_utils/ExecutionReport.h"
 #include "spark/orderman/orderman_utils/orderman_common.h"
+#include "spark/os/os.h"
 #include "spark/tracing/bts_trace.h"
 #include "spark/utils/bts_assert.h"
 #include "spark/utils/datetime.h"
 //#include "spark/utils/helper_macros.h"
 #include "sparkd/md/platforms/emumba/EmMDListener.h"
 #include "sparkd/md/platforms/emumba/IEMOrderMuxer.h"
-#include "system/logging/logging.h"
-#include "system/os.h"

 #include "spark/api/service_helpers/AdminServiceUtils.h"
 #include "spark/config/config.h"
@@ -4,9 +4,8 @@
 #include "spark/api/public/ApiUtils.h"
 #include "spark/api/services/AdminService.client.pb.h"
 #include "spark/api/services/AuthService.client.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/utils/bts_assert.h"
-#include "system/logging/logging.h"
-
 #include <mutex>
 #include <unordered_map>
 #include <vector>
@@ -1,6 +1,5 @@
 #include "MultipartFormDecoder.h"
-#include "system/logging/logging.h"
-
+#include "spark/logging/logging.h"
 #include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/iter_find.hpp>

@@ -5,12 +5,12 @@
 #include "MultipartFormDecoder.h"
 #include "spark/api/apigateway.pb.h"
 #include "spark/api/core/ApiClientInterop.h"
+#include "spark/logging/logging.h"
+#include "spark/os/os.h"
 #include "spark/tracing/PerfLog.h"
 #include "spark/utils/Dispatcher.h"
 #include "spark/utils/TimeStamp.h"
 #include "spark/utils/bts_chrono.h"
-#include "system/logging/logging.h"
-#include "system/os.h"

 #include <websocketpp/config/asio_no_tls.hpp>
 #include <websocketpp/server.hpp>
@@ -1,6 +1,6 @@
 #include "../MultipartFormDecoder.h"
+#include "catch/catch.hpp"

-#include <catch2/catch_all.hpp>
 #include <string>

 namespace bts {
@@ -49,4 +49,4 @@ TEST_CASE("MultiPart form decoder", "[http]") {

 } // namespace test
 } // namespace http
-} // namespace bts
+} // namespace bts
@@ -1,28 +1,25 @@
 #include "spark/testing/catch_test_runner.h"
 #include "spark/utils/init_spark_lib.h"
+// #include "py-market-data/py_exec.h"
 #include "sparkd/cmds/AdminReport.h"
 #include "sparkd/cmds/ConfigCmd.h"
 #include "sparkd/cmds/KeyCmd.h"
 #include "sparkd/cmds/Md.h"
 #include "sparkd/cmds/QuickTest.h"
 #include "sparkd/cmds/Run.h"
-#include "sparkd/cmds/TestPythonMdProviderCmd.h"
 #include "sparkd/cmds/Web.h"
 #include "sparkd/cmds/crypto/CryptoCmd.h"
-
+#include "sparkd/md/MarketDataGateway.h"
 #include <CLI11/CLI11.hpp>
-#include <csignal>

 using namespace bts;

-void signal_handler(int signum) {
-   std::cerr << "Interrupt signal (" << signum << "). Exiting" << std::endl;
-   exit(signum);
-}
-
 int main(int argc, char **argv) {
-   std::signal(SIGINT, signal_handler);
-   std::signal(SIGTERM, signal_handler);
+   if (argc < 40) {
+      // md::py_interop::PyExec::exec("/home/bts/spark/tools/py");
+      md::MarketDataGateway::test_py();
+      return 0;
+   }
   utils::SparkLibInit init;

   // The catch testing framework has its own command line parser so
@@ -40,7 +37,6 @@ int main(int argc, char **argv) {
      app.add_subcommand(CLI::App_p(new cmds::Run()));
      app.add_subcommand(CLI::App_p(new cmds::KeyCmd()));
      app.add_subcommand(CLI::App_p(new cmds::Web()));
-      app.add_subcommand(CLI::App_p(new cmds::TestPythonMdProviderCmd()));
      app.add_subcommand(
          "test", "Run unit test(s)"); // so this shows up in help
      CLI11_PARSE(app, argc, argv);