webapp check pt?

Instead of using magic strings

.vscode/launch.json

@@ -21,16 +21,6 @@
         ],
         "cwd": "build/Debug/bin/net7.0",
       },
-      "linux": {
-        "preLaunchTask": "Build Api Server (Debug)",
-        "program": "spark_api_server",
-        "args": [
-          "run",
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -48,15 +38,6 @@
         ],
         "cwd": "build/Debug/bin/net7.0",
       },
-      "linux": {
-        "program": "spark_api_server",
-        "args": [
-          "run",
-          "--sparkd",
-          "http://127.0.0.1:8000/config"
-        ],
-        "cwd": "build/Debug/bin/net7.0",
-      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -284,34 +265,6 @@
         }
       ]
     },
-    {
-      "name": "Sparkd Web (Skip Build)",
-      "type": "cppdbg",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd/sparkd",
-      },
-      "args": [
-        "web"
-      ],
-      "cwd": "${workspaceFolder}/build/sparkd-debug-clang/src/app_sparkd",
-      "stopAtEntry": false,
-      "externalConsole": false,
-      "MIMode": "gdb",
-      "setupCommands": [
-        {
-          "description": "Enable pretty-printing for gdb",
-          "text": "-enable-pretty-printing",
-          "ignoreFailures": true
-        },
-        {
-          "description": "Set Disassembly Flavor to Intel",
-          "text": "-gdb-set disassembly-flavor intel",
-          "ignoreFailures": true
-        }
-      ]
-    },
   ],
   "inputs": [
     {

CMakeLists.txt

@@ -33,6 +33,9 @@ else()
   set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
 endif()
 
+include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
+link_directories(${EXTERNALS_LIB})
+
 add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
 if(DEFINED APP_DATA_DIR)
   add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")

CMakePresets.json

@@ -169,7 +169,7 @@
           "value": "ON"
         },
         "CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf",
-        "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
+        "DOTNET_BINARY_DIR": "${sourceDir}/build/Spark.Api.Server/$<CONFIG>/bin/net7.0",
         "EXPORT_MD_CONSUMER": {
           "type": "BOOL",
           "value": "ON"

Spark.Api.Server.sln

@@ -14,16 +14,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\DataStreamSuite\Bts.Net\Bts.Net.csproj", "{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{05995ED5-650A-4955-9DFC-3242B737E77A}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{EFA9F78D-083E-457A-BABB-9386C88C32C0}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -80,46 +70,6 @@ Global
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
 		{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Debug|x64.Build.0 = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.ActiveCfg = Release|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.Release|x64.Build.0 = Release|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{CCDCB00A-CABF-4D82-8A5F-33EEC8ACE51D}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Debug|x64.Build.0 = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.ActiveCfg = Release|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.Release|x64.Build.0 = Release|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{CB62D5D2-3495-4488-BFBC-7C7A3725A1EE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.Debug|x64.Build.0 = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.ActiveCfg = Release|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.Release|x64.Build.0 = Release|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{05995ED5-650A-4955-9DFC-3242B737E77A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Debug|x64.Build.0 = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.ActiveCfg = Release|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.Release|x64.Build.0 = Release|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{A8B861DE-E098-4291-98EE-76AA0F5F5A7B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Debug|x64.Build.0 = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.ActiveCfg = Release|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.Release|x64.Build.0 = Release|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{EFA9F78D-083E-457A-BABB-9386C88C32C0}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

cmake/InitLibrary.cmake

@@ -72,14 +72,6 @@ function(init_library)
   else()
     add_library(${_name} ${${_name}_SOURCES} ${${_name}_CXX_PROTO_SOURCES})
   endif()
-  target_include_directories(
-    ${_name}
-    SYSTEM
-    BEFORE
-    PUBLIC
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/include/zlib>
-  )
   target_include_directories(
     ${_name}
     PUBLIC
@@ -98,7 +90,6 @@ function(init_library)
     ${_name}
     PUBLIC
       $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-      $<BUILD_INTERFACE:${EXTERNALS_ROOT}/lib/$<CONFIG>>
       $<INSTALL_INTERFACE:${CMAKE_INSTALL_LIBDIR}>
   )
   # Install main lib

cmake/SymLink.cmake

@@ -20,14 +20,16 @@ endfunction()
 
 function(sym_link_file)
   set(_options)
-  set(_one_value_args SOURCE DEST INSTALL_DEST)
+  set(_one_value_args TARGET_NAME SOURCE DEST INSTALL_DEST)
   set(_multi_value_args)
   cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
-  file(
-    CREATE_LINK
-      ${_ARGS_SOURCE}
-      ${_ARGS_DEST}
-    SYMBOLIC
+  add_custom_command(
+    OUTPUT ${_ARGS_DEST}
+    COMMAND ln ARGS -s ${_ARGS_SOURCE} ${_ARGS_DEST}
+  )
+  add_custom_target(
+    ${_ARGS_TARGET_NAME} ALL
+    DEPENDS ${_ARGS_DEST}
   )
   if (DEFINED _ARGS_INSTALL_DEST)
     install(

compose.yaml

@@ -1,15 +1,9 @@
 services:
   sparkd:
     build:
-      dockerfile: src/app_sparkd/sparkd/Dockerfile
-      args:
-        - SOURCE_DIR=install/sparkd-release-gcc
-        - CONF_FILE=src/app_sparkd/spark.current.conf
+      dockerfile: src/server/sparkd/Dockerfile
     command: [ "./sparkd", "run" ]
-    ports:
-      - 8000:8000
-      - 8001:8001
-      - 8002:8002
+    ports: [ "8000:8000", "8001:8001", "8002:8002" ]
 
   pubsub:
     environment:

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.env.development

@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.env.production

@@ -1 +1 @@
-VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:8443/ws
+VITE_DATASTREAM_WEBSOCKET_URL=wss://localhost:443

src/DataStreamSuite/Bts.DataStreamWebApp.Client/.eslintrc.cjs

@@ -0,0 +1,32 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.3.1' } },
+  plugins: ['react-refresh'],
+  rules: {
+    "import/order": [
+      "error",
+      {
+        "groups": [
+          "builtin",
+          "external",
+          "internal",
+          ["parent", "sibling", "index"]
+        ],
+        "newlines-between": "always"
+      }
+    ],
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}

src/DataStreamSuite/Bts.DataStreamWebApp.Client/eslint.config.js

@@ -1,36 +0,0 @@
-import importPlugin from "eslint-plugin-import";
-import reactPkg from "eslint-plugin-react";
-const { configs: reactConfigs } = reactPkg;
-import js from "@eslint/js";
-import defineConfig from "./vite.config.js";
-
-export default [
-  js.configs.recommended,
-  importPlugin.flatConfigs.recommended,
-  reactConfigs.flat.recommended,
-  reactConfigs.flat["jsx-runtime"],
-  {
-    files: ["**/*.{js,jsx}"],
-    languageOptions: {
-      ecmaVersion: "latest",
-      sourceType: "module",
-      globals: {
-        console: "readonly"
-      }
-    },
-    settings: {
-      "import/resolver": {
-        vite: {
-          viteConfig: defineConfig({ mode: "development" })
-        }
-      }
-    },
-    rules: {
-      semi: "error",
-      "prefer-const": "error",
-      "no-unused-vars": "warn",
-      "import/no-dynamic-require": "warn",
-      "import/no-nodejs-modules": "warn",
-    }
-  }
-];

src/DataStreamSuite/Bts.DataStreamWebApp.Client/package.json

@@ -10,37 +10,35 @@
     "preview": "vite preview"
   },
   "dependencies": {
-    "@angular/material": "^18.2.8",
-    "@emotion/react": "^11.13.3",
-    "@emotion/styled": "^11.13.0",
-    "@fontsource/noto-sans": "^5.1.0",
-    "@material/data-table": "^15.0.0-canary.423edc3dc.0",
-    "@mui/icons-material": "^6.1.4",
-    "@mui/material": "^6.1.4",
-    "@mui/x-data-grid": "^7.20.0",
-    "bulma": "^1.0.2",
+    "@angular/material": "^18.0.4",
+    "@emotion/react": "^11.11.4",
+    "@emotion/styled": "^11.11.5",
+    "@fontsource/noto-sans": "^5.0.22",
+    "@material/data-table": "^15.0.0-canary.7f224ddd4.0",
+    "@mui/icons-material": "^5.15.20",
+    "@mui/material": "^5.15.20",
+    "@mui/x-data-grid": "^7.7.1",
+    "bulma": "^1.0.1",
     "classnames": "^2.5.1",
     "react": "^18.3.1",
-    "react-data-grid": "^7.0.0-beta.47",
+    "react-data-grid": "^7.0.0-beta.44",
     "react-data-table-component": "^7.6.2",
     "react-dom": "^18.3.1",
-    "react-json-view-lite": "^1.5.0",
+    "react-json-view-lite": "^1.4.0",
     "react-pro-sidebar": "^1.1.0",
-    "react-use-websocket": "^4.9.0",
-    "rpc-websockets": "^9.0.4"
+    "react-use-websocket": "^4.8.1",
+    "rpc-websockets": "^7.11.2"
   },
   "devDependencies": {
-    "@types/react": "^18.3.11",
-    "@types/react-dom": "^18.3.1",
-    "@vitejs/plugin-react": "^4.3.2",
-    "eslint": "^9.12.0",
-    "@eslint/js": "^9.12.0",
-    "eslint-plugin-import": "^2.31.0",
-    "eslint-plugin-react": "^7.37.1",
-    "eslint-plugin-react-hooks": "^5.0.0",
-    "eslint-plugin-react-refresh": "^0.4.12",
-    "eslint-import-resolver-vite": "^2.0.1",
-    "react-mui-sidebar": "^1.3.8",
-    "vite": "^5.4.9"
+    "@types/react": "^18.3.3",
+    "@types/react-dom": "^18.3.0",
+    "@vitejs/plugin-react": "^4.3.1",
+    "eslint": "^8.57.0",
+    "eslint-plugin-import": "^2.29.1",
+    "eslint-plugin-react": "^7.34.2",
+    "eslint-plugin-react-hooks": "^4.6.2",
+    "eslint-plugin-react-refresh": "^0.4.7",
+    "react-mui-sidebar": "^1.3.5",
+    "vite": "^5.2.13"
   }
 }

src/DataStreamSuite/Bts.DataStreamWebApp.Client/vite.config.js

@@ -1,5 +1,5 @@
 import { fileURLToPath, URL } from 'node:url';
-import { defineConfig } from 'vite';
+import { defineConfig, loadEnv } from 'vite';
 import plugin from '@vitejs/plugin-react';
 import fs from 'fs';
 import path from 'path';

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -17,8 +17,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <Proto Include="../../lib_spark/spark/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
-      <OutDir>$(ProtoOutputPath)/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/data_stream_config.proto" CSName="DataStreamConfig">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
     </Proto>
   </ItemGroup>
 

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionTokens.lex

@@ -9,15 +9,7 @@
 Letter                   [a-zA-Z]
 BooleanConstant          'true'|'false'
 CSStringConstant         \"([^\"\\]*|\\.)*\"
-YearInt                  [0-9]{4}
-MonthInt                 [0-9]{1,2}
-DayInt                   [0-9]{1,2}
-/**
- * DateTimeConstant supports American (mm/dd/yyyy) and European (yyyy/mm/dd)
- * formats. Month and day can be 1 or 2 digits. It does not validate that the
- * numbers represent an actual date.
- */
-DateTimeConstant         ({MonthInt}\/{DayInt}\/{YearInt}|{YearInt}\/{MonthInt}\/{DayInt})
+DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
 IntegerConstant          -?[0-9]+
 NullConstant             'null'
 RealConstant             -?[0-9]+\.[0-9]+

src/Directory.Build.props

@@ -43,7 +43,7 @@
       Condition="Exists($(ComputerSpecificConfig))" />
 
   <PropertyGroup Label="ProtoBuf">
-    <ProtocPluginBase>$(SparkRoot)\src\lib_spark\spark\api\service_compiler</ProtocPluginBase>
+    <ProtocPluginBase>$(SparkRoot)\src\lib\api\service_compiler</ProtocPluginBase>
   </PropertyGroup>
   <Choose>
     <When Condition="'$(OS)' == 'Windows_NT'">

src/Directory.Build.targets

@@ -24,9 +24,7 @@
     <PropertyGroup>
       <GenProtosCmd>$(protoc) %(Proto.FullPath)</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)\lib_system</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)\lib_domain</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)\lib_spark</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-spark-csharp=$(ProtocPlugin)&quot;</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --spark-csharp_out=lang=c#,templates=immutable.pb.cs,strip_out_path=1:%(Proto.OutDir)</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(Proto.OutDir)</GenProtosCmd>
@@ -86,9 +84,7 @@
     <PropertyGroup>
       <GenProtosCmd>$(protoc) %(ProtoService.FullPath)</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --error_format=msvs</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)\lib_system</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)\lib_domain</GenProtosCmd>
-      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)\lib_spark</GenProtosCmd>
+      <GenProtosCmd>$(GenProtosCmd) --proto_path=$(SourceRoot)</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --plugin=&quot;protoc-gen-service=$(ProtocPlugin)&quot;</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --service_out=lang=c#,strip_out_path=1:%(ProtoService.OutDir)</GenProtosCmd>
       <GenProtosCmd>$(GenProtosCmd) --csharp_out=%(ProtoService.OutDir)</GenProtosCmd>

src/Spark.Admin.WebApp/CMakeLists.txt

@@ -176,14 +176,4 @@ add_custom_target(
   COMMENT "Bundling Spark.Admin.WebApp"
 )
 
-# Exclude 'public' dir because it is a symlink
-install(
-  DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
-  DESTINATION "${CMAKE_INSTALL_BINDIR}"
-  PATTERN "public" EXCLUDE
-)
-# Install actual (non-symlinked) 'public' dir
-install(
-  DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/public"
-  DESTINATION "${CMAKE_INSTALL_BINDIR}/${PROJECT_NAME}"
-)
+install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" DESTINATION "${CMAKE_INSTALL_BINDIR}")

src/app_sparkd/CMakeLists.txt

@@ -122,6 +122,7 @@ include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
 
 sym_link_local_lib(python_md)
 sym_link_file(
+  TARGET_NAME link_python_md_module
   SOURCE ${CMAKE_BINARY_DIR}/src/lib_python_md/python_md_module.cpython-310-x86_64-linux-gnu.so
   DEST ${CMAKE_CURRENT_BINARY_DIR}/python_md_module.cpython-310-x86_64-linux-gnu.so
   INSTALL_DEST ${CMAKE_INSTALL_BINDIR}/
@@ -142,6 +143,9 @@ target_link_libraries(
   databento::databento
   ${emumba-deribit-connector-library_LIBRARIES}
   tbb
+  onixs-cmemdh-all-4.8.0
+  OnixS.IceImpactMulticastPriceFeedHandlerCpp
+  ACE
   quickfix
   benchmark
   Common++
@@ -153,19 +157,6 @@ target_link_libraries(
   -static-libstdc++
   -pthread)
 
-# Onixs
-target_link_libraries(
-  ${PROJECT_NAME}
-  onixs-cmemdh
-  OnixS.IceImpactMulticastPriceFeedHandlerCpp
-  # gcc doesn't resolve onixs-fix-engine and ACE in runtime path unless we specify
-  # --no-as-needed. These are onixs deps that aren't called directly by our code.
-  # Note: this isn't an issue for clang
-  -Wl,--no-as-needed
-  onixs-fix-engine
-  ACE
-)
-
 # Catch2
 find_package(Catch2 CONFIG REQUIRED)
 target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
@@ -173,13 +164,7 @@ target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
 install_basic()
 install(
   FILES
-    "${EXTERNALS_LIB}/libonixs-cmemdh.so"
-    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4"
-    "${EXTERNALS_LIB}/libonixs-cmemdh.so.4.8.0"
     "${EXTERNALS_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
-    "${EXTERNALS_LIB}/libonixs-fix-engine.so"
-    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4"
-    "${EXTERNALS_LIB}/libonixs-fix-engine.so.4.10.1"
     "${EXTERNALS_LIB}/libACE-5.7.5.so"
   DESTINATION ${CMAKE_INSTALL_LIBDIR}
 )

src/app_sparkd/sparkd/cmds/Run.cpp

@@ -24,7 +24,6 @@
 #include "spark/utils/Dispatcher.h"
 #include "sparkd/algo/AlgoDefinitionServiceServer.h"
 #include "sparkd/analytics/AnalyticsServiceServer.h"
-#include "sparkd/cmds/md/cmds/Cleanup.h"
 #include "sparkd/http/WebServer.h"
 #include "sparkd/md/MarketDataAuxiliaryServiceServer.h"
 #include "sparkd/md/MarketDataGateway.h"
@@ -64,7 +63,6 @@ static void execute() {
 
    bts::md::MBOProcessor mbo_processor;
    bts::md::TradeSummarySignalDispatcher tss_dispatcher;
-   md::cmds::Cleanup::execute();
    bts::md::MarketDataGateway md_gateway{
        config::get<api::md::MarketDataConfig>(),
        persistence::IDGenerator::instance_ptr()};

src/app_sparkd/sparkd/md/MarketDataGateway.cpp

@@ -21,7 +21,6 @@
 #include "system/FMT.h"
 #include "system/logging/logging.h"
 #include "system/os.h"
-#include "system/threading/dispatcher.h"
 
 #include <bits/stdint-uintn.h>
 #include <memory>
@@ -51,17 +50,12 @@ struct MarketDataGateway::PImpl {
       shared_ptr<GeneralSecurityIndex> sec_index;
       shared_ptr<MDProducer> md_producer;
       shared_ptr<IMarketDataListener> listener;
-      utils::DispatcherTimer timer;
 
       MdSink(api::Venue venue, shared_ptr<IDGenerator> sec_id_store)
           : sec_index(std::make_shared<GeneralSecurityIndex>()),
             md_producer(std::make_shared<MDProducer>()),
             listener(std::make_shared<GeneralMarketDataListener>(
-                sec_index, md_producer, venue, sec_id_store)),
-            timer(
-                utils::Dispatcher::background(),
-                200,
-                [producer = this->md_producer]() { producer->heartbeat(); }) {}
+                sec_index, md_producer, venue, sec_id_store)) {}
    };
 
    MarketDataConfig config;

src/app_sparkd/sparkd/md/databento/DatabentoDataHandler.cpp

@@ -224,10 +224,10 @@ void DatabentoDatahandler::run(
         LOG(md, Error, "Timed out waiting for data");
         break;
     }
-    if (record->GetIf<databento::Mbp1Msg>() != nullptr) {
+    if (auto* bbo = record->GetIf<databento::Mbp1Msg>()) {
         process_bbo(*record);
     }
-    if (record->GetIf<databento::InstrumentDefMsg>() != nullptr) {
+    if (auto* def = record->GetIf<databento::InstrumentDefMsg>()) {
         process_security(*record);
     }
   } catch (const databento::TcpError& error) {

src/app_sparkd/sparkd/md/taifex/TaifexDataProcessors.h

@@ -1,249 +0,0 @@
-#pragma once
-#include "TaifexMessages.h"
-#include "TaifexSecurityDetailsMapper.h"
-#include "domain/book.h"
-#include "domain/protobufs/security_type.pb.h"
-#include "domain/security.h"
-#include "domain/trade_record.h"
-#include "spark/md/IMarketDataListener.h"
-#include "spark/persistence/IDGenerator.h"
-#include "spark/utils/bts_assert.h"
-
-namespace bts::md::taifex {
-
-class TaifexDataProcessors {
- public:
-   static bool processBookSnapShotMessage(
-       const TaifexMessages::I083Data &i083_message,
-       std::shared_ptr<bts::md::IMarketDataListener> listener,
-       std::shared_ptr<TaifexSecurityDetailsMapper> security_details) {
-
-      auto [sec_id, dec_loc] = security_details->getSecurityDetails(
-          TaifexUtils::remove_spaces(i083_message.prod_id));
-      if (!sec_id || !dec_loc)
-         return false;
-      domain::Book book;
-      auto &bids = book.bids();
-      auto &asks = book.asks();
-      book.set_security_id(sec_id.value());
-
-      for (const auto &entry : i083_message.entries) {
-         bts::domain::Price price(
-             entry.price *
-             pow(10,
-                 9 - dec_loc
-                         .value())); // Assuming the price needs to be adjusted by a factor of -2
-         bts::domain::Quantity quantity(entry.quantity);
-
-         // Process each MD_ENTRY
-         if (entry.md_entry_type == '0') { // Buy or derived buy
-            // Update the corresponding bid level
-            if (entry.level <= bids.size()) {
-               bids[entry.level - 1].price = price;
-               bids[entry.level - 1].total_quantity = quantity;
-            }
-         } else if (
-             entry.md_entry_type == '1') { // Sell or derived sell
-            // Update the corresponding ask level
-            if (entry.level <= asks.size()) {
-               asks[entry.level - 1].price = price;
-               asks[entry.level - 1].total_quantity = quantity;
-            }
-         }
-      }
-      std::stringstream book_after;
-      book_after << "Book after snapshot update:\n";
-      book_after << "Bids:\n";
-      for (const auto &level : bids) {
-         book_after << "Price: " << level.price.to_double()
-                    << ", Quantity: " << level.total_quantity.value() << "\n";
-      }
-      book_after << "Asks:\n";
-      for (const auto &level : asks) {
-         book_after << "Price: " << level.price.to_double()
-                    << ", Quantity: " << level.total_quantity.value() << "\n";
-      }
-      book_after<<"MSG:\n"<<i083_message;
-      std::string book_snapshot_details = book_after.str() + "\n";
-      bts_assert_msgf(
-          book.is_sorted(), "book not sorted {}", book_snapshot_details);
-      listener->on_book(book);
-      return true;
-   }
-   static void processBookIncrements(
-       const TaifexMessages::I081Data &msg,
-       const std::shared_ptr<bts::md::IMarketDataListener> &listener,
-       const std::shared_ptr<TaifexSecurityDetailsMapper> &security_details) {
-
-      auto [sec_id, dec_loc] = security_details->getSecurityDetails(
-          TaifexUtils::remove_spaces(msg.prod_id));
-      if (!sec_id || !dec_loc)
-         return;
-
-      for (const auto &entry : msg.entries) {
-         IMarketDataListener::BookOp inc;
-         inc.id = *sec_id;
-         inc.price =
-             bts::domain::Price(entry.price * std::pow(10, 9 - *dec_loc));
-         inc.quantity = bts::domain::Quantity(entry.quantity);
-         inc.side = (entry.md_entry_type == '0' || entry.md_entry_type == 'E')
-                        ? bts::domain::Side::Buy
-                        : bts::domain::Side::Sell;
-
-         switch (entry.md_update_action) {
-         case '0':
-            inc.op_type = IMarketDataListener::BookOpType::Add;
-            break;
-         case '1':
-            inc.op_type = IMarketDataListener::BookOpType::MODIFY;
-            break;
-         case '2':
-            inc.op_type = IMarketDataListener::BookOpType::REMOVE;
-            break;
-         default:
-            continue;
-         }
-         listener->on_book_increment(inc);
-      }
-   }
-   // Refactored trade processing function
-   static void processTradeRecords(
-       const TaifexMessages::I024Data &msg,
-       const std::shared_ptr<bts::md::IMarketDataListener> &listener,
-       const std::shared_ptr<TaifexSecurityDetailsMapper> &security_details) {
-
-      auto [sec_id, dec_loc] = security_details->getSecurityDetails(
-          TaifexUtils::remove_spaces(msg.prod_id));
-      if (!sec_id || !dec_loc)
-         return;
-
-      std::vector<bts::domain::TradeRecord> trades;
-      for (const auto &trade_record : msg.trades) {
-         auto &record = trades.emplace_back();
-         record.sec_id = *sec_id;
-         record.time_stamp = bts::core::TimeStamp::from_iso_string(
-             TaifexUtils::convertToISO(msg.match_time, "datetime_no_timezone"));
-         record.trade_quantity = bts::domain::Quantity(trade_record.match_qty);
-         record.trade_price = bts::domain::Price(
-             trade_record.match_price * std::pow(10, 9 - *dec_loc));
-         // TODO: Set the aggressor side (record.agg_side)
-      }
-
-      std::vector<const domain::TradeRecord *> trade_ptrs;
-      trade_ptrs.reserve(trades.size());
-      for (const auto &trade : trades) {
-         trade_ptrs.push_back(&trade);
-      }
-      //   for (const auto *trade_ptr : trade_ptrs) {
-      //          if (trade_ptr != nullptr) {
-      //             LOG(md, Error, "Trade Quantity: {}, Trade Price: {}",
-      //                 trade_ptr->trade_quantity.value(),
-      //                 trade_ptr->trade_price);
-      //          }
-      //       }
-      listener->on_trades(trade_ptrs);
-   }
-   static bts::domain::Security initializeSecurity(
-       const std::string &id,
-       const bts::domain::SecurityID &sec_id,
-       const std::string &symbol,
-       const std::shared_ptr<TaifexSecurityDetailsMapper> &security_details) {
-
-      bts::domain::Security sec;
-      sec.security_common.venue = domain::protobufs::VENUE_TAIFEX;
-      sec.security_common.id = sec_id;
-      sec.security_common.exchange_sec_id = id;
-      sec.security_common.desc = id;
-      sec.security_common.symbol = symbol;
-      sec.security_common.asset = id;
-      sec.security_common.group = id;
-      sec.security_common.min_price_increment =
-          security_details->get_min_inc(id);
-      sec.security_common.min_quantity = security_details->get_min_inc(id);
-      sec.security_common.min_quantity_increment =
-          security_details->get_min_inc(id);
-      sec.security_common.currency = security_details->get_currency(id);
-
-      return sec;
-   }
-
-   // Process I010Data
-   static void processI010Data(
-       const TaifexMessages::I010Data &msg,
-       const std::shared_ptr<bts::md::IMarketDataListener> &listener,
-       const std::shared_ptr<persistence::IDGenerator> &sec_id_store,
-       const std::shared_ptr<TaifexSecurityDetailsMapper> &security_details) {
-
-      const auto id = TaifexUtils::remove_spaces(msg.prod_id);
-      const auto symbol = id.substr(0, 3);
-
-      const auto sec_type = convert_sec_type(msg.parsedProductInfo.productType);
-      bts::domain::SecurityID sec_id{sec_id_store->get_or_add(
-          domain::protobufs::VENUE_TAIFEX, sec_type, id)};
-      auto check =
-          security_details->add_i010_mapping(id, sec_id, msg.decimal_locator);
-      if (!check) {
-         return;
-      }
-
-      // Initialize the security object
-      bts::domain::Security sec =
-          initializeSecurity(id, sec_id, symbol, security_details);
-
-      // Handle product-specific details
-      switch (msg.parsedProductInfo.productType) {
-      case TaifexMessages::I010Data::ProductInfo::ProductType::FUTURES: {
-         auto &futureSec = sec.security.emplace<domain::FutureSecurity>();
-         std::tm tm = {};
-         std::istringstream iss(msg.end_date);
-         iss >> std::get_time(&tm, "%Y%m%d");
-         std::time_t time = std::mktime(&tm);
-         futureSec.expiration =
-             bts::core::TimeStamp(static_cast<uint64_t>(time));
-         break;
-      }
-      case TaifexMessages::I010Data::ProductInfo::ProductType::OPTIONS: {
-         auto &optionSec = sec.security.emplace<domain::OptionSecurity>();
-         std::tm tm = {};
-         std::istringstream iss(msg.end_date);
-         iss >> std::get_time(&tm, "%Y%m%d");
-         std::time_t time = std::mktime(&tm);
-         optionSec.expiration =
-             bts::core::TimeStamp(static_cast<uint64_t>(time));
-         optionSec.strike =
-             domain::Price::from_double(msg.parsedProductInfo.strikePrice);
-         break;
-      }
-      }
-
-      std::vector<const domain::Security *> const_security_list;
-      const_security_list.push_back(&sec);
-      listener->on_securities(const_security_list);
-   }
-
-   // Process I011Data
-   static void processI011Data(
-       const TaifexMessages::I011Data &msg,
-       const std::shared_ptr<TaifexSecurityDetailsMapper> &security_details) {
-
-      const auto id = TaifexUtils::remove_spaces(msg.kind_id);
-      security_details->add_i011_mapping(
-          id, msg.contract_size, msg.getCurrencyString());
-   }
-
- private:
-   static domain::protobufs::SecurityType convert_sec_type(
-       TaifexMessages::I010Data::ProductInfo::ProductType type) {
-      switch (type) {
-      case TaifexMessages::I010Data::ProductInfo::ProductType::FUTURES:
-         return domain::protobufs::SecurityType::SECURITY_TYPE_FUTURE;
-      case TaifexMessages::I010Data::ProductInfo::ProductType::OPTIONS:
-         return domain::protobufs::SecurityType::SECURITY_TYPE_OPTION;
-      default:
-         bts_assert_msg(
-             false, "Unexpected security type (must be FUTURES or OPTIONS)");
-      }
-   }
-};
-
-} //namespace bts::md::taifex

src/app_sparkd/sparkd/md/taifex/TaifexMessages.h

@@ -1,5 +1,3 @@
-#pragma once
-
 #include "TaifexUtils.h"
 #include "spark/utils/bts_assert.h"
 #include "spark/utils/helper_macros.h"
@@ -11,7 +9,6 @@
 #include <vector>
 
 namespace bts::md::taifex {
-   using UNPROCESSED_MSG = std::vector<unsigned char>;
 class TaifexMessages {
  public:
    struct MessageIdentifier {
@@ -256,7 +253,7 @@ class TaifexMessages {
       uint Body_Length;
       bool valid = false;
       Header_Fields(
-          const UNPROCESSED_MSG &buffer, const size_t &offset)
+          const std::vector<unsigned char> &buffer, const size_t &offset)
           : Esc_Code(buffer[offset]),
             message_type(buffer[offset + 1], buffer[offset + 2]) {
          // creates header feild struct
@@ -301,7 +298,7 @@ class TaifexMessages {
          float match_qty;    // keeping str for simplicity
 
          trade(
-             const UNPROCESSED_MSG &buffer,
+             const std::vector<unsigned char> &buffer,
              const size_t &offset = 0,
              const size_t match_qty_size = 2) {
             const size_t SIGN_SIZE = 1;
@@ -339,7 +336,7 @@ class TaifexMessages {
       std::string match_buy_count;
       std::string match_sell_count;
       I024Data(
-          const UNPROCESSED_MSG &buffer,
+          const std::vector<unsigned char> &buffer,
           const size_t &offset = 0,
           const size_t &length = -1) {
          const size_t PROD_ID_SIZE = 20;
@@ -405,7 +402,7 @@ class TaifexMessages {
          // match sell count
          match_sell_count = TaifexUtils::extractBCD_STRING(
              buffer, current_position, MATCH_SELL_COUNT_SIZE);
-         // std::cout << "trades count: " << trades_count << std::endl;
+         std::cout << "trades count: " << trades_count << std::endl;
          assert(
              buffer[current_position + 1] == 0x0D &&
              buffer[current_position + 2] == 0x0A);
@@ -557,7 +554,7 @@ class TaifexMessages {
       } parsedProductInfo;
       // Constructor to initialize from a buffer
       I010Data(
-          const UNPROCESSED_MSG &buffer,
+          const std::vector<unsigned char> &buffer,
           size_t offset = 0,
           size_t length = -1) {
          const size_t PROD_ID_SIZE = 10;
@@ -700,7 +697,7 @@ class TaifexMessages {
 
       // Constructor to initialize from a buffer
       I011Data(
-          const UNPROCESSED_MSG &buffer,
+          const std::vector<unsigned char> &buffer,
           size_t offset = 0,
           size_t length = -1) {
          const size_t KIND_ID_SIZE = 4;
@@ -811,7 +808,7 @@ class TaifexMessages {
                << "Quantity: " << entry.quantity << "\n";
             return os;
          }
-         MD_ENTRY(const UNPROCESSED_MSG &buffer, size_t offset = 0) {
+         MD_ENTRY(const std::vector<unsigned char> &buffer, size_t offset = 0) {
             md_update_action = buffer[offset];
             md_entry_type = buffer[offset + 1];
             sign = static_cast<Sign>(buffer[offset + 2]);
@@ -848,7 +845,7 @@ class TaifexMessages {
          return os;
       }
       I081Data(
-          const UNPROCESSED_MSG &buffer, const size_t &offset = 0) {
+          const std::vector<unsigned char> &buffer, const size_t &offset = 0) {
          const size_t ENTRIES_COUNT_SIZE = 1;
          const size_t PROD_ID_SIZE = 20;
          const size_t SEQ_NUMBER_SIZE = 5;
@@ -899,7 +896,7 @@ class TaifexMessages {
             return os;
          }
 
-         MD_ENTRY(const UNPROCESSED_MSG &buffer, size_t offset = 0) {
+         MD_ENTRY(const std::vector<unsigned char> &buffer, size_t offset = 0) {
             md_entry_type = buffer[offset];
             sign = static_cast<Sign>(buffer[offset + 1]);
 
@@ -931,14 +928,14 @@ class TaifexMessages {
             << "Entries Count: " << data.entries_count << "\n"
             << "MD entries: ";
          for (auto entry : data.entries) {
-            os << entry;
+            std::cout << entry;
          }
 
          return os;
       }
 
       I083Data(
-          const UNPROCESSED_MSG &buffer,
+          const std::vector<unsigned char> &buffer,
           const size_t &offset = 0,
           size_t length = -1,
           bool ignore_calc_flag=false) {
@@ -1014,7 +1011,7 @@ class TaifexMessages {
         return os;
     }
 
-    I084Data(const UNPROCESSED_MSG &buffer, const size_t &offset = 0, const bool skip_o =false) {
+    I084Data(const std::vector<unsigned char> &buffer, const size_t &offset = 0, const bool skip_o =false) {
         auto current_position = offset;
         message_type = charToMessageType(buffer[current_position]);
         current_position += 1;

src/app_sparkd/sparkd/md/taifex/TaifexSecurityDetailsMapper.h

@@ -1,7 +1,6 @@
 #pragma once
 
 #include "domain/security_id.h"
-#include "system/logging/logging.h"
 
 #include <optional>
 #include <sstream>
@@ -9,135 +8,92 @@
 
 namespace bts::md::taifex {
 class TaifexSecurityDetailsMapper {
-
-
-   struct product_details {
-      std::string instrument_id; // use this as 
-      bts::domain::SecurityID sec_id;//uint64_t
-      uint dec_loc;
-   };
-   struct contract_details{
-      std::unordered_map<std::string,product_details> products;
-      std::string kind_id;
+   struct security_details {
       double min_increment;
       std::string currency;
+      bool has_i010 = false;
+      bool has_i011 = false;
+      uint dec_loc;
+      bts::domain::SecurityID sec_id;
    };
 
  public:
-   static std::string get_kind_id(const std::string &prod_id ){
-      return prod_id.substr(0,3);
-   }
    // Add a mapping from instrument_id to sec_id
-   bool add_i010_mapping(
+   void add_i010_mapping(
        std::string instrument_id,
-       const bts::domain::SecurityID sec_id,
+       bts::domain::SecurityID sec_id,
        uint decimal_locator) {
-      auto kind_id = get_kind_id(instrument_id);
-      if(!sec_deets_map.contains(kind_id)){
-         return false;
-      }
-      auto &contract = sec_deets_map[kind_id];
-      auto &details = contract.products[instrument_id];
+      auto &details = sec_deets_map[instrument_id];
       details.sec_id = sec_id;
       details.dec_loc = decimal_locator;
-      details.instrument_id = instrument_id;
-      auto s = this->to_string();
-      LOG(md, Info, "add_i010_mapping: SEC_ID STORE= {}",s);
-      return true;
+      details.has_i010 = true;
    }
 
    // Add a mapping for i011
    void add_i011_mapping(
-       std::string kind_id, double min_increment, std::string currency) {
-       auto &contract = sec_deets_map[kind_id];
-       contract.currency= currency!=""?currency:"usd";
-       contract.kind_id=kind_id;
-       contract.min_increment=min_increment;
+       std::string instrument_id, double min_increment, std::string currency) {
+      auto &details = sec_deets_map[instrument_id];
+      details.min_increment = min_increment;
+      details.currency = currency;
+      details.has_i011 = true;
    }
 
    // Find sec_id by instrument_id
    std::optional<bts::domain::SecurityID> get_sec_id(
        std::string instrument_id) const {
-      auto kind_id = get_kind_id(instrument_id);
-      auto it = sec_deets_map.find(kind_id);
-      if (it != sec_deets_map.end()) {
-         auto &contract = it->second;
-         auto it2 = contract.products.find(instrument_id);
-         if (it2 != contract.products.end()) 
-            return it2->second.sec_id;
+      auto it = sec_deets_map.find(instrument_id);
+      if (it != sec_deets_map.end() && it->second.has_i010) {
+         return it->second.sec_id;
       }
       return std::nullopt;
    }
 
    // Find dec_loc by instrument_id
    std::optional<uint> get_dec_loc(std::string instrument_id) const {
-      auto kind_id = get_kind_id(instrument_id);
-      auto it = sec_deets_map.find(kind_id);
-      if (it != sec_deets_map.end()) {
-         auto &contract = it->second;
-         auto it2 = contract.products.find(instrument_id);
-         if (it2 != contract.products.end()) 
-            return it2->second.dec_loc;
+      auto it = sec_deets_map.find(instrument_id);
+      if (it != sec_deets_map.end() && it->second.has_i010) {
+         return it->second.dec_loc;
       }
       return std::nullopt;
    }
    std::string get_currency(std::string instrument_id) const {
-      auto kind_id = get_kind_id(instrument_id);
-      auto it = sec_deets_map.find(kind_id);
-      if (it != sec_deets_map.end()) {
+      auto it = sec_deets_map.find(instrument_id);
+      if (it != sec_deets_map.end() && it->second.has_i010 &&
+          it->second.has_i011) {
          return it->second.currency;
-         
       }
       return "usd";
    }
 
    double get_min_inc(std::string instrument_id) const {
-      auto kind_id = get_kind_id(instrument_id);
-      auto it = sec_deets_map.find(kind_id);
-      if (it != sec_deets_map.end()) {
+      auto it = sec_deets_map.find(instrument_id);
+      if (it != sec_deets_map.end() && it->second.has_i010 &&
+          it->second.has_i011) {
          return it->second.min_increment;
       }
       return 1;
    }
-   std::pair<std::optional<bts::domain::SecurityID>, std::optional<int>> getSecurityDetails(
-    const std::string &prod_id) {
-    auto sec_id = get_sec_id(prod_id);
-    auto dec_loc = get_dec_loc(prod_id);
-
-    if (!sec_id.has_value() || !dec_loc.has_value()) {
-        if (sec_id.has_value() != dec_loc.has_value())
-            LOG(md, Error, "Mismatch in sec_id: {}, dec_loc: {}", sec_id.has_value(), dec_loc.has_value());
-        return {std::nullopt, std::nullopt};
-    }
-    return {sec_id, dec_loc};
-   }
    // to_string function to print all data in both maps
    std::string to_string() const {
-    std::stringstream ss;
-    for (const auto &[kind_id, contract] : sec_deets_map) {
-        ss << "Kind ID: '" << kind_id << "'\n"
-           << "  Currency: '" << contract.currency << "'\n"
-           << "  Min Increment: " << contract.min_increment << "\n";
-
-        for (const auto &[instrument_id, details] : contract.products) {
-            ss << "    Instrument ID: '" << instrument_id << "'\n"
-               << "      Sec ID: " << details.sec_id << "\n"
-               << "      Dec Loc: " << details.dec_loc << "\n";
-        }
-    }
-    return ss.str();
-}
-
+      std::stringstream ss;
+      for (const auto &[instrument_id, details] : sec_deets_map) {
+         ss << "Instrument ID: " << instrument_id << "\n"
+            << "  Sec ID: " << details.sec_id << "\n"
+            << "  Dec Loc: " << details.dec_loc << "\n"
+            << "  Min Increment: " << details.min_increment << "\n"
+            << "  Currency: " << details.currency << "\n"
+            << "  Has I010: " << (details.has_i010 ? "Yes" : "No") << "\n"
+            << "  Has I011: " << (details.has_i011 ? "Yes" : "No") << "\n";
+      }
+      return ss.str();
+   }
    // Reset function to clear the data map
    void reset() {
-      for(auto &contract:sec_deets_map){
-         contract.second.products.clear();
-      }
       sec_deets_map.clear();
    }
 
  private:
-   std::unordered_map<std::string, contract_details> sec_deets_map;
+   std::unordered_map<std::string, security_details> sec_deets_map;
 };
 
 } // namespace bts::md::taifex

src/app_sparkd/sparkd/md/taifex/TaifexUtils.h

@@ -14,11 +14,9 @@
 #include <vector>
 
 namespace bts::md::taifex {
-
 class TaifexUtils {
-   using UNPROCESSED_MSG = std::vector<unsigned char>;
  public:
-   static int recv_socket_data(int socket, UNPROCESSED_MSG& buffer, const size_t FEED_BUFFER_SIZE, const std::string & socket_desc ="") {
+   static int recv_socket_data(int socket, std::vector<unsigned char>& buffer, const size_t FEED_BUFFER_SIZE, const std::string & socket_desc ="") {
     int bytes_received = -1;
     try {
         bytes_received = recvfrom(socket, buffer.data(), FEED_BUFFER_SIZE, 0, nullptr, nullptr);
@@ -99,10 +97,10 @@ class TaifexUtils {
       }
       LOG(md, Info, "Joined multicast group on IP: {}, Port: {}", multicast_ip,
           port);
-      // struct timeval tv;
-      // tv.tv_sec = 5;
-      // tv.tv_usec = 0;
-      // setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tv, sizeof tv);
+      struct timeval tv;
+      tv.tv_sec = 5;
+      tv.tv_usec = 0;
+      setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tv, sizeof tv);
       return sock;
    }
    static int create_multicast_socket_assert(
@@ -124,57 +122,8 @@ class TaifexUtils {
       }
       sockets.clear();
    }
-   static std::string getIP(const std::string &ipType) {
-   auto isLastTradingDay=[](){
-      return false;
-   };
-    const std::string regular_referenceIP = "225.0.140.141";
-    const std::string regular_refreshIP = "225.0.140.147";
-    const std::string regular_realTimeIP = "225.0.140.140";
-
-    const std::string afterHours_referenceIP = "225.10.140.141";
-    const std::string afterHours_refreshIP = "225.10.140.147";
-    const std::string afterHours_realTimeIP = "225.10.140.140";
-
-    // Get current time
-    std::time_t now = std::time(nullptr);
-    std::tm *localTime = std::localtime(&now);
-
-    int hour = localTime->tm_hour;
-   //  int minute = localTime->tm_min;
-
-    bool lastTradingDay = isLastTradingDay();
-
-    // Check if it's regular trading hours
-   //  bool isRegularTrading = (hour == 8 && minute >= 45) ||
-   //                          (hour > 8 && hour < 14 && (lastTradingDay ? minute < 30 : minute < 45));
-
-   //  // Check if it's after-hours trading
-    bool isAfterHoursTrading = !lastTradingDay && 
-                               ((hour == 15) || (hour > 15 && hour < 24) || (hour < 5));
-
-    // Determine the IP based on the time of day and type of IP requested
-    if (isAfterHoursTrading) {
-        if (ipType == "reference")
-            return afterHours_referenceIP;
-        if (ipType == "refresh")
-            return afterHours_refreshIP;
-        if (ipType == "realtime")
-            return afterHours_realTimeIP;
-    } //else if (isAfterHoursTrading) {
-   //  }
-        if (ipType == "reference")
-            return regular_referenceIP;
-        if (ipType == "refresh")
-            return regular_refreshIP;
-        if (ipType == "realtime")
-            return regular_realTimeIP;
-
-    // Return an error message if an invalid IP type is passed
-    return "Invalid IP type!";
-}
    static std::string getBufferString(
-       const UNPROCESSED_MSG &buffer,
+       const std::vector<unsigned char> &buffer,
        size_t offset = 0,
        size_t length = std::numeric_limits<size_t>::max()) {
 
@@ -200,7 +149,7 @@ class TaifexUtils {
    }
 
    static void printBuffer(
-       const UNPROCESSED_MSG &buffer,
+       const std::vector<unsigned char> &buffer,
        size_t offset = 0,
        size_t length = std::numeric_limits<size_t>::max()) {
       // Adjust length if it exceeds buffer size
@@ -431,7 +380,7 @@ class TaifexUtils {
       return result.empty() ? "0" : result;
    }
    static std::string extractString(
-       const UNPROCESSED_MSG &buffer,
+       const std::vector<unsigned char> &buffer,
        size_t &position,
        size_t length) {
       // Extract the substring from the buffer
@@ -446,7 +395,7 @@ class TaifexUtils {
    // Helper function to extract a BCD value from the buffer and convert to
    // unsigned long
    static unsigned long extractBCD_STOUL(
-       const UNPROCESSED_MSG &buffer,
+       const std::vector<unsigned char> &buffer,
        size_t &position,
        size_t length) {
       std::string bcd_str = decodePackBCD(&buffer[position], length);
@@ -454,7 +403,7 @@ class TaifexUtils {
       return std::stoul(bcd_str);
    }
    static std::string extractBCD_STRING(
-       const UNPROCESSED_MSG &buffer,
+       const std::vector<unsigned char> &buffer,
        size_t &position,
        size_t length) {
       std::string bcd_str = decodePackBCD(&buffer[position], length);
@@ -463,14 +412,14 @@ class TaifexUtils {
    }
    // Helper function to extract a single character and convert to bool
    static bool extractBool(
-       const UNPROCESSED_MSG &buffer, size_t &position) {
+       const std::vector<unsigned char> &buffer, size_t &position) {
       bool result = (buffer[position] == 'Y');
       position += 1;
       return result;
    }
 
    static std::string extractBCDDate(
-       const UNPROCESSED_MSG &buffer,
+       const std::vector<unsigned char> &buffer,
        size_t &current_position,
        size_t size) {
       if (size != 4) {
@@ -494,18 +443,16 @@ class TaifexUtils {
       return date;
    }
 
-   static std::string remove_spaces(const std::string &name) {
-    std::string removed;
-    removed.reserve(name.size());  // Reserve space to avoid reallocations
-    const char space = ' ';
-    for (auto c : name) {
-        if (c != space) {
+   static auto remove_spaces(std::string &name) {
+      std::string removed = "";
+      const char space = ' ';
+      for (auto c : name) {
+         if (c != space) {
             removed += c;
-        }
-    }
-    return removed;
-   }
-   
+         }
+      }
+      return removed;
+   };
 };
 
 } //namespace bts::md::taifex

src/client/spark_api/Domain/SecurityMetadataExtensions.cs

@@ -1,6 +1,6 @@
 #nullable enable
 namespace Spark.Api.Domain {
-   using bts.domain.protobufs;
+   using bts.api.domain;
    using bts.utils;
 
    using mbts;

src/client/spark_api/MarketData/BookTuple.cs

@@ -1,7 +1,7 @@
 #nullable enable
 namespace Spark.Api.MarketData {
    using bts.api;
-   using bts.domain.protobufs;
+   using bts.api.domain;
    using bts.md;
 
    using System;

src/client/spark_api/MarketData/PriceTicker.cs

@@ -1,7 +1,7 @@
 #nullable enable
 namespace Spark.Api.MarketData {
    using bts.api;
-   using bts.domain.protobufs;
+   using bts.api.domain;
 
    using Spark.Core.Collections;
 

src/client/spark_api/MarketData/SecurityMD.cs

@@ -2,7 +2,7 @@
 namespace mbts {
    using bts;
    using bts.api;
-   using bts.domain.protobufs;
+   using bts.api.domain;
    using bts.utils;
 
    using mbts.co2;

src/client/spark_api/MarketData/TradeInfo.cs

@@ -1,6 +1,6 @@
 using bts.api;
+using bts.api.domain;
 using bts.co2;
-using bts.domain.protobufs;
 
 using mbts.analytics;
 

src/client/spark_api/Orders/OrdersExtensions.cs

@@ -3,7 +3,6 @@ namespace Spark.Api.Orders {
    using bts.api;
    using bts.api.orderman;
    using bts.co2;
-   using bts.domain.protobufs;
    using bts.spark;
    using bts.utils;
 
@@ -218,7 +217,7 @@ namespace Spark.Api.Orders {
       public static string full_description(
          this OrderRecordImmutable order,
          SecurityImmutable security,
-         bts.domain.protobufs.TickDisplaySettingsImmutable? qty_display = null) {
+         bts.api.domain.TickDisplaySettingsImmutable? qty_display = null) {
          StringBuilder sb = new StringBuilder();
          string sec_key_text;
          if (order.HasDef) {
@@ -274,8 +273,8 @@ namespace Spark.Api.Orders {
       public static string price_quantity_description(
          double price,
          double quantity,
-         bts.domain.protobufs.TickDisplaySettingsImmutable? price_display = null,
-         bts.domain.protobufs.TickDisplaySettingsImmutable? quantity_display = null) {
+         bts.api.domain.TickDisplaySettingsImmutable? price_display = null,
+         bts.api.domain.TickDisplaySettingsImmutable? quantity_display = null) {
          price_display ??= Security.Defaults.price_display_settingsi;
          quantity_display ??= Security.Defaults.quantity_display_settingsi;
          return $"{quantity_display.Value.format(quantity)} @ {price_display.Value.format(price)}";
@@ -284,7 +283,7 @@ namespace Spark.Api.Orders {
       public static string filled_status_description(
          this OrderRecordImmutable order,
          ContractSpecificationImmutable spec,
-         bts.domain.protobufs.TickDisplaySettingsImmutable? qty_display = null
+         bts.api.domain.TickDisplaySettingsImmutable? qty_display = null
       ) {
          qty_display ??= Security.Defaults.quantity_display_settingsi;
          var sb = new StringBuilder();
@@ -303,8 +302,8 @@ namespace Spark.Api.Orders {
          OrderRecordImmutable order,
          double? new_price = null,
          double? new_quantity = null,
-         bts.domain.protobufs.TickDisplaySettingsImmutable? price_display = null,
-         bts.domain.protobufs.TickDisplaySettingsImmutable? quantity_display = null
+         bts.api.domain.TickDisplaySettingsImmutable? price_display = null,
+         bts.api.domain.TickDisplaySettingsImmutable? quantity_display = null
       ) {
          var price = new_price ?? order.price();
          var quantity = new_quantity ?? order.working_quantity();

src/client/spark_api/data/ISecurityMetadataStore.cs

@@ -1,7 +1,7 @@
 #nullable enable
 namespace Spark.Core.Data {
    using bts.api;
-   using bts.domain.protobufs;
+   using bts.api.domain;
    using bts.utils;
 
    using Spark.Api.Domain;

src/client/spark_api/data/SecurityCriteriaExtensions.cs

@@ -348,7 +348,7 @@ namespace mbts.spark {
       /// </remarks>
       /// <returns>The number of names that were resolved</returns>
       public static int resolve_names(
-         this bts.domain.protobufs.SecurityCriteriaClause api_clause, Func<string, uint?> try_resolve) {
+         this bts.api.domain.SecurityCriteriaClause api_clause, Func<string, uint?> try_resolve) {
          var orig_num_names = api_clause.Names.Count;
          for (var i = 0; i < api_clause.Names.Count;) {
             var name = api_clause.Names[i];
@@ -381,9 +381,9 @@ namespace mbts.spark {
       //    return n_resolved;
       // }
 
-      public static bts.domain.protobufs.SecurityCriteriaClause to_clause(
+      public static bts.api.domain.SecurityCriteriaClause to_clause(
          this SecurityCriteriaClause clause) {
-         var api_clause = new bts.domain.protobufs.SecurityCriteriaClause();
+         var api_clause = new bts.api.domain.SecurityCriteriaClause();
          api_clause.Venues.AddRange(
             clause.exchanges.empty_if_null().Select(e => e.to_api_venue()));
          api_clause.Symbols.AddRange(clause.symbols.empty_if_null());
@@ -395,11 +395,11 @@ namespace mbts.spark {
       }
 
       public static SecurityCriteriaClause to_clause(
-         this bts.domain.protobufs.SecurityCriteriaClause api_clause) =>
+         this bts.api.domain.SecurityCriteriaClause api_clause) =>
          api_clause.to_clause(null);
 
       public static SecurityCriteriaClause to_clause(
-         this bts.domain.protobufs.SecurityCriteriaClause api_clause, Func<string, uint?>? try_resolve) {
+         this bts.api.domain.SecurityCriteriaClause api_clause, Func<string, uint?>? try_resolve) {
          return new(
             api_clause.Venues.WrapSelectorRO(v => v.to_venue()).ToArray(),
          api_clause.SecurityTypes.WrapSelectorRO(t => t.to_security_type()).ToArray(),
@@ -413,7 +413,7 @@ namespace mbts.spark {
       }
 
       public static SecurityCriteriaClause to_clause(
-         this bts.domain.protobufs.SecurityCriteriaClauseImmutable api_clause) =>
+         this bts.api.domain.SecurityCriteriaClauseImmutable api_clause) =>
          new SecurityCriteriaClause(
             api_clause.Venues.Select(v => v.to_venue()).ToArray(),
             api_clause.SecurityTypes.Select(t => t.to_security_type()).ToArray(),
@@ -426,8 +426,8 @@ namespace mbts.spark {
             api_clause.Names.ToArray()
             );
 
-      public static bts.domain.protobufs.SecurityCriteria to_criteria(this SecurityCriteria criteria) {
-         var api_criteria = new bts.domain.protobufs.SecurityCriteria();
+      public static bts.api.domain.SecurityCriteria to_criteria(this SecurityCriteria criteria) {
+         var api_criteria = new bts.api.domain.SecurityCriteria();
          if (criteria is null) {
             return api_criteria;
          }
@@ -436,11 +436,11 @@ namespace mbts.spark {
       }
 
       public static SecurityCriteria to_criteria(
-         this bts.domain.protobufs.SecurityCriteria api_criteria) =>
+         this bts.api.domain.SecurityCriteria api_criteria) =>
          new(api_criteria.Clauses.Select(to_clause));
 
       public static SecurityCriteria to_criteria(
-         this bts.domain.protobufs.SecurityCriteriaImmutable api_criteria) =>
+         this bts.api.domain.SecurityCriteriaImmutable api_criteria) =>
          new(api_criteria.Clauses.Select(to_clause));
 
       static IEnumerable<Func<SecurityImmutable, bool>> _to_predicates(

src/client/spark_api/data/SecurityMetadataStore.cs

@@ -3,8 +3,8 @@
 namespace Spark.Core.Data {
    using bts;
    using bts.api;
+   using bts.api.domain;
    using bts.collections;
-   using bts.domain.protobufs;
 
    using mbts;
    using mbts.config;

src/client/spark_api/data/ticks/TickExtensions.cs

@@ -17,22 +17,22 @@ namespace mbts {
       static string _nan_text => TickDisplaySettings.Default.nan_text;
 
       public static TickFactors to_tick_factors(
-         this bts.domain.protobufs.TickFactorsImmutable api_tick_factors) =>
+         this bts.api.domain.TickFactorsImmutable api_tick_factors) =>
          new(api_tick_factors.TickBase, api_tick_factors.TickUnit);
 
-      public static bts.domain.protobufs.TickFactors to_api_tick_factors(
+      public static bts.api.domain.TickFactors to_api_tick_factors(
          this TickFactors tick_factors) =>
          new() {
             TickBase = tick_factors.tick_base,
             TickUnit = tick_factors.tick_unit
          };
 
-      public static bts.domain.protobufs.TickFactorsImmutable to_api_tick_factorsi(
+      public static bts.api.domain.TickFactorsImmutable to_api_tick_factorsi(
          this TickFactors tick_factors) =>
          new(tick_factors.to_api_tick_factors());
 
       public static TickDisplaySettings to_tick_display_settings(
-         this bts.domain.protobufs.TickDisplaySettingsImmutable api_tick_display_settings) =>
+         this bts.api.domain.TickDisplaySettingsImmutable api_tick_display_settings) =>
          new() {
             pad_with_leading_zeros = api_tick_display_settings.PadWithLeadingZeros,
             display_multiplier = api_tick_display_settings.DisplayMultiplier,
@@ -44,14 +44,14 @@ namespace mbts {
                api_tick_display_settings.TickFactors.to_tick_factors() : null
          };
 
-      public static bts.domain.protobufs.TickDisplaySettings to_api_tick_display_settings(
+      public static bts.api.domain.TickDisplaySettings to_api_tick_display_settings(
          this TickFactors tick_factors) {
          var api_tds = Security.Defaults.price_display_settings;
          api_tds.TickFactors = tick_factors.to_api_tick_factors();
          return api_tds;
       }
 
-      public static bts.domain.protobufs.TickDisplaySettingsImmutable to_api_tick_display_settingsi(
+      public static bts.api.domain.TickDisplaySettingsImmutable to_api_tick_display_settingsi(
          this TickFactors tick_factors) =>
          new(tick_factors.to_api_tick_display_settings());
 
@@ -79,7 +79,7 @@ namespace mbts {
       public static double tick_unit_frac(this TickFactors tick) =>
          tick.tick_unit / tick.tick_base;
 
-      public static double tick_unit_frac(this bts.domain.protobufs.TickFactorsImmutable tick) =>
+      public static double tick_unit_frac(this bts.api.domain.TickFactorsImmutable tick) =>
          tick.TickUnit / tick.TickBase;
 
       public static T tick_unit_frac<T>(this TickFactors tick) where T : IFloatingPoint<T> =>
@@ -156,7 +156,7 @@ namespace mbts {
          format(settings.tick_factors, value, settings);
 
       public static string format<T>(
-         this bts.domain.protobufs.TickDisplaySettingsImmutable settings, T value)
+         this bts.api.domain.TickDisplaySettingsImmutable settings, T value)
          where T : IFloatingPointIeee754<T> =>
          settings.to_tick_display_settings().format(value);
 
@@ -241,7 +241,7 @@ namespace mbts {
       }
 
       public static double from_display_value(
-         this bts.domain.protobufs.TickDisplaySettingsImmutable display_settings,
+         this bts.api.domain.TickDisplaySettingsImmutable display_settings,
          double display_value,
          bool round = true) {
          //
@@ -257,7 +257,7 @@ namespace mbts {
       }
 
       public static double to_display_value(
-         this bts.domain.protobufs.TickDisplaySettingsImmutable display_settings,
+         this bts.api.domain.TickDisplaySettingsImmutable display_settings,
          double actual_value,
          bool round = true) {
          //
@@ -314,12 +314,12 @@ namespace mbts {
          where T : IFloatingPointIeee754<T> =>
          count < 0 ? tick.tickdown(value, (uint)-count) : tick.tickup(value, (ulong)count);
 
-      public static T tick<T>(this bts.domain.protobufs.TickFactorsImmutable tick, T value, long count)
+      public static T tick<T>(this bts.api.domain.TickFactorsImmutable tick, T value, long count)
          where T : IFloatingPointIeee754<T> =>
          tick.to_tick_factors().tick(value, count);
 
       public static T tick<T>(
-         this bts.domain.protobufs.TickDisplaySettingsImmutable settings, T value, long count)
+         this bts.api.domain.TickDisplaySettingsImmutable settings, T value, long count)
          where T : IFloatingPointIeee754<T> =>
          settings.HasTickFactors ?
             settings.TickFactors.tick(value, count) :
@@ -558,7 +558,7 @@ namespace mbts {
       public static double round(this TickFactors tick, double input) =>
          _round_tick(tick.tick_unit_frac(), input);
 
-      public static double round(this bts.domain.protobufs.TickFactorsImmutable tick, double input) =>
+      public static double round(this bts.api.domain.TickFactorsImmutable tick, double input) =>
          _round_tick(tick.tick_unit_frac(), input);
 
       public static double round(this RegionalTickFactors regional_tick, double input) {

src/client/spark_api/data/ticks/TickFactorsFactory.cs

@@ -38,7 +38,7 @@ namespace bts.data.ticks {
       }
 
       static RegionalTickFactors _create_rtf(
-         IList<domain.protobufs.TickDisplayRuleImmutable> rules) {
+         IList<api.domain.TickDisplayRuleImmutable> rules) {
          if (rules.is_null_or_empty()) {
             throw new ArgumentException("Rules is null or empty", nameof(rules));
          }

src/client/spark_api/security/Security.cs

@@ -6,8 +6,8 @@ using static bts.api.Security;
 using security_key = System.UInt32;
 
 namespace bts.api {
-   using TickDisplaySettings = bts.domain.protobufs.TickDisplaySettings;
-   using TickDisplaySettingsImmutable = bts.domain.protobufs.TickDisplaySettingsImmutable;
+   using TickDisplaySettings = domain.TickDisplaySettings;
+   using TickDisplaySettingsImmutable = domain.TickDisplaySettingsImmutable;
 
    public partial class Security {
       public static security_key key(Security s) { return s.Id; }
@@ -75,13 +75,13 @@ namespace bts.api {
          public static readonly TickDisplaySettingsImmutable quantity_display_settingsi =
             new(quantity_display_settings);
 
-         public static bts.domain.protobufs.SecurityMetadata security_metadata =>
+         public static domain.SecurityMetadata security_metadata =>
             new() {
                PriceDisplay = price_display_settings,
                QuantityDisplay = quantity_display_settings
             };
 
-         public static bts.domain.protobufs.SecurityMetadataImmutable security_metadata_immutable =>
+         public static domain.SecurityMetadataImmutable security_metadata_immutable =>
             new(security_metadata);
       }
    }

src/client/spark_api/security/SecurityExtensions.cs

@@ -124,60 +124,62 @@ namespace bts.api {
             _ => side,
          };
 
-      public static SecurityType to_security_type(this bts.domain.protobufs.SecurityType api_security_type) =>
+      public static SecurityType to_security_type(this domain.SecurityType api_security_type) =>
          api_security_type switch {
-            bts.domain.protobufs.SecurityType.Spot => SecurityType.Spot,
-            bts.domain.protobufs.SecurityType.Future => SecurityType.Future,
-            bts.domain.protobufs.SecurityType.Option => SecurityType.Option,
-            bts.domain.protobufs.SecurityType.Spread => SecurityType.Spread,
-            bts.domain.protobufs.SecurityType.Unspecified => SecurityType.Unset,
+            domain.SecurityType.Spot => SecurityType.Spot,
+            domain.SecurityType.Future => SecurityType.Future,
+            domain.SecurityType.Option => SecurityType.Option,
+            domain.SecurityType.Spread => SecurityType.Spread,
+            domain.SecurityType.Unspecified => SecurityType.Unset,
             _ => throw new NotImplementedException(),
          };
 
-      public static bts.domain.protobufs.SecurityType to_api_security_type(this SecurityType api_security_type) =>
+      public static domain.SecurityType to_api_security_type(this SecurityType api_security_type) =>
          api_security_type switch {
-            SecurityType.Spot => bts.domain.protobufs.SecurityType.Spot,
-            SecurityType.Future => bts.domain.protobufs.SecurityType.Future,
-            SecurityType.Option => bts.domain.protobufs.SecurityType.Option,
-            SecurityType.Spread => bts.domain.protobufs.SecurityType.Spread,
-            SecurityType.Unset => bts.domain.protobufs.SecurityType.Unspecified,
+            SecurityType.Spot => domain.SecurityType.Spot,
+            SecurityType.Future => domain.SecurityType.Future,
+            SecurityType.Option => domain.SecurityType.Option,
+            SecurityType.Spread => domain.SecurityType.Spread,
+            SecurityType.Unset => domain.SecurityType.Unspecified,
             _ => throw new NotImplementedException(),
          };
 
-      public static Venue to_venue(this bts.domain.protobufs.Venue api_venue) =>
+      public static Venue to_venue(this domain.Venue api_venue) =>
          api_venue switch {
-            bts.domain.protobufs.Venue.Bts => Venue.Bts,
-            bts.domain.protobufs.Venue.Sim => Venue.Sim,
-            bts.domain.protobufs.Venue.Deribit => Venue.Deribit,
-            bts.domain.protobufs.Venue.Ftx => Venue.Ftx,
-            bts.domain.protobufs.Venue.Binance => Venue.Binance,
-            bts.domain.protobufs.Venue.BinanceUs => Venue.BinanceUs,
-            bts.domain.protobufs.Venue.BinanceUsdsFutures => Venue.BinanceUsdsFutures,
-            bts.domain.protobufs.Venue.Glbx => Venue.Glbx,
-            bts.domain.protobufs.Venue.Coinbase => Venue.Coinbase,
-            bts.domain.protobufs.Venue.Okx => Venue.Okx,
-            bts.domain.protobufs.Venue.Cboe => Venue.Cboe,
-            bts.domain.protobufs.Venue.Taifex => Venue.Taifex,
-            bts.domain.protobufs.Venue.Unspecified => Venue.Unset,
+            domain.Venue.Bts => Venue.Bts,
+            domain.Venue.Sim => Venue.Sim,
+            domain.Venue.Deribit => Venue.Deribit,
+            domain.Venue.Ftx => Venue.Ftx,
+            domain.Venue.Binance => Venue.Binance,
+            domain.Venue.BinanceUs => Venue.BinanceUs,
+            domain.Venue.BinanceUsdsFutures => Venue.BinanceUsdsFutures,
+            domain.Venue.Glbx => Venue.Glbx,
+            domain.Venue.Coinbase => Venue.Coinbase,
+            domain.Venue.Okx => Venue.Okx,
+                        domain.Venue.Cboe => Venue.Cboe,
+            domain.Venue.Taifex => Venue.Taifex,
+
+
+            domain.Venue.Unspecified => Venue.Unset,
             _ => throw new NotImplementedException(),
          };
 
-      public static bts.domain.protobufs.Venue to_api_venue(this Venue venue) =>
+      public static domain.Venue to_api_venue(this Venue venue) =>
          venue switch {
-            Venue.Bts => bts.domain.protobufs.Venue.Bts,
-            Venue.Sim => bts.domain.protobufs.Venue.Sim,
-            Venue.Deribit => bts.domain.protobufs.Venue.Deribit,
-            Venue.Ftx => bts.domain.protobufs.Venue.Ftx,
-            Venue.Binance => bts.domain.protobufs.Venue.Binance,
-            Venue.BinanceUs => bts.domain.protobufs.Venue.BinanceUs,
-            Venue.BinanceUsdsFutures => bts.domain.protobufs.Venue.BinanceUsdsFutures,
-            Venue.Coinbase => bts.domain.protobufs.Venue.Coinbase,
-            Venue.Glbx => bts.domain.protobufs.Venue.Glbx,
-            Venue.Okx => bts.domain.protobufs.Venue.Okx,
-            Venue.Cboe => bts.domain.protobufs.Venue.Cboe,
-            Venue.Taifex => bts.domain.protobufs.Venue.Taifex,
+            Venue.Bts => domain.Venue.Bts,
+            Venue.Sim => domain.Venue.Sim,
+            Venue.Deribit => domain.Venue.Deribit,
+            Venue.Ftx => domain.Venue.Ftx,
+            Venue.Binance => domain.Venue.Binance,
+            Venue.BinanceUs => domain.Venue.BinanceUs,
+            Venue.BinanceUsdsFutures => domain.Venue.BinanceUsdsFutures,
+            Venue.Coinbase => domain.Venue.Coinbase,
+            Venue.Glbx => domain.Venue.Glbx,
+            Venue.Okx => domain.Venue.Okx,
+            Venue.Cboe => domain.Venue.Cboe,
+            Venue.Taifex => domain.Venue.Taifex,
 
-            Venue.Unset => bts.domain.protobufs.Venue.Unspecified,
+            Venue.Unset => domain.Venue.Unspecified,
             _ => throw new NotImplementedException(),
          };
 

src/client/spark_api/spark_api.csproj

@@ -33,174 +33,174 @@
     <Proto Include="**/*.proto" />
 
     <!-- Protos in lib/api -->
-    <Proto Include="../../lib_spark/spark/api/apicore.proto" CSName="Apicore">
-      <OutDir>$(ProtoOutputPath)/spark/api</OutDir>
+    <Proto Include="../../lib/api/apicore.proto" CSName="Apicore">
+      <OutDir>$(ProtoOutputPath)/lib/api</OutDir>
     </Proto>
-    <Proto Include="../../lib_spark/spark/api/apioptions.proto" CSName="Apioptions">
-      <OutDir>$(ProtoOutputPath)/spark/api</OutDir>
+    <Proto Include="../../lib/api/apioptions.proto" CSName="Apioptions">
+      <OutDir>$(ProtoOutputPath)/lib/api</OutDir>
     </Proto>
-    <Proto Include="../../lib_spark/spark/api/apigateway.proto" CSName="Apigateway">
-      <OutDir>$(ProtoOutputPath)/spark/api</OutDir>
+    <Proto Include="../../lib/api/apigateway.proto" CSName="Apigateway">
+      <OutDir>$(ProtoOutputPath)/lib/api</OutDir>
     </Proto>
 
-    <Proto Include="../../lib_spark/spark/api/analytics/ExpirationDriver.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/analytics</OutDir>
+    <Proto Include="../../lib/api/analytics/ExpirationDriver.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api</OutDir>
     </Proto>
-    <Proto Include="../../lib_spark/spark/api/analytics/TimeWeightedInterpolatedDriver.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/analytics</OutDir>
-    </Proto>
-    <Proto Include="../../lib_spark/spark/api/domain/SecurityExpiration.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/domain</OutDir>
+    <Proto Include="../../lib/api/analytics/TimeWeightedInterpolatedDriver.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api</OutDir>
     </Proto>
 
     <!-- Protos in lib_domain/domain -->
-    <Proto Include="../../lib_domain/domain/protobufs/aggressor_side.proto" CSName="AggressorSide">
-      <OutDir>$(ProtoOutputPath)/domain/protobufs</OutDir>
+    <Proto Include="../../lib_domain/domain/Currency.proto">
+      <OutDir>$(ProtoOutputPath)/domain</OutDir>
     </Proto>
-    <Proto Include="../../lib_domain/domain/protobufs/currency.proto" CSName="Currency">
-      <OutDir>$(ProtoOutputPath)/domain/protobufs</OutDir>
+    <Proto Include="../../lib_domain/domain/SecurityCriteria.proto">
+      <OutDir>$(ProtoOutputPath)/domain</OutDir>
     </Proto>
-    <Proto Include="../../lib_domain/domain/protobufs/security_criteria.proto" CSName="SecurityCriteria">
-      <OutDir>$(ProtoOutputPath)/domain/protobufs</OutDir>
+    <Proto Include="../../lib_domain/domain/SecurityCriteriaClause.proto">
+      <OutDir>$(ProtoOutputPath)/domain</OutDir>
     </Proto>
-    <Proto Include="../../lib_domain/domain/protobufs/security_criteria_clause.proto" CSName="SecurityCriteriaClause">
-      <OutDir>$(ProtoOutputPath)/domain/protobufs</OutDir>
+    <Proto Include="../../lib_domain/domain/SecurityCriteriaClause.proto">
+      <OutDir>$(ProtoOutputPath)/domain</OutDir>
     </Proto>
-    <Proto Include="../../lib_domain/domain/protobufs/security_metadata.proto" CSName="SecurityMetadata">
-      <OutDir>$(ProtoOutputPath)/domain/protobufs</OutDir>
+    <Proto Include="../../lib_domain/domain/SecurityExpiration.proto">
+      <OutDir>$(ProtoOutputPath)/domain</OutDir>
     </Proto>
-    <Proto Include="../../lib_domain/domain/protobufs/security_metadata_collection.proto" CSName="SecurityMetadataCollection">
-      <OutDir>$(ProtoOutputPath)/domain/protobufs</OutDir>
+    <Proto Include="../../lib_domain/domain/SecurityMetadata.proto">
+      <OutDir>$(ProtoOutputPath)/domain</OutDir>
     </Proto>
-    <Proto Include="../../lib_domain/domain/protobufs/security_metadata_item.proto" CSName="SecurityMetadataItem">
-      <OutDir>$(ProtoOutputPath)/domain/protobufs</OutDir>
+    <Proto Include="../../lib_domain/domain/SecurityMetadataCollection.proto">
+      <OutDir>$(ProtoOutputPath)/domain</OutDir>
     </Proto>
-    <Proto Include="../../lib_domain/domain/protobufs/security_type.proto" CSName="SecurityType">
-      <OutDir>$(ProtoOutputPath)/domain/protobufs</OutDir>
+    <Proto Include="../../lib_domain/domain/SecurityMetadataItem.proto">
+      <OutDir>$(ProtoOutputPath)/domain</OutDir>
     </Proto>
-    <Proto Include="../../lib_domain/domain/protobufs/tick_display_rule.proto" CSName="TickDisplayRule">
-      <OutDir>$(ProtoOutputPath)/domain/protobufs</OutDir>
+    <Proto Include="../../lib_domain/domain/SecurityType.proto">
+      <OutDir>$(ProtoOutputPath)/domain</OutDir>
     </Proto>
-    <Proto Include="../../lib_domain/domain/protobufs/tick_display_settings.proto" CSName="TickDisplaySettings">
-      <OutDir>$(ProtoOutputPath)/domain/protobufs</OutDir>
+    <Proto Include="../../lib_domain/domain/TickDisplayRule.proto">
+      <OutDir>$(ProtoOutputPath)/domain</OutDir>
     </Proto>
-    <Proto Include="../../lib_domain/domain/protobufs/tick_factors.proto" CSName="TickFactors">
-      <OutDir>$(ProtoOutputPath)/domain/protobufs</OutDir>
+    <Proto Include="../../lib_domain/domain/TickDisplaySettings.proto">
+      <OutDir>$(ProtoOutputPath)/domain</OutDir>
     </Proto>
-    <Proto Include="../../lib_domain/domain/protobufs/venue.proto" CSName="Venue">
-      <OutDir>$(ProtoOutputPath)/domain/protobufs</OutDir>
+    <Proto Include="../../lib_domain/domain/TickFactors.proto">
+      <OutDir>$(ProtoOutputPath)/domain</OutDir>
+    </Proto>
+    <Proto Include="../../lib_domain/domain/Venue.proto">
+      <OutDir>$(ProtoOutputPath)/domain</OutDir>
     </Proto>
 
     <!-- Protos in lib/api/services -->
-    <ProtoService Include="../../lib_spark/spark/api/services/AdminService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/AdminService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/AlgoDefinitionService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/AlgoDefinitionService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/AlgoService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/AlgoService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/AnalyticsService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/AnalyticsService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/ApiUnitTestService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/ApiUnitTestService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/AuthService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/AuthService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/BasePositionService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/BasePositionService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/ErrorReportService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/ErrorReportService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/HelloService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/HelloService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/MarginService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/MarginService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/MarketDataAuxiliaryService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/MarketDataAuxiliaryService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/MarketDataRelay.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/MarketDataRelay.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/MarketDataService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/MarketDataService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/MBOService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/MBOService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/MoneyFileService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/MoneyFileService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/OrderService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/OrderService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/OutOfProcessURLListener.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/OutOfProcessURLListener.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/PersistenceService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/PersistenceService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/PositionService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/PositionService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/ShellDiagnosticService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/ShellDiagnosticService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/ShellLatencyService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/ShellLatencyService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Include="../../lib_spark/spark/api/services/SyntheticSecurityService.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Include="../../lib/api/services/SyntheticSecurityService.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
-    <ProtoService Update="../../lib_spark/spark/api/services/OutOfProcessURLListener.proto">
-      <OutDir>$(ProtoOutputPath)/spark/api/services</OutDir>
+    <ProtoService Update="../../lib/api/services/OutOfProcessURLListener.proto">
+      <OutDir>$(ProtoOutputPath)/lib/api/services</OutDir>
     </ProtoService>
 
-    <Proto Include="../../lib_spark/spark/md/protos/co2.proto">
-      <OutDir>$(ProtoOutputPath)/spark/md/protos</OutDir>
+    <Proto Include="../../lib/md/protos/co2.proto">
+      <OutDir>$(ProtoOutputPath)/lib/md/protos</OutDir>
       <CSName>Co2</CSName>
     </Proto>
 
-    <Proto Include="../../lib_spark/spark/domain_model/Security.proto">
-      <OutDir>$(ProtoOutputPath)/spark/domain_model</OutDir>
+    <Proto Include="../../lib/domain_model/Security.proto">
+      <OutDir>$(ProtoOutputPath)/lib/domain_model</OutDir>
     </Proto>
-    <Proto Include="../../lib_spark/spark/sparkproto/SparkProtos.proto">
-      <OutDir>$(ProtoOutputPath)/spark/sparkproto</OutDir>
+    <Proto Include="../../lib/sparkproto/SparkProtos.proto">
+      <OutDir>$(ProtoOutputPath)/lib/sparkproto</OutDir>
     </Proto>
 
     <!-- Protos in lib/config/protos -->
-    <Proto Include="../../lib_spark/spark/config/protos/algo_types.proto" CSName="AlgoTypes">
-      <OutDir>$(ProtoOutputPath)/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/algo_types.proto" CSName="AlgoTypes">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
     </Proto>
-    <Proto Include="../../lib_spark/spark/config/protos/market_data.proto" CSName="MarketData">
-      <OutDir>$(ProtoOutputPath)/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/market_data.proto" CSName="MarketData">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
     </Proto>
-    <Proto Include="../../lib_spark/spark/config/protos/orders_fills_cleanup.proto" CSName="OrdersFillsCleanup">
-      <OutDir>$(ProtoOutputPath)/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/orders_fills_cleanup.proto" CSName="OrdersFillsCleanup">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
     </Proto>
-    <Proto Include="../../lib_spark/spark/config/protos/ports.proto" CSName="Ports">
-      <OutDir>$(ProtoOutputPath)/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/ports.proto" CSName="Ports">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
     </Proto>
-    <Proto Include="../../lib_spark/spark/config/protos/position_cfg.proto" CSName="PositionCfg">
-      <OutDir>$(ProtoOutputPath)/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/position_cfg.proto" CSName="PositionCfg">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
     </Proto>
-    <Proto Include="../../lib_spark/spark/config/protos/quickbus_cfg.proto" CSName="QuickbusCfg">
-      <OutDir>$(ProtoOutputPath)/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/quickbus_cfg.proto" CSName="QuickbusCfg">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
     </Proto>
-    <Proto Include="../../lib_spark/spark/config/protos/symbols.proto" CSName="Symbols">
-      <OutDir>$(ProtoOutputPath)/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/symbols.proto" CSName="Symbols">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
     </Proto>
-    <Proto Include="../../lib_spark/spark/config/protos/toggles.proto" CSName="Toggles">
-      <OutDir>$(ProtoOutputPath)/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/toggles.proto" CSName="Toggles">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
     </Proto>
-    <Proto Include="../../lib_spark/spark/config/protos/ui.proto" CSName="Ui">
-      <OutDir>$(ProtoOutputPath)/spark/config/protos</OutDir>
+    <Proto Include="../../lib/config/protos/ui.proto" CSName="Ui">
+      <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
     </Proto>
   </ItemGroup>
 

src/client/spark_api_server/Dockerfile

@@ -1,10 +1,5 @@
 FROM bluetradingsystems/linux_dotnet7
 
-ARG CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
-ARG CPP_SOURCE_DIR="install/spark-api-server-linux"
-ARG CONF_FILE="src/client/spark_api_server/spark.current.conf"
-
-COPY --chown=bts:bts "${CPP_SOURCE_DIR}/lib" "bin"
-COPY --chown=bts:bts "${CS_SOURCE_DIR}" "bin"
+COPY --chown=bts:bts "build/Release/bin/net7.0/linux-x64/publish" "bin"
 COPY --chown=bts:bts "src/client/spark_api_server/spark.conf" ".local/share/bts/spark_api_server/conf/spark.conf"
 WORKDIR "/home/bts/bin"

src/client/spark_api_server/Program.cs

@@ -1,9 +1,6 @@
 using bts;
 using bts.websocket_core;
 
-using Bts.DataStreamCore.Services;
-using Bts.PubSub.Services;
-
 using CommandLine;
 
 using Microsoft.AspNetCore.Builder;
@@ -89,9 +86,6 @@ static void ConfigureServices(IServiceCollection services, IConfiguration config
       .AddLogging(ConfigureLogging)
       .AddSpark(config.GetSection("Spark"))
       .AddSparkApi(config.GetSection("SparkApi"))
-      .AddPubSubClient(config.GetSection("PubSub"))
-      .AddViewStreamDatabase()
-      .AddMasterWorker(config.GetSection("MasterWorker"))
       .AddWebSocketServer();
 
    static void ConfigureLogging(ILoggingBuilder loggingBuilder) =>

src/client/spark_api_server/appsettings.json

@@ -25,62 +25,5 @@
       "User": "bts",
       "Password": ""
     }
-  },
-  "MasterWorker": {
-    "Interval": "00:00:00.1"
-  },
-  "PubSub": {
-    "HeartbeatInterval": 15000,
-    "MissingHeartbeatsBeforeGc": 50,
-    "DisposedConnectionCheckPeriod": 1000,
-    "ServerPort": 22019,
-    "Routing": {
-      "PublisherRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://pubsub:22019",
-          "TagRegExp": ".*"
-        }
-      ],
-      "SubscriberRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://pubsub:22019",
-          "TagRegExp": ".*"
-        }
-      ]
-    },
-    "TagTypeToAssemblyMap": {
-      "Entries": [
-        {
-          "HandlerClass": "BCG.PubSubServerTagHandler",
-          "Assembly": "DataStreamCore.dll"
-        }
-      ]
-    },
-    "TagHandlerFactory": {
-      "Handlers": [
-        {
-          "TagType": "DataStream",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
-        },
-        {
-          "TagType": "System_Sub",
-          "Assembly": "PubSub",
-          "Class": "BCG.SystemSubscriptionMapPublisher"
-        },
-        {
-          "TagType": "System_Stats",
-          "Assembly": "PubSub",
-          "Class": "BCG.SystemStatsPublisher"
-        },
-        {
-          "TagType": ".*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerGenericDataHandler"
-        }
-      ]
-    }
-  },
+  }
 }

src/client/spark_core/logging/LogUtil.cs

@@ -12,7 +12,7 @@ namespace mbts
 {
    public static class LogUtil {
       static string log_path = Path.Combine(
-         Configuration.get<bts.system.protobufs.SystemConfig>().DataPath, "logging", "log");
+         Configuration.get<bts.config.SystemConfig>().DataPath, "logging", "log");
 
       public static ByteString read_logs_compressed() {
          return FileUtils.read_and_gzip(log_path);

src/client/spark_core/spark_core.csproj

@@ -21,13 +21,13 @@
   </ItemGroup>
 
   <ItemGroup>
-    <Proto Include="../../lib_system/system/protobufs/logging.proto" CSName="Logging">
+    <Proto Include="../../lib/config/protos/logging.proto" CSName="Logging">
       <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
     </Proto>
-    <Proto Include="../../lib_system/system/protobufs/config/system_config.proto" CSName="SystemConfig">
+    <Proto Include="../../lib/config/protos/system_config.proto" CSName="SystemConfig">
       <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
     </Proto>
-    <Proto Include="../../lib_spark/spark/config/protos/deployment_type.proto" CSName="DeploymentType">
+    <Proto Include="../../lib/config/protos/deployment_type.proto" CSName="DeploymentType">
       <OutDir>$(ProtoOutputPath)/lib/config/protos</OutDir>
     </Proto>
   </ItemGroup>

src/client/spark_websocket/Core/WebsocketJsonParsers.cs

@@ -142,14 +142,14 @@ namespace bts.websocket_core {
          }
 
       }
-      public static bool TryParse<T>(JObject filter, string key, out T[]? list, out string error) where T : struct {
+      public static bool TryParse<T>(JObject filter, string key, out T[]? list, out string error) {
          list = null;
          if (TryParse<StringListWrapper>(filter, key, out StringListWrapper? wrap, out error)) {
             if (wrap != null) {
                 var values = new List<T>();
                foreach (var item in wrap) {
                   if (TryParse<T>(item, key, out T? obj, out error) && obj != null) {
-                     values.Add(obj.Value);
+                     values.Add(obj);
                   } else {
                      return false;
                   }
@@ -167,13 +167,13 @@ namespace bts.websocket_core {
             error = $"{key} not present";
             return true;
          }
-         return TryParse<T>(value, key, out obj, out error);
+         return TryParse(value, key, out obj, out error);
 
       }
-      public static bool TryParse<T>(string value, string key, [NotNullWhen(true)] out T? obj, [NotNull] out string error) where T : struct {
+      public static bool TryParse<T>(string value, string key, [MaybeNullWhen(false)] out T obj, [NotNullWhen(false)] out string error) {
          error = "";
-         if (typeof(T).IsEnum && Enum.TryParse<T>(value, true, out var result)) {
-            obj = result;
+         if (typeof(T).IsEnum && Enum.TryParse(typeof(T), value, true, out var result)) {
+            obj = (T)result;
             return true;
          }
          if (typeof(T) == typeof(double) && double.TryParse(value, out var d)) {
@@ -215,10 +215,20 @@ namespace bts.websocket_core {
 
          }
          error = $"{key} is not in the correct format";
-         obj = null;
+         obj = default;
          return false;
       }
       
 
+      public static bool TryParse<T>(
+         JObject filter, string key, [MaybeNull]out T obj, [NotNullWhen(false)] out string error) {
+         obj = default;
+         var value = filter.GetValueOrDefault(key)?.ToString();
+         if (value == null) {
+            error = $"{key} not present";
+            return true;
+         }
+         return TryParse(value, key, out obj, out error);
+      }
    }
-}
+}

src/client/spark_websocket/Core/WebsocketRequest.cs

@@ -35,17 +35,12 @@ namespace bts.websocket_core {
             message = $"Incorrect Json: {ex.Path}";
             return false;
          }
-         if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(
-            request,
-            "id",
-            out WebsocketJsonParsers.StringWrapper? message_id,
-            out message)
+         if (!WebsocketJsonParsers.TryParse(request, "id", out string? message_id, out message)
             || message_id == null) {
             return false;
          }
-         _message_id = message_id.Value.Val ?? "missing";
-         REQUEST_TYPES? rt;
-         if (!WebsocketJsonParsers.TryParse<REQUEST_TYPES>(request, "method", out rt, out message)) {
+         _message_id = message_id ?? "missing";
+         if (!WebsocketJsonParsers.TryParse(request, "method", out REQUEST_TYPES? rt, out message)) {
             return false;
          }
          type = rt ?? REQUEST_TYPES.UNKOWN;
@@ -78,4 +73,4 @@ namespace bts.websocket_core {
          (parameters, type) = (para, other.type);
       }
    }
-}
+}

src/client/spark_websocket/Core/WebsocketRestHandler.cs

@@ -51,19 +51,19 @@ namespace bts.websocket_core {
             case DynamicOrderTypeDescriptor.DescriptorOneofCase.Primitive:
                switch (descriptor.ParamType.Primitive) {
                   case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveBool:
-                     if (!WebsocketJsonParsers.TryParse<bool>(custom, key, out bool? b, out error) || b == null) {
+                     if (!WebsocketJsonParsers.TryParse(custom, key, out bool? b, out error) || b == null) {
                         return (false, null, error);
                      }
                      rv.AsBool = (bool)b; 
                      return (true,rv,error);
                   case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveDouble:
-                     if (!WebsocketJsonParsers.TryParse<double>(custom, key, out double? d, out error) || d == null) {
+                     if (!WebsocketJsonParsers.TryParse(custom, key, out double? d, out error) || d == null) {
                         return (false, null, error);
                      }
                      rv.AsDouble = (double)d; 
                      return (true,rv,error);
                   case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveInt: 
-                  if (!WebsocketJsonParsers.TryParse<int>(custom, key, out int? i, out error) || i == null) {
+                  if (!WebsocketJsonParsers.TryParse(custom, key, out int? i, out error) || i == null) {
                         return (false, null, error);
                      }
                      rv.AsInt32 = (int)i; 
@@ -79,25 +79,25 @@ namespace bts.websocket_core {
                   rv.AsOrderDef = child_custom_def;
                   return(true,rv,error);
                   case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveString: 
-                  if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(custom,key,out WebsocketJsonParsers.StringWrapper? s,out error)|| s == null) {
+                  if (!WebsocketJsonParsers.TryParse(custom,key,out WebsocketJsonParsers.StringWrapper? s,out error)|| s == null) {
                      return (false, null, error);
                   }
                   rv.AsString = s.Value.Val; 
                   return (true,rv,error);
                   case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveSecurityKey:
                   case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveUint: 
-                  if (!WebsocketJsonParsers.TryParse<uint>(custom, key, out uint? u, out error) || u == null) {
+                  if (!WebsocketJsonParsers.TryParse(custom, key, out uint? u, out error) || u == null) {
                         return (false, null, error);
                      }
                      rv.AsUint32 = (uint)u; 
                      return (true,rv,error);
                   case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveQuantity:
-                  if (WebsocketJsonParsers.TryParse<uint>(custom, key, out uint? q, out error) && q != null) {
+                  if (WebsocketJsonParsers.TryParse(custom, key, out uint? q, out error) && q != null) {
                         rv.AsQuantity = new QuantityMeasure() {
                            Absolute = (uint)q
                         };
                         return (true, rv, error);
-                  } else if (WebsocketJsonParsers.TryParse<double>(custom, key, out double? qd, out error) && qd != null){
+                  } else if (WebsocketJsonParsers.TryParse(custom, key, out double? qd, out error) && qd != null){
                      rv.AsQuantity = new QuantityMeasure() {
                            Ratio = (double)qd
                         };
@@ -145,7 +145,7 @@ namespace bts.websocket_core {
       private static (bool, CustomOrderDefinition?, string) TryGetCustomDef(JObject custom, IReadOnlyDictionary<string, DynamicOrderDescriptor> Dynamic_Descriptors) {
          CustomOrderDefinition customdef;
 
-         if (!WebsocketJsonParsers.TryParse<CustomOrderDefinition.CustomOneofCase>(custom, "order_type", out CustomOrderDefinition.CustomOneofCase? order_type, out string status) || order_type == null) {
+         if (!WebsocketJsonParsers.TryParse(custom, "order_type", out CustomOrderDefinition.CustomOneofCase? order_type, out string status) || order_type == null) {
             return (false, null, status);
          }
          switch (order_type) {
@@ -156,7 +156,7 @@ namespace bts.websocket_core {
                   break;
             }
             case CustomOrderDefinition.CustomOneofCase.StopLimit:{
-               if (!WebsocketJsonParsers.TryParse<double>(custom, "stop_price", out double? sp, out status) || sp == null) {
+               if (!WebsocketJsonParsers.TryParse(custom, "stop_price", out double? sp, out status) || sp == null) {
                      return (false, null, status);
                   }
                customdef = new CustomOrderDefinition() {
@@ -167,7 +167,7 @@ namespace bts.websocket_core {
                   break;
             }
             case CustomOrderDefinition.CustomOneofCase.Iceberg: {
-                  if (!WebsocketJsonParsers.TryParse<double>(custom, "show_quantity", out double? sq, out status) || sq == null) {
+                  if (!WebsocketJsonParsers.TryParse(custom, "show_quantity", out double? sq, out status) || sq == null) {
                      return (false, null, status);
                   }
                   if (custom["child"] is not JObject child) {
@@ -234,11 +234,11 @@ namespace bts.websocket_core {
             }
             case CustomOrderDefinition.CustomOneofCase.Cover: {
                if (
-                  !WebsocketJsonParsers.TryParse<uint>( custom, "hedge_ticks", out uint? ht, out status) || ht == null || 
-                  !WebsocketJsonParsers.TryParse<bool>(custom, "cancel_on_external_hedge_cancel", out bool? coehc, out status) || coehc == null ||
-                  !WebsocketJsonParsers.TryParse<bool>(custom, "cancel_hedges_on_cancel", out bool? choc, out status) || choc == null ||
-                  !WebsocketJsonParsers.TryParse<bool>(custom, "detach_hedges", out bool? dh, out status) || dh == null
-                  ) {
+                  !WebsocketJsonParsers.TryParse(custom, "hedge_ticks", out uint? ht, out status) || ht == null || 
+                  !WebsocketJsonParsers.TryParse(custom, "cancel_on_external_hedge_cancel", out bool? coehc, out status) || coehc == null ||
+                  !WebsocketJsonParsers.TryParse(custom, "cancel_hedges_on_cancel", out bool? choc, out status) || choc == null ||
+                  !WebsocketJsonParsers.TryParse(custom, "detach_hedges", out bool? dh, out status) || dh == null
+               ) {
                   return (false, null, status);
                }
                if (custom["child"] is not JObject child) {
@@ -268,7 +268,7 @@ namespace bts.websocket_core {
                break;
             }
             case CustomOrderDefinition.CustomOneofCase.Offset:{
-               if (!WebsocketJsonParsers.TryParse<uint>(custom, "tick_offset", out uint? to, out status) || to == null) {
+               if (!WebsocketJsonParsers.TryParse(custom, "tick_offset", out uint? to, out status) || to == null) {
                      return (false, null, status);
                   }
                   if (custom["child"] is not JObject child) {
@@ -297,7 +297,7 @@ namespace bts.websocket_core {
                return (false,null,"Composite not enabled on api");
             }
             case CustomOrderDefinition.CustomOneofCase.GlbxIceberg:{
-               if (!WebsocketJsonParsers.TryParse<uint>(custom, "max_show_quantity", out uint? msq, out status) || msq == null) {
+               if (!WebsocketJsonParsers.TryParse(custom, "max_show_quantity", out uint? msq, out status) || msq == null) {
                      return (false, null, status);
                   }
                   // mbts.spark.utilities.OrderDataHelper 
@@ -333,7 +333,7 @@ namespace bts.websocket_core {
                   if (custom == null) {
                      return (false, null, "custom not present");
                   }
-                  if(!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(custom, "dynamic_order_type", out WebsocketJsonParsers.StringWrapper? dynamic_order_type, out status) || dynamic_order_type == null ||
+                  if(!WebsocketJsonParsers.TryParse(custom, "dynamic_order_type", out WebsocketJsonParsers.StringWrapper? dynamic_order_type, out status) || dynamic_order_type == null ||
                      !Dynamic_Descriptors.TryGetValue(dynamic_order_type.Value.Val, out var desc)){
                      return (false, null, status);
                   }
@@ -377,11 +377,11 @@ namespace bts.websocket_core {
             return (false, null, "parameters not present");
          }
          if (
-            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(req.parameters,"user",out WebsocketJsonParsers.StringWrapper? usr,out string status) || usr == null ||
-            !WebsocketJsonParsers.TryParse<Venue>(req.parameters, "venue", out Venue? exchange, out status) || exchange == null || 
-            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(req.parameters, "account", out WebsocketJsonParsers.StringWrapper? act, out status) || act == null ||
-            !WebsocketJsonParsers.TryParse<Side>(req.parameters, "side", out Side? side, out status) || side == null || 
-            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(req.parameters, "exchange_security_name", out WebsocketJsonParsers.StringWrapper? exchange_securty_name, out status) || exchange_securty_name == null
+            !WebsocketJsonParsers.TryParse(req.parameters,"user",out WebsocketJsonParsers.StringWrapper? usr,out string status) || usr == null ||
+            !WebsocketJsonParsers.TryParse(req.parameters, "venue", out Venue? exchange, out status) || exchange == null || 
+            !WebsocketJsonParsers.TryParse(req.parameters, "account", out WebsocketJsonParsers.StringWrapper? act, out status) || act == null ||
+            !WebsocketJsonParsers.TryParse(req.parameters, "side", out Side? side, out status) || side == null || 
+            !WebsocketJsonParsers.TryParse(req.parameters, "exchange_security_name", out WebsocketJsonParsers.StringWrapper? exchange_securty_name, out status) || exchange_securty_name == null
             ) {
             return (false, null, status);
          }
@@ -436,7 +436,7 @@ namespace bts.websocket_core {
       }
       static bool TryGetPoints(WebsocketRequest req, [NotNullWhen(true)] out List<PrecomputedTheoPoint>? theos, out string message, IReadOnlyDictionary<uint, SecurityMD> securities) {
          theos = null;
-         if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.PrecomputedTheoPointWrapper>(req.parameters, "points", out WebsocketJsonParsers.PrecomputedTheoPointWrapper[]? wrapper_theos, out message) || wrapper_theos == null) {
+         if (!WebsocketJsonParsers.TryParse(req.parameters, "points", out WebsocketJsonParsers.PrecomputedTheoPointWrapper[]? wrapper_theos, out message) || wrapper_theos == null) {
             return false;
          }
          theos = wrapper_theos.Select(wrapper => {
@@ -469,7 +469,7 @@ namespace bts.websocket_core {
          if (req.parameters == null) {
             return (false, null, "parameters not present");
          }
-         if (!WebsocketJsonParsers.TryParse<uint>(req.parameters, "order_key", out uint? key, out string status) || key == null) {
+         if (!WebsocketJsonParsers.TryParse(req.parameters, "order_key", out uint? key, out string status) || key == null) {
             return (false, null, status);
          }
          Response res = await oc.TryDeleteOrderAsync((uint)key);
@@ -481,7 +481,7 @@ namespace bts.websocket_core {
             return (false, null, "parameters not present");
          }
          if (
-            !WebsocketJsonParsers.TryParse<uint>(request.parameters, "order_key", out uint? key, out string status) || key == null ||
+            !WebsocketJsonParsers.TryParse(request.parameters, "order_key", out uint? key, out string status) || key == null ||
             !WebsocketJsonParsers.TryParse(request.parameters, "quantity", out double? quantity, out status) || quantity == null ||
             !WebsocketJsonParsers.TryParse(request.parameters, "price", out double? price, out status) || price == null
             ) {

src/client/spark_websocket/Core/WebsocketServerConfigExtensions.cs

@@ -3,11 +3,11 @@ namespace bts.websocket_core;
 using Microsoft.AspNetCore.Builder;
 
 public static class WebsocketServerConfigExtensions {
-   // public static string Uri(this WebsocketServerConfig config) =>
-   //    $"{config.Scheme}://{config.Host}:{config.Port}";
+   public static string Uri(this WebsocketServerConfig config) =>
+      $"{config.Scheme}://{config.Host}:{config.Port}";
 
-   // public static WebSocketOptions get_ws_options(this WebsocketServerConfig config) =>
-   //    new() {
-   //       KeepAliveInterval = TimeSpan.FromSeconds(config.KeepAliveIntervalSecs)
-   //    };
+   public static WebSocketOptions get_ws_options(this WebsocketServerConfig config) =>
+      new() {
+         KeepAliveInterval = TimeSpan.FromSeconds(config.KeepAliveIntervalSecs)
+      };
 }

src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs

@@ -1,21 +1,14 @@
 namespace bts.websocket_core {
-   using BCG;
-
    using bts.api;
    using bts.api.orderman;
    using bts.md;
 
-   using Bts.DataStream;
-   using Bts.DataStreamCore;
-   using Bts.DataStreamExpressions;
-
    using mbts.spark;
 
    using Newtonsoft.Json.Linq;
 
    using Spark.Api.MarketData;
    using Spark.Api.Orders;
-   using Spark.Core.Collections;
 
    using System;
    using System.Diagnostics.CodeAnalysis;
@@ -29,7 +22,6 @@ namespace bts.websocket_core {
          PRICE_STATS,
          PRICE_TICKERS,
          FILLS,
-         DATA_STREAM,
          UNKNOWN = 0
       }
       public bool Valid { get; set; } = true;
@@ -99,47 +91,6 @@ namespace bts.websocket_core {
                hdlr.SendMessage(WebsocketJsonParsers.to_json<OrderRecordTupleWithRoot, uint>(upd)));
          };
       }
-
-      private static Action<WebsocketSubscriptionHandler> GetDataStreamSubscriber(ViewStream vs, string subId) {
-         return Subscribe;
-
-         void Subscribe(WebsocketSubscriptionHandler hdlr) {
-            // Declare dictionaries here to reduce allocations in the callback. Use a lock
-            // to be safe (callbacks are likely single-threaded, this is just insurance).
-            var syncRoot = new object();
-            var removedRows = new Dictionary<int, object>();
-            var changedRows = new Dictionary<int, object>();
-            var addedRows = new Dictionary<int, object>();
-
-            hdlr.dis = vs.AsObservable().Subscribe(OnData);
-
-            void OnData(DataPacket dp) {
-               lock (syncRoot) {
-                  removedRows.Clear();
-                  changedRows.Clear();
-                  addedRows.Clear();
-                  foreach (IBlueRow r in dp.deletedRows) {
-                     var id = r.Id();
-                     removedRows[id] = r;
-                  }
-                  foreach (IBlueRow r in dp.addedRows) {
-                     var id = r.Id();
-                     if (removedRows.Remove(id)) {
-                        changedRows.Add(id, r);
-                     } else {
-                        addedRows.Add(id, r);
-                     }
-                  }
-                  hdlr.SendMessage(new {
-                     added = addedRows.Values,
-                     changed = changedRows.Values,
-                     removed = removedRows.Values,
-                  });
-               }
-            }
-         }
-      }
-
       private static Action<WebsocketSubscriptionHandler> GetFillsSubscriber(FillsSubscriptionFilter cri) {
          return (WebsocketSubscriptionHandler hdlr) => {
             hdlr.dis = hdlr._OrdersClient.fills(cri).Subscribe(
@@ -198,35 +149,6 @@ namespace bts.websocket_core {
          return true;
       }
 
-      static bool TryGetDataStream(WebsocketRequest req, [NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(false)] out string error) {
-         stream = null;
-         error = "";
-         var query = (req.parameters?["query"] as JValue)?.Value<string>();
-         if (query is null) {
-            error = "'params.query' field not found (expected a string)";
-            return false;
-         }
-         var exp = DataStreamExpressionsInterpreter.Evaluate(query);
-         if (exp is null) {
-            error = "'params.query' expression has invalid syntax";
-            return false;
-         }
-         Exception? exception = null;
-         try {
-            stream = exp.getView(new Dictionary<string, StatementExpression>(0));
-         } catch (Exception ex) {
-            exception = ex;
-         }
-         if (stream is null) {
-            error = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)";
-            if (exception is not null) {
-               error += ". " + exception.Message;
-            }
-            return false;
-         }
-         return true;
-      }
-
       static bool TryGetFilter(WebsocketRequest req, [NotNullWhen(true)] out SecurityCriteria? cri, out string message) {
          cri = null;
          if (req.parameters?["filter"] is not JObject filter) {
@@ -234,13 +156,13 @@ namespace bts.websocket_core {
             return false;
          }
          if (
-            !WebsocketJsonParsers.TryParse<Venue>(filter, "venue", out Venue[]? exchanges, out message) ||
-            !WebsocketJsonParsers.TryParse<SecurityType>(filter, "kind", out SecurityType[]? types, out message) ||
-            !WebsocketJsonParsers.TryParse<PutOrCall>(filter, "call_or_put", out PutOrCall[]? callputs, out message) ||
-            !WebsocketJsonParsers.TryParse<DateTime>(filter, "expiration", out DateTime[]? expiration, out message) || 
-            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringListWrapper>(filter, "symbol", out WebsocketJsonParsers.StringListWrapper? symbols, out message) ||
-            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringListWrapper>(filter, "name", out WebsocketJsonParsers.StringListWrapper? names, out message) ||
-            !WebsocketJsonParsers.TryParse<double>(filter, "strikes", out double[]? strikes, out message)
+            !WebsocketJsonParsers.TryParse(filter, "venue", out Venue[]? exchanges, out message) ||
+            !WebsocketJsonParsers.TryParse(filter, "kind", out SecurityType[]? types, out message) ||
+            !WebsocketJsonParsers.TryParse(filter, "call_or_put", out PutOrCall[]? callputs, out message) ||
+            !WebsocketJsonParsers.TryParse(filter, "expiration", out DateTime[]? expiration, out message) || 
+            !WebsocketJsonParsers.TryParse(filter, "symbol", out WebsocketJsonParsers.StringListWrapper? symbols, out message) ||
+            !WebsocketJsonParsers.TryParse(filter, "name", out WebsocketJsonParsers.StringListWrapper? names, out message) ||
+            !WebsocketJsonParsers.TryParse(filter, "strikes", out double[]? strikes, out message)
          ) {
             return false;
          }
@@ -268,8 +190,8 @@ namespace bts.websocket_core {
          hdlr = null;
 
          if (
-            !WebsocketJsonParsers.TryParse<SUBSCRIPTION_TYPE>(req.parameters, "type", out SUBSCRIPTION_TYPE? type, out status) || status != "" ||
-            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(req.parameters, "subscription_id", out WebsocketJsonParsers.StringWrapper? id, out status) || id == null) {
+            !WebsocketJsonParsers.TryParse(req.parameters, "type", out SUBSCRIPTION_TYPE? type, out status) || status != "" ||
+            !WebsocketJsonParsers.TryParse(req.parameters, "subscription_id", out WebsocketJsonParsers.StringWrapper? id, out status) || id == null) {
             return false;
          }
         
@@ -325,14 +247,6 @@ namespace bts.websocket_core {
                   sub = GetOrdersSubscriber(filter);
                   break;
                }
-            case SUBSCRIPTION_TYPE.DATA_STREAM: {
-                  // TODO: make this nullable
-                  if (!TryGetDataStream(req, out var filter, out status)) {
-                     return false;
-                  }
-                  sub = GetDataStreamSubscriber(filter, id.Value.Val);
-                  break;
-               }
             default: {
                   status = "Incorrect subscription type";
                   return false;

src/client/spark_websocket/WebsocketServerConfig.proto

@@ -0,0 +1,13 @@
+syntax = "proto3";
+
+option csharp_namespace = "bts.websocket_core";
+
+package bts.websocket_core;
+
+message WebsocketServerConfig {
+  string host = 1;
+  uint32 port = 2;
+  string scheme = 3;
+  string default_end_point = 4;
+  double keep_alive_interval_secs = 5;
+}

src/client/spark_websocket/spark_websocket.csproj

@@ -26,8 +26,12 @@
   <ItemGroup>
     <ProjectReference Include="..\spark_core\spark_core.csproj" />
     <ProjectReference Include="..\spark_api\spark_api.csproj" />
-    <ProjectReference Include="..\..\DataStreamSuite\DataStream\DataStream.csproj" />
-    <ProjectReference Include="..\..\DataStreamSuite\SharedStreams\SharedStreams.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Proto Include="WebsocketServerConfig.proto">
+      <OutDir>$(ProtoOutputPath)/spark_websocket</OutDir>
+    </Proto>
   </ItemGroup>
 
 </Project>

src/lib_domain/CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.23)
 project(domain)
 include(${CMAKE_SOURCE_DIR}/cmake/InitLibrary.cmake)
 
-init_library(STATIC)
+init_library(SHARED)
 target_link_libraries(
   ${PROJECT_NAME}
   PUBLIC

src/lib_http/http/concurren_runtime.h

@@ -1,11 +1,32 @@
 #pragma once
 
-#include "system/threading/atomic_singleton.h"
+#include "system/logging/logging.h"
 
-#include <concurrencpp/runtime/runtime.h>
+#include <curlpp/cURLpp.hpp>
 
 namespace bts::http {
 
-using ConcurrenRuntime = bts::system::AtomicSingleton<concurrencpp::runtime>;
+/**
+ * Manages the (global) lifetime of curlpp for the program. An atomic reference
+ * counter is used to synchronize initialization, and eventual termination.
+ */
+struct ConcurrenRuntime {
+   explicit ConcurrenRuntime() {
+      if (reference_count_++ == 0) {
+         LOG(Gen, Info, "Initializing curl");
+         curlpp::initialize();
+      }
+   }
+
+   ~ConcurrenRuntime() {
+      if (--reference_count_ == 0) {
+         LOG(Gen, Info, "Terminating curl");
+         curlpp::terminate();
+      }
+   }
+
+ private:
+   static inline std::atomic<int> reference_count_{0};
+};
 
 } // namespace bts::http

src/lib_http/http/http_client.h

@@ -1,12 +1,13 @@
 #pragma once
 
-#include "http/concurren_runtime.h"
 #include "http/http_response.h"
 #include "http/i_http_response.h"
 
 #include <concurrencpp/concurrencpp.h>
 #include <concurrencpp/executors/executor.h>
+#include <concurrencpp/runtime/runtime.h>
 #include <cstdint>
+#include <mutex>
 #include <string>
 #include <tuple>
 #include <unordered_map>
@@ -23,16 +24,10 @@ class HttpClient {
        const std::unordered_map<std::string, std::string> &headers,
        IHttpResponse *response);
 
-   /**
-    * Performs an http GET request asynchronously
-    */
    static concurrencpp::result<HttpResponse> get_async(
        const std::string &url,
        const std::unordered_map<std::string, std::string> &headers);
 
-   /**
-    * Performs an http GET request asynchronously
-    */
    static concurrencpp::result<HttpResponse> get_async(
        const std::string &url,
        const std::unordered_map<std::string, std::string> &headers,
@@ -56,6 +51,10 @@ class HttpClient {
        const std::string &url,
        const std::string &client_id,
        const std::string &client_secret);
+
+ private:
+   static std::mutex global_runtime_mx_;
+   static std::optional<concurrencpp::runtime> global_runtime_;
 };
 
 //////////////////////////////////////////////////
@@ -65,8 +64,11 @@ class HttpClient {
 inline concurrencpp::result<HttpResponse> HttpClient::get_async(
     const std::string &url,
     const std::unordered_map<std::string, std::string> &headers) {
-   static ConcurrenRuntime global_runtime;
-   return get_async(url, headers, global_runtime.value().thread_executor());
+   if (!global_runtime_) {
+      std::lock_guard<std::mutex> lock{global_runtime_mx_};
+      global_runtime_.emplace();
+   }
+   return get_async(url, headers, global_runtime_->thread_executor());
 }
 
 inline concurrencpp::result<HttpResponse> HttpClient::get_async(

src/lib_python_md/CMakeLists.txt

@@ -11,6 +11,7 @@ cmake_minimum_required(VERSION 3.23)
 #       - https://docs.python.org/3/c-api/module.html#c.PyModuleDef
 #   3) python_md_providers
 #     * Copies py md provider plugins to a runtime dir where sparkd can find them
+#       load them as plugins.
 #     * Doesn't output a c++ lib or exe
 
 project(python_md)

src/lib_spark/cmake/GetLinuxClientLinkLibraries.cmake

@@ -2,9 +2,7 @@ function(get_link_libraries dest)
   find_package(fmt CONFIG REQUIRED)
   find_package(OpenSSL REQUIRED)
   find_package(cryptopp CONFIG REQUIRED)
-  find_package(Catch2 CONFIG REQUIRED COMPONENTS Catch2)
   set(${dest}
-    Catch2::Catch2
     system
     domain
     http

src/lib_spark/cmake/GetLinuxClientSources.cmake

@@ -5,12 +5,6 @@ function(get_proto_sources cfg_srcs non_cfg_srcs service_srcs)
     ${PROJECT_NAME}/config/protos/*.proto
     ${PROJECT_NAME}/api/md/*.proto
   )
-  list(APPEND _cfg_srcs
-    system/protobufs/config/system_config.proto
-    system/protobufs/config/threads.proto
-    system/protobufs/logging.proto
-    domain/protobufs/security_metadata_collection.proto
-  )
   file(
     GLOB_RECURSE _service_srcs
     RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
@@ -32,7 +26,6 @@ function(get_cpp_sources dest)
   file(
     GLOB_RECURSE _dest
     RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
-    ${PROJECT_NAME}/spark_data_directory.cpp
     ${PROJECT_NAME}/api/*.cpp
     ${PROJECT_NAME}/config/*.cpp
     ${PROJECT_NAME}/domain/*.cpp

src/lib_spark/spark/all_cpps.txt

@@ -0,0 +1,348 @@
+./api/core/ApiClientInterop.cpp
+./api/core/ApiServerInterop.cpp
+./api/md/MarketDataConfigUtils.cpp
+./api/public/ApiServer.cpp
+./api/public/ClientChannel.cpp
+./api/public/QuickbusClientChannel.cpp
+./api/public/ServerChannel.cpp
+./api/public/ServiceContext.cpp
+./api/service_helpers/AdminServiceUtils.cpp
+./api/service_helpers/AlgoServiceUtils.cpp
+./api/service_helpers/PositionUtilities.cpp
+./api/test/ApiTests.cpp
+./api/test/HelloServiceServer.cpp
+./communincation/websockets/tests/WebsocketSyncClientTests.cpp
+./config/ConfigDownloader.cpp
+./config/ConfigErrorCollector.cpp
+./config/ConfigPropertyEvaluator.cpp
+./config/ConfigQuerier.cpp
+./config/ConfigValueQueue.cpp
+./config/ConfigurationException.cpp
+./config/ConfigurationState.cpp
+./config/PreprocessInputStream.cpp
+./config/config.cpp
+./config/tests/HTTPServer.cpp
+./config/tests/config_ref_tests.cpp
+./config/tests/config_tests.cpp
+./config/text_format/text_format.cpp
+./datastreams/Tables/tests/stream_tests.cpp
+./domain/DomainSecurityUtils.cpp
+./domain/NewOrder.cpp
+./domain/OrderDefinition.cpp
+./domain/OrderStatus.cpp
+./domain/OrderUpdate.cpp
+./domain/TimeInForce.cpp
+./domain_model/Security.cpp
+./domain_model/SecurityTests.cpp
+./domain_model/sod.cpp
+./domain_model/tests/domain_tests.cpp
+./expression/BooleanGrammar.cpp
+./expression/ExpressionEvaluator.cpp
+./expression/ParserExceptions.cpp
+./expression/Value.cpp
+./expression/tests/expression_parser_tests.cpp
+./io/QueryParams.cpp
+./io/URL.cpp
+./io/http/CurlHandle.cpp
+./io/http/HTTPClient.cpp
+./io/http/HTTPHeaders.cpp
+./io/http/HTTPRequest.cpp
+./io/http/HTTPResponse.cpp
+./io/websocket/IJsonRpcHandler.cpp
+./io/websocket/JsonRpcHandler.cpp
+./io/websocket/WebsocketClient.cpp
+./io/websocket/WebsocketClientConfiguration.cpp
+./io/websocket/WebsocketError.cpp
+./io/websocket/WebsocketMessage.cpp
+./io/websocket/WebsocketSSLClientInternal.cpp
+./io/websocket/tests/WebsocketClientTests.cpp
+./jetfix/codec/FIXArray.cpp
+./jetfix/common/OutFIX.cpp
+./jetfix/connection/AsyncReader.cpp
+./jetfix/connection/IConnection.cpp
+./jetfix/connection/SocketConnection.cpp
+./jetfix/gateway/CoreGateway.cpp
+./jetfix/gateway/JetFIXGateway.cpp
+./jetfix/gateway/Session.cpp
+./jetfix/store/FIXFileStore.cpp
+./jetfix/store/session_name.cpp
+./jetfix/tests/async_reader_tests.cpp
+./jetfix/tests/decoding_tests.cpp
+./jetfix/tests/fix_encoding_stream_tests.cpp
+./jetfix/tests/fix_fragment_tests.cpp
+./jetfix/tests/jetfix_performance_tests.cpp
+./jetfix/tests/sample_fix.cpp
+./jetfix/tests/store_tests.cpp
+./logging/LoggerHub.cpp
+./logging/Sink.cpp
+./logging/logging.cpp
+./logging/tests/logging_tests.cpp
+./md/Diag/MDPeriodicDumper.cpp
+./md/Diag/ShmDumper.cpp
+./md/GeneralMarketDataListener.cpp
+./md/MBOProcessor.cpp
+./md/MBOServiceServer.cpp
+./md/MDClient.cpp
+./md/MDConsumer.cpp
+./md/MDConsumerHandle.cpp
+./md/MDConsumerInterop.cpp
+./md/MDProducer.cpp
+./md/MDProducerHandle.cpp
+./md/MemorySegmentUtils.cpp
+./md/PriceSize.cpp
+./md/Product.cpp
+./md/SecurityCache.cpp
+./md/SecurityDB.cpp
+./md/SecurityIDRecorder.cpp
+./md/SecurityMapper.cpp
+./md/SecurityMarketState.cpp
+./md/SecurityOracle.cpp
+./md/SecurityStore.cpp
+./md/SettlementRecorder.cpp
+./md/Shm/FixedSecurity.cpp
+./md/ShmSegmentName.cpp
+./md/SyntheticSecurityService.cpp
+./md/cme/MDP3Utils.cpp
+./md/common.cpp
+./md/ice/ICEUtils.cpp
+./md/relay/QueryServer.cpp
+./md/relay/RelayClient.cpp
+./md/relay/RelayRecorder.cpp
+./md/service/MarketDataServiceServer.cpp
+./md/test/Co2ManagedTestHelper.cpp
+./md/test/Co2TestInstrument.cpp
+./md/test/mbo_tests.cpp
+./md/test/md2_tests.cpp
+./md/test/md_test_utils.cpp
+./md/test/synthetic_book_merge_tests.cpp
+./md/vendor/OnixsUtils.cpp
+./mercury/actors/IPollableActor.cpp
+./mercury/broker/Broker.cpp
+./mercury/broker/BrokerProxy.cpp
+./mercury/broker/ILocalBroker.cpp
+./mercury/broker/IRemoteBroker.cpp
+./mercury/common/mercury_common.cpp
+./mercury/events/TimerEvent.cpp
+./mercury/executor/Executor.cpp
+./mercury/topics/Topic.cpp
+./metrics/metrics.cpp
+./metrics/system_metrics.cpp
+./orderman/ChildOrder.cpp
+./orderman/Governor.cpp
+./orderman/OrderBuilder.cpp
+./orderman/OrderRouter.cpp
+./orderman/OrderServiceServer.cpp
+./orderman/OrderServices.cpp
+./orderman/actors/AsyncFactory.cpp
+./orderman/actors/FillStore.cpp
+./orderman/actors/MDSupplementarySource.cpp
+./orderman/actors/OrderStore.cpp
+./orderman/actors/SODPosition.cpp
+./orderman/actors/SecuritySrc.cpp
+./orderman/actors/ServiceBridge.cpp
+./orderman/actors/order_manager/OrderManager.cpp
+./orderman/actors/order_manager/OrderServicesData.cpp
+./orderman/actors/order_manager/PositionIndex.cpp
+./orderman/calculations/TheoCalculator.cpp
+./orderman/ccapi/CCAPIDeribitHelpers.cpp
+./orderman/ccapi/CCAPIDeribitOrderMux.cpp
+./orderman/ccapi/CCAPIOrderCallbackMsgs.cpp
+./orderman/ccapi/CCAPIOrderState.cpp
+./orderman/ccapi/CCAPISingleOrderManager.cpp
+./orderman/events/CancelRejectEvent.cpp
+./orderman/events/FilledPositionEvent.cpp
+./orderman/events/NewExecutionReportEvent.cpp
+./orderman/gateways/CCAPIExecutionGateway.cpp
+./orderman/gateways/ExecutionGatewayFactory.cpp
+./orderman/gateways/FakeExecutionGateway.cpp
+./orderman/gateways/GLBXExecutionGateway.cpp
+./orderman/gateways/GLBXiLink3ExecutionGateway.cpp
+./orderman/gateways/ICEExecutionGateway.cpp
+./orderman/gateways/IFIXGenericAdapter.cpp
+./orderman/gateways/sim/SimExecutionGateway.cpp
+./orderman/gateways/sim/SimExecutionGatewayFactory.cpp
+./orderman/gateways/sim/SimExecutionGatewayUniqueWrapper.cpp
+./orderman/ilink3/IL3ConnectionHandler.cpp
+./orderman/ilink3/IL3GW.cpp
+./orderman/ilink3/IL3Parser.cpp
+./orderman/ilink3/IL3Reactor.cpp
+./orderman/ilink3/ILink3Utils.cpp
+./orderman/ilink3/SessionStore.cpp
+./orderman/ilink3/test/ILinkGWTests.cpp
+./orderman/limiter/LimitStore.cpp
+./orderman/limiter/Limiter.cpp
+./orderman/limiter/ProductLimitNode.cpp
+./orderman/limiter/ProductLimitTree.cpp
+./orderman/limiter/SecurityLimitNode.cpp
+./orderman/limiter/SecurityLimitTree.cpp
+./orderman/limiter/tests/limit_tests.cpp
+./orderman/order_factory/DynamicOrderRegistry.cpp
+./orderman/order_factory/OrderFactory.cpp
+./orderman/orderman_utils/BookUtils.cpp
+./orderman/orderman_utils/ExecutionReport.cpp
+./orderman/orderman_utils/fix_order_codec.cpp
+./orderman/orderman_utils/orderman_utils.cpp
+./orderman/orders/CompositeOrder.cpp
+./orderman/orders/DynamicOrder.cpp
+./orderman/orders/EMNativeOrder.cpp
+./orderman/orders/GLBXNativeOrder.cpp
+./orderman/orders/ICENativeOrder.cpp
+./orderman/orders/Order.cpp
+./orderman/orders/TimeBoxOrder.cpp
+./orderman/orders/TriggerOrder.cpp
+./orderman/orders/algo/AlgoWrapperOrder.cpp
+./orderman/orders/algo/DynamicHelpers.cpp
+./orderman/orders/algo/IAlgoServices.cpp
+./orderman/orders/algo/IBeforeLaunchAlgoServices.cpp
+./orderman/orders/algo/PositionUpdate.cpp
+./orderman/orders/algo/algo_orders/CoverAlgo.cpp
+./orderman/orders/algo/algo_orders/DeleteAndCoverAlgo.cpp
+./orderman/orders/algo/algo_orders/DormanAlgo.cpp
+./orderman/orders/algo/algo_orders/IcebergAlgo.cpp
+./orderman/orders/algo/algo_orders/JoinAlgo.cpp
+./orderman/orders/algo/algo_orders/LeanAlgo.cpp
+./orderman/orders/algo/algo_orders/LimitCoverAlgo.cpp
+./orderman/orders/algo/algo_orders/OffsetAlgo.cpp
+./orderman/orders/algo/algo_orders/OnSignalAlgo.cpp
+./orderman/orders/algo/algo_orders/ProRataAlgo.cpp
+./orderman/orders/algo/algo_orders/PyramidAlgo.cpp
+./orderman/orders/algo/algo_orders/QuotingAlgo.cpp
+./orderman/orders/algo/algo_orders/RegisterDorman.cpp
+./orderman/orders/algo/algo_orders/ScratchAlgo.cpp
+./orderman/orders/algo/algo_orders/SniperAlgo.cpp
+./orderman/orders/algo/algo_orders/SplitterAlgo.cpp
+./orderman/orders/algo/algo_orders/TASAlgo.cpp
+./orderman/orders/algo/algo_orders/TASCounterAlgo.cpp
+./orderman/orders/algo/algo_orders/TaskingEyeAlgo.cpp
+./orderman/orders/algo/algo_orders/TopAlgo.cpp
+./orderman/orders/algo/algo_orders/VolAlgo.cpp
+./orderman/orders/algo/algo_orders/limit_cover/LimitCoverAdapter.cpp
+./orderman/orders/algo/algo_orders/tests/AlgoTestDriver.cpp
+./orderman/orders/algo/algo_orders/tests/ExecutiontTesterAlgo.cpp
+./orderman/orders/algo/algo_orders/tests/LimitCoverTests.cpp
+./orderman/orders/algo/algo_orders/tests/dorman_tests.cpp
+./orderman/orders/algo/algo_orders/tests/limit_cover/LimitCoverState.cpp
+./orderman/orders/algo/algo_orders/tests/limit_cover/TestSteps.cpp
+./orderman/orders/algo/algo_orders/tests/splitter_tests.cpp
+./orderman/orders/book_algos/BookAlgo.cpp
+./orderman/orders/book_algos/BookOrderBase.cpp
+./orderman/orders/book_algos/OCOOrder.cpp
+./orderman/orders/book_algos/PricerOrder.cpp
+./orderman/orders/glbx/IGLBXOrderEncoder.cpp
+./orderman/orders/glbx/iLink2OrderEncoder.cpp
+./orderman/orders/glbx/iLink3OrderEncoder.cpp
+./orderman/orders/queue_holder/QueueHolderOrder.cpp
+./orderman/orders/queue_holder/state_workers/QHOEndStateWorker.cpp
+./orderman/orders/queue_holder/state_workers/QHOStartStateWorker.cpp
+./orderman/orders/queue_holder/state_workers/QHOWaitingForAckStateWorker.cpp
+./orderman/orders/queue_holder/state_workers/QHOWaitingForCancelAllStateWorker.cpp
+./orderman/orders/queue_holder/state_workers/QHOWorkingStateWorker.cpp
+./orderman/orders/spreader_orders/SpreaderOrder.cpp
+./orderman/orders/spreader_orders/spreader/LegFillState.cpp
+./orderman/orders/spreader_orders/spreader/LegPropertyData.cpp
+./orderman/orders/spreader_orders/spreader/NLegsHedgeRecompData.cpp
+./orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp
+./orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp
+./orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp
+./orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp
+./orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp
+./orderman/orders/spreader_orders/spreader/SpreaderFractionalQtyHelper.cpp
+./orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp
+./orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp
+./orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp
+./orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp
+./orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp
+./orderman/orders/spreader_orders/spreader/hedger/HedgerManager.cpp
+./orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp
+./orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp
+./orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp
+./orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp
+./orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp
+./orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp
+./orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp
+./orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp
+./orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp
+./orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp
+./orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp
+./orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp
+./orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp
+./orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp
+./orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp
+./orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp
+./orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp
+./orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp
+./orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp
+./orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp
+./orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp
+./orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.cpp
+./orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp
+./orderman/orders/spreader_orders/tests/spreader_tests.cpp
+./orderman/tests/CCApiExecutionTests.cpp
+./orderman/tests/fat_wrapper_tests.cpp
+./os/os.cpp
+./os/tests/os_tests.cpp
+./persistence/IDGenerator.cpp
+./persistence/IStore.cpp
+./persistence/PersistenceServiceServer.cpp
+./persistence/SqliteStore.cpp
+./persistence/persistence_utils.cpp
+./persistence/tests/PersistenceServiceTests.cpp
+./plugin/SparkServices.cpp
+./plugin/plugin.cpp
+./quickbus/core/LocalNode.cpp
+./quickbus/core/Quickbus.cpp
+./quickbus/core/SubscriptionID.cpp
+./quickbus/core/TableID.cpp
+./quickbus/message_model/Update.cpp
+./quickbus/message_model/merge_updates.cpp
+./quickbus/network/QBConnection.cpp
+./quickbus/network/QBConnectionManager.cpp
+./quickbus/network/endpoint_description.cpp
+./quickbus/pubsub/TablePublisher.cpp
+./quickbus/pubsub/TableSubscriber.cpp
+./quickbus/pubsub/tests/qb_pubsub_tests.cpp
+./quickbus/routing/INode.cpp
+./quickbus/routing/NodeID.cpp
+./quickbus/routing/Router.cpp
+./quickbus/rpc/RPCClient.cpp
+./quickbus/rpc/RPCServer.cpp
+./quickbus/rpc/RPCSessionID.cpp
+./quickbus/rpc/rpc_utils.cpp
+./quickbus/rpc/tests/qb_rpc_tests.cpp
+./quickbus/tests/quickbus_tests.cpp
+./testing/catch_test_runner.cpp
+./tracing/PerfLog.cpp
+./tracing/bts_trace.cpp
+./utils/AuthSessionManager.cpp
+./utils/AuthToken.cpp
+./utils/Dispatcher.cpp
+./utils/FileUtils.cpp
+./utils/GrowableSPSCQueue.cpp
+./utils/HTTPStream.cpp
+./utils/MathUtils.cpp
+./utils/PBUtils.cpp
+./utils/Puid.cpp
+./utils/SecureLogon.cpp
+./utils/SharedMemory.cpp
+./utils/Sqlite_utils.cpp
+./utils/Uuid.cpp
+./utils/crypto/CipherType.cpp
+./utils/crypto/CryptoInit.cpp
+./utils/crypto/Digest.cpp
+./utils/crypto/KeyCodeHelper.cpp
+./utils/crypto/PasswordCallback.cpp
+./utils/crypto/PrivateKey.cpp
+./utils/crypto/PublicKey.cpp
+./utils/crypto/iostreambio.cpp
+./utils/datetime.cpp
+./utils/download.cpp
+./utils/file_utils.cpp
+./utils/init_spark_lib.cpp
+./utils/is_unittest_environment.cpp
+./utils/sftp.cpp
+./utils/stats.cpp
+./utils/tests/AuthTokenTests.cpp
+./utils/tests/IDGeneratorTests.cpp
+./utils/tests/MathUtilsTests.cpp
+./utils/tests/SingleWriterConcurrentVectorTests.cpp
+./utils/thread_manager.cpp

src/lib_spark/spark/md/GeneralMarketDataListener.cpp

@@ -77,78 +77,73 @@ struct IncrementalBookData {
    std::map<domain::SecurityID, domain::Book> book_map;
 
    bool update_incremental(const IMarketDataListener::BookOp & op) {
-    // Retrieve the Book from the map and update it
-    domain::Book* book = get_book(op.id);
-    if (book == nullptr)
-        return false;
+      // TODO: Arham/Adeel, implement this
+      // retreive the Book from the map and udpate it. domain::Book has a lot of functions
+      // already, add more if you want to. but the idea is to update the book in place, ie
+      // if a level needs to be added push the lower levels out
 
-    // Choosing the correct side of the book (bids or asks)
-    auto& book_side = (op.side == domain::Side::Buy) ? book->bids() : book->asks();
-    std::string side_str = (op.side == domain::Side::Buy) ? "Buy" : "Sell";
+      //getting book
+      domain::Book* book = get_book(op.id);
+      if(book == nullptr)
+         return false;
+      
+      //checking if side is Buy or Sell
+      auto& book_side = (op.side == api::Side::SIDE_BUY) ? book->bids() : book->asks();
 
-    // Capturing the book state before the update
-    std::stringstream book_before;
-    book_before << "Book before update (" << side_str << " side):\n";
-    for (const auto& level : book_side) {
-        book_before << "Price: " << level.price << ", Quantity: " << level.direct_quantity << "\n";
-    }
+      size_t insert_level = 0;
+      bool level_found = false;
 
-    size_t insert_level = 0;
-    bool level_found = false;
-
-    // Logic to find where to insert or update the level
-    if (op.side == domain::Side::Buy) {
-        while (insert_level < book_side.size()) {
-            if (op.price == book_side[insert_level].price) {
-                level_found = true;
-                break;
+      if(op.side == api::Side::SIDE_BUY)
+      {
+         //going through book
+         while(insert_level < book_side.size())
+         {
+            //op.price_size?
+            if(op.price == book_side[insert_level].price)
+            {
+               level_found = true;
+               break;
             }
-            if (op.price > book_side[insert_level].price) {
-                break;
+            //if its not equal and its not larger, we go to next level
+            if(op.price > book_side[insert_level].price)
+            {
+               break;
+            }
+            //moving to next level 
+            insert_level++;
+         }
+      }else{
+         while(insert_level < book_side.size())
+         {
+            if(op.price == book_side[insert_level].price)
+            {
+               level_found = true;
+               break;
+            }
+            if (op.price < book_side[insert_level].price)
+            {
+               break;
             }
             insert_level++;
+         }
+      }
+      
+      //inserting and pushing levels
+      if(!level_found)
+      {
+         // shift all lower levels down
+        for (size_t i = book_side.size() - 1; i > insert_level; i--) 
+        {
+            book_side[i] = book_side[i - 1];
         }
-    } else {
-        while (insert_level < book_side.size()) {
-            if (op.price == book_side[insert_level].price) {
-                level_found = true;
-                break;
-            }
-            if (op.price < book_side[insert_level].price) {
-                break;
-            }
-            insert_level++;
-        }
-    }
-
-    // Insert or update the book level
-    if (!level_found) {
-        book_side.insert(book_side.begin() + insert_level, {op.price, op.quantity});
-    } else {
-        book_side[insert_level].direct_quantity = op.quantity;
-    }
-
-    // Capturing the book state after the update
-    std::stringstream book_after;
-    book_after << "Book after update (" << side_str << " side):\n";
-    for (const auto& level : book_side) {
-        book_after << "Price: " << level.price << ", Quantity: " << level.direct_quantity << "\n";
-    }
-
-    // Capturing the operation details
-    std::stringstream op_details;
-    op_details << "Operation: Price: " << op.price << ", Quantity: " << op.quantity 
-               << ", Side: " << side_str << "\n";
-
-    // Combine the before/after and operation details into one string
-    std::string book_before_after_op = book_before.str() + "\n" + book_after.str() + "\n" + op_details.str();
-
-    // Assertion to check the book is sorted
-    bts_assert_msgf(book->is_sorted(), "book not sorted {}", book_before_after_op);
-
-    return true;
-}
+        // insert the new level with its price
+        //book_side[insert_level].price.to_double() = op.price_size.price();
 
+      }
+        book_side[insert_level].price =op.price;
+        book_side[insert_level].direct_quantity =op.quantity;
+      return true;
+   }
    domain::Book const *get_const_book(domain::SecurityID id) const {
       auto it = book_map.find(id);
       if (it == book_map.end()) {

src/lib_spark/spark/md/IMarketDataListener.h

@@ -24,7 +24,7 @@ class IMarketDataListener {
    enum BookOpType { Add = 0, MODIFY, REMOVE  };
    struct BookOp {
       domain::SecurityID id;
-      domain::Side side;
+      bts::api::Side side;
       BookOpType op_type;
       // for op == REMOVE size in price_size is no-op
       domain::Price price;

src/lib_spark/spark/md/MDProducer.cpp

@@ -50,9 +50,6 @@ class MDProducerImpl {
       // We should monitor and adjust based on production use for optimal
       // performance
       _cold.header().has_depth = enable_depth(symbol, type);
-      // Initialize the heartbeat timestamp so MDConsumer doesn't detect a dropped hb
-      // before the hb timer, which runs on a background thread in a 200ms loop, ticks.
-      heartbeat();
 
       // static_assert(sizeof(HotEntry) <= 32,"HotEntry should be less than 32
       // bytes");

src/lib_spark/spark/md/shm/cold_entry.cpp

@@ -1,4 +1,5 @@
 #include "cold_entry.h"
+#include "system/bool_name.h"
 
 #include <nlohmann/json.hpp>
 
@@ -26,7 +27,7 @@ void to_json(nlohmann::json &j, const ColdEntry &c) {
        c.misc.previous_settlement_date.day, &prev_settle_date);
    j["prev_settle_date"] = prev_settle_date.to_iso_string();
    j["status"] = domain::protobufs::TradingStatus_Name(c.exchange_status);
-   j["deleted"] = c.deleted;
+   j["deleted"] = core::BoolName::yes_no(c.deleted);
 }
 
 } // namespace bts::md

src/lib_spark/spark/md/shm/cold_header.cpp

@@ -1,10 +1,12 @@
 #include "cold_header.h"
+#include "system/bool_name.h"
 #include "system/time_stamp.h"
 
 #include <nlohmann/json.hpp>
 
 namespace bts::md {
 
+using bts::core::BoolName;
 using std::size_t;
 
 void to_json(nlohmann::json &j, const ColdHeader *c) { to_json(j, *c); }
@@ -17,11 +19,11 @@ void to_json(nlohmann::json &j, const ColdHeader &c) {
        {"create_time",
         core::TimeStamp{c.create_time.to_uint64()}.to_iso_string()},
        {"hb", core::TimeStamp{c.heartbeat.to_uint64()}.to_iso_string()},
-       {"disable_hb", c.disable_heartbeat},
+       {"disable_hb", BoolName::yes_no(c.disable_heartbeat)},
        {"hb_ms", c.heartbeat_ms},
        {"hb_boot_secs", c.heartbeat_boot_seconds},
        {"sec_count", c.security_count},
-       {"has_depth", c.has_depth},
+       {"has_depth", BoolName::yes_no(c.has_depth)},
        {"trade_count", static_cast<size_t>(c.trade_count)},
        {"coconut_status_count", static_cast<size_t>(c.coconut_status_count)},
        {"exchange_status_count", static_cast<size_t>(c.exchange_status_count)},

src/lib_spark/spark/utils/TimeStamp.h

@@ -11,9 +11,7 @@
 #include "bts_assert.h"
 #include "system/os.h"
 
-#include <chrono>
 #include <cstdint>
-#include <fmt/chrono.h>
 #include <limits>
 
 namespace bts {
@@ -189,10 +187,6 @@ inline bool operator<=(const TimeStamp &t1, const TimeStamp &t2) {
    return t1._time <= t2._time;
 }
 
-inline auto format_as(TimeStamp ts) {
-   return std::chrono::nanoseconds{ts.to_uint64()};
-}
-
 } // namespace utils
 } // namespace bts
 

src/lib_spark/spark/utils/init_spark_lib.cpp

@@ -5,7 +5,6 @@
 #include "system/logging/protobufs/logging.h"
 #include "system/output_switch.h"
 #include "system/protobufs/config/system_config.pb.h"
-#include "system/protobufs/logging.pb.h"
 
 namespace bts {
 namespace utils {
@@ -17,7 +16,7 @@ using bts::system::threading::ThreadManager;
 static LoggingConfigPoller *poller = nullptr;
 
 static void init_spark_lib(
-    std::optional<logging::protobufs::Logging> logging_cfg,
+    const logging::protobufs::Logging &logging_cfg,
     bool hot_reload,
     bool quiet_console) {
    system::OutputSwitch cout{&std::cout, quiet_console};
@@ -30,23 +29,20 @@ static void init_spark_lib(
    logging::start_default_logging_thread();
    cout << "Initializing LoggingConfigPoller" << std::endl;
    ThreadManager thread_man{ThreadManager::default_thread_config()};
-   const auto logger_thread_cfg = thread_man.get_thread_cfg("logger");
-   if (!logging_cfg) {
-      logging_cfg = config::get<logging::protobufs::Logging>();
-   }
+   auto lcfg = thread_man.get_thread_cfg("logger");
    const spark::SparkDataDirectory data_dir;
    if (hot_reload) {
       poller = new LoggingConfigPoller(
-          *logging_cfg, thread_man.get_thread_cfg("logger"),
+          logging_cfg, thread_man.get_thread_cfg("logger"),
           data_dir.log_file().string(), data_dir.log_cache_file().string());
    } else {
       logging::configure_logging(
-          *logging_cfg, logger_thread_cfg, data_dir.log_file().string());
+          logging_cfg, lcfg, data_dir.log_file().string());
    }
 }
 
 SparkLibInit::SparkLibInit(
-    std::optional<logging::protobufs::Logging> logging_cfg,
+    const logging::protobufs::Logging &logging_cfg,
     bool hot_reload,
     bool quiet_console) {
    init_spark_lib(logging_cfg, hot_reload, quiet_console);
@@ -54,7 +50,9 @@ SparkLibInit::SparkLibInit(
 
 SparkLibInit::~SparkLibInit() { finit_spark_lib(); }
 
-void init_spark_lib() { init_spark_lib(std::nullopt, true, false); }
+void init_spark_lib() {
+   init_spark_lib(logging::protobufs::default_logging_config(), true, false);
+}
 
 void finit_spark_lib() {
    LOG(Gen, Diag, "finit_spark_lib");

src/lib_spark/spark/utils/init_spark_lib.h

@@ -1,9 +1,8 @@
 #pragma once
 #include "bts_export.h"
+#include "system/logging/protobufs/logging.h"
 #include "system/protobufs/logging.pb.h"
 
-#include <optional>
-
 // this is needed because we cannot relly on static initialization do
 // work reliably in DLL's on Windows. See:
 // https://msdn.microsoft.com/en-us/library/windows/desktop/dn633971(v=vs.85).aspx
@@ -17,7 +16,8 @@ extern "C" BTS_EXPORT void finit_spark_lib();
 
 struct SparkLibInit {
    explicit SparkLibInit(
-       std::optional<logging::protobufs::Logging> logging_cfg = std::nullopt,
+       const logging::protobufs::Logging &logging_cfg =
+           logging::protobufs::default_logging_config(),
        bool hot_reload = true,
        bool quiet_console = false);
    ~SparkLibInit();

src/lib_system/CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.23)
 project(system)
 include(${CMAKE_SOURCE_DIR}/cmake/InitLibrary.cmake)
 
-init_library(STATIC)
+init_library(SHARED)
 target_link_libraries(
   ${PROJECT_NAME}
   PUBLIC

src/lib_system/system/bool_name.h

@@ -0,0 +1,50 @@
+#pragma once
+#include <string>
+
+namespace bts::core {
+
+class BoolName {
+ public:
+   explicit constexpr BoolName(bool value);
+
+   [[nodiscard]] static constexpr const std::string &true_false(bool value);
+   [[nodiscard]] static constexpr const std::string &yes_no(bool value);
+   [[nodiscard]] static constexpr const std::string &y_n(bool value);
+   [[nodiscard]] static constexpr const std::string &on_off(bool value);
+
+ private:
+   bool value_;
+
+   struct Names {
+      static inline const std::string _TRUE = "true";
+      static inline const std::string _FALSE = "false";
+      static inline const std::string YES = "yes";
+      static inline const std::string NO = "no";
+      static inline const std::string Y = "y";
+      static inline const std::string N = "n";
+      static inline const std::string ON = "on";
+      static inline const std::string OFF = "off";
+   };
+};
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+constexpr const std::string &BoolName::true_false(bool value) {
+   return value ? Names::_TRUE : Names::_FALSE;
+}
+
+constexpr const std::string &BoolName::yes_no(bool value) {
+   return value ? Names::YES : Names::NO;
+}
+
+constexpr const std::string &BoolName::y_n(bool value) {
+   return value ? Names::Y : Names::N;
+}
+
+constexpr const std::string &BoolName::on_off(bool value) {
+   return value ? Names::ON : Names::OFF;
+}
+
+} // namespace bts::core

src/lib_system/system/formatted_bool.h

@@ -1,129 +0,0 @@
-#pragma once
-
-#include <fmt/format.h>
-#include <nlohmann/json.hpp>
-#include <ostream>
-
-namespace bts::system {
-
-enum class FormattedBool_Format {
-   TRUE_FALSE,
-   TRUE_FALSE_UPPER,
-   YES_NO,
-   YES_NO_UPPER,
-   Y_N,
-   Y_N_UPPER,
-   ON_OFF,
-   ON_OFF_UPPER,
-};
-
-template <FormattedBool_Format F = FormattedBool_Format::TRUE_FALSE>
-struct FormattedBool {
-   constexpr FormattedBool() = default;
-   constexpr FormattedBool(bool value) : value(value) {}
-   bool value = false;
-
-   constexpr bool operator()() const noexcept;
-};
-
-typedef FormattedBool<FormattedBool_Format::TRUE_FALSE> fbool;
-typedef FormattedBool<FormattedBool_Format::TRUE_FALSE> fbool_true_false;
-typedef FormattedBool<FormattedBool_Format::TRUE_FALSE> fbool_TRUE_FALSE;
-typedef FormattedBool<FormattedBool_Format::YES_NO> fbool_yes_no;
-typedef FormattedBool<FormattedBool_Format::YES_NO_UPPER> fbool_YES_NO;
-typedef FormattedBool<FormattedBool_Format::Y_N> fbool_y_n;
-typedef FormattedBool<FormattedBool_Format::Y_N_UPPER> fbool_Y_N;
-typedef FormattedBool<FormattedBool_Format::ON_OFF> fbool_on_off;
-typedef FormattedBool<FormattedBool_Format::ON_OFF_UPPER> fbool_ON_OFF;
-
-template <FormattedBool_Format F>
-std::ostream &operator<<(std::ostream &out, FormattedBool<F> rhs) noexcept;
-
-//////////////////////////////////////////////////
-// Definition
-//////////////////////////////////////////////////
-
-template <FormattedBool_Format F>
-constexpr bool FormattedBool<F>::operator()() const noexcept {
-   return value;
-}
-
-} // namespace bts::system
-
-template <bts::system::FormattedBool_Format F>
-inline std::ostream &operator<<(
-    std::ostream &out, bts::system::FormattedBool<F> fb) noexcept {
-   out << fb.value ? "true" : "false";
-   return out;
-}
-
-#define FORMATTED_BOOL_OUTPUT_IMPL(enum_name, t, f)                            \
-   template <>                                                                 \
-   inline std::ostream &operator<<(                                            \
-       std::ostream &out,                                                      \
-       bts::system::FormattedBool<                                             \
-           bts::system::FormattedBool_Format::enum_name> fb) noexcept {        \
-      if (fb.value) {                                                          \
-         out << t;                                                             \
-      } else {                                                                 \
-         out << f;                                                             \
-      }                                                                        \
-      return out;                                                              \
-   }
-
-#define FORMATTED_BOOL_JSON_IMPL(enum_name, t, f)                              \
-   inline void from_json(                                                      \
-       const nlohmann::json &j,                                                \
-       bts::system::FormattedBool<                                             \
-           bts::system::FormattedBool_Format::enum_name> &fb) {                \
-      switch (j.type()) {                                                      \
-      case nlohmann::detail::value_t::string: {                                \
-         const auto &s = j.get<std::string>();                                 \
-         if (s == t) {                                                         \
-            fb.value = true;                                                   \
-         } else if (s == f) {                                                  \
-            fb.value = false;                                                  \
-         }                                                                     \
-         break;                                                                \
-      }                                                                        \
-      case nlohmann::detail::value_t::boolean:                                 \
-         fb.value = j.get<bool>();                                             \
-         break;                                                                \
-      default:                                                                 \
-         break;                                                                \
-      }                                                                        \
-   }
-
-#define FORMATTED_BOOL_FMT_IMPL(enum_name, t, f)                               \
-   template <>                                                                 \
-   struct fmt::formatter<bts::system::FormattedBool<                           \
-       bts::system::FormattedBool_Format::enum_name>>                          \
-       : formatter<string_view> {                                              \
-      template <typename FormatContext>                                        \
-      auto format(                                                             \
-          const bts::system::FormattedBool<                                    \
-              bts::system::FormattedBool_Format::enum_name> &fb,               \
-          FormatContext &ctx) const -> format_context::iterator {              \
-         string_view name = fb.value ? t : f;                                  \
-         return formatter<string_view>::format(name, ctx);                     \
-      }                                                                        \
-   }
-
-#define FORMATTED_BOOL_IMPL(enum_name, t, f)                                   \
-   FORMATTED_BOOL_OUTPUT_IMPL(enum_name, t, f)                                 \
-   FORMATTED_BOOL_JSON_IMPL(enum_name, t, f)                                   \
-   FORMATTED_BOOL_FMT_IMPL(enum_name, t, f)
-
-FORMATTED_BOOL_IMPL(TRUE_FALSE, "true", "false");
-FORMATTED_BOOL_IMPL(TRUE_FALSE_UPPER, "TRUE", "FALSE");
-FORMATTED_BOOL_IMPL(YES_NO, "yes", "no");
-FORMATTED_BOOL_IMPL(YES_NO_UPPER, "YES", "NO");
-FORMATTED_BOOL_IMPL(Y_N, "yes", "no");
-FORMATTED_BOOL_IMPL(Y_N_UPPER, "YES", "NO");
-FORMATTED_BOOL_IMPL(ON_OFF, "on", "off");
-FORMATTED_BOOL_IMPL(ON_OFF_UPPER, "ON", "OFF");
-
-#undef FORMATTED_BOOL_OUTPUT_IMPL
-#undef FORMATTED_BOOL_JSON_IMPL
-#undef FORMATTED_BOOL_FMT_IMPL
-#undef FORMATTED_BOOL_IMPL

src/lib_system/system/threading/atomic_singleton.h

@@ -1,73 +0,0 @@
-#pragma once
-
-#include "system/logging/logging.h"
-
-#include <atomic>
-#include <thread>
-
-namespace bts::system {
-
-/**
- * Manages the (global) lifetime of a static singleton `T`. An atomic reference
- * counter is used to synchronize initialization, and eventual destruction.
- */
-template <class T>
-struct AtomicSingleton {
-   explicit AtomicSingleton() {
-      if (global_reference_count_++ == 0) {
-         LOG(Gen, Info, "Constructing static singleton of type {}",
-             std::string{typeid(T).name()});
-         global_value_.emplace();
-         global_initialized_ = true;
-      }
-   }
-
-   ~AtomicSingleton() {
-      if (--global_reference_count_ == 0) {
-         LOG(Gen, Info, "Destructing static singleton of type {}");
-         global_value_.reset();
-         global_initialized_ = false;
-      }
-   }
-
-   T &value() {
-      if (!local_initialized_ && !global_initialized_) {
-         // Global init is triggered by an atomic reference count, and is lockless. If this class
-         // is simultaneously instantiated on multiple threads, it's possible that a "non-first"
-         // instance (according to `reference_count_`) will have to wait for the "first" instance to
-         // finish initialization.
-         std::this_thread::yield();
-         for (int i = 0; i < 100 && !global_initialized_; i++) {
-            using namespace std::chrono_literals;
-            std::this_thread::sleep_for(10ms);
-         }
-         if (!global_initialized_) {
-            // This should never happen because we initialize the global conrrencpp::runtime
-            // instance in the constructor according to an atomic reference count.
-            std::cerr << "FATAL: Timed out waiting for global "
-                         "concurrencpp::runtime. Exiting immediately."
-                      << std::endl;
-            exit(1);
-         }
-         local_initialized_ = true;
-      }
-      return *global_value_;
-   }
-
-   const T &value() const { return *global_value_; }
-
-   T &operator->() { return value(); }
-   const T &operator->() const { return value(); }
-
- private:
-   static inline std::atomic<bool> global_initialized_{false};
-   static inline std::atomic<int> global_reference_count_{0};
-   static inline std::optional<T> global_value_;
-
-   // When true, this indicates that the atomic `initialized_` variable has been observed true (by
-   // this instance), and thus does not need to be checked again. It is a subtle optimization that
-   // allows individual instances to optimisitically read a regular bool instead of an atomic one.
-   bool local_initialized_ = false;
-};
-
-} // namespace bts::system