diff --git a/.env b/.env new file mode 100644 index 000000000..44d9ec838 --- /dev/null +++ b/.env @@ -0,0 +1,80 @@ +# This file defines the global default configuration for docker deployments. +# It is tracked in source control. For temporary modifications, please create +# a separate file called ".env.local" (it's in .gitignore, so won't be tracked). +# If a variable is defined in both files, the value in .env.local will be used. +# +# About the deployment procedure: +# We use Docker Compose to build images and launch containers for various services +# that make up our software suite: +# - sparkd +# - pubsub +# - md-publisher +# - webapp (DSQA on the web) +# - api-server (websocket based) +# From inside a devcontainer, you can launch all services in separate containers +# on your machine by running: +# +# tools/deploy.sh +# +# This script builds release versions for each service, packaged (generally) in +# self-contained directories. Next, it runs a few "docker compose" commands to +# build docker images loaded with the self-contained binaries from the previous +# step, and start containers with those images to run each service. To view the +# commands that deploy.sh runs internally (without actually running them), run: +# +# tools/deploy.sh --dry-run +# +# For additional information/help, run: +# +# tools/deploy.sh --help +# + +### Sparkd ### +# The internal image for sparkd to use. For more info, view: +# src/app_sparkd/Dockerfile +SPARKD__IMAGE=sparkd +# The 'spark.conf' for this service to use +SPARKD__CONF_FILE=src/app_sparkd/spark.conf +# To run replay md, put recorded md files in SPARKD__MD_RECORDING_DIR, and use: +# SPARKD__IMAGE=sparkd_md_replay +# SPARKD__CONF_FILE=src/app_sparkd/spark.local.conf +# (configure which md to replay in the new SPARKD__CONF_FILE) +SPARKD__MD_RECORDING_DIR=.md_recording +SPARKD__SOURCE_DIR=install/sparkd-release-gcc +SPARKD__CONFIG_PORT=8000 +SPARKD__QUICKBUS_PORT=8001 +SPARKD__SIM_FIX_PORT=8002 + +### Pubsub ### +# The 'spark.conf' for this service to use +PUBSUB__CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf +PUBSUB__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish +# The appsettings to load at runtime. Expects a file named: +# src/DataStreamSuite/PubSub/appsettings.${PUBSUB__DOTNET_ENVIRONMENT}.json +PUBSUB__DOTNET_ENVIRONMENT=Staging + +### Md Publisher ### +# The 'spark.conf' for this service to use +MD_PUBLISHER__CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf +MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish +# Determines which appsettings to load at runtime. Expects a file named: +# src/DataStreamSuite/DataStreamPublisher/appsettings.${MD_PUBLISHER__DOTNET_ENVIRONMENT}.json +MD_PUBLISHER__DOTNET_ENVIRONMENT=MarketData + +### Webapp (DSQA on the web) ### +# The 'spark.conf' for this service to use +WEBAPP__CONF_FILE=src/client/spark_api_server/spark.conf +WEBAPP__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish +WEBAPP__WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist +# Determines which appsettings to load at runtime. Expects a file named: +# src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.${WEBAPP__ASPNETCORE_ENVIRONMENT}.json +WEBAPP__ASPNETCORE_ENVIRONMENT=Staging + +### Api Server ### +# The 'spark.conf' for this service to use +API_SERVER__CONF_FILE=src/client/spark_api_server/spark.conf +API_SERVER__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish +API_SERVER__CPP_SOURCE_DIR=install/spark-api-server-linux +# Determines which appsettings to load at runtime. Expects a file named: +# src/client/spark_api_server/appsettings.${API_SERVER__ASPNETCORE_ENVIRONMENT}.json +API_SERVER__ASPNETCORE_ENVIRONMENT=Staging diff --git a/.gitignore b/.gitignore index 568999f35..9fe8a9548 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,6 @@ trace_data/ .devcontainer/ *.Generated.cs *.lst +src/app_sparkd/spark.local.conf +.env.local +.md_recording diff --git a/compose.yaml b/compose.yaml index a955744d3..9e3c7de8b 100644 --- a/compose.yaml +++ b/compose.yaml @@ -3,8 +3,10 @@ services: build: dockerfile: src/app_sparkd/Dockerfile args: - - SOURCE_DIR=install/sparkd-release-gcc - - CONF_FILE=src/app_sparkd/spark.conf + - SPARKD_IMAGE=${SPARKD_IMAGE:-sparkd} + - SPARKD_SOURCE_DIR=${SPARKD_SOURCE_DIR:-install/sparkd-release-gcc} + - SPARKD_CONF_FILE=${SPARKD_CONF_FILE:-src/app_sparkd/spark.conf} + - SPARKD_MD_RECORDING_DIR=${SPARKD_MD_RECORDING_DIR:-src/app_sparkd/.md_recording} command: [ "./sparkd", "run" ] ports: - 8000:8000 diff --git a/src/DataStreamSuite/DataStreamPublisher/Dockerfile b/src/DataStreamSuite/DataStreamPublisher/Dockerfile index 10549960b..416a1da31 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Dockerfile +++ b/src/DataStreamSuite/DataStreamPublisher/Dockerfile @@ -1,8 +1,8 @@ FROM bluetradingsystems/linux_dotnet7 -ARG CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish" -ARG CPP_SOURCE_DIR="install/data-stream-suite-linux" -ARG CONF_FILE="src/DataStreamSuite/DataStreamPublisher/spark.conf" +ARG DS_PUBLISHER_CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish" +ARG DS_PUBLISHER_CPP_SOURCE_DIR="install/data-stream-suite-linux" +ARG DS_PUBLISHER_CONF_FILE="src/DataStreamSuite/DataStreamPublisher/spark.conf" ENV DOCKERIZE_VERSION=v0.7.0 USER root @@ -12,7 +12,7 @@ RUN apt-get update \ && apt-get autoremove -yqq --purge wget && rm -rf /var/lib/apt/lists/* USER bts -COPY --chown=bts:bts "${CPP_SOURCE_DIR}/lib" "bin" -COPY --chown=bts:bts "${CS_SOURCE_DIR}" "bin" -COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/DataStreamPublisher/conf/spark.conf" +COPY --chown=bts:bts "${DS_PUBLISHER_CPP_SOURCE_DIR}/lib" "bin" +COPY --chown=bts:bts "${DS_PUBLISHER_CS_SOURCE_DIR}" "bin" +COPY --chown=bts:bts "${DS_PUBLISHER_CONF_FILE}" ".local/share/bts/DataStreamPublisher/conf/spark.conf" WORKDIR "/home/bts/bin" diff --git a/src/app_sparkd/Dockerfile b/src/app_sparkd/Dockerfile index e308db9a4..f6be1e13b 100644 --- a/src/app_sparkd/Dockerfile +++ b/src/app_sparkd/Dockerfile @@ -1,7 +1,10 @@ -FROM bluetradingsystems/linux_dotnet7 - -ARG SOURCE_DIR="install/sparkd-release-gcc" +ARG IMAGE="sparkd" ARG CONF_FILE="src/app_sparkd/spark.conf" +ARG SOURCE_DIR="install/sparkd-release-gcc" + +FROM bluetradingsystems/linux_dotnet7 AS sparkd_base +ARG CONF_FILE +ARG SOURCE_DIR USER "root" RUN apt-get update \ @@ -17,3 +20,11 @@ USER "bts" COPY --chown=bts:bts "${SOURCE_DIR}" "." COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/sparkd/conf/spark.conf" WORKDIR "/home/bts/bin" + +FROM sparkd_base AS sparkd + +FROM sparkd_base AS sparkd_md_replay +ARG MD_RECORDING_DIR="src/app_sparkd/.md_recording" +COPY --chown=bts:bts "${MD_RECORDING_DIR}" ".local/share/bts/sparkd/data/md_recording" + +FROM ${IMAGE} diff --git a/tools/py/deploy.py b/tools/py/deploy.py index 1c5a9a530..b5b0580f0 100644 --- a/tools/py/deploy.py +++ b/tools/py/deploy.py @@ -5,8 +5,8 @@ import publish_api_server import publish_data_stream import publish_sparkd - from core.argparse_utils import add_toggle_opt +from core.file_utils import SparkPaths def _get_arg_parser(): @@ -24,6 +24,9 @@ def _get_arg_parser(): action="store_true", help="Perform a clean build (remove artifacts from previous build)", ) + parser.add_argument( + "--env-files", action="append", help=".env files to pass to docker compose" + ) parser.add_argument( "--verbosity", metavar="{0-5}", @@ -49,6 +52,11 @@ def _get_arg_parser(): return parser +def _get_env_files(): + root = SparkPaths.root() + files = [root.joinpath(".env"), root.joinpath(".env.local")] + return list(filter(lambda p: p.exists(), files)) + def deploy(args): def run(cmd): print(cmd) @@ -92,9 +100,12 @@ def deploy(args): ) ) service_args = " ".join(services) - compose_cmd = "sudo docker compose" + compose_cmd = "sudo docker compose" if args.compose_file: compose_cmd += f" -f {args.compose_file}" + env_files = _get_env_files() + if len(env_files) > 0: + compose_cmd += " " + " ".join([f"--env-file {f}" for f in env_files]) run(f"{compose_cmd} down {service_args}") run(f"{compose_cmd} build {service_args}") run(f"{compose_cmd} up {service_args} -d")