Incorporate env files into docker compose
In order to support local (ephemeral) config. For example, to run sparkd with replay md to troubleshoot an issue.
This commit is contained in:
1 parent
5ae46fef77
commit
fd6926eee8
6 files changed
+120
-13
No files matched your search
@@ -0,0 +1,80 @@
|
||||
# This file defines the global default configuration for docker deployments.
|
||||
# It is tracked in source control. For temporary modifications, please create
|
||||
# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
|
||||
# If a variable is defined in both files, the value in .env.local will be used.
|
||||
#
|
||||
# About the deployment procedure:
|
||||
# We use Docker Compose to build images and launch containers for various services
|
||||
# that make up our software suite:
|
||||
# - sparkd
|
||||
# - pubsub
|
||||
# - md-publisher
|
||||
# - webapp (DSQA on the web)
|
||||
# - api-server (websocket based)
|
||||
# From inside a devcontainer, you can launch all services in separate containers
|
||||
# on your machine by running:
|
||||
#
|
||||
# tools/deploy.sh
|
||||
#
|
||||
# This script builds release versions for each service, packaged (generally) in
|
||||
# self-contained directories. Next, it runs a few "docker compose" commands to
|
||||
# build docker images loaded with the self-contained binaries from the previous
|
||||
# step, and start containers with those images to run each service. To view the
|
||||
# commands that deploy.sh runs internally (without actually running them), run:
|
||||
#
|
||||
# tools/deploy.sh --dry-run
|
||||
#
|
||||
# For additional information/help, run:
|
||||
#
|
||||
# tools/deploy.sh --help
|
||||
#
|
||||
|
||||
### Sparkd ###
|
||||
# The internal image for sparkd to use. For more info, view:
|
||||
# src/app_sparkd/Dockerfile
|
||||
SPARKD__IMAGE=sparkd
|
||||
# The 'spark.conf' for this service to use
|
||||
SPARKD__CONF_FILE=src/app_sparkd/spark.conf
|
||||
# To run replay md, put recorded md files in SPARKD__MD_RECORDING_DIR, and use:
|
||||
# SPARKD__IMAGE=sparkd_md_replay
|
||||
# SPARKD__CONF_FILE=src/app_sparkd/spark.local.conf
|
||||
# (configure which md to replay in the new SPARKD__CONF_FILE)
|
||||
SPARKD__MD_RECORDING_DIR=.md_recording
|
||||
SPARKD__SOURCE_DIR=install/sparkd-release-gcc
|
||||
SPARKD__CONFIG_PORT=8000
|
||||
SPARKD__QUICKBUS_PORT=8001
|
||||
SPARKD__SIM_FIX_PORT=8002
|
||||
|
||||
### Pubsub ###
|
||||
# The 'spark.conf' for this service to use
|
||||
PUBSUB__CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
|
||||
PUBSUB__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
|
||||
# The appsettings to load at runtime. Expects a file named:
|
||||
# src/DataStreamSuite/PubSub/appsettings.${PUBSUB__DOTNET_ENVIRONMENT}.json
|
||||
PUBSUB__DOTNET_ENVIRONMENT=Staging
|
||||
|
||||
### Md Publisher ###
|
||||
# The 'spark.conf' for this service to use
|
||||
MD_PUBLISHER__CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
|
||||
MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
|
||||
# Determines which appsettings to load at runtime. Expects a file named:
|
||||
# src/DataStreamSuite/DataStreamPublisher/appsettings.${MD_PUBLISHER__DOTNET_ENVIRONMENT}.json
|
||||
MD_PUBLISHER__DOTNET_ENVIRONMENT=MarketData
|
||||
|
||||
### Webapp (DSQA on the web) ###
|
||||
# The 'spark.conf' for this service to use
|
||||
WEBAPP__CONF_FILE=src/client/spark_api_server/spark.conf
|
||||
WEBAPP__SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
|
||||
WEBAPP__WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
|
||||
# Determines which appsettings to load at runtime. Expects a file named:
|
||||
# src/DataStreamSuite/Bts.DataStreamWebApp.Server/appsettings.${WEBAPP__ASPNETCORE_ENVIRONMENT}.json
|
||||
WEBAPP__ASPNETCORE_ENVIRONMENT=Staging
|
||||
|
||||
### Api Server ###
|
||||
# The 'spark.conf' for this service to use
|
||||
API_SERVER__CONF_FILE=src/client/spark_api_server/spark.conf
|
||||
API_SERVER__CS_SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
|
||||
API_SERVER__CPP_SOURCE_DIR=install/spark-api-server-linux
|
||||
# Determines which appsettings to load at runtime. Expects a file named:
|
||||
# src/client/spark_api_server/appsettings.${API_SERVER__ASPNETCORE_ENVIRONMENT}.json
|
||||
API_SERVER__ASPNETCORE_ENVIRONMENT=Staging
|
||||
@@ -59,3 +59,6 @@ trace_data/
|
||||
.devcontainer/
|
||||
*.Generated.cs
|
||||
*.lst
|
||||
src/app_sparkd/spark.local.conf
|
||||
.env.local
|
||||
.md_recording
|
||||
+4
-2
@@ -3,8 +3,10 @@ services:
|
||||
build:
|
||||
dockerfile: src/app_sparkd/Dockerfile
|
||||
args:
|
||||
- SOURCE_DIR=install/sparkd-release-gcc
|
||||
- CONF_FILE=src/app_sparkd/spark.conf
|
||||
- SPARKD_IMAGE=${SPARKD_IMAGE:-sparkd}
|
||||
- SPARKD_SOURCE_DIR=${SPARKD_SOURCE_DIR:-install/sparkd-release-gcc}
|
||||
- SPARKD_CONF_FILE=${SPARKD_CONF_FILE:-src/app_sparkd/spark.conf}
|
||||
- SPARKD_MD_RECORDING_DIR=${SPARKD_MD_RECORDING_DIR:-src/app_sparkd/.md_recording}
|
||||
command: [ "./sparkd", "run" ]
|
||||
ports:
|
||||
- 8000:8000
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
FROM bluetradingsystems/linux_dotnet7
|
||||
|
||||
ARG CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
|
||||
ARG CPP_SOURCE_DIR="install/data-stream-suite-linux"
|
||||
ARG CONF_FILE="src/DataStreamSuite/DataStreamPublisher/spark.conf"
|
||||
ARG DS_PUBLISHER_CS_SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
|
||||
ARG DS_PUBLISHER_CPP_SOURCE_DIR="install/data-stream-suite-linux"
|
||||
ARG DS_PUBLISHER_CONF_FILE="src/DataStreamSuite/DataStreamPublisher/spark.conf"
|
||||
|
||||
ENV DOCKERIZE_VERSION=v0.7.0
|
||||
USER root
|
||||
@@ -12,7 +12,7 @@ RUN apt-get update \
|
||||
&& apt-get autoremove -yqq --purge wget && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
USER bts
|
||||
COPY --chown=bts:bts "${CPP_SOURCE_DIR}/lib" "bin"
|
||||
COPY --chown=bts:bts "${CS_SOURCE_DIR}" "bin"
|
||||
COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/DataStreamPublisher/conf/spark.conf"
|
||||
COPY --chown=bts:bts "${DS_PUBLISHER_CPP_SOURCE_DIR}/lib" "bin"
|
||||
COPY --chown=bts:bts "${DS_PUBLISHER_CS_SOURCE_DIR}" "bin"
|
||||
COPY --chown=bts:bts "${DS_PUBLISHER_CONF_FILE}" ".local/share/bts/DataStreamPublisher/conf/spark.conf"
|
||||
WORKDIR "/home/bts/bin"
|
||||
@@ -1,7 +1,10 @@
|
||||
FROM bluetradingsystems/linux_dotnet7
|
||||
|
||||
ARG SOURCE_DIR="install/sparkd-release-gcc"
|
||||
ARG IMAGE="sparkd"
|
||||
ARG CONF_FILE="src/app_sparkd/spark.conf"
|
||||
ARG SOURCE_DIR="install/sparkd-release-gcc"
|
||||
|
||||
FROM bluetradingsystems/linux_dotnet7 AS sparkd_base
|
||||
ARG CONF_FILE
|
||||
ARG SOURCE_DIR
|
||||
|
||||
USER "root"
|
||||
RUN apt-get update \
|
||||
@@ -17,3 +20,11 @@ USER "bts"
|
||||
COPY --chown=bts:bts "${SOURCE_DIR}" "."
|
||||
COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/sparkd/conf/spark.conf"
|
||||
WORKDIR "/home/bts/bin"
|
||||
|
||||
FROM sparkd_base AS sparkd
|
||||
|
||||
FROM sparkd_base AS sparkd_md_replay
|
||||
ARG MD_RECORDING_DIR="src/app_sparkd/.md_recording"
|
||||
COPY --chown=bts:bts "${MD_RECORDING_DIR}" ".local/share/bts/sparkd/data/md_recording"
|
||||
|
||||
FROM ${IMAGE}
|
||||
+13
-2
@@ -5,8 +5,8 @@ import publish_api_server
|
||||
import publish_data_stream
|
||||
import publish_sparkd
|
||||
|
||||
|
||||
from core.argparse_utils import add_toggle_opt
|
||||
from core.file_utils import SparkPaths
|
||||
|
||||
|
||||
def _get_arg_parser():
|
||||
@@ -24,6 +24,9 @@ def _get_arg_parser():
|
||||
action="store_true",
|
||||
help="Perform a clean build (remove artifacts from previous build)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--env-files", action="append", help=".env files to pass to docker compose"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbosity",
|
||||
metavar="{0-5}",
|
||||
@@ -49,6 +52,11 @@ def _get_arg_parser():
|
||||
return parser
|
||||
|
||||
|
||||
def _get_env_files():
|
||||
root = SparkPaths.root()
|
||||
files = [root.joinpath(".env"), root.joinpath(".env.local")]
|
||||
return list(filter(lambda p: p.exists(), files))
|
||||
|
||||
def deploy(args):
|
||||
def run(cmd):
|
||||
print(cmd)
|
||||
@@ -92,9 +100,12 @@ def deploy(args):
|
||||
)
|
||||
)
|
||||
service_args = " ".join(services)
|
||||
compose_cmd = "sudo docker compose"
|
||||
compose_cmd = "sudo docker compose"
|
||||
if args.compose_file:
|
||||
compose_cmd += f" -f {args.compose_file}"
|
||||
env_files = _get_env_files()
|
||||
if len(env_files) > 0:
|
||||
compose_cmd += " " + " ".join([f"--env-file {f}" for f in env_files])
|
||||
run(f"{compose_cmd} down {service_args}")
|
||||
run(f"{compose_cmd} build {service_args}")
|
||||
run(f"{compose_cmd} up {service_args} -d")
|
||||
|
||||
Reference in new issue
Block a user