Bugs (fixed): * md-publisher silently fails if it tries to connect before those services are "ready". The Docker compose "depends_on" property ensures sparkd and pubsub containers are up, but doesn't prevent startup race bugs (e.g., if md-publisher and pubsub start around the same time, and md-publisher tries to connect before pubsub has started accepting connections). To address these race conditions, we use a nifty tool called "dockerize" References: * https://github.com/jwilder/dockerize
156 lines
5.1 KiB
YAML
156 lines
5.1 KiB
YAML
services:
|
|
sparkd:
|
|
volumes:
|
|
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
|
|
build:
|
|
dockerfile: src/app_sparkd/Dockerfile
|
|
args:
|
|
- IMAGE=${SPARKD__IMAGE}
|
|
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
|
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
|
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
|
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
|
command: >
|
|
sh -c "
|
|
echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
|
|
if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
|
|
mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
|
|
fi
|
|
if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
|
|
echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
|
|
set -a
|
|
. /home/bts/sparkdata/conf/.env.local
|
|
set +a
|
|
fi
|
|
if [ -d /home/bts/sparkdata/conf/plugins ]; then
|
|
echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
|
|
cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
|
|
fi
|
|
./sparkd run"
|
|
env_file:
|
|
- .env
|
|
- ${SPARKD__CONFIG_DIR}/.env
|
|
ports:
|
|
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
|
|
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
|
|
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
memory: "${SPARKD__MEMORY:-4gb}"
|
|
|
|
sparkd-symbols:
|
|
volumes:
|
|
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
|
|
- ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
|
|
build:
|
|
dockerfile: src/app_sparkd/Dockerfile
|
|
args:
|
|
- IMAGE=${SPARKD__IMAGE}
|
|
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
|
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
|
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
|
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
|
command: >
|
|
sh -c "
|
|
echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
|
|
if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
|
|
mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
|
|
fi
|
|
ls -alh /home/bts/sparkdata/conf/
|
|
cat /home/bts/sparkdata/conf/.env.local
|
|
if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
|
|
echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
|
|
set -a
|
|
. /home/bts/sparkdata/conf/.env.local
|
|
set +a
|
|
fi
|
|
if [ -d /home/bts/sparkdata/conf/plugins ]; then
|
|
echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
|
|
cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
|
|
fi
|
|
./sparkd run"
|
|
env_file:
|
|
- .env
|
|
- ${SPARKD__CONFIG_DIR}/.env
|
|
ports:
|
|
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
|
|
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
|
|
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
|
|
deploy:
|
|
restart_policy:
|
|
condition: any
|
|
delay: 3s
|
|
resources:
|
|
limits:
|
|
memory: "${SPARKD__MEMORY:-4gb}"
|
|
|
|
sparkd-tail:
|
|
image: bluetradingsystems/linux_dotnet7
|
|
volumes:
|
|
- type: volume
|
|
read_only: true
|
|
source: sparkd_data
|
|
target: /home/bts/sparkdata/data/logging.sparkd
|
|
volume:
|
|
subpath: logging.sparkd
|
|
command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
|
|
|
|
sparkd-ping-test:
|
|
depends_on:
|
|
- sparkd
|
|
profiles:
|
|
- ping-test
|
|
build:
|
|
context: .
|
|
dockerfile_inline: |
|
|
FROM bluetradingsystems/linux_dotnet7
|
|
USER "root"
|
|
RUN apt-get update && apt-get -y install iputils-ping
|
|
command: [ "ping", "sparkd" ]
|
|
|
|
pubsub:
|
|
environment:
|
|
- DOTNET_ENVIRONMENT=Staging
|
|
build:
|
|
dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
|
|
args:
|
|
- SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
|
|
- CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
|
|
ports:
|
|
- 22019:22019
|
|
command: [ "./PubSubServer" ]
|
|
|
|
md-publisher:
|
|
build:
|
|
dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
|
|
args:
|
|
- SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
|
|
- CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
|
|
depends_on:
|
|
- pubsub
|
|
- sparkd
|
|
command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
|
|
environment:
|
|
- DOTNET_ENVIRONMENT=MarketData
|
|
|
|
webapp:
|
|
ports:
|
|
- 8080:8080
|
|
- 8443:8443
|
|
build:
|
|
dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
|
|
args:
|
|
- SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
|
|
- WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
|
|
- CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
|
|
command: [ "./Bts.DataStreamWebApp.Server" ]
|
|
environment:
|
|
- ASPNETCORE_HTTP_PORT=8080
|
|
- ASPNETCORE_HTTPS_PORT=8443
|
|
- ASPNETCORE_ENVIRONMENT=Staging
|
|
|
|
volumes:
|
|
sparkd_data:
|
|
name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
|