Files
spark/compose.yaml
T
John Colvin ec37426c35 fix: Ensure md-publisher waits for pubsub and sparkd
Bugs (fixed):
  * md-publisher silently fails if it tries to connect before those services are "ready".

The Docker compose "depends_on" property ensures sparkd and pubsub
containers are up, but doesn't prevent startup race bugs (e.g., if
md-publisher and pubsub start around the same time, and md-publisher
tries to connect before pubsub has started accepting connections). To
address these race conditions, we use a nifty tool called "dockerize"

References:
  * https://github.com/jwilder/dockerize
2025-07-23 16:31:31 -04:00

156 lines
5.1 KiB
YAML

services:
sparkd:
volumes:
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
build:
dockerfile: src/app_sparkd/Dockerfile
args:
- IMAGE=${SPARKD__IMAGE}
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
command: >
sh -c "
echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
fi
if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
set -a
. /home/bts/sparkdata/conf/.env.local
set +a
fi
if [ -d /home/bts/sparkdata/conf/plugins ]; then
echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
fi
./sparkd run"
env_file:
- .env
- ${SPARKD__CONFIG_DIR}/.env
ports:
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
deploy:
resources:
limits:
memory: "${SPARKD__MEMORY:-4gb}"
sparkd-symbols:
volumes:
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
- ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
build:
dockerfile: src/app_sparkd/Dockerfile
args:
- IMAGE=${SPARKD__IMAGE}
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
command: >
sh -c "
echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
fi
ls -alh /home/bts/sparkdata/conf/
cat /home/bts/sparkdata/conf/.env.local
if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
set -a
. /home/bts/sparkdata/conf/.env.local
set +a
fi
if [ -d /home/bts/sparkdata/conf/plugins ]; then
echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
fi
./sparkd run"
env_file:
- .env
- ${SPARKD__CONFIG_DIR}/.env
ports:
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
deploy:
restart_policy:
condition: any
delay: 3s
resources:
limits:
memory: "${SPARKD__MEMORY:-4gb}"
sparkd-tail:
image: bluetradingsystems/linux_dotnet7
volumes:
- type: volume
read_only: true
source: sparkd_data
target: /home/bts/sparkdata/data/logging.sparkd
volume:
subpath: logging.sparkd
command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
sparkd-ping-test:
depends_on:
- sparkd
profiles:
- ping-test
build:
context: .
dockerfile_inline: |
FROM bluetradingsystems/linux_dotnet7
USER "root"
RUN apt-get update && apt-get -y install iputils-ping
command: [ "ping", "sparkd" ]
pubsub:
environment:
- DOTNET_ENVIRONMENT=Staging
build:
dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
args:
- SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
- CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
ports:
- 22019:22019
command: [ "./PubSubServer" ]
md-publisher:
build:
dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
args:
- SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
- CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
depends_on:
- pubsub
- sparkd
command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
environment:
- DOTNET_ENVIRONMENT=MarketData
webapp:
ports:
- 8080:8080
- 8443:8443
build:
dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
args:
- SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
- WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
- CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
command: [ "./Bts.DataStreamWebApp.Server" ]
environment:
- ASPNETCORE_HTTP_PORT=8080
- ASPNETCORE_HTTPS_PORT=8443
- ASPNETCORE_ENVIRONMENT=Staging
volumes:
sparkd_data:
name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"