spark/compose.yaml

services:
  sparkd:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
    build:
      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: >
      sh -c "
        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
        fi
        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
          set -a
          . /home/bts/sparkdata/conf/.env.local
          set +a
        fi
        if [ -d /home/bts/sparkdata/conf/plugins ]; then
          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
        fi
        ./sparkd run"
    env_file:
      - .env
      - ${SPARKD__CONFIG_DIR}/.env
    ports:
      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
    deploy:
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-4gb}"

  sparkd-symbols:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
    build:
      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: >
      sh -c "
        echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
        if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
          mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
        fi
        ls -alh /home/bts/sparkdata/conf/
        cat /home/bts/sparkdata/conf/.env.local
        if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
          echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
          set -a
          . /home/bts/sparkdata/conf/.env.local
          set +a
        fi
        if [ -d /home/bts/sparkdata/conf/plugins ]; then
          echo 'Copying /home/bts/sparkdata/conf/plugins to runtime dir (/home/bts/bin)'
          cp -r /home/bts/sparkdata/conf/plugins /home/bts/bin/
        fi
        ./sparkd run"
    env_file:
      - .env
      - ${SPARKD__CONFIG_DIR}/.env
    ports:
      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
    deploy:
      restart_policy:
        condition: any
        delay: 3s
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-4gb}"

  sparkd-tail:
    image: bluetradingsystems/linux_dotnet7
    volumes:
      - type: volume
        read_only: true
        source: sparkd_data
        target: /home/bts/sparkdata/data/logging.sparkd
        volume:
          subpath: logging.sparkd
    command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]

  sparkd-ping-test:
    depends_on:
      - sparkd
    profiles:
      - ping-test
    build:
      context: .
      dockerfile_inline: |
        FROM bluetradingsystems/linux_dotnet7
        USER "root"
        RUN apt-get update && apt-get -y install iputils-ping
    command: [ "ping", "sparkd" ]

  pubsub:
    environment:
      - DOTNET_ENVIRONMENT=Staging
    build:
      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
      args:
        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
        - CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
    ports:
      - 22019:22019
    command: [ "./PubSubServer" ]

  md-publisher:
    build:
      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
      args:
        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
        - CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
    depends_on:
      - pubsub
      - sparkd
    command: [ "dockerize", "-wait", "tcp://pubsub:22019", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./DataStreamPublisher" ]
    environment:
      - DOTNET_ENVIRONMENT=MarketData

  webapp:
    ports:
      - 8080:8080
      - 8443:8443
    build:
      dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
      args:
        - SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
        - WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
        - CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
    command: [ "./Bts.DataStreamWebApp.Server" ]
    environment:
      - ASPNETCORE_ENVIRONMENT=Staging

  api-server:
    depends_on:
      - sparkd
    ports:
      - 8008:8008
      - 9443:9443
    build:
      dockerfile: src/client/spark_api_server/Dockerfile
    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
    environment:
      - ASPNETCORE_ENVIRONMENT=Staging

volumes:
  sparkd_data:
    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"