spark/compose.yaml

services:
  sparkd:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
    build:
      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: >
      sh -c "
        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd/data
        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
        fi
        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
          set -a
          . /home/bts/.local/share/bts/sparkd/conf/.env.local
          set +a
        fi
        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
        fi
        ./sparkd run"
    env_file:
      - .env
      - ${SPARKD__CONFIG_DIR}/.env
    ports:
      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
    deploy:
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-4gb}"

  sparkd-symbols:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
    build:
      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: >
      sh -c "
        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd
        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
        fi
        ls -alh /home/bts/.local/share/bts/sparkd/conf/
        cat /home/bts/.local/share/bts/sparkd/conf/.env.local
        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
          set -a
          . /home/bts/.local/share/bts/sparkd/conf/.env.local
          set +a
        fi
        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
        fi
        ./sparkd run"
    env_file:
      - .env
      - ${SPARKD__CONFIG_DIR}/.env
    ports:
      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
    deploy:
      restart_policy:
        condition: any
        delay: 3s
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-4gb}"

  sparkd-tail:
    image: bluetradingsystems/linux_dotnet7
    volumes:
      - type: volume
        read_only: true
        source: sparkd_data
        target: /home/bts/.local/share/bts/sparkd/data/logging.sparkd
        volume:
          subpath: logging
    command: ["tail", "-f", "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log"]

  sparkd-ping-test:
    depends_on:
      - sparkd
    profiles:
      - ping-test
    build:
      context: .
      dockerfile_inline: |
        FROM bluetradingsystems/linux_dotnet7
        USER "root"
        RUN apt-get update && apt-get -y install iputils-ping
    command: [ "ping", "sparkd" ]

  pubsub:
    volumes:
      - ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/PubSubServer/data
    build:
      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
      args:
        - SOURCE_DIR=${PUBSUB__SOURCE_DIR}
        - CONFIG_DIR=${PUBSUB__CONFIG_DIR}
    ports:
      - 22019:22019
    command: >
      sh -c "
        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/PubSubServer
        if [ -e "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log" ]; then
          mv "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')"
        fi
        if [ -e "/home/bts/.local/share/bts/PubSubServer/conf/.env" ]; then
          echo 'Sourcing /home/bts/.local/share/bts/PubSubServer/conf/.env'
          set -a
          . /home/bts/.local/share/bts/PubSubServer/conf/.env
          set +a
        fi
        if [ -e "/home/bts/.local/share/bts/PubSubServer/conf/.env.local" ]; then
          echo 'Sourcing /home/bts/.local/share/bts/PubSubServer/conf/.env.local'
          set -a
          . /home/bts/.local/share/bts/PubSubServer/conf/.env.local
          set +a
        fi
        ./PubSubServer"
    env_file:
      - .env
      - ${PUBSUB__CONFIG_DIR}/.env

  md-publisher:
    volumes:
      - ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/DataStreamPublisher/data
      - historic_md:/home/bts/.local/share/bts/databento
    build:
      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
      args:
        - SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
        - CONFIG_DIR=${MD_PUBLISHER__CONFIG_DIR}
    command: >
      sh -c "
        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/DataStreamPublisher
        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log" ]; then
          mv "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')"
        fi
        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/conf/.env" ]; then
          echo 'Sourcing /home/bts/.local/share/bts/DataStreamPublisher/conf/.env'
          set -a
          . /home/bts/.local/share/bts/DataStreamPublisher/conf/.env
          set +a
        fi
        if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local" ]; then
          echo 'Sourcing /home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local'
          set -a
          . /home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local
          set +a
        fi
      dockerize -wait tcp://pubsub:22019 -timeout 10s ./DataStreamPublisher"
    env_file:
      - .env
      - ${MD_PUBLISHER__CONFIG_DIR}/.env

  api-server:
    depends_on:
      - sparkd
    ports:
      - 8008:8008
      - 9443:9443
    build:
      dockerfile: src/client/spark_api_server/Dockerfile
    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
    environment:
      - ASPNETCORE_ENVIRONMENT=Staging

volumes:
  sparkd_data:
    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
  pubsub_data:
    name: "pubsub_22019"
  md_publisher_data:
    name: "md_publisher_22019"
  historic_md:
    name: "historic_md"