spark/compose.yaml

services:
  sparkd:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
    build:
      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    env_file:
      - .env
      - ${SPARKD__CONFIG_DIR}/.env
    ports:
      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
    deploy:
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-4gb}"

  sparkd-shell:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
    build:
      dockerfile: docker/images/bluetradingsystems/linux_shell/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    command: "do sleep 1; done"
    env_file:
      - .env
      - ${SPARKD__CONFIG_DIR}/.env
    ports:
      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
    deploy:
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-4gb}"

  sparkd-symbols:
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
    build:
      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
    env_file:
      - .env
      - ${SPARKD__CONFIG_DIR}/.env
    ports:
      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
    deploy:
      restart_policy:
        condition: any
        delay: 3s
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-4gb}"
  sparkd-multicast:
    environment:
      - TZ=America/Chicago
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
    build:
      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
    env_file:
      - .env
      - ${SPARKD__CONFIG_DIR}/.env
    network_mode: "host"
    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
    deploy:
      restart_policy:
        condition: any
        delay: 3s
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-16gb}"
  
  
  sparkd-multicast-cert:
    environment:
      - TZ=America/Chicago
    volumes:
      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
    build:
      dockerfile: src/app_sparkd/Dockerfile
      args:
        - IMAGE=${SPARKD__IMAGE}
        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
    env_file:
      - .env
      - ${SPARKD__CONFIG_DIR}/.env
    network_mode: "host"
    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
    deploy:
      restart_policy:
        condition: any
        delay: 3s
      resources:
        limits:
          memory: "${SPARKD__MEMORY:-16gb}"
  
  sparkd-tail:
    image: bluetradingsystems/linux_dotnet8
    volumes:
      - type: volume
        read_only: true
        source: sparkd_data
        target: /home/bts/.local/share/bts/sparkd/data/logging.sparkd
        volume:
          subpath: logging.sparkd
    command: ["tail", "-f", "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log"]

  sparkd-ping-test:
    depends_on:
      - sparkd
    profiles:
      - ping-test
    build:
      context: .
      dockerfile_inline: |
        FROM bluetradingsystems/linux_dotnet8
        USER "root"
        RUN apt-get update && apt-get -y install iputils-ping
    command: [ "ping", "sparkd" ]

  pubsub:
    volumes:
      - ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/pubsub/data
    build:
      dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
      args:
        - SOURCE_DIR=${PUBSUB__SOURCE_DIR}
        - CONFIG_DIR=${PUBSUB__CONFIG_DIR}
    ports:
      - 22019:22019
    command: >
      sh -c "
        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/pubsub/data
        if [ -e "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" ]; then
          mv "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/pubsub/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')"
        fi
        if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env" ]; then
          echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env'
          set -a
          . /home/bts/.local/share/bts/pubsub/conf/.env
          set +a
        fi
        if [ -e "/home/bts/.local/share/bts/pubsub/conf/.env.local" ]; then
          echo 'Sourcing /home/bts/.local/share/bts/pubsub/conf/.env.local'
          set -a
          . /home/bts/.local/share/bts/pubsub/conf/.env.local
          set +a
        fi
        ./PubSubServer"
    env_file:
      - .env
      - ${PUBSUB__CONFIG_DIR}/.env

  md-publisher:
    volumes:
      - ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/md_publisher/data
      - historic_md:/home/bts/.local/share/bts/md_publisher/data/databento
    build:
      dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
      args:
        - SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
        - CONFIG_DIR=${MD_PUBLISHER__CONFIG_DIR}
    command: >
      sh -c "
        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/md_publisher/data
        if [ -e "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" ]; then
          mv "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/md_publisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')"
        fi
        if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env" ]; then
          echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env'
          set -a
          . /home/bts/.local/share/bts/md_publisher/conf/.env
          set +a
        fi
        if [ -e "/home/bts/.local/share/bts/md_publisher/conf/.env.local" ]; then
          echo 'Sourcing /home/bts/.local/share/bts/md_publisher/conf/.env.local'
          set -a
          . /home/bts/.local/share/bts/md_publisher/conf/.env.local
          set +a
        fi
      dockerize -wait tcp://pubsub:22019 -timeout 10s ./DataStreamPublisher"
    env_file:
      - .env
      - ${MD_PUBLISHER__CONFIG_DIR}/.env

  api-server:
    depends_on:
      - sparkd
    ports:
      - 8008:8008
      - 9443:9443
    build:
      dockerfile: src/client/spark_api_server/Dockerfile
    command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
    environment:
      - ASPNETCORE_ENVIRONMENT=Staging

volumes:
  sparkd_data:
    name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
  pubsub_data:
    name: "pubsub_22019"
  md_publisher_data:
    name: "md_publisher_22019"
  historic_md:
    name: "historic_md"
  md_recording:
    name: "sparkd_md_recording_${SPARKD__CONFIG_PORT:-8000}"
    external: true