From 7eb8cd5eadbab199022829d1413b5dad53fe8f1d Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 16 Mar 2026 17:46:10 +0000 Subject: [PATCH] Allow md recording dir to be mounted as volume in docker compose --- .env | 3 ++ compose.yaml | 2 +- src/app_sparkd/configurations/cme_replay/.env | 4 ++ .../cme_replay/requirements.txt | 1 + .../configurations/cme_replay/spark.conf | 38 +++++++++++++++ .../configurations/cme_replay/symbols.conf | 48 +++++++++++++++++++ 6 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 src/app_sparkd/configurations/cme_replay/.env create mode 100644 src/app_sparkd/configurations/cme_replay/requirements.txt create mode 100644 src/app_sparkd/configurations/cme_replay/spark.conf create mode 100644 src/app_sparkd/configurations/cme_replay/symbols.conf diff --git a/.env b/.env index 2679534a2..5c78843cd 100644 --- a/.env +++ b/.env @@ -41,6 +41,9 @@ SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev # Mount a specific volume or dir to "~/sparkdata/data" inside the docker. # Defaults to a volume named "sparkd_" (e.g., sparkd_8000). # SPARKD__DATA_DIR=/home/bts/sparkdata/data +# Mount volume or dir to "/md_recording" inside the docker. Defaults to volume +# "sparkd_md_recording_" (e.g., sparkd_md_recording_8000). +# SPARKD__MD_RECORDING_DIR=/home/bts/md_recording SPARKD__SOURCE_DIR=install/sparkd-release-gcc SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/ SPARKD__CONFIG_PORT=8000 diff --git a/compose.yaml b/compose.yaml index 334fcf039..01fa44944 100644 --- a/compose.yaml +++ b/compose.yaml @@ -163,5 +163,5 @@ volumes: historic_md: name: "historic_md" md_recording: - name: "md_recording" + name: "sparkd_md_recording_${SPARKD__CONFIG_PORT:-8000}" external: true diff --git a/src/app_sparkd/configurations/cme_replay/.env b/src/app_sparkd/configurations/cme_replay/.env new file mode 100644 index 000000000..817a8672b --- /dev/null +++ b/src/app_sparkd/configurations/cme_replay/.env @@ -0,0 +1,4 @@ +SPARKD__CONFIG_DIR=src/app_sparkd/configurations/cme_replay +SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/cme_replay/requirements.txt +SPARKD__ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh +TZ=America/Chicago diff --git a/src/app_sparkd/configurations/cme_replay/requirements.txt b/src/app_sparkd/configurations/cme_replay/requirements.txt new file mode 100644 index 000000000..2c24336eb --- /dev/null +++ b/src/app_sparkd/configurations/cme_replay/requirements.txt @@ -0,0 +1 @@ +requests==2.31.0 diff --git a/src/app_sparkd/configurations/cme_replay/spark.conf b/src/app_sparkd/configurations/cme_replay/spark.conf new file mode 100644 index 000000000..db56dd41c --- /dev/null +++ b/src/app_sparkd/configurations/cme_replay/spark.conf @@ -0,0 +1,38 @@ +PublisherOptions { + value { + network_interface: "0.0.0.0" + heartbeat_ms: 5000 + heartbeat_boot_seconds: 60 + skip_options: true + hot_initial: 1024 + hot_growth: 1024 + cold_initial: 8192 + cold_growth: 2048 + channel_prefix: "" + disable_heartbeat: true + } +} + +MarketDataConfig { + value { + # glbx{ + # use_databento: true + # databento_settings { + # read_from_file: true + # key: "db-DgxpuTXneNgVvd7sJCAEKYDhyCApW" + # bbo_file_name: "glbx-mdp3-20251103.mbp-1.dbn.zst" + # contracts_file_name: "glbx-mdp3-20251103.definition.dbn.zst" + # } + # } + # recording{ + # venue: VENUE_GLBX + # num_of_seconds_to_run_file_writer: 240 + # num_of_seconds_to_run_db_writer : 0 + # n_book_levels: 10 + # batch_size: 100 #for the db writer + # } + replay { + venue: VENUE_GLBX + } + } +} diff --git a/src/app_sparkd/configurations/cme_replay/symbols.conf b/src/app_sparkd/configurations/cme_replay/symbols.conf new file mode 100644 index 000000000..c919c1aa2 --- /dev/null +++ b/src/app_sparkd/configurations/cme_replay/symbols.conf @@ -0,0 +1,48 @@ +Symbols { + value { + venues { + name: "VENUE_GLBX" + future: "ES" + # option: "E1A" + # option: "E1B" + # option: "E1C" + # option: "E1D" + # option: "E2A" + # option: "E2B" + # option: "E2C" + # option: "E2D" + # option: "E3A" + # option: "E3B" + # option: "E3C" + # option: "E3D" + # option: "E4A" + # option: "E4B" + # option: "E4C" + # option: "E4D" + # option: "E5A" + # option: "E5B" + # option: "E5C" + # option: "E5D" + # option: "EW1" + # option: "EW2" + # option: "EW3" + # option: "EW4" + # option: "ESH9" + # option: "ESZ6" + # option: "ESH0" + # option: "ESM6" + # option: "ESZ7" + # option: "ESH6" + # option: "ESM7" + # option: "ESZ8" + # option: "ESH7" + # option: "ESH8" + # option: "ESU6" + # option: "ESZ5" + # option: "ESZ9" + # option: "E1D" + # option: "ESZ5" + # option: "ES" + } + } +}