From 4deeacda4a8aa25536a1c7ca5e111d67dd2fd1e1 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 18 Jul 2024 17:26:10 +0000 Subject: [PATCH] Add scripts to deploy data stream apps --- docker/ubuntu-22.04/dotnet-7/Dockerfile | 39 ++++ .../Configurations/AwsProd.json | 118 +++++++++++ .../PubSubServer/Configurations/AwsProd.json | 64 ++++++ tools/bounce_data_stream_app.sh | 183 ++++++++++++++++++ tools/deploy_data_stream_app.sh | 170 ++++++++++++++++ 5 files changed, 574 insertions(+) create mode 100644 docker/ubuntu-22.04/dotnet-7/Dockerfile create mode 100644 src/DataStreamSuite/DataStreamPublisher/Configurations/AwsProd.json create mode 100644 src/DataStreamSuite/PubSubServer/Configurations/AwsProd.json create mode 100644 tools/bounce_data_stream_app.sh create mode 100755 tools/deploy_data_stream_app.sh diff --git a/docker/ubuntu-22.04/dotnet-7/Dockerfile b/docker/ubuntu-22.04/dotnet-7/Dockerfile new file mode 100644 index 000000000..ce39caee6 --- /dev/null +++ b/docker/ubuntu-22.04/dotnet-7/Dockerfile @@ -0,0 +1,39 @@ +FROM ubuntu:22.04 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get install -y pkg-config \ + && apt-get install -y locales \ + && apt-get install -y ca-certificates \ + && apt-get install -y libssh2-1-dev \ + && apt-get install -y software-properties-common \ + && apt-get install -y sudo + +# GCC +ARG gcc_major=13 +RUN add-apt-repository ppa:ubuntu-toolchain-r/test \ + && apt-get update \ + && apt-get install -y g++-$gcc_major + +RUN apt-get update \ + && apt-get install -y dotnet7 + +RUN locale-gen en_US.UTF-8 + +ARG docker_user="bts" +ARG docker_uid="1000" +ARG docker_pass="bts" + +RUN \ + groupadd -g "$docker_uid" "$docker_user" \ + && useradd \ + -m \ + -u "$docker_uid" \ + -g "$docker_uid" \ + -G sudo \ + --password "$(openssl passwd -1 "$docker_pass")" \ + "$docker_user" + +USER bts +WORKDIR /home/bts diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/AwsProd.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/AwsProd.json new file mode 100644 index 000000000..db789da16 --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/Configurations/AwsProd.json @@ -0,0 +1,118 @@ +{ + "Spark": { + "ConfigServer": "http://localhost:8000/config" + }, + "SparkApi": { + "Credentials": { + "User": "bts", + "Password": "" + }, + "EnableAdmin": true, + "EnableOrders": true, + "EnablePosition": true + }, + "MasterWorker": { + "Interval": "00:00:00.1" + }, + "ViewStreamDatabase": { + "Interval": "00:00:00.1" + }, + "PubSub": { + "HeartbeatInterval": 15000, + "MissingHeartbeatsBeforeGc": 50, + "DisposedConnectionCheckPeriod": 1000, + "ServerPort": 22019, + "Routing": { + "PublisherRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://34.237.179.93:22019", + "TagRegExp": ".*" + } + ], + "SubscriberRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://34.237.179.93:22019", + "TagRegExp": ".*" + } + ] + }, + "TagTypeToAssemblyMap": { + "Entries": [ + { + "HandlerClass": "BCG.PubSubServerTagHandler", + "Assembly": "DataStreamCore.dll" + } + ] + }, + "TagHandlerFactory": { + "Handlers": [ + { + "TagType": "DataStream", + "Assembly": "DataStreamCore", + "Class": "bts.PubsubServerStreamTagHandler" + }, + { + "TagType": "System_Sub", + "Assembly": "PubSub", + "Class": "BCG.SystemSubscriptionMapPublisher" + }, + { + "TagType": "System_Stats", + "Assembly": "PubSub", + "Class": "BCG.SystemStatsPublisher" + }, + { + "TagType": ".*", + "Assembly": "DataStreamCore", + "Class": "bts.PubsubServerGenericDataHandler" + } + ] + } + }, + "DataStreamPublisher": { + "PublisherNodes": [ + { + "Name": "market_data", + "Streams": [ + { + "Name": "bbo", + "Expression": "bbo()", + "IsOnWebsocket": true + }, + { + "Name": "instruments", + "Expression": "instruments()", + "IsOnWebsocket": true + }, + { + "Name": "timesales", + "Expression": "timesales(10000)", + "IsOnWebsocket": true + }, + { + "Name": "books", + "Expression": "books()", + "IsOnWebsocket": true + } + ] + }, + { + "Name": "test_data", + "Streams": [ + { + "Name": "milliseconds_1000_100", + "Expression": "milliseconds(3000,5)", + "IsOnWebsocket": true + }, + { + "Name": "test_literal", + "Expression": "literal(\"x\", \"420.0\")", + "IsOnWebsocket": true + } + ] + } + ] + } +} diff --git a/src/DataStreamSuite/PubSubServer/Configurations/AwsProd.json b/src/DataStreamSuite/PubSubServer/Configurations/AwsProd.json new file mode 100644 index 000000000..f2639ee9a --- /dev/null +++ b/src/DataStreamSuite/PubSubServer/Configurations/AwsProd.json @@ -0,0 +1,64 @@ +{ + "Spark": {}, + "MasterWorker": { + "Interval": "00:00:00.1" + }, + "PubSub": { + "HeartbeatInterval": 15000, + "MissingHeartbeatsBeforeGc": 50, + "DisposedConnectionCheckPeriod": 1000, + "ServerPort": 22019, + "Routing": { + "PublisherRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://34.237.179.93:22019", + "TagRegExp": ".*" + } + ], + "SubscriberRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://34.237.179.93:22019", + "TagRegExp": ".*" + } + ] + }, + "TagTypeToAssemblyMap": { + "Entries": [ + { + "HandlerClass": "BCG.PubSubServerTagHandler", + "Assembly": "DataStreamCore.dll" + } + ] + }, + "TagHandlerFactory": { + "Handlers": [ + { + "TagType": "DataStream", + "Assembly": "DataStreamCore", + "Class": "bts.PubsubServerStreamTagHandler" + }, + { + "TagType": "System_Sub", + "Assembly": "PubSub", + "Class": "BCG.SystemSubscriptionMapPublisher" + }, + { + "TagType": "System_Stats", + "Assembly": "PubSub", + "Class": "BCG.SystemStatsPublisher" + }, + { + "TagType": ".*", + "Assembly": "DataStreamCore", + "Class": "bts.PubsubServerGenericDataHandler" + } + ] + } + }, + "GarbageCollector": { + "ActivationThreshold": 0, + "Interval": "00:00:01" + } +} diff --git a/tools/bounce_data_stream_app.sh b/tools/bounce_data_stream_app.sh new file mode 100644 index 000000000..d854e6c40 --- /dev/null +++ b/tools/bounce_data_stream_app.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash + +print_help_and_exit() { + cat < + +Positionals: + app The application to bounce [PubSubServer | DataStreamPublisher | Bts.DataStreamWebApp.Server] + container_prefix The container prefix to use. Existing dockers with this prefix will be terminated. + release The release id. This will be appended to the container prefix to form the new container name. + +Options: + --app The application to bounce [PubSubServer | DataStreamPublisher | Bts.DataStreamWebApp.Server] + --container-prefix The container prefix to use. Existing dockers with this prefix will be terminated. + --release The release id. This will be appended to the container prefix to form the new container name. + +Examples: + Deploy PubSubServer : deploy_data_stream_app.sh PubSubServer pubsub 00d0d6ade + Deploy DataStreamPublisher : deploy_data_stream_app.sh DataStreamPublisher ds-publisher 00d0d6ade +EOF +exit "$1" +} + +stop_containers() { + if [ $# -ne 1 ]; then + echo "stop_containers expects exactly 1 arg (the container prefix). Received $#: '$@'" + exit 1 + fi + declare -a containers="$(docker ps -q --filter "name=^$1-[a-z0-9]{9}\$")" + for c in ${containers[@]}; do + echo "Removing old container $c" + docker stop "$c" + status=$? + if [ $status -ne 0 ]; then + echo "Failed to remove container $c" + exit 1 + fi + done +} + +remove_containers() { + if [ $# -ne 1 ]; then + echo "remove_containers expects exactly 1 arg (the container prefix). Received $#: '$@'" + exit 1 + fi + declare -a containers="$(docker container ls --all -q --filter "name=^$1-[a-z0-9]{9}\$")" + for c in ${containers[@]}; do + echo "Removing old container $c" + docker rm "$c" + status=$? + if [ $status -ne 0 ]; then + echo "Failed to remove container $c" + exit 1 + fi + done +} + +kill_containers() { + stop_containers $1 + remove_containers $1 +} + +app_name=$1 +container_prefix=$2 +release=$3 + +while [[ $# -gt 0 ]]; do + case "$1" in + --app) + if [[ -z "$2" ]]; then + echo "App (--app) must be non-empty" + print_help_and_exit 1 + fi + app_name="$2" + shift 2 + ;; + --container-prefix) + if [[ -z "$2" ]]; then + echo "Container prefix (--container-prefix) must be non-empty" + print_help_and_exit 1 + fi + container_prefix="$2" + shift 2 + ;; + --release) + if [[ -z "$2" ]]; then + echo "Release (--release) must be non-empty" + print_help_and_exit 1 + fi + release="$2" + shift 2 + ;; + -h) + ;& + --help) + print_help_and_exit 0 + ;; + -*) + echo "Unexpected option '$1'" + print_help_and_exit 1 + ;; + *) + if [[ -z "$1" ]]; then + echo "Invalid position argument '$1'" + print_help_and_exit 1 + elif [[ -z "$app_name" ]]; then + app_name="$1" + elif [[ -z "$container_prefix" ]]; then + container_prefix="$1" + elif [[ -z "$release" ]]; then + release="$1" + fi + shift 1 + ;; + esac +done + +# Validate args +if [ "$release" == "" ]; then + echo "Release not defined" + print_help_and_exit 1 +fi + +if [ "$app_name" == "" ]; then + echo "App name not defined" + print_help_and_exit 1 +fi + +if [ "$container_prefix" == "" ]; then + echo "Container prefix not defined" + print_help_and_exit 1 +fi + +container="$container_prefix-$release" +image="$container_prefix-$release" + +cat < + +Positionals: + app The application to bounce [PubSubServer | DataStreamPublisher | Bts.DataStreamWebApp.Server] + container_prefix The container prefix to use. Existing dockers with this prefix will be terminated. + release The release id. This will be appended to the container prefix to form the new container name. + +Options: + --app The application to bounce [PubSubServer | DataStreamPublisher | Bts.DataStreamWebApp.Server] + --container-prefix The container prefix to use. Existing dockers with this prefix will be terminated. + --release The release id. This will be appended to the container prefix to form the new container name. + +EOF +exit "$1" +} + +app_name="" +container_prefix="" +release="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --app) + if [[ -z "$2" ]]; then + echo "App (--app) must be non-empty" + print_help_and_exit 1 + fi + app_name="$2" + shift 2 + ;; + --container-prefix) + if [[ -z "$2" ]]; then + echo "Container prefix (--container-prefix) must be non-empty" + print_help_and_exit 1 + fi + container_prefix="$2" + shift 2 + ;; + --release) + if [[ -z "$2" ]]; then + echo "Release (--release) must be non-empty" + print_help_and_exit 1 + fi + release="$2" + shift 2 + ;; + -h) + ;& + --help) + print_help_and_exit 0 + ;; + -*) + echo "Unexpected option '$1'" + print_help_and_exit 1 + ;; + *) + if [[ -z "$1" ]]; then + echo "Invalid position argument '$1'" + print_help_and_exit 1 + elif [[ -z "$app_name" ]]; then + app_name="$1" + elif [[ -z "$container_prefix" ]]; then + container_prefix="$1" + elif [[ -z "$release" ]]; then + release="$1" + fi + shift 1 + ;; + esac +done + +# Validate args +if [ "$app_name" == "" ]; then + echo "App name not defined" + print_help_and_exit 1 +fi + +if [ "$container_prefix" == "" ]; then + echo "Container prefix not defined" + print_help_and_exit 1 +fi + +if [ "$release" == "" ]; then + # Use commit sha + release="$(git rev-parse --short HEAD)" +fi + +if [ "$release" == "" ]; then + echo "Release not defined" + print_help_and_exit 1 +fi + +this_dir="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" +cd $this_dir/.. +cfg_profile="AwsProd" +cfg_profiles_dir="src/DataStreamSuite/$app_name/Configurations" +cfg_profile_json="$cfg_profiles_dir/$cfg_profile.json" +if [ ! -e "$cfg_profile_json" ]; then + echo "Profile '${cfg_profile}' not found. Looked for file '${cfg_profile_json}'" +fi + +release_dir=".tmp/$release" +mkdir -p "$release_dir" +release_bin="$release_dir/bin" +publish_sh="$this_dir/publish_data_stream.sh" + +# Publish self-contained app(s) +echo "Publishing $release" +"$publish_sh" -o "$release_bin" +status=$? +if [ $status -ne 0 ]; then + echo "Publish exited with code $status. Full command below:" + echo "Command: $publish_sh -o $release_bin" + exit 1 +fi + +# Copy configuration file to release_bin +echo "Copying configuration to appsettings.json" +cp "$cfg_profile_json" "$release_bin/appsettings.json" +status=$? +if [ $status -ne 0 ]; then + echo "Copy configuration exited with code $status. Full command below:" + echo "cp $cfg_profile_json $release_bin/appsettings.json" + exit 1 +fi + +# Copy Dockerfile to release_dir +cp "docker/ubuntu-22.04/dotnet-7/Dockerfile" "$release_dir/" +status=$? +if [ $status -ne 0 ]; then + echo "Copy Dockerfile exited with code $status. Full command below:" + echo "cp docker/ubuntu-22.04/dotnet-7/Dockerfile $release_dir/" + exit 1 +fi + +cat <> "$release_dir/Dockerfile" +COPY "bin" "$app_name/bin" +WORKDIR /home/bts/$app_name/bin +ENTRYPOINT [ "./$app_name" ] + +EOF + +# Compress +echo "Compressing $release" +cd .tmp +tar -czf "$release.tar.gz" "$release" +status=$? +if [ $status -ne 0 ]; then + echo "Compress exited with code $status. Full command below:" + echo "tar -czf $release.tar.gz $release" + exit 1 +fi + +# Clean +echo "Removing $release (uncompressed dir)" +rm -r "$release" + +# Upload +remote=bts-cloud-streams +echo "Uploading $release.tar.gz to $remote" +scp "$release.tar.gz" $remote:~/DataStream/releases/ + +cat <