Add scripts to deploy data stream apps

This commit is contained in:
John Colvin committed 2024-11-25 15:26:58 +00:00
1 parent 9a0bf98b2c
commit 4deeacda4a
5 files changed
+574

No files matched your search

+39
View File
@@ -0,0 +1,39 @@
FROM ubuntu:22.04
ARG DEBIAN_FRONTEND=noninteractive
RUN apt-get update \
&& apt-get install -y pkg-config \
&& apt-get install -y locales \
&& apt-get install -y ca-certificates \
&& apt-get install -y libssh2-1-dev \
&& apt-get install -y software-properties-common \
&& apt-get install -y sudo
# GCC
ARG gcc_major=13
RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
&& apt-get update \
&& apt-get install -y g++-$gcc_major
RUN apt-get update \
&& apt-get install -y dotnet7
RUN locale-gen en_US.UTF-8
ARG docker_user="bts"
ARG docker_uid="1000"
ARG docker_pass="bts"
RUN \
groupadd -g "$docker_uid" "$docker_user" \
&& useradd \
-m \
-u "$docker_uid" \
-g "$docker_uid" \
-G sudo \
--password "$(openssl passwd -1 "$docker_pass")" \
"$docker_user"
USER bts
WORKDIR /home/bts
@@ -0,0 +1,118 @@
{
"Spark": {
"ConfigServer": "http://localhost:8000/config"
},
"SparkApi": {
"Credentials": {
"User": "bts",
"Password": ""
},
"EnableAdmin": true,
"EnableOrders": true,
"EnablePosition": true
},
"MasterWorker": {
"Interval": "00:00:00.1"
},
"ViewStreamDatabase": {
"Interval": "00:00:00.1"
},
"PubSub": {
"HeartbeatInterval": 15000,
"MissingHeartbeatsBeforeGc": 50,
"DisposedConnectionCheckPeriod": 1000,
"ServerPort": 22019,
"Routing": {
"PublisherRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://34.237.179.93:22019",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://34.237.179.93:22019",
"TagRegExp": ".*"
}
]
},
"TagTypeToAssemblyMap": {
"Entries": [
{
"HandlerClass": "BCG.PubSubServerTagHandler",
"Assembly": "DataStreamCore.dll"
}
]
},
"TagHandlerFactory": {
"Handlers": [
{
"TagType": "DataStream",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerStreamTagHandler"
},
{
"TagType": "System_Sub",
"Assembly": "PubSub",
"Class": "BCG.SystemSubscriptionMapPublisher"
},
{
"TagType": "System_Stats",
"Assembly": "PubSub",
"Class": "BCG.SystemStatsPublisher"
},
{
"TagType": ".*",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerGenericDataHandler"
}
]
}
},
"DataStreamPublisher": {
"PublisherNodes": [
{
"Name": "market_data",
"Streams": [
{
"Name": "bbo",
"Expression": "bbo()",
"IsOnWebsocket": true
},
{
"Name": "instruments",
"Expression": "instruments()",
"IsOnWebsocket": true
},
{
"Name": "timesales",
"Expression": "timesales(10000)",
"IsOnWebsocket": true
},
{
"Name": "books",
"Expression": "books()",
"IsOnWebsocket": true
}
]
},
{
"Name": "test_data",
"Streams": [
{
"Name": "milliseconds_1000_100",
"Expression": "milliseconds(3000,5)",
"IsOnWebsocket": true
},
{
"Name": "test_literal",
"Expression": "literal(\"x\", \"420.0\")",
"IsOnWebsocket": true
}
]
}
]
}
}
@@ -0,0 +1,64 @@
{
"Spark": {},
"MasterWorker": {
"Interval": "00:00:00.1"
},
"PubSub": {
"HeartbeatInterval": 15000,
"MissingHeartbeatsBeforeGc": 50,
"DisposedConnectionCheckPeriod": 1000,
"ServerPort": 22019,
"Routing": {
"PublisherRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://34.237.179.93:22019",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://34.237.179.93:22019",
"TagRegExp": ".*"
}
]
},
"TagTypeToAssemblyMap": {
"Entries": [
{
"HandlerClass": "BCG.PubSubServerTagHandler",
"Assembly": "DataStreamCore.dll"
}
]
},
"TagHandlerFactory": {
"Handlers": [
{
"TagType": "DataStream",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerStreamTagHandler"
},
{
"TagType": "System_Sub",
"Assembly": "PubSub",
"Class": "BCG.SystemSubscriptionMapPublisher"
},
{
"TagType": "System_Stats",
"Assembly": "PubSub",
"Class": "BCG.SystemStatsPublisher"
},
{
"TagType": ".*",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerGenericDataHandler"
}
]
}
},
"GarbageCollector": {
"ActivationThreshold": 0,
"Interval": "00:00:01"
}
}
+183
View File
@@ -0,0 +1,183 @@
#!/usr/bin/env bash
print_help_and_exit() {
cat <<EOF
Usage: bounce_data_stream_app.sh [OPTIONS] <app> <container_prefix> <release>
Positionals:
app The application to bounce [PubSubServer | DataStreamPublisher | Bts.DataStreamWebApp.Server]
container_prefix The container prefix to use. Existing dockers with this prefix will be terminated.
release The release id. This will be appended to the container prefix to form the new container name.
Options:
--app The application to bounce [PubSubServer | DataStreamPublisher | Bts.DataStreamWebApp.Server]
--container-prefix The container prefix to use. Existing dockers with this prefix will be terminated.
--release The release id. This will be appended to the container prefix to form the new container name.
Examples:
Deploy PubSubServer : deploy_data_stream_app.sh PubSubServer pubsub 00d0d6ade
Deploy DataStreamPublisher : deploy_data_stream_app.sh DataStreamPublisher ds-publisher 00d0d6ade
EOF
exit "$1"
}
stop_containers() {
if [ $# -ne 1 ]; then
echo "stop_containers expects exactly 1 arg (the container prefix). Received $#: '$@'"
exit 1
fi
declare -a containers="$(docker ps -q --filter "name=^$1-[a-z0-9]{9}\$")"
for c in ${containers[@]}; do
echo "Removing old container $c"
docker stop "$c"
status=$?
if [ $status -ne 0 ]; then
echo "Failed to remove container $c"
exit 1
fi
done
}
remove_containers() {
if [ $# -ne 1 ]; then
echo "remove_containers expects exactly 1 arg (the container prefix). Received $#: '$@'"
exit 1
fi
declare -a containers="$(docker container ls --all -q --filter "name=^$1-[a-z0-9]{9}\$")"
for c in ${containers[@]}; do
echo "Removing old container $c"
docker rm "$c"
status=$?
if [ $status -ne 0 ]; then
echo "Failed to remove container $c"
exit 1
fi
done
}
kill_containers() {
stop_containers $1
remove_containers $1
}
app_name=$1
container_prefix=$2
release=$3
while [[ $# -gt 0 ]]; do
case "$1" in
--app)
if [[ -z "$2" ]]; then
echo "App (--app) must be non-empty"
print_help_and_exit 1
fi
app_name="$2"
shift 2
;;
--container-prefix)
if [[ -z "$2" ]]; then
echo "Container prefix (--container-prefix) must be non-empty"
print_help_and_exit 1
fi
container_prefix="$2"
shift 2
;;
--release)
if [[ -z "$2" ]]; then
echo "Release (--release) must be non-empty"
print_help_and_exit 1
fi
release="$2"
shift 2
;;
-h)
;&
--help)
print_help_and_exit 0
;;
-*)
echo "Unexpected option '$1'"
print_help_and_exit 1
;;
*)
if [[ -z "$1" ]]; then
echo "Invalid position argument '$1'"
print_help_and_exit 1
elif [[ -z "$app_name" ]]; then
app_name="$1"
elif [[ -z "$container_prefix" ]]; then
container_prefix="$1"
elif [[ -z "$release" ]]; then
release="$1"
fi
shift 1
;;
esac
done
# Validate args
if [ "$release" == "" ]; then
echo "Release not defined"
print_help_and_exit 1
fi
if [ "$app_name" == "" ]; then
echo "App name not defined"
print_help_and_exit 1
fi
if [ "$container_prefix" == "" ]; then
echo "Container prefix not defined"
print_help_and_exit 1
fi
container="$container_prefix-$release"
image="$container_prefix-$release"
cat <<EOF
Bouncing with the following parameters:
Application : $app_name
Container Prefix : $container_prefix
Container : $container
Image : $image
Release : $release
EOF
cd $HOME/DataStream/releases
if [ ! -e $release ]; then
if [ -e $release.tar.gz ]; then
tar -xf $release.tar.gz
status=$?
if [ $status -ne 0 ]; then
echo "Failed to extract $release.tar.gz"
exit 1
fi
fi
if [ ! -e $release ]; then
echo "Did not find release '$release'"
exit 1
fi
fi
cd $release
echo "Building new image"
docker build -t "$image" .
status=$?
if [ $status -ne 0 ]; then
echo "Build image failed. Full command below:"
echo "docker build -t $image ."
exit 1
fi
cd ..
echo "Killing existing $app_name containers (matching prefix '$container_prefix')"
kill_containers "$container_prefix"
echo "Starting container $container"
docker run \
--name "$container" \
--net host \
-w "/home/bts/$app_name/bin" \
--shm-size=4024m \
-d \
"$image"
+170
View File
@@ -0,0 +1,170 @@
#!/usr/bin/env bash
print_help_and_exit() {
cat <<EOF
Usage: deploy_data_stream_app.sh [OPTIONS] <app> <container_prefix> <release>
Positionals:
app The application to bounce [PubSubServer | DataStreamPublisher | Bts.DataStreamWebApp.Server]
container_prefix The container prefix to use. Existing dockers with this prefix will be terminated.
release The release id. This will be appended to the container prefix to form the new container name.
Options:
--app The application to bounce [PubSubServer | DataStreamPublisher | Bts.DataStreamWebApp.Server]
--container-prefix The container prefix to use. Existing dockers with this prefix will be terminated.
--release The release id. This will be appended to the container prefix to form the new container name.
EOF
exit "$1"
}
app_name=""
container_prefix=""
release=""
while [[ $# -gt 0 ]]; do
case "$1" in
--app)
if [[ -z "$2" ]]; then
echo "App (--app) must be non-empty"
print_help_and_exit 1
fi
app_name="$2"
shift 2
;;
--container-prefix)
if [[ -z "$2" ]]; then
echo "Container prefix (--container-prefix) must be non-empty"
print_help_and_exit 1
fi
container_prefix="$2"
shift 2
;;
--release)
if [[ -z "$2" ]]; then
echo "Release (--release) must be non-empty"
print_help_and_exit 1
fi
release="$2"
shift 2
;;
-h)
;&
--help)
print_help_and_exit 0
;;
-*)
echo "Unexpected option '$1'"
print_help_and_exit 1
;;
*)
if [[ -z "$1" ]]; then
echo "Invalid position argument '$1'"
print_help_and_exit 1
elif [[ -z "$app_name" ]]; then
app_name="$1"
elif [[ -z "$container_prefix" ]]; then
container_prefix="$1"
elif [[ -z "$release" ]]; then
release="$1"
fi
shift 1
;;
esac
done
# Validate args
if [ "$app_name" == "" ]; then
echo "App name not defined"
print_help_and_exit 1
fi
if [ "$container_prefix" == "" ]; then
echo "Container prefix not defined"
print_help_and_exit 1
fi
if [ "$release" == "" ]; then
# Use commit sha
release="$(git rev-parse --short HEAD)"
fi
if [ "$release" == "" ]; then
echo "Release not defined"
print_help_and_exit 1
fi
this_dir="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
cd $this_dir/..
cfg_profile="AwsProd"
cfg_profiles_dir="src/DataStreamSuite/$app_name/Configurations"
cfg_profile_json="$cfg_profiles_dir/$cfg_profile.json"
if [ ! -e "$cfg_profile_json" ]; then
echo "Profile '${cfg_profile}' not found. Looked for file '${cfg_profile_json}'"
fi
release_dir=".tmp/$release"
mkdir -p "$release_dir"
release_bin="$release_dir/bin"
publish_sh="$this_dir/publish_data_stream.sh"
# Publish self-contained app(s)
echo "Publishing $release"
"$publish_sh" -o "$release_bin"
status=$?
if [ $status -ne 0 ]; then
echo "Publish exited with code $status. Full command below:"
echo "Command: $publish_sh -o $release_bin"
exit 1
fi
# Copy configuration file to release_bin
echo "Copying configuration to appsettings.json"
cp "$cfg_profile_json" "$release_bin/appsettings.json"
status=$?
if [ $status -ne 0 ]; then
echo "Copy configuration exited with code $status. Full command below:"
echo "cp $cfg_profile_json $release_bin/appsettings.json"
exit 1
fi
# Copy Dockerfile to release_dir
cp "docker/ubuntu-22.04/dotnet-7/Dockerfile" "$release_dir/"
status=$?
if [ $status -ne 0 ]; then
echo "Copy Dockerfile exited with code $status. Full command below:"
echo "cp docker/ubuntu-22.04/dotnet-7/Dockerfile $release_dir/"
exit 1
fi
cat <<EOF >> "$release_dir/Dockerfile"
COPY "bin" "$app_name/bin"
WORKDIR /home/bts/$app_name/bin
ENTRYPOINT [ "./$app_name" ]
EOF
# Compress
echo "Compressing $release"
cd .tmp
tar -czf "$release.tar.gz" "$release"
status=$?
if [ $status -ne 0 ]; then
echo "Compress exited with code $status. Full command below:"
echo "tar -czf $release.tar.gz $release"
exit 1
fi
# Clean
echo "Removing $release (uncompressed dir)"
rm -r "$release"
# Upload
remote=bts-cloud-streams
echo "Uploading $release.tar.gz to $remote"
scp "$release.tar.gz" $remote:~/DataStream/releases/
cat <<EOF | ssh $remote /usr/bin/env bash
rm -r /home/ubuntu/DataStream/releases/$release
/home/ubuntu/DataStream/scripts/bounce_data_stream_app.sh $app_name $container_prefix $release
EOF