diff --git a/.vscode/tasks.json b/.vscode/tasks.json
index 23f56f6f4..8e9de2a28 100644
--- a/.vscode/tasks.json
+++ b/.vscode/tasks.json
@@ -422,12 +422,11 @@
"windows": {
"command": "DataStreamPublisher.exe",
},
- "args": [
- "--config",
- "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
- ],
"options": {
- "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+ "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
+ "env": {
+ "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+ },
},
"icon": {
"id": "run",
@@ -445,12 +444,11 @@
"windows": {
"command": "DataStreamPublisher.exe",
},
- "args": [
- "--config",
- "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
- ],
"options": {
- "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+ "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+ "env": {
+ "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+ },
},
"icon": {
"id": "run",
@@ -468,12 +466,11 @@
"windows": {
"command": "DataStreamPublisher.exe",
},
- "args": [
- "--config",
- "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
- ],
"options": {
- "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+ "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+ "env": {
+ "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
+ },
},
"icon": {
"id": "run",
@@ -491,12 +488,11 @@
"windows": {
"command": "PubSubServer.exe",
},
- "args": [
- "--config",
- "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
- ],
"options": {
- "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
+ "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
+ "env": {
+ "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+ },
},
"icon": {
"id": "run",
@@ -514,12 +510,11 @@
"windows": {
"command": "PubSubServer.exe",
},
- "args": [
- "--config",
- "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
- ],
"options": {
- "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+ "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+ "env": {
+ "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+ },
},
"icon": {
"id": "run",
@@ -536,12 +531,11 @@
"windows": {
"command": "PubSubServer.exe",
},
- "args": [
- "--config",
- "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
- ],
"options": {
- "cwd": "${workspaceFolder}/build/Debug/bin/net8.0"
+ "cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
+ "env": {
+ "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+ },
},
"icon": {
"id": "run",
@@ -559,12 +553,11 @@
"windows": {
"command": "PubSubServer.exe",
},
- "args": [
- "--config",
- "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
- ],
"options": {
- "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+ "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+ "env": {
+ "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+ },
},
"icon": {
"id": "run",
@@ -581,12 +574,11 @@
"windows": {
"command": "PubSubServer.exe",
},
- "args": [
- "--config",
- "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
- ],
"options": {
- "cwd": "${workspaceFolder}/build/Release/bin/net8.0"
+ "cwd": "${workspaceFolder}/build/Release/bin/net8.0",
+ "env": {
+ "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+ },
},
"icon": {
"id": "run",
@@ -885,26 +877,30 @@
"default": "sparkd-debug-clang",
},
{
- "id": "dataStreamPublisherConfig",
+ "id": "dataStreamPublisherEnvironment",
"type": "pickString",
"description": "Publisher Configuration",
"options": [
- "MarketDataPublisher",
- "SimpleEchoClient",
- "SimpleTestDataPublisher",
- "TestDataPublisher",
+ "Development",
+ "MarketData",
+ "Production",
+ "ProductionAws",
+ "Staging",
],
"default": "MarketDataPublisher",
},
{
- "id": "pubSubServerConfig",
+ "id": "pubSubServerEnvironment",
"type": "pickString",
"description": "PubSub Configuration",
"options": [
- "PubSubServer",
- "SimpleEchoClient",
+ "Development",
+ "EchoServer",
+ "Production",
+ "ProductionAws",
+ "Staging",
],
- "default": "PubSubServer",
+ "default": "Development",
},
{
"id": "sparkConfigQuery",
diff --git a/src/DataStreamSuite/Bts.Extensions.Hosting/Bts.Extensions.Hosting.csproj b/src/DataStreamSuite/Bts.Extensions.Hosting/Bts.Extensions.Hosting.csproj
new file mode 100644
index 000000000..988058984
--- /dev/null
+++ b/src/DataStreamSuite/Bts.Extensions.Hosting/Bts.Extensions.Hosting.csproj
@@ -0,0 +1,13 @@
+
+
+
+ net8.0
+ enable
+ enable
+
+
+
+
+
+
+
diff --git a/src/DataStreamSuite/Bts.Extensions.Hosting/HostBuilderExtensions.cs b/src/DataStreamSuite/Bts.Extensions.Hosting/HostBuilderExtensions.cs
new file mode 100644
index 000000000..1b70b345b
--- /dev/null
+++ b/src/DataStreamSuite/Bts.Extensions.Hosting/HostBuilderExtensions.cs
@@ -0,0 +1,59 @@
+namespace Bts.Extensions.Hosting;
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Hosting;
+
+using System.Diagnostics;
+
+public static class HostBuilderExtensions {
+ public static IHostBuilder CreateBtsBuilder(string[] args) {
+ // Obtain calling assembly name here, rather than in the lambda where it is actually needed,
+ // because want the name of the assembly that called this function, not the assembly that will eventually invoke
+ // the lambda...that will be the Microsoft.Extensions.Hosting assembly.
+ // Get the name of the assembly that called this function. That will act as the "appsettings"
+ // json file prefix
+ var exeName = GetCallingAssemblyName();
+ return Host.CreateDefaultBuilder(args)
+ .ConfigureAppConfiguration((context, configBuilder) =>
+ ConfigureApp(configBuilder, context.HostingEnvironment.EnvironmentName, exeName));
+ }
+
+ public static void ConfigureApp(IConfigurationBuilder configBuilder, string? environment = null, string? prefix = null) {
+ var baseFileName = prefix is null ? "appsettings" : $"{prefix}.appsettings";
+ AddJsonFile($"{baseFileName}.json");
+ if (environment is not null) {
+ AddJsonFile($"{baseFileName}.{environment}.json");
+ }
+
+ var cfg = configBuilder.Build().GetSection("config")?.Value;
+ if (!SetBasePath(cfg)) {
+ AddJsonFile(cfg);
+ }
+
+ bool SetBasePath(string? path) {
+ if (!Directory.Exists(path)) {
+ return false;
+ }
+ configBuilder.SetBasePath(path);
+ return true;
+ }
+
+ bool AddJsonFile(string? path) {
+ if (!File.Exists(path)) {
+ return false;
+ }
+ configBuilder.AddJsonFile(path);
+ return true;
+ }
+ }
+
+ public static string GetCallingAssemblyName() =>
+ new StackTrace()
+ .GetFrame(2)
+ ?.GetMethod()
+ ?.Module
+ ?.Assembly
+ ?.GetName()
+ ?.Name
+ ?? throw new InvalidOperationException("Failed to determine name of calling assembly (calling method is null)");
+}
diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/SimpleEchoClient.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/SimpleEchoClient.json
deleted file mode 100644
index d4ef92277..000000000
--- a/src/DataStreamSuite/DataStreamPublisher/Configurations/SimpleEchoClient.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "user": "bts",
- "simple_echo_client": true
-}
diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/SimpleTestDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/SimpleTestDataPublisher.json
deleted file mode 100644
index 64ec4dd76..000000000
--- a/src/DataStreamSuite/DataStreamPublisher/Configurations/SimpleTestDataPublisher.json
+++ /dev/null
@@ -1,73 +0,0 @@
-{
- "user": "bts",
- "pubsub": {
- "milliseconds_between_sub_info_updates": 1000,
- "milliseconds_between_time_slice_updates": 5000,
- "milliseconds_between_no_buffer_space_retries": 100,
- "retries_on_no_buffer_space": 1,
- "do_it_all_seconds_between_logs": 0,
- "heartbeat_interval": 15000,
- "missing_heartbeats_before_gc": 50,
- "use_managed_sockets": true,
- "state_update_period": 5000,
- "statistics_update_period": 5000,
- "disposed_connection_check_period": 1000,
- "message_processing_warning_threshold": 1000,
- "server_port": 12345,
- "publisher_routes": [
- {
- "name": "default",
- "server_address": "bts-cloud://localhost:12345",
- "tag": ".*"
- }
- ],
- "subscriber_routes": [
- {
- "name": "default",
- "server_address": "bts-cloud://localhost:12345",
- "tag": ".*"
- }
- ],
- "max_packet_size": 500000000,
- "blue_comm_must_die": true,
- "crushable_tags": [],
- "tag_type_to_assembly_map": [
- {
- "handler_class": "BCG.PubSubServerTagHandler",
- "assembly": "DataStreamCore.dll"
- }
- ],
- "server_queue": "PubSubQueue",
- "port_range": {
- "first": 30400,
- "last": 31700
- },
- "bind_to_random_port": false,
- "debug_tags": "^NOTAGS$",
- "latency_report_threshold_ms": 1500,
- "use_substations": false,
- "use_dynamic_clients": false,
- "is_substation": false,
- "internally_publish_sub_info": true,
- "use_managed_clients_in_native_code": false,
- "compressed_tags": "^NOTAGS$",
- "tags_to_report_compression_ratio": "^NOTAGS$",
- "disable_msmq": false,
- "use_prepublished_server_state": false,
- "ip_translation": null,
- "gc_activation_threshold": 0,
- "wait_period_after_gc": 1000
- },
- "publishers": [
- {
- "name": "test_data",
- "streams": [
- {
- "name": "test_literal",
- "expression": "literal(\"x\", \"420.0\")",
- "is_on_websocket": true
- }
- ]
- }
- ]
-}
diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json
deleted file mode 100644
index 736a893e0..000000000
--- a/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json
+++ /dev/null
@@ -1,40 +0,0 @@
-{
- "user": "bts",
- "publishers": [
- {
- "name": "test_data",
- "streams": [
- {
- "name": "seconds_1_10",
- "expression": "milliseconds(1000,10)",
- "is_on_websocket": true
- },
- {
- "name": "benzinga_news",
- "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")",
- "is_on_websocket": true
- },
- {
- "name": "benzinga_news_symbols",
- "expression": "benzingaNewsSymbols(benzinga_news)",
- "is_on_websocket": true
- },
- {
- "name": "benzinga_news_symbol_state",
- "expression": "benzingaNewsSymbolState(benzinga_news)",
- "is_on_websocket": true
- },
- {
- "name": "test_literal",
- "expression": "literal(\"x\", \"420.0\")",
- "is_on_websocket": true
- },
- {
- "name": "test_not_on_websocket",
- "expression": "literal(\"x\", \"840.0\")",
- "is_on_websocket": false
- }
- ]
- }
- ]
-}
\ No newline at end of file
diff --git a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj
index 015e92e14..65b09dc93 100644
--- a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj
+++ b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj
@@ -15,9 +15,19 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/src/DataStreamSuite/DataStreamPublisher/Dockerfile b/src/DataStreamSuite/DataStreamPublisher/Dockerfile
new file mode 100644
index 000000000..91acfb74c
--- /dev/null
+++ b/src/DataStreamSuite/DataStreamPublisher/Dockerfile
@@ -0,0 +1,16 @@
+FROM bluetradingsystems/linux_dotnet7
+
+ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
+ARG CONF_FILE="src/DataStreamSuite/DataStreamPublisher/spark.conf"
+
+ENV DOCKERIZE_VERSION=v0.7.0
+USER root
+RUN apt-get update \
+ && apt-get install -y wget \
+ && wget -O - https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz | tar xzf - -C /usr/local/bin \
+ && apt-get autoremove -yqq --purge wget && rm -rf /var/lib/apt/lists/*
+
+USER bts
+COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
+COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/DataStreamPublisher/conf/spark.conf"
+WORKDIR "/home/bts/bin"
diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs
index 555631256..bcd1b6089 100644
--- a/src/DataStreamSuite/DataStreamPublisher/Program.cs
+++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs
@@ -1,16 +1,14 @@
using Bts.DataStreamPublisher.Services;
+using Bts.Extensions.Hosting;
-using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using System;
-var host = Host.CreateDefaultBuilder(args)
- .ConfigureAppConfiguration(ConfigureApp)
+var host = HostBuilderExtensions.CreateBtsBuilder(args)
.ConfigureServices(ConfigureServices)
.Build();
-
try {
Console.Write("Starting DataStreamPublisher");
host.Run();
@@ -21,14 +19,5 @@ try {
}
return 0;
-static void ConfigureApp(HostBuilderContext context, IConfigurationBuilder configBuilder) {
- configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
- var cfgRoot = configBuilder.Build();
- var cfg = cfgRoot["config"];
- if (cfg is not null) {
- configBuilder.AddJsonFile(cfg, false);
- }
-}
-
static void ConfigureServices(HostBuilderContext context, IServiceCollection services) =>
services.AddDataStreamPublisher(context.Configuration);
diff --git a/src/DataStreamSuite/DataStreamPublisher/appsettings.Development.json b/src/DataStreamSuite/DataStreamPublisher/appsettings.Development.json
new file mode 100644
index 000000000..dd820fc8d
--- /dev/null
+++ b/src/DataStreamSuite/DataStreamPublisher/appsettings.Development.json
@@ -0,0 +1,39 @@
+{
+ "PubSub": {
+ "Routing": {
+ "PublisherRoutes": [
+ {
+ "Name": "default",
+ "ServerAddress": "bts-cloud://127.0.0.1:22019",
+ "TagRegExp": ".*"
+ }
+ ],
+ "SubscriberRoutes": [
+ {
+ "Name": "default",
+ "ServerAddress": "bts-cloud://127.0.0.1:22019",
+ "TagRegExp": ".*"
+ }
+ ]
+ }
+ },
+ "DataStreamPublisher": {
+ "PublisherNodes": [
+ {
+ "Name": "test_data",
+ "Streams": [
+ {
+ "Name": "seconds_1_10",
+ "Expression": "milliseconds(1000,10)",
+ "IsOnWebsocket": true
+ },
+ {
+ "Name": "test_literal",
+ "Expression": "literal(\"x\", \"420.0\")",
+ "IsOnWebsocket": true
+ }
+ ]
+ }
+ ]
+ }
+}
diff --git a/src/DataStreamSuite/DataStreamPublisher/appsettings.MarketData.json b/src/DataStreamSuite/DataStreamPublisher/appsettings.MarketData.json
new file mode 100644
index 000000000..df9df695f
--- /dev/null
+++ b/src/DataStreamSuite/DataStreamPublisher/appsettings.MarketData.json
@@ -0,0 +1,63 @@
+{
+ "Spark": {
+ "ConfigServer": "http://sparkd:8000/config"
+ },
+ "SparkApi": {
+ "Credentials": {
+ "User": "bts",
+ "Password": ""
+ },
+ "EnableAdmin": true,
+ "EnableOrders": true,
+ "EnablePosition": true
+ },
+ "DataStreamPublisher": {
+ "PublisherNodes": [
+ {
+ "Name": "market_data",
+ "Streams": [
+ {
+ "Name": "bbo",
+ "Expression": "bbo()",
+ "IsOnWebsocket": true
+ },
+ {
+ "Name": "instruments",
+ "Expression": "instruments()",
+ "IsOnWebsocket": true
+ },
+ {
+ "Name": "timesales",
+ "Expression": "timesales(10000)",
+ "IsOnWebsocket": true
+ },
+ {
+ "Name": "books",
+ "Expression": "books()",
+ "IsOnWebsocket": true
+ }
+ ]
+ },
+ {
+ "Name": "test_data",
+ "Streams": [
+ {
+ "Name": "milliseconds_1000_100",
+ "Expression": "milliseconds(3000,5)",
+ "IsOnWebsocket": true
+ },
+ {
+ "Name": "test_literal",
+ "Expression": "literal(\"x\", \"420.0\")",
+ "IsOnWebsocket": true
+ },
+ {
+ "Name": "test_not_on_websocket",
+ "Expression": "literal(\"x\", \"840.0\")",
+ "IsOnWebsocket": false
+ }
+ ]
+ }
+ ]
+ }
+}
diff --git a/src/DataStreamSuite/DataStreamPublisher/appsettings.Production.json b/src/DataStreamSuite/DataStreamPublisher/appsettings.Production.json
new file mode 100644
index 000000000..0967ef424
--- /dev/null
+++ b/src/DataStreamSuite/DataStreamPublisher/appsettings.Production.json
@@ -0,0 +1 @@
+{}
diff --git a/src/DataStreamSuite/DataStreamPublisher/appsettings.ProductionAws.json b/src/DataStreamSuite/DataStreamPublisher/appsettings.ProductionAws.json
new file mode 100644
index 000000000..d8a8c8c57
--- /dev/null
+++ b/src/DataStreamSuite/DataStreamPublisher/appsettings.ProductionAws.json
@@ -0,0 +1,91 @@
+{
+ "Spark": {
+ "ConfigServer": "http://localhost:8000/config"
+ },
+ "SparkApi": {
+ "Credentials": {
+ "User": "bts",
+ "Password": ""
+ },
+ "EnableAdmin": true,
+ "EnableOrders": true,
+ "EnablePosition": true
+ },
+ "PubSub": {
+ "Routing": {
+ "PublisherRoutes": [
+ {
+ "Name": "default",
+ "ServerAddress": "bts-cloud://172.30.2.181:22019",
+ "TagRegExp": ".*"
+ }
+ ],
+ "SubscriberRoutes": [
+ {
+ "Name": "default",
+ "ServerAddress": "bts-cloud://172.30.2.181:22019",
+ "TagRegExp": ".*"
+ }
+ ]
+ }
+ },
+ "DataStreamPublisher": {
+ "PublisherNodes": [
+ {
+ "Name": "market_data",
+ "Streams": [
+ {
+ "Name": "bbo",
+ "Expression": "bbo()",
+ "IsOnWebsocket": true
+ },
+ {
+ "Name": "instruments",
+ "Expression": "instruments()",
+ "IsOnWebsocket": true
+ },
+ {
+ "Name": "timesales",
+ "Expression": "timesales(10000)",
+ "IsOnWebsocket": true
+ },
+ {
+ "Name": "books",
+ "Expression": "books()",
+ "IsOnWebsocket": true
+ }
+ ]
+ },
+ {
+ "Name": "test_data",
+ "Streams": [
+ {
+ "Name": "seconds_1_10",
+ "Expression": "milliseconds(1000,10)",
+ "IsOnWebsocket": true
+ },
+ {
+ "Name": "benzinga_news",
+ "Expression": "benzingaNews(\"mongodb://mongoadmin:bts@172.30.2.181:27017/?directConnection=true&serverSelectionTimeoutMS=2000&authSource=admin&appName=mongosh+2.2.12\" ,\"benzinga_news\",\"news_headlines\", \"30s\", \"5d\")",
+ "IsOnWebsocket": true
+ },
+ {
+ "Name": "benzinga_news_symbols",
+ "Expression": "benzingaNewsSymbols(benzinga_news)",
+ "IsOnWebsocket": true
+ },
+ {
+ "Name": "benzinga_news_symbol_state",
+ "Expression": "benzingaNewsSymbolState(benzinga_news)",
+ "IsOnWebsocket": true
+ },
+ {
+ "Name": "test_literal",
+ "Expression": "literal(\"x\", \"420.0\")",
+ "IsOnWebsocket": true
+ }
+ ]
+ }
+ ]
+ }
+}
diff --git a/src/DataStreamSuite/DataStreamPublisher/appsettings.Staging.json b/src/DataStreamSuite/DataStreamPublisher/appsettings.Staging.json
new file mode 100644
index 000000000..0967ef424
--- /dev/null
+++ b/src/DataStreamSuite/DataStreamPublisher/appsettings.Staging.json
@@ -0,0 +1 @@
+{}
diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/appsettings.json
similarity index 55%
rename from src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json
rename to src/DataStreamSuite/DataStreamPublisher/appsettings.json
index ae9a3e5a1..8c5581afe 100644
--- a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json
+++ b/src/DataStreamSuite/DataStreamPublisher/appsettings.json
@@ -1,16 +1,11 @@
{
- "Spark": {
- "ConfigServer": "http://localhost:8000/config"
- },
- "SparkApi": {
- "Credentials": {
- "User": "bts",
- "Password": ""
- },
- "EnableAdmin": true,
- "EnableOrders": true,
- "EnablePosition": true
+ "Logging": {
+ "LogLevel": {
+ "Default": "Information",
+ "Microsoft.AspNetCore": "Warning"
+ }
},
+ "Spark": {},
"MasterWorker": {
"Interval": "00:00:00.1"
},
@@ -21,19 +16,19 @@
"HeartbeatInterval": 15000,
"MissingHeartbeatsBeforeGc": 50,
"DisposedConnectionCheckPeriod": 1000,
- "ServerPort": 12345,
+ "ServerPort": 22019,
"Routing": {
"PublisherRoutes": [
{
"Name": "default",
- "ServerAddress": "bts-cloud://127.0.0.1:12345",
+ "ServerAddress": "bts-cloud://pubsub:22019",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
- "ServerAddress": "bts-cloud://127.0.0.1:12345",
+ "ServerAddress": "bts-cloud://pubsub:22019",
"TagRegExp": ".*"
}
]
@@ -73,48 +68,13 @@
},
"DataStreamPublisher": {
"PublisherNodes": [
- {
- "Name": "market_data",
- "Streams": [
- {
- "Name": "bbo",
- "Expression": "bbo()",
- "IsOnWebsocket": true
- },
- {
- "Name": "instruments",
- "Expression": "instruments()",
- "IsOnWebsocket": true
- },
- {
- "Name": "timesales",
- "Expression": "timesales(10000)",
- "IsOnWebsocket": true
- },
- {
- "Name": "books",
- "Expression": "books()",
- "IsOnWebsocket": true
- }
- ]
- },
{
"Name": "test_data",
"Streams": [
- {
- "Name": "milliseconds_1000_100",
- "Expression": "milliseconds(3000,5)",
- "IsOnWebsocket": true
- },
{
"Name": "test_literal",
"Expression": "literal(\"x\", \"420.0\")",
"IsOnWebsocket": true
- },
- {
- "Name": "test_not_on_websocket",
- "Expression": "literal(\"x\", \"840.0\")",
- "IsOnWebsocket": false
}
]
}
diff --git a/src/DataStreamSuite/DataStreamPublisher/spark.conf b/src/DataStreamSuite/DataStreamPublisher/spark.conf
new file mode 100644
index 000000000..81b3e4f58
--- /dev/null
+++ b/src/DataStreamSuite/DataStreamPublisher/spark.conf
@@ -0,0 +1,15 @@
+Logging {
+ value {
+ console {
+ default_level: "Info"
+ }
+ syslog {
+ default_level: "Info"
+ }
+ file {
+ default_level: "Info"
+ }
+ initial_queue_size: 120001
+ max_queue_growth: 4
+ }
+}
diff --git a/src/DataStreamSuite/DataStreamSuite.sln b/src/DataStreamSuite/DataStreamSuite.sln
index 5a6420d6b..c7358f552 100644
--- a/src/DataStreamSuite/DataStreamSuite.sln
+++ b/src/DataStreamSuite/DataStreamSuite.sln
@@ -38,6 +38,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions",
EndProject
/*Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamQueryAnalyzer", "DataStreamQueryAnalyzer\DataStreamQueryAnalyzer.csproj", "{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}"
EndProject*/
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{5C528A65-E43C-48A2-8CA9-ED2D0E356A37}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -119,5 +121,9 @@ Global
{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Debug|Any CPU.Build.0 = Debug|Any CPU
{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Release|Any CPU.ActiveCfg = Release|Any CPU
{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Release|Any CPU.Build.0 = Release|Any CPU
+ {5C528A65-E43C-48A2-8CA9-ED2D0E356A37}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {5C528A65-E43C-48A2-8CA9-ED2D0E356A37}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {5C528A65-E43C-48A2-8CA9-ED2D0E356A37}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {5C528A65-E43C-48A2-8CA9-ED2D0E356A37}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal
diff --git a/src/DataStreamSuite/PubSubServer/Configurations/PubsubServer_Linux.json b/src/DataStreamSuite/PubSubServer/Configurations/PubsubServer_Linux.json
deleted file mode 100644
index 4c9a17a45..000000000
--- a/src/DataStreamSuite/PubSubServer/Configurations/PubsubServer_Linux.json
+++ /dev/null
@@ -1,61 +0,0 @@
-{
- "user": "bts",
- "pubsub": {
- "milliseconds_between_sub_info_updates": 1000,
- "milliseconds_between_time_slice_updates": 5000,
- "milliseconds_between_no_buffer_space_retries": 100,
- "retries_on_no_buffer_space": 1,
- "do_it_all_seconds_between_logs": 0,
- "heartbeat_interval": 15000,
- "missing_heartbeats_before_gc": 50,
- "use_managed_sockets": true,
- "state_update_period": 5000,
- "statistics_update_period": 5000,
- "disposed_connection_check_period": 1000,
- "message_processing_warning_threshold": 1000,
- "server_port": 12345,
- "publisher_routes": [
- {
- "name": "default",
- "server_address": "bts-cloud://192.168.1.80:12345",
- "tag": ".*"
- }
- ],
- "subscriber_routes": [
- {
- "name": "default",
- "server_address": "bts-cloud://192.168.1.80:12345",
- "tag": ".*"
- }
- ],
- "max_packet_size": 500000000,
- "blue_comm_must_die": true,
- "crushable_tags": [],
- "tag_type_to_assembly_map": [
- {
- "handler_class": "BCG.PubSubServerTagHandler",
- "assembly": "DataStreamCore.dll"
- }
- ],
- "server_queue": "PubSubQueue",
- "port_range": {
- "first": 30400,
- "last": 31700
- },
- "bind_to_random_port": false,
- "debug_tags": "^NOTAGS$",
- "latency_report_threshold_ms": 1500,
- "use_substations": false,
- "use_dynamic_clients": false,
- "is_substation": false,
- "internally_publish_sub_info": true,
- "use_managed_clients_in_native_code": false,
- "compressed_tags": "^NOTAGS$",
- "tags_to_report_compression_ratio": "^NOTAGS$",
- "disable_msmq": false,
- "use_prepublished_server_state": false,
- "ip_translation": null,
- "gc_activation_threshold": 0,
- "wait_period_after_gc": 1000
- }
-}
\ No newline at end of file
diff --git a/src/DataStreamSuite/PubSubServer/Dockerfile b/src/DataStreamSuite/PubSubServer/Dockerfile
new file mode 100644
index 000000000..b45188a37
--- /dev/null
+++ b/src/DataStreamSuite/PubSubServer/Dockerfile
@@ -0,0 +1,8 @@
+FROM bluetradingsystems/linux_dotnet7
+
+ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
+ARG CONF_FILE="src/DataStreamSuite/PubSubServer/spark.conf"
+
+COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
+COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/PubSubServer/conf/spark.conf"
+WORKDIR "/home/bts/bin"
diff --git a/src/DataStreamSuite/PubSubServer/Program.cs b/src/DataStreamSuite/PubSubServer/Program.cs
index 59726464f..d7b663942 100644
--- a/src/DataStreamSuite/PubSubServer/Program.cs
+++ b/src/DataStreamSuite/PubSubServer/Program.cs
@@ -1,3 +1,4 @@
+using Bts.Extensions.Hosting;
using Bts.PubSub.Services;
using Microsoft.Extensions.Configuration;
@@ -8,8 +9,7 @@ using Spark.Core.Services;
using System;
-var host = Host.CreateDefaultBuilder(args)
- .ConfigureAppConfiguration(ConfigureApp)
+var host = HostBuilderExtensions.CreateBtsBuilder(args)
.ConfigureServices(ConfigureServices)
.Build();
try {
@@ -22,15 +22,6 @@ try {
}
return 0;
-static void ConfigureApp(HostBuilderContext context, IConfigurationBuilder configBuilder) {
- configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
- var cfgRoot = configBuilder.Build();
- var cfg = cfgRoot["config"];
- if (cfg is not null) {
- configBuilder.AddJsonFile(cfg, false);
- }
-}
-
static void ConfigureServices(HostBuilderContext context, IServiceCollection services) {
var echoServerCfg = context.Configuration.GetSection("SimpleEchoServer");
if (echoServerCfg.Get() is not null) {
diff --git a/src/DataStreamSuite/PubSubServer/PubSubServer.csproj b/src/DataStreamSuite/PubSubServer/PubSubServer.csproj
index 666dd989e..49e107ae2 100644
--- a/src/DataStreamSuite/PubSubServer/PubSubServer.csproj
+++ b/src/DataStreamSuite/PubSubServer/PubSubServer.csproj
@@ -12,7 +12,17 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/src/DataStreamSuite/PubSubServer/appsettings.Development.json b/src/DataStreamSuite/PubSubServer/appsettings.Development.json
new file mode 100644
index 000000000..f83647ef7
--- /dev/null
+++ b/src/DataStreamSuite/PubSubServer/appsettings.Development.json
@@ -0,0 +1,20 @@
+{
+ "PubSub": {
+ "Routing": {
+ "PublisherRoutes": [
+ {
+ "Name": "default",
+ "ServerAddress": "bts-cloud://localhost:22019",
+ "TagRegExp": ".*"
+ }
+ ],
+ "SubscriberRoutes": [
+ {
+ "Name": "default",
+ "ServerAddress": "bts-cloud://localhost:22019",
+ "TagRegExp": ".*"
+ }
+ ]
+ }
+ }
+}
diff --git a/src/DataStreamSuite/PubSubServer/Configurations/SimpleEchoServer.json b/src/DataStreamSuite/PubSubServer/appsettings.EchoServer.json
similarity index 91%
rename from src/DataStreamSuite/PubSubServer/Configurations/SimpleEchoServer.json
rename to src/DataStreamSuite/PubSubServer/appsettings.EchoServer.json
index d898b5837..b7a63f076 100644
--- a/src/DataStreamSuite/PubSubServer/Configurations/SimpleEchoServer.json
+++ b/src/DataStreamSuite/PubSubServer/appsettings.EchoServer.json
@@ -1,6 +1,6 @@
-{
- "PubSubEchoServer": {
- "Echo": true,
- "Port": 12345
- }
-}
+{
+ "PubSubEchoServer": {
+ "Echo": true,
+ "Port": 12345
+ }
+}
diff --git a/src/DataStreamSuite/PubSubServer/appsettings.Production.json b/src/DataStreamSuite/PubSubServer/appsettings.Production.json
new file mode 100644
index 000000000..0967ef424
--- /dev/null
+++ b/src/DataStreamSuite/PubSubServer/appsettings.Production.json
@@ -0,0 +1 @@
+{}
diff --git a/src/DataStreamSuite/PubSubServer/appsettings.ProductionAws.json b/src/DataStreamSuite/PubSubServer/appsettings.ProductionAws.json
new file mode 100644
index 000000000..d54f0f942
--- /dev/null
+++ b/src/DataStreamSuite/PubSubServer/appsettings.ProductionAws.json
@@ -0,0 +1,20 @@
+{
+ "PubSub": {
+ "Routing": {
+ "PublisherRoutes": [
+ {
+ "Name": "default",
+ "ServerAddress": "bts-cloud://34.237.179.93:22019",
+ "TagRegExp": ".*"
+ }
+ ],
+ "SubscriberRoutes": [
+ {
+ "Name": "default",
+ "ServerAddress": "bts-cloud://34.237.179.93:22019",
+ "TagRegExp": ".*"
+ }
+ ]
+ }
+ }
+}
diff --git a/src/DataStreamSuite/PubSubServer/appsettings.Staging.json b/src/DataStreamSuite/PubSubServer/appsettings.Staging.json
new file mode 100644
index 000000000..0967ef424
--- /dev/null
+++ b/src/DataStreamSuite/PubSubServer/appsettings.Staging.json
@@ -0,0 +1 @@
+{}
diff --git a/src/DataStreamSuite/PubSubServer/Configurations/PubSubServer.json b/src/DataStreamSuite/PubSubServer/appsettings.json
similarity index 85%
rename from src/DataStreamSuite/PubSubServer/Configurations/PubSubServer.json
rename to src/DataStreamSuite/PubSubServer/appsettings.json
index ba43a5956..13f360bff 100644
--- a/src/DataStreamSuite/PubSubServer/Configurations/PubSubServer.json
+++ b/src/DataStreamSuite/PubSubServer/appsettings.json
@@ -1,5 +1,5 @@
{
- "Spark": { },
+ "Spark": {},
"MasterWorker": {
"Interval": "00:00:00.1"
},
@@ -7,19 +7,19 @@
"HeartbeatInterval": 15000,
"MissingHeartbeatsBeforeGc": 50,
"DisposedConnectionCheckPeriod": 1000,
- "ServerPort": 12345,
+ "ServerPort": 22019,
"Routing": {
"PublisherRoutes": [
{
"Name": "default",
- "ServerAddress": "bts-cloud://127.0.0.1:12345",
+ "ServerAddress": "bts-cloud://pubsub:22019",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
- "ServerAddress": "bts-cloud://127.0.0.1:12345",
+ "ServerAddress": "bts-cloud://pubsub:22019",
"TagRegExp": ".*"
}
]
@@ -52,7 +52,7 @@
{
"TagType": ".*",
"Assembly": "DataStreamCore",
- "Class": "bts.PubsubServerGenericDataHandler"
+ "Class": "bts.PubsubServerGenericDataHandler"
}
]
}
diff --git a/src/DataStreamSuite/PubSubServer/spark.conf b/src/DataStreamSuite/PubSubServer/spark.conf
new file mode 100644
index 000000000..81b3e4f58
--- /dev/null
+++ b/src/DataStreamSuite/PubSubServer/spark.conf
@@ -0,0 +1,15 @@
+Logging {
+ value {
+ console {
+ default_level: "Info"
+ }
+ syslog {
+ default_level: "Info"
+ }
+ file {
+ default_level: "Info"
+ }
+ initial_queue_size: 120001
+ max_queue_growth: 4
+ }
+}
diff --git a/src/DataStreamSuite/PubSubServer/staging.env b/src/DataStreamSuite/PubSubServer/staging.env
new file mode 100644
index 000000000..7a19a5a7a
--- /dev/null
+++ b/src/DataStreamSuite/PubSubServer/staging.env
@@ -0,0 +1 @@
+ASPNETCORE_ENVIRONMENT=Staging
diff --git a/src/Directory.Build.props b/src/Directory.Build.props
index 87b8de912..a7e2e217d 100644
--- a/src/Directory.Build.props
+++ b/src/Directory.Build.props
@@ -88,4 +88,13 @@
+
+
+ $(OutputPath)\Configurations\$(MSBuildProjectName)
+ $(OutputPath)\$(MSBuildProjectName).%(FileName)%(Extension)
+ $(OutputPath)\publish\Configurations\$(MSBuildProjectName)
+ $(OutputPath)\publish\$(MSBuildProjectName).%(FileName)%(Extension)
+
+
+
diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets
index 3409f22ba..6ea2693db 100644
--- a/src/Directory.Build.targets
+++ b/src/Directory.Build.targets
@@ -36,6 +36,34 @@
+
+
+
+
+
+
+
+
+
+
+
+
+