diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 23f56f6f4..8e9de2a28 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -422,12 +422,11 @@ "windows": { "command": "DataStreamPublisher.exe", }, - "args": [ - "--config", - "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json", - ], "options": { - "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0" + "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0", + "env": { + "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}", + }, }, "icon": { "id": "run", @@ -445,12 +444,11 @@ "windows": { "command": "DataStreamPublisher.exe", }, - "args": [ - "--config", - "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json", - ], "options": { - "cwd": "${workspaceFolder}/build/Debug/bin/net8.0" + "cwd": "${workspaceFolder}/build/Debug/bin/net8.0", + "env": { + "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}", + }, }, "icon": { "id": "run", @@ -468,12 +466,11 @@ "windows": { "command": "DataStreamPublisher.exe", }, - "args": [ - "--config", - "${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json", - ], "options": { - "cwd": "${workspaceFolder}/build/Release/bin/net8.0" + "cwd": "${workspaceFolder}/build/Release/bin/net8.0", + "env": { + "DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}", + }, }, "icon": { "id": "run", @@ -491,12 +488,11 @@ "windows": { "command": "PubSubServer.exe", }, - "args": [ - "--config", - "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json", - ], "options": { - "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0" + "cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0", + "env": { + "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}", + }, }, "icon": { "id": "run", @@ -514,12 +510,11 @@ "windows": { "command": "PubSubServer.exe", }, - "args": [ - "--config", - "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json", - ], "options": { - "cwd": "${workspaceFolder}/build/Debug/bin/net8.0" + "cwd": "${workspaceFolder}/build/Debug/bin/net8.0", + "env": { + "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}", + }, }, "icon": { "id": "run", @@ -536,12 +531,11 @@ "windows": { "command": "PubSubServer.exe", }, - "args": [ - "--config", - "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json", - ], "options": { - "cwd": "${workspaceFolder}/build/Debug/bin/net8.0" + "cwd": "${workspaceFolder}/build/Debug/bin/net8.0", + "env": { + "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}", + }, }, "icon": { "id": "run", @@ -559,12 +553,11 @@ "windows": { "command": "PubSubServer.exe", }, - "args": [ - "--config", - "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json", - ], "options": { - "cwd": "${workspaceFolder}/build/Release/bin/net8.0" + "cwd": "${workspaceFolder}/build/Release/bin/net8.0", + "env": { + "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}", + }, }, "icon": { "id": "run", @@ -581,12 +574,11 @@ "windows": { "command": "PubSubServer.exe", }, - "args": [ - "--config", - "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json", - ], "options": { - "cwd": "${workspaceFolder}/build/Release/bin/net8.0" + "cwd": "${workspaceFolder}/build/Release/bin/net8.0", + "env": { + "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}", + }, }, "icon": { "id": "run", @@ -885,26 +877,30 @@ "default": "sparkd-debug-clang", }, { - "id": "dataStreamPublisherConfig", + "id": "dataStreamPublisherEnvironment", "type": "pickString", "description": "Publisher Configuration", "options": [ - "MarketDataPublisher", - "SimpleEchoClient", - "SimpleTestDataPublisher", - "TestDataPublisher", + "Development", + "MarketData", + "Production", + "ProductionAws", + "Staging", ], "default": "MarketDataPublisher", }, { - "id": "pubSubServerConfig", + "id": "pubSubServerEnvironment", "type": "pickString", "description": "PubSub Configuration", "options": [ - "PubSubServer", - "SimpleEchoClient", + "Development", + "EchoServer", + "Production", + "ProductionAws", + "Staging", ], - "default": "PubSubServer", + "default": "Development", }, { "id": "sparkConfigQuery", diff --git a/src/DataStreamSuite/Bts.Extensions.Hosting/Bts.Extensions.Hosting.csproj b/src/DataStreamSuite/Bts.Extensions.Hosting/Bts.Extensions.Hosting.csproj new file mode 100644 index 000000000..988058984 --- /dev/null +++ b/src/DataStreamSuite/Bts.Extensions.Hosting/Bts.Extensions.Hosting.csproj @@ -0,0 +1,13 @@ + + + + net8.0 + enable + enable + + + + + + + diff --git a/src/DataStreamSuite/Bts.Extensions.Hosting/HostBuilderExtensions.cs b/src/DataStreamSuite/Bts.Extensions.Hosting/HostBuilderExtensions.cs new file mode 100644 index 000000000..1b70b345b --- /dev/null +++ b/src/DataStreamSuite/Bts.Extensions.Hosting/HostBuilderExtensions.cs @@ -0,0 +1,59 @@ +namespace Bts.Extensions.Hosting; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; + +using System.Diagnostics; + +public static class HostBuilderExtensions { + public static IHostBuilder CreateBtsBuilder(string[] args) { + // Obtain calling assembly name here, rather than in the lambda where it is actually needed, + // because want the name of the assembly that called this function, not the assembly that will eventually invoke + // the lambda...that will be the Microsoft.Extensions.Hosting assembly. + // Get the name of the assembly that called this function. That will act as the "appsettings" + // json file prefix + var exeName = GetCallingAssemblyName(); + return Host.CreateDefaultBuilder(args) + .ConfigureAppConfiguration((context, configBuilder) => + ConfigureApp(configBuilder, context.HostingEnvironment.EnvironmentName, exeName)); + } + + public static void ConfigureApp(IConfigurationBuilder configBuilder, string? environment = null, string? prefix = null) { + var baseFileName = prefix is null ? "appsettings" : $"{prefix}.appsettings"; + AddJsonFile($"{baseFileName}.json"); + if (environment is not null) { + AddJsonFile($"{baseFileName}.{environment}.json"); + } + + var cfg = configBuilder.Build().GetSection("config")?.Value; + if (!SetBasePath(cfg)) { + AddJsonFile(cfg); + } + + bool SetBasePath(string? path) { + if (!Directory.Exists(path)) { + return false; + } + configBuilder.SetBasePath(path); + return true; + } + + bool AddJsonFile(string? path) { + if (!File.Exists(path)) { + return false; + } + configBuilder.AddJsonFile(path); + return true; + } + } + + public static string GetCallingAssemblyName() => + new StackTrace() + .GetFrame(2) + ?.GetMethod() + ?.Module + ?.Assembly + ?.GetName() + ?.Name + ?? throw new InvalidOperationException("Failed to determine name of calling assembly (calling method is null)"); +} diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/SimpleEchoClient.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/SimpleEchoClient.json deleted file mode 100644 index d4ef92277..000000000 --- a/src/DataStreamSuite/DataStreamPublisher/Configurations/SimpleEchoClient.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "user": "bts", - "simple_echo_client": true -} diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/SimpleTestDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/SimpleTestDataPublisher.json deleted file mode 100644 index 64ec4dd76..000000000 --- a/src/DataStreamSuite/DataStreamPublisher/Configurations/SimpleTestDataPublisher.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "user": "bts", - "pubsub": { - "milliseconds_between_sub_info_updates": 1000, - "milliseconds_between_time_slice_updates": 5000, - "milliseconds_between_no_buffer_space_retries": 100, - "retries_on_no_buffer_space": 1, - "do_it_all_seconds_between_logs": 0, - "heartbeat_interval": 15000, - "missing_heartbeats_before_gc": 50, - "use_managed_sockets": true, - "state_update_period": 5000, - "statistics_update_period": 5000, - "disposed_connection_check_period": 1000, - "message_processing_warning_threshold": 1000, - "server_port": 12345, - "publisher_routes": [ - { - "name": "default", - "server_address": "bts-cloud://localhost:12345", - "tag": ".*" - } - ], - "subscriber_routes": [ - { - "name": "default", - "server_address": "bts-cloud://localhost:12345", - "tag": ".*" - } - ], - "max_packet_size": 500000000, - "blue_comm_must_die": true, - "crushable_tags": [], - "tag_type_to_assembly_map": [ - { - "handler_class": "BCG.PubSubServerTagHandler", - "assembly": "DataStreamCore.dll" - } - ], - "server_queue": "PubSubQueue", - "port_range": { - "first": 30400, - "last": 31700 - }, - "bind_to_random_port": false, - "debug_tags": "^NOTAGS$", - "latency_report_threshold_ms": 1500, - "use_substations": false, - "use_dynamic_clients": false, - "is_substation": false, - "internally_publish_sub_info": true, - "use_managed_clients_in_native_code": false, - "compressed_tags": "^NOTAGS$", - "tags_to_report_compression_ratio": "^NOTAGS$", - "disable_msmq": false, - "use_prepublished_server_state": false, - "ip_translation": null, - "gc_activation_threshold": 0, - "wait_period_after_gc": 1000 - }, - "publishers": [ - { - "name": "test_data", - "streams": [ - { - "name": "test_literal", - "expression": "literal(\"x\", \"420.0\")", - "is_on_websocket": true - } - ] - } - ] -} diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json deleted file mode 100644 index 736a893e0..000000000 --- a/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "user": "bts", - "publishers": [ - { - "name": "test_data", - "streams": [ - { - "name": "seconds_1_10", - "expression": "milliseconds(1000,10)", - "is_on_websocket": true - }, - { - "name": "benzinga_news", - "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")", - "is_on_websocket": true - }, - { - "name": "benzinga_news_symbols", - "expression": "benzingaNewsSymbols(benzinga_news)", - "is_on_websocket": true - }, - { - "name": "benzinga_news_symbol_state", - "expression": "benzingaNewsSymbolState(benzinga_news)", - "is_on_websocket": true - }, - { - "name": "test_literal", - "expression": "literal(\"x\", \"420.0\")", - "is_on_websocket": true - }, - { - "name": "test_not_on_websocket", - "expression": "literal(\"x\", \"840.0\")", - "is_on_websocket": false - } - ] - } - ] -} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj index 015e92e14..65b09dc93 100644 --- a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj +++ b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj @@ -15,9 +15,19 @@ + + + + + + + + + + diff --git a/src/DataStreamSuite/DataStreamPublisher/Dockerfile b/src/DataStreamSuite/DataStreamPublisher/Dockerfile new file mode 100644 index 000000000..91acfb74c --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/Dockerfile @@ -0,0 +1,16 @@ +FROM bluetradingsystems/linux_dotnet7 + +ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish" +ARG CONF_FILE="src/DataStreamSuite/DataStreamPublisher/spark.conf" + +ENV DOCKERIZE_VERSION=v0.7.0 +USER root +RUN apt-get update \ + && apt-get install -y wget \ + && wget -O - https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz | tar xzf - -C /usr/local/bin \ + && apt-get autoremove -yqq --purge wget && rm -rf /var/lib/apt/lists/* + +USER bts +COPY --chown=bts:bts "${SOURCE_DIR}" "bin" +COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/DataStreamPublisher/conf/spark.conf" +WORKDIR "/home/bts/bin" diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs index 555631256..bcd1b6089 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Program.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs @@ -1,16 +1,14 @@ using Bts.DataStreamPublisher.Services; +using Bts.Extensions.Hosting; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using System; -var host = Host.CreateDefaultBuilder(args) - .ConfigureAppConfiguration(ConfigureApp) +var host = HostBuilderExtensions.CreateBtsBuilder(args) .ConfigureServices(ConfigureServices) .Build(); - try { Console.Write("Starting DataStreamPublisher"); host.Run(); @@ -21,14 +19,5 @@ try { } return 0; -static void ConfigureApp(HostBuilderContext context, IConfigurationBuilder configBuilder) { - configBuilder.AddCommandLine(Environment.GetCommandLineArgs()); - var cfgRoot = configBuilder.Build(); - var cfg = cfgRoot["config"]; - if (cfg is not null) { - configBuilder.AddJsonFile(cfg, false); - } -} - static void ConfigureServices(HostBuilderContext context, IServiceCollection services) => services.AddDataStreamPublisher(context.Configuration); diff --git a/src/DataStreamSuite/DataStreamPublisher/appsettings.Development.json b/src/DataStreamSuite/DataStreamPublisher/appsettings.Development.json new file mode 100644 index 000000000..dd820fc8d --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/appsettings.Development.json @@ -0,0 +1,39 @@ +{ + "PubSub": { + "Routing": { + "PublisherRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://127.0.0.1:22019", + "TagRegExp": ".*" + } + ], + "SubscriberRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://127.0.0.1:22019", + "TagRegExp": ".*" + } + ] + } + }, + "DataStreamPublisher": { + "PublisherNodes": [ + { + "Name": "test_data", + "Streams": [ + { + "Name": "seconds_1_10", + "Expression": "milliseconds(1000,10)", + "IsOnWebsocket": true + }, + { + "Name": "test_literal", + "Expression": "literal(\"x\", \"420.0\")", + "IsOnWebsocket": true + } + ] + } + ] + } +} diff --git a/src/DataStreamSuite/DataStreamPublisher/appsettings.MarketData.json b/src/DataStreamSuite/DataStreamPublisher/appsettings.MarketData.json new file mode 100644 index 000000000..df9df695f --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/appsettings.MarketData.json @@ -0,0 +1,63 @@ +{ + "Spark": { + "ConfigServer": "http://sparkd:8000/config" + }, + "SparkApi": { + "Credentials": { + "User": "bts", + "Password": "" + }, + "EnableAdmin": true, + "EnableOrders": true, + "EnablePosition": true + }, + "DataStreamPublisher": { + "PublisherNodes": [ + { + "Name": "market_data", + "Streams": [ + { + "Name": "bbo", + "Expression": "bbo()", + "IsOnWebsocket": true + }, + { + "Name": "instruments", + "Expression": "instruments()", + "IsOnWebsocket": true + }, + { + "Name": "timesales", + "Expression": "timesales(10000)", + "IsOnWebsocket": true + }, + { + "Name": "books", + "Expression": "books()", + "IsOnWebsocket": true + } + ] + }, + { + "Name": "test_data", + "Streams": [ + { + "Name": "milliseconds_1000_100", + "Expression": "milliseconds(3000,5)", + "IsOnWebsocket": true + }, + { + "Name": "test_literal", + "Expression": "literal(\"x\", \"420.0\")", + "IsOnWebsocket": true + }, + { + "Name": "test_not_on_websocket", + "Expression": "literal(\"x\", \"840.0\")", + "IsOnWebsocket": false + } + ] + } + ] + } +} diff --git a/src/DataStreamSuite/DataStreamPublisher/appsettings.Production.json b/src/DataStreamSuite/DataStreamPublisher/appsettings.Production.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/appsettings.Production.json @@ -0,0 +1 @@ +{} diff --git a/src/DataStreamSuite/DataStreamPublisher/appsettings.ProductionAws.json b/src/DataStreamSuite/DataStreamPublisher/appsettings.ProductionAws.json new file mode 100644 index 000000000..d8a8c8c57 --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/appsettings.ProductionAws.json @@ -0,0 +1,91 @@ +{ + "Spark": { + "ConfigServer": "http://localhost:8000/config" + }, + "SparkApi": { + "Credentials": { + "User": "bts", + "Password": "" + }, + "EnableAdmin": true, + "EnableOrders": true, + "EnablePosition": true + }, + "PubSub": { + "Routing": { + "PublisherRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://172.30.2.181:22019", + "TagRegExp": ".*" + } + ], + "SubscriberRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://172.30.2.181:22019", + "TagRegExp": ".*" + } + ] + } + }, + "DataStreamPublisher": { + "PublisherNodes": [ + { + "Name": "market_data", + "Streams": [ + { + "Name": "bbo", + "Expression": "bbo()", + "IsOnWebsocket": true + }, + { + "Name": "instruments", + "Expression": "instruments()", + "IsOnWebsocket": true + }, + { + "Name": "timesales", + "Expression": "timesales(10000)", + "IsOnWebsocket": true + }, + { + "Name": "books", + "Expression": "books()", + "IsOnWebsocket": true + } + ] + }, + { + "Name": "test_data", + "Streams": [ + { + "Name": "seconds_1_10", + "Expression": "milliseconds(1000,10)", + "IsOnWebsocket": true + }, + { + "Name": "benzinga_news", + "Expression": "benzingaNews(\"mongodb://mongoadmin:bts@172.30.2.181:27017/?directConnection=true&serverSelectionTimeoutMS=2000&authSource=admin&appName=mongosh+2.2.12\" ,\"benzinga_news\",\"news_headlines\", \"30s\", \"5d\")", + "IsOnWebsocket": true + }, + { + "Name": "benzinga_news_symbols", + "Expression": "benzingaNewsSymbols(benzinga_news)", + "IsOnWebsocket": true + }, + { + "Name": "benzinga_news_symbol_state", + "Expression": "benzingaNewsSymbolState(benzinga_news)", + "IsOnWebsocket": true + }, + { + "Name": "test_literal", + "Expression": "literal(\"x\", \"420.0\")", + "IsOnWebsocket": true + } + ] + } + ] + } +} diff --git a/src/DataStreamSuite/DataStreamPublisher/appsettings.Staging.json b/src/DataStreamSuite/DataStreamPublisher/appsettings.Staging.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/appsettings.Staging.json @@ -0,0 +1 @@ +{} diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/appsettings.json similarity index 55% rename from src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json rename to src/DataStreamSuite/DataStreamPublisher/appsettings.json index ae9a3e5a1..8c5581afe 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json +++ b/src/DataStreamSuite/DataStreamPublisher/appsettings.json @@ -1,16 +1,11 @@ { - "Spark": { - "ConfigServer": "http://localhost:8000/config" - }, - "SparkApi": { - "Credentials": { - "User": "bts", - "Password": "" - }, - "EnableAdmin": true, - "EnableOrders": true, - "EnablePosition": true + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } }, + "Spark": {}, "MasterWorker": { "Interval": "00:00:00.1" }, @@ -21,19 +16,19 @@ "HeartbeatInterval": 15000, "MissingHeartbeatsBeforeGc": 50, "DisposedConnectionCheckPeriod": 1000, - "ServerPort": 12345, + "ServerPort": 22019, "Routing": { "PublisherRoutes": [ { "Name": "default", - "ServerAddress": "bts-cloud://127.0.0.1:12345", + "ServerAddress": "bts-cloud://pubsub:22019", "TagRegExp": ".*" } ], "SubscriberRoutes": [ { "Name": "default", - "ServerAddress": "bts-cloud://127.0.0.1:12345", + "ServerAddress": "bts-cloud://pubsub:22019", "TagRegExp": ".*" } ] @@ -73,48 +68,13 @@ }, "DataStreamPublisher": { "PublisherNodes": [ - { - "Name": "market_data", - "Streams": [ - { - "Name": "bbo", - "Expression": "bbo()", - "IsOnWebsocket": true - }, - { - "Name": "instruments", - "Expression": "instruments()", - "IsOnWebsocket": true - }, - { - "Name": "timesales", - "Expression": "timesales(10000)", - "IsOnWebsocket": true - }, - { - "Name": "books", - "Expression": "books()", - "IsOnWebsocket": true - } - ] - }, { "Name": "test_data", "Streams": [ - { - "Name": "milliseconds_1000_100", - "Expression": "milliseconds(3000,5)", - "IsOnWebsocket": true - }, { "Name": "test_literal", "Expression": "literal(\"x\", \"420.0\")", "IsOnWebsocket": true - }, - { - "Name": "test_not_on_websocket", - "Expression": "literal(\"x\", \"840.0\")", - "IsOnWebsocket": false } ] } diff --git a/src/DataStreamSuite/DataStreamPublisher/spark.conf b/src/DataStreamSuite/DataStreamPublisher/spark.conf new file mode 100644 index 000000000..81b3e4f58 --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/spark.conf @@ -0,0 +1,15 @@ +Logging { + value { + console { + default_level: "Info" + } + syslog { + default_level: "Info" + } + file { + default_level: "Info" + } + initial_queue_size: 120001 + max_queue_growth: 4 + } +} diff --git a/src/DataStreamSuite/DataStreamSuite.sln b/src/DataStreamSuite/DataStreamSuite.sln index 5a6420d6b..c7358f552 100644 --- a/src/DataStreamSuite/DataStreamSuite.sln +++ b/src/DataStreamSuite/DataStreamSuite.sln @@ -38,6 +38,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", EndProject /*Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamQueryAnalyzer", "DataStreamQueryAnalyzer\DataStreamQueryAnalyzer.csproj", "{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}" EndProject*/ +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{5C528A65-E43C-48A2-8CA9-ED2D0E356A37}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -119,5 +121,9 @@ Global {BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Debug|Any CPU.Build.0 = Debug|Any CPU {BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Release|Any CPU.ActiveCfg = Release|Any CPU {BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Release|Any CPU.Build.0 = Release|Any CPU + {5C528A65-E43C-48A2-8CA9-ED2D0E356A37}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5C528A65-E43C-48A2-8CA9-ED2D0E356A37}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5C528A65-E43C-48A2-8CA9-ED2D0E356A37}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5C528A65-E43C-48A2-8CA9-ED2D0E356A37}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/src/DataStreamSuite/PubSubServer/Configurations/PubsubServer_Linux.json b/src/DataStreamSuite/PubSubServer/Configurations/PubsubServer_Linux.json deleted file mode 100644 index 4c9a17a45..000000000 --- a/src/DataStreamSuite/PubSubServer/Configurations/PubsubServer_Linux.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "user": "bts", - "pubsub": { - "milliseconds_between_sub_info_updates": 1000, - "milliseconds_between_time_slice_updates": 5000, - "milliseconds_between_no_buffer_space_retries": 100, - "retries_on_no_buffer_space": 1, - "do_it_all_seconds_between_logs": 0, - "heartbeat_interval": 15000, - "missing_heartbeats_before_gc": 50, - "use_managed_sockets": true, - "state_update_period": 5000, - "statistics_update_period": 5000, - "disposed_connection_check_period": 1000, - "message_processing_warning_threshold": 1000, - "server_port": 12345, - "publisher_routes": [ - { - "name": "default", - "server_address": "bts-cloud://192.168.1.80:12345", - "tag": ".*" - } - ], - "subscriber_routes": [ - { - "name": "default", - "server_address": "bts-cloud://192.168.1.80:12345", - "tag": ".*" - } - ], - "max_packet_size": 500000000, - "blue_comm_must_die": true, - "crushable_tags": [], - "tag_type_to_assembly_map": [ - { - "handler_class": "BCG.PubSubServerTagHandler", - "assembly": "DataStreamCore.dll" - } - ], - "server_queue": "PubSubQueue", - "port_range": { - "first": 30400, - "last": 31700 - }, - "bind_to_random_port": false, - "debug_tags": "^NOTAGS$", - "latency_report_threshold_ms": 1500, - "use_substations": false, - "use_dynamic_clients": false, - "is_substation": false, - "internally_publish_sub_info": true, - "use_managed_clients_in_native_code": false, - "compressed_tags": "^NOTAGS$", - "tags_to_report_compression_ratio": "^NOTAGS$", - "disable_msmq": false, - "use_prepublished_server_state": false, - "ip_translation": null, - "gc_activation_threshold": 0, - "wait_period_after_gc": 1000 - } -} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSubServer/Dockerfile b/src/DataStreamSuite/PubSubServer/Dockerfile new file mode 100644 index 000000000..b45188a37 --- /dev/null +++ b/src/DataStreamSuite/PubSubServer/Dockerfile @@ -0,0 +1,8 @@ +FROM bluetradingsystems/linux_dotnet7 + +ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish" +ARG CONF_FILE="src/DataStreamSuite/PubSubServer/spark.conf" + +COPY --chown=bts:bts "${SOURCE_DIR}" "bin" +COPY --chown=bts:bts "${CONF_FILE}" ".local/share/bts/PubSubServer/conf/spark.conf" +WORKDIR "/home/bts/bin" diff --git a/src/DataStreamSuite/PubSubServer/Program.cs b/src/DataStreamSuite/PubSubServer/Program.cs index 59726464f..d7b663942 100644 --- a/src/DataStreamSuite/PubSubServer/Program.cs +++ b/src/DataStreamSuite/PubSubServer/Program.cs @@ -1,3 +1,4 @@ +using Bts.Extensions.Hosting; using Bts.PubSub.Services; using Microsoft.Extensions.Configuration; @@ -8,8 +9,7 @@ using Spark.Core.Services; using System; -var host = Host.CreateDefaultBuilder(args) - .ConfigureAppConfiguration(ConfigureApp) +var host = HostBuilderExtensions.CreateBtsBuilder(args) .ConfigureServices(ConfigureServices) .Build(); try { @@ -22,15 +22,6 @@ try { } return 0; -static void ConfigureApp(HostBuilderContext context, IConfigurationBuilder configBuilder) { - configBuilder.AddCommandLine(Environment.GetCommandLineArgs()); - var cfgRoot = configBuilder.Build(); - var cfg = cfgRoot["config"]; - if (cfg is not null) { - configBuilder.AddJsonFile(cfg, false); - } -} - static void ConfigureServices(HostBuilderContext context, IServiceCollection services) { var echoServerCfg = context.Configuration.GetSection("SimpleEchoServer"); if (echoServerCfg.Get() is not null) { diff --git a/src/DataStreamSuite/PubSubServer/PubSubServer.csproj b/src/DataStreamSuite/PubSubServer/PubSubServer.csproj index 666dd989e..49e107ae2 100644 --- a/src/DataStreamSuite/PubSubServer/PubSubServer.csproj +++ b/src/DataStreamSuite/PubSubServer/PubSubServer.csproj @@ -12,7 +12,17 @@ + + + + + + + + + + diff --git a/src/DataStreamSuite/PubSubServer/appsettings.Development.json b/src/DataStreamSuite/PubSubServer/appsettings.Development.json new file mode 100644 index 000000000..f83647ef7 --- /dev/null +++ b/src/DataStreamSuite/PubSubServer/appsettings.Development.json @@ -0,0 +1,20 @@ +{ + "PubSub": { + "Routing": { + "PublisherRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://localhost:22019", + "TagRegExp": ".*" + } + ], + "SubscriberRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://localhost:22019", + "TagRegExp": ".*" + } + ] + } + } +} diff --git a/src/DataStreamSuite/PubSubServer/Configurations/SimpleEchoServer.json b/src/DataStreamSuite/PubSubServer/appsettings.EchoServer.json similarity index 91% rename from src/DataStreamSuite/PubSubServer/Configurations/SimpleEchoServer.json rename to src/DataStreamSuite/PubSubServer/appsettings.EchoServer.json index d898b5837..b7a63f076 100644 --- a/src/DataStreamSuite/PubSubServer/Configurations/SimpleEchoServer.json +++ b/src/DataStreamSuite/PubSubServer/appsettings.EchoServer.json @@ -1,6 +1,6 @@ -{ - "PubSubEchoServer": { - "Echo": true, - "Port": 12345 - } -} +{ + "PubSubEchoServer": { + "Echo": true, + "Port": 12345 + } +} diff --git a/src/DataStreamSuite/PubSubServer/appsettings.Production.json b/src/DataStreamSuite/PubSubServer/appsettings.Production.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/src/DataStreamSuite/PubSubServer/appsettings.Production.json @@ -0,0 +1 @@ +{} diff --git a/src/DataStreamSuite/PubSubServer/appsettings.ProductionAws.json b/src/DataStreamSuite/PubSubServer/appsettings.ProductionAws.json new file mode 100644 index 000000000..d54f0f942 --- /dev/null +++ b/src/DataStreamSuite/PubSubServer/appsettings.ProductionAws.json @@ -0,0 +1,20 @@ +{ + "PubSub": { + "Routing": { + "PublisherRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://34.237.179.93:22019", + "TagRegExp": ".*" + } + ], + "SubscriberRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://34.237.179.93:22019", + "TagRegExp": ".*" + } + ] + } + } +} diff --git a/src/DataStreamSuite/PubSubServer/appsettings.Staging.json b/src/DataStreamSuite/PubSubServer/appsettings.Staging.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/src/DataStreamSuite/PubSubServer/appsettings.Staging.json @@ -0,0 +1 @@ +{} diff --git a/src/DataStreamSuite/PubSubServer/Configurations/PubSubServer.json b/src/DataStreamSuite/PubSubServer/appsettings.json similarity index 85% rename from src/DataStreamSuite/PubSubServer/Configurations/PubSubServer.json rename to src/DataStreamSuite/PubSubServer/appsettings.json index ba43a5956..13f360bff 100644 --- a/src/DataStreamSuite/PubSubServer/Configurations/PubSubServer.json +++ b/src/DataStreamSuite/PubSubServer/appsettings.json @@ -1,5 +1,5 @@ { - "Spark": { }, + "Spark": {}, "MasterWorker": { "Interval": "00:00:00.1" }, @@ -7,19 +7,19 @@ "HeartbeatInterval": 15000, "MissingHeartbeatsBeforeGc": 50, "DisposedConnectionCheckPeriod": 1000, - "ServerPort": 12345, + "ServerPort": 22019, "Routing": { "PublisherRoutes": [ { "Name": "default", - "ServerAddress": "bts-cloud://127.0.0.1:12345", + "ServerAddress": "bts-cloud://pubsub:22019", "TagRegExp": ".*" } ], "SubscriberRoutes": [ { "Name": "default", - "ServerAddress": "bts-cloud://127.0.0.1:12345", + "ServerAddress": "bts-cloud://pubsub:22019", "TagRegExp": ".*" } ] @@ -52,7 +52,7 @@ { "TagType": ".*", "Assembly": "DataStreamCore", - "Class": "bts.PubsubServerGenericDataHandler" + "Class": "bts.PubsubServerGenericDataHandler" } ] } diff --git a/src/DataStreamSuite/PubSubServer/spark.conf b/src/DataStreamSuite/PubSubServer/spark.conf new file mode 100644 index 000000000..81b3e4f58 --- /dev/null +++ b/src/DataStreamSuite/PubSubServer/spark.conf @@ -0,0 +1,15 @@ +Logging { + value { + console { + default_level: "Info" + } + syslog { + default_level: "Info" + } + file { + default_level: "Info" + } + initial_queue_size: 120001 + max_queue_growth: 4 + } +} diff --git a/src/DataStreamSuite/PubSubServer/staging.env b/src/DataStreamSuite/PubSubServer/staging.env new file mode 100644 index 000000000..7a19a5a7a --- /dev/null +++ b/src/DataStreamSuite/PubSubServer/staging.env @@ -0,0 +1 @@ +ASPNETCORE_ENVIRONMENT=Staging diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 87b8de912..a7e2e217d 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -88,4 +88,13 @@ + + + $(OutputPath)\Configurations\$(MSBuildProjectName) + $(OutputPath)\$(MSBuildProjectName).%(FileName)%(Extension) + $(OutputPath)\publish\Configurations\$(MSBuildProjectName) + $(OutputPath)\publish\$(MSBuildProjectName).%(FileName)%(Extension) + + + diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets index 3409f22ba..6ea2693db 100644 --- a/src/Directory.Build.targets +++ b/src/Directory.Build.targets @@ -36,6 +36,34 @@ + + + + + + + + + + + + +