Configure apps as a distributed system via Docker compose

This commit is contained in:
John Colvin committed 2024-11-25 15:26:58 +00:00
1 parent 8f50979f90
commit 5606e690e6
47 files changed
+687 -528

No files matched your search

+35 -31
View File
@@ -12,15 +12,11 @@
"request": "launch",
"preLaunchTask": "Build DataStream WebApp (Debug)",
"program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
],
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
"env": {
"ASPNETCORE_HTTP_PORT": "8080",
"ASPNETCORE_HTTPS_PORT": "8443",
"ASPNETCORE_ENVIRONMENT": "Development",
"ASPNETCORE_ENVIRONMENT": "${input:dataStreamWebAppServerEnvironment}",
"ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
}
},
@@ -29,15 +25,11 @@
"type": "coreclr",
"request": "launch",
"program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
],
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
"env": {
"ASPNETCORE_HTTP_PORT": "8080",
"ASPNETCORE_HTTPS_PORT": "8443",
"ASPNETCORE_ENVIRONMENT": "Development",
"ASPNETCORE_ENVIRONMENT": "${input:dataStreamWebAppServerEnvironment}",
"ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
}
},
@@ -53,10 +45,9 @@
"windows": {
"program": "DataStreamPublisher.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
],
"env": {
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
},
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
"stopAtEntry": false,
"console": "integratedTerminal"
@@ -72,10 +63,9 @@
"windows": {
"program": "DataStreamPublisher.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
],
"env": {
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
},
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
"stopAtEntry": false,
"console": "integratedTerminal"
@@ -124,11 +114,10 @@
"windows": {
"program": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
],
"cwd": "build/Debug/bin/net7.0",
"env": {
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
},
"stopAtEntry": false,
"console": "integratedTerminal"
},
@@ -185,26 +174,41 @@
],
"inputs": [
{
"id": "dataStreamPublisherConfig",
"id": "dataStreamPublisherEnvironment",
"type": "pickString",
"description": "Publisher Configuration",
"options": [
"MarketDataPublisher",
"SimpleEchoClient",
"SimpleTestDataPublisher",
"TestDataPublisher",
"Development",
"MarketData",
"Production",
"ProductionAws",
"Staging",
],
"default": "MarketDataPublisher",
},
{
"id": "pubSubServerConfig",
"id": "pubSubServerEnvironment",
"type": "pickString",
"description": "PubSub Configuration",
"options": [
"PubSubServer",
"SimpleEchoClient",
"Development",
"EchoServer",
"Production",
"ProductionAws",
"Staging",
],
"default": "PubSubServer",
"default": "Development",
},
{
"id": "dataStreamWebAppServerEnvironment",
"type": "pickString",
"description": "DataStream WebApp Environment",
"options": [
"Development",
"Production",
"ProductionAws",
"Staging",
]
},
{
"id": "sparkConfigServer",
+59
View File
@@ -0,0 +1,59 @@
services:
sparkd:
build:
dockerfile: src/server/sparkd/Dockerfile
args:
- SOURCE_DIR=install/Release/bin
- CONF_FILE=src/server/sparkd/spark.conf
command: [ "./sparkd", "run" ]
ports:
- 8000:8000
- 8001:8001
- 8002:8002
pubsub:
environment:
- DOTNET_ENVIRONMENT=Staging
build:
dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
args:
- SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
- CONF_FILE=src/DataStreamSuite/PubSubServer/spark.conf
ports:
- 22019:22019
command: [ "./PubSubServer" ]
md-publisher:
build:
dockerfile: src/DataStreamSuite/Dockerfile
args:
- SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
- CONF_FILE=src/DataStreamSuite/DataStreamPublisher/spark.conf
command: [ "./DataStreamPublisher" ]
environment:
- DOTNET_ENVIRONMENT=MarketData
webapp:
ports:
- 8080:8080
- 8443:8443
build:
dockerfile: src/DataStreamSuite/Bts.DataStreamWebApp.Server/Dockerfile
args:
- SOURCE_DIR=build/Release/bin/net7.0/linux-x64/publish
- WEB_ROOT=build/Bts.DataStreamWebApp.Client/production/dist
- CONF_FILE=src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf
command: [ "./Bts.DataStreamWebApp.Server" ]
environment:
- ASPNETCORE_HTTP_PORT=8080
- ASPNETCORE_HTTPS_PORT=8443
- ASPNETCORE_ENVIRONMENT=Staging
pingtest:
build:
context: .
dockerfile_inline: |
FROM bluetradingsystems/linux_dotnet7
USER "root"
RUN apt-get update && apt-get -y install iputils-ping
command: [ "ping", "sparkd" ]
@@ -30,10 +30,11 @@
</ItemGroup>
<ItemGroup>
<ConfigurationProfile Include="Configurations\*.json" />
<ConfigurationProfile Include="appsettings.Development.json" />
<ConfigurationProfile Include="appsettings.Production.json" />
<ConfigurationProfile Include="appsettings.ProductionAws.json" />
<ConfigurationProfile Include="appsettings.Staging.json" />
<ConfigurationProfile Include="appsettings.json" />
</ItemGroup>
<!-- <ItemGroup>
<Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
</ItemGroup> -->
</Project>
@@ -1,97 +0,0 @@
{
"MasterWorker": {
"Interval": "00:00:00.1"
},
"ViewStreamDatabase": {
"Interval": "00:00:00.1"
},
"Spark": {
"ConfigServer": "http://localhost:8000/config"
},
"SparkApi": {
"Credentials": {
"User": "bts",
"Password": ""
},
"EnableAdmin": true,
"EnableOrders": true,
"EnablePosition": true
},
"PubSub": {
"HeartbeatInterval": 15000,
"MissingHeartbeatsBeforeGc": 50,
"ServerPort": 12345,
"Routing": {
"PublisherRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://127.0.0.1:12345",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://127.0.0.1:12345",
"TagRegExp": ".*"
}
]
},
"TagTypeToAssemblyMap": {
"Entries": [
{
"HandlerClass": "BCG.PubSubServerTagHandler",
"Assembly": "DataStreamCore.dll"
}
]
},
"TagHandlerFactory": {
"Handlers": [
{
"TagType": "DataStream",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerStreamTagHandler"
},
{
"TagType": "System_Sub",
"Assembly": "PubSub",
"Class": "BCG.SystemSubscriptionMapPublisher"
},
{
"TagType": "System_Stats",
"Assembly": "PubSub",
"Class": "BCG.SystemStatsPublisher"
},
{
"TagType": ".*",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerGenericDataHandler"
}
]
}
},
"Kestrel": {
"Endpoints": {
"Http": {
"Url": "http://localhost:8080"
},
"Https": {
"Url": "https://localhost:8443"
}
}
},
"CannedQueries": [
{
"Name": "BBO",
"Query": [
"bbo"
]
},
{
"Name": "Trades",
"Query": [
"timesales"
]
}
]
}
@@ -1,4 +0,0 @@
{
"content_root_path": "build/Bts.DataStreamWebApp.Client/production",
"web_root_path": "dist"
}
@@ -0,0 +1,11 @@
FROM bluetradingsystems/linux_dotnet7
ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
ARG WEB_ROOT="build/Bts.DataStreamWebApp.Client/production/dist"
ARG CONF_FILE="src/DataStreamSuite/Bts.DataStreamWebApp.Server/spark.conf"
RUN dotnet dev-certs https --trust
COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
COPY --chown=bts:bts "${WEB_ROOT}" "bin/wwwroot"
COPY --chown=bts:bts "${CONF_FILE}" ".local/share/DataStream/conf/spark.conf"
WORKDIR "/home/bts/bin"
@@ -5,13 +5,13 @@ using Spark.Api.Services;
using Spark.Core.Services;
using System.Reactive.Linq;
using System.Text.Json;
using System.Reflection;
var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
Args = args,
ApplicationName = "Bts.DataStreamWebApp.Server",
});
ConfigureApp(builder.Configuration);
ConfigureApp(builder.Configuration, builder.Environment.EnvironmentName);
ConfigureServices(builder.Services, builder.Configuration);
var app = builder.Build();
@@ -39,10 +39,32 @@ try {
}
return 0;
static void ConfigureApp(ConfigurationManager configBuilder) {
var cfgRoot = configBuilder.GetSection("config");
if (cfgRoot?.Value is not null) {
configBuilder.AddJsonFile(cfgRoot.Value);
static void ConfigureApp(ConfigurationManager configBuilder, string environment) {
// We prefix appsettings with the exe name because multiple exes will usually be in the build dir
// (this way, each exe can still load its own appsettings files from the current directory).
var exeName = Assembly.GetCallingAssembly().GetName().Name;
AddJsonFile($"{exeName}.appsettings.json");
AddJsonFile($"{exeName}.appsettings.{environment}.json");
var cfg = configBuilder.GetSection("config")?.Value;
if (!SetBasePath(cfg)) {
AddJsonFile(cfg);
}
bool SetBasePath(string? path) {
if (!Directory.Exists(path)) {
return false;
}
configBuilder.SetBasePath(path);
return true;
}
bool AddJsonFile(string? path) {
if (!File.Exists(path)) {
return false;
}
configBuilder.AddJsonFile(path);
return true;
}
}
@@ -0,0 +1,52 @@
{
"PubSub": {
"Routing": {
"PublisherRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://localhost:22019",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://localhost:22019",
"TagRegExp": ".*"
}
]
}
},
"CannedQueries": [
{
"Name": "BBO",
"Query": [
"bbo"
]
},
{
"Name": "Trades",
"Query": [
"timesales"
]
},
{
"Name": "Instruments",
"Query": [
"instruments"
]
},
{
"Name": "Books",
"Query": [
"books"
]
},
{
"Name": "Benzinga News",
"Query": [
"benzinga_news"
]
}
]
}
@@ -0,0 +1,8 @@
{
"Spark": {
"ConfigServer": "http://sparkd:8000/config"
},
"SparkApi": {
"IsEnabled": true
}
}
@@ -0,0 +1,44 @@
{
"CannedQueries": [
{
"Name": "BBO",
"Query": [
"bbo"
]
},
{
"Name": "Trades",
"Query": [
"timesales"
]
},
{
"Name": "Instruments",
"Query": [
"instruments"
]
},
{
"Name": "Books",
"Query": [
"books"
]
},
{
"Name": "Benzinga News",
"Query": [
"benzinga_news"
]
}
],
"Spark": {
"ConfigServer": "http://sparkd:8000/config"
},
"SparkApi": {
"IsEnabled": true,
"Credentials": {
"User": "bts",
"Password": ""
}
}
}
@@ -9,11 +9,77 @@
"Kestrel": {
"Endpoints": {
"Http": {
"Url": "http://localhost:8080"
"Url": "http://0.0.0.0:8080"
},
"Https": {
"Url": "https://localhost:8443"
"Url": "https://0.0.0.0:8443"
}
}
},
"MasterWorker": {
"Interval": "00:00:00.1"
},
"ViewStreamDatabase": {
"Interval": "00:00:00.1"
},
"SparkApi": {
"IsEnabled": false,
"Credentials": {
"User": "bts",
"Password": ""
}
},
"PubSub": {
"HeartbeatInterval": 15000,
"MissingHeartbeatsBeforeGc": 50,
"ServerPort": 22019,
"Routing": {
"PublisherRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://pubsub:22019",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://pubsub:22019",
"TagRegExp": ".*"
}
]
},
"TagTypeToAssemblyMap": {
"Entries": [
{
"HandlerClass": "BCG.PubSubServerTagHandler",
"Assembly": "DataStreamCore.dll"
}
]
},
"TagHandlerFactory": {
"Handlers": [
{
"TagType": "DataStream",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerStreamTagHandler"
},
{
"TagType": "System_Sub",
"Assembly": "PubSub",
"Class": "BCG.SystemSubscriptionMapPublisher"
},
{
"TagType": "System_Stats",
"Assembly": "PubSub",
"Class": "BCG.SystemStatsPublisher"
},
{
"TagType": ".*",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerGenericDataHandler"
}
]
}
}
}
@@ -0,0 +1,15 @@
Logging {
value {
console {
default_level: "Info"
}
syslog {
default_level: "Info"
}
file {
default_level: "Info"
}
initial_queue_size: 120001
max_queue_growth: 4
}
}
@@ -0,0 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
</ItemGroup>
</Project>
@@ -0,0 +1,59 @@
namespace Bts.Extensions.Hosting;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using System.Diagnostics;
public static class HostBuilderExtensions {
public static IHostBuilder CreateBtsBuilder(string[] args) {
// Obtain calling assembly name here, rather than in the lambda where it is actually needed,
// because want the name of the assembly that called this function, not the assembly that will eventually invoke
// the lambda...that will be the Microsoft.Extensions.Hosting assembly.
// Get the name of the assembly that called this function. That will act as the "appsettings"
// json file prefix
var exeName = GetCallingAssemblyName();
return Host.CreateDefaultBuilder(args)
.ConfigureAppConfiguration((context, configBuilder) =>
ConfigureApp(configBuilder, context.HostingEnvironment.EnvironmentName, exeName));
}
public static void ConfigureApp(IConfigurationBuilder configBuilder, string? environment = null, string? prefix = null) {
var baseFileName = prefix is null ? "appsettings" : $"{prefix}.appsettings";
AddJsonFile($"{baseFileName}.json");
if (environment is not null) {
AddJsonFile($"{baseFileName}.{environment}.json");
}
var cfg = configBuilder.Build().GetSection("config")?.Value;
if (!SetBasePath(cfg)) {
AddJsonFile(cfg);
}
bool SetBasePath(string? path) {
if (!Directory.Exists(path)) {
return false;
}
configBuilder.SetBasePath(path);
return true;
}
bool AddJsonFile(string? path) {
if (!File.Exists(path)) {
return false;
}
configBuilder.AddJsonFile(path);
return true;
}
}
public static string GetCallingAssemblyName() =>
new StackTrace()
.GetFrame(2)
?.GetMethod()
?.Module
?.Assembly
?.GetName()
?.Name
?? throw new InvalidOperationException("Failed to determine name of calling assembly (calling method is null)");
}
@@ -1,123 +0,0 @@
{
"Spark": {
"ConfigServer": "http://localhost:8000/config"
},
"SparkApi": {
"Credentials": {
"User": "bts",
"Password": ""
},
"EnableAdmin": true,
"EnableOrders": true,
"EnablePosition": true
},
"MasterWorker": {
"Interval": "00:00:00.1"
},
"ViewStreamDatabase": {
"Interval": "00:00:00.1"
},
"PubSub": {
"HeartbeatInterval": 15000,
"MissingHeartbeatsBeforeGc": 50,
"DisposedConnectionCheckPeriod": 1000,
"ServerPort": 12345,
"Routing": {
"PublisherRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://127.0.0.1:12345",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://127.0.0.1:12345",
"TagRegExp": ".*"
}
]
},
"TagTypeToAssemblyMap": {
"Entries": [
{
"HandlerClass": "BCG.PubSubServerTagHandler",
"Assembly": "DataStreamCore.dll"
}
]
},
"TagHandlerFactory": {
"Handlers": [
{
"TagType": "DataStream",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerStreamTagHandler"
},
{
"TagType": "System_Sub",
"Assembly": "PubSub",
"Class": "BCG.SystemSubscriptionMapPublisher"
},
{
"TagType": "System_Stats",
"Assembly": "PubSub",
"Class": "BCG.SystemStatsPublisher"
},
{
"TagType": ".*",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerGenericDataHandler"
}
]
}
},
"DataStreamPublisher": {
"PublisherNodes": [
{
"Name": "market_data",
"Streams": [
{
"Name": "bbo",
"Expression": "bbo()",
"IsOnWebsocket": true
},
{
"Name": "instruments",
"Expression": "instruments()",
"IsOnWebsocket": true
},
{
"Name": "timesales",
"Expression": "timesales(10000)",
"IsOnWebsocket": true
},
{
"Name": "books",
"Expression": "books()",
"IsOnWebsocket": true
}
]
},
{
"Name": "test_data",
"Streams": [
{
"Name": "milliseconds_1000_100",
"Expression": "milliseconds(3000,5)",
"IsOnWebsocket": true
},
{
"Name": "test_literal",
"Expression": "literal(\"x\", \"420.0\")",
"IsOnWebsocket": true
},
{
"Name": "test_not_on_websocket",
"Expression": "literal(\"x\", \"840.0\")",
"IsOnWebsocket": false
}
]
}
]
}
}
@@ -1,4 +0,0 @@
{
"user": "bts",
"simple_echo_client": true
}
@@ -1,73 +0,0 @@
{
"user": "bts",
"pubsub": {
"milliseconds_between_sub_info_updates": 1000,
"milliseconds_between_time_slice_updates": 5000,
"milliseconds_between_no_buffer_space_retries": 100,
"retries_on_no_buffer_space": 1,
"do_it_all_seconds_between_logs": 0,
"heartbeat_interval": 15000,
"missing_heartbeats_before_gc": 50,
"use_managed_sockets": true,
"state_update_period": 5000,
"statistics_update_period": 5000,
"disposed_connection_check_period": 1000,
"message_processing_warning_threshold": 1000,
"server_port": 12345,
"publisher_routes": [
{
"name": "default",
"server_address": "bts-cloud://localhost:12345",
"tag": ".*"
}
],
"subscriber_routes": [
{
"name": "default",
"server_address": "bts-cloud://localhost:12345",
"tag": ".*"
}
],
"max_packet_size": 500000000,
"blue_comm_must_die": true,
"crushable_tags": [],
"tag_type_to_assembly_map": [
{
"handler_class": "BCG.PubSubServerTagHandler",
"assembly": "DataStreamCore.dll"
}
],
"server_queue": "PubSubQueue",
"port_range": {
"first": 30400,
"last": 31700
},
"bind_to_random_port": false,
"debug_tags": "^NOTAGS$",
"latency_report_threshold_ms": 1500,
"use_substations": false,
"use_dynamic_clients": false,
"is_substation": false,
"internally_publish_sub_info": true,
"use_managed_clients_in_native_code": false,
"compressed_tags": "^NOTAGS$",
"tags_to_report_compression_ratio": "^NOTAGS$",
"disable_msmq": false,
"use_prepublished_server_state": false,
"ip_translation": null,
"gc_activation_threshold": 0,
"wait_period_after_gc": 1000
},
"publishers": [
{
"name": "test_data",
"streams": [
{
"name": "test_literal",
"expression": "literal(\"x\", \"420.0\")",
"is_on_websocket": true
}
]
}
]
}
@@ -1,40 +0,0 @@
{
"user": "bts",
"publishers": [
{
"name": "test_data",
"streams": [
{
"name": "seconds_1_10",
"expression": "milliseconds(1000,10)",
"is_on_websocket": true
},
{
"name": "benzinga_news",
"expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")",
"is_on_websocket": true
},
{
"name": "benzinga_news_symbols",
"expression": "benzingaNewsSymbols(benzinga_news)",
"is_on_websocket": true
},
{
"name": "benzinga_news_symbol_state",
"expression": "benzingaNewsSymbolState(benzinga_news)",
"is_on_websocket": true
},
{
"name": "test_literal",
"expression": "literal(\"x\", \"420.0\")",
"is_on_websocket": true
},
{
"name": "test_not_on_websocket",
"expression": "literal(\"x\", \"840.0\")",
"is_on_websocket": false
}
]
}
]
}
@@ -15,13 +15,19 @@
<ItemGroup>
<ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
<ProjectReference Include="..\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj" />
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
<ProjectReference Include="..\DataStream\DataStream.csproj" />
<ProjectReference Include="..\PubSub\PubSub.csproj" />
</ItemGroup>
<ItemGroup>
<ConfigurationProfile Include="Configurations\*.json" />
<ConfigurationProfile Include="appsettings.Development.json" />
<ConfigurationProfile Include="appsettings.MarketData.json" />
<ConfigurationProfile Include="appsettings.Production.json" />
<ConfigurationProfile Include="appsettings.ProductionAws.json" />
<ConfigurationProfile Include="appsettings.Staging.json" />
<ConfigurationProfile Include="appsettings.json" />
</ItemGroup>
</Project>
@@ -0,0 +1,8 @@
FROM bluetradingsystems/linux_dotnet7
ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
ARG CONF_FILE="src/DataStreamSuite/DataStreamPublisher/spark.conf"
COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
COPY --chown=bts:bts "${CONF_FILE}" ".local/share/DataStream/conf/spark.conf"
WORKDIR "/home/bts/bin"
@@ -1,16 +1,14 @@
using Bts.DataStreamPublisher.Services;
using Bts.Extensions.Hosting;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using System;
var host = Host.CreateDefaultBuilder(args)
.ConfigureAppConfiguration(ConfigureApp)
var host = HostBuilderExtensions.CreateBtsBuilder(args)
.ConfigureServices(ConfigureServices)
.Build();
try {
Console.Write("Starting DataStreamPublisher");
host.Run();
@@ -21,14 +19,5 @@ try {
}
return 0;
static void ConfigureApp(HostBuilderContext context, IConfigurationBuilder configBuilder) {
configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
var cfgRoot = configBuilder.Build();
var cfg = cfgRoot["config"];
if (cfg is not null) {
configBuilder.AddJsonFile(cfg, false);
}
}
static void ConfigureServices(HostBuilderContext context, IServiceCollection services) =>
services.AddDataStreamPublisher(context.Configuration);
@@ -0,0 +1,39 @@
{
"PubSub": {
"Routing": {
"PublisherRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://127.0.0.1:22019",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://127.0.0.1:22019",
"TagRegExp": ".*"
}
]
}
},
"DataStreamPublisher": {
"PublisherNodes": [
{
"Name": "test_data",
"Streams": [
{
"Name": "seconds_1_10",
"Expression": "milliseconds(1000,10)",
"IsOnWebsocket": true
},
{
"Name": "test_literal",
"Expression": "literal(\"x\", \"420.0\")",
"IsOnWebsocket": true
}
]
}
]
}
}
@@ -0,0 +1,63 @@
{
"Spark": {
"ConfigServer": "http://sparkd:8000/config"
},
"SparkApi": {
"Credentials": {
"User": "bts",
"Password": ""
},
"EnableAdmin": true,
"EnableOrders": true,
"EnablePosition": true
},
"DataStreamPublisher": {
"PublisherNodes": [
{
"Name": "market_data",
"Streams": [
{
"Name": "bbo",
"Expression": "bbo()",
"IsOnWebsocket": true
},
{
"Name": "instruments",
"Expression": "instruments()",
"IsOnWebsocket": true
},
{
"Name": "timesales",
"Expression": "timesales(10000)",
"IsOnWebsocket": true
},
{
"Name": "books",
"Expression": "books()",
"IsOnWebsocket": true
}
]
},
{
"Name": "test_data",
"Streams": [
{
"Name": "milliseconds_1000_100",
"Expression": "milliseconds(3000,5)",
"IsOnWebsocket": true
},
{
"Name": "test_literal",
"Expression": "literal(\"x\", \"420.0\")",
"IsOnWebsocket": true
},
{
"Name": "test_not_on_websocket",
"Expression": "literal(\"x\", \"840.0\")",
"IsOnWebsocket": false
}
]
}
]
}
}
@@ -11,17 +11,7 @@
"EnableOrders": true,
"EnablePosition": true
},
"MasterWorker": {
"Interval": "00:00:00.1"
},
"ViewStreamDatabase": {
"Interval": "00:00:00.1"
},
"PubSub": {
"HeartbeatInterval": 15000,
"MissingHeartbeatsBeforeGc": 50,
"DisposedConnectionCheckPeriod": 1000,
"ServerPort": 22019,
"Routing": {
"PublisherRoutes": [
{
@@ -37,38 +27,6 @@
"TagRegExp": ".*"
}
]
},
"TagTypeToAssemblyMap": {
"Entries": [
{
"HandlerClass": "BCG.PubSubServerTagHandler",
"Assembly": "DataStreamCore.dll"
}
]
},
"TagHandlerFactory": {
"Handlers": [
{
"TagType": "DataStream",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerStreamTagHandler"
},
{
"TagType": "System_Sub",
"Assembly": "PubSub",
"Class": "BCG.SystemSubscriptionMapPublisher"
},
{
"TagType": "System_Stats",
"Assembly": "PubSub",
"Class": "BCG.SystemStatsPublisher"
},
{
"TagType": ".*",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerGenericDataHandler"
}
]
}
},
"DataStreamPublisher": {
@@ -0,0 +1 @@
{}
@@ -1,25 +1,34 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning"
}
},
"Spark": {},
"MasterWorker": {
"Interval": "00:00:00.1"
},
"ViewStreamDatabase": {
"Interval": "00:00:00.1"
},
"PubSub": {
"HeartbeatInterval": 15000,
"MissingHeartbeatsBeforeGc": 50,
"DisposedConnectionCheckPeriod": 1000,
"ServerPort": 12345,
"ServerPort": 22019,
"Routing": {
"PublisherRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://127.0.0.1:12345",
"ServerAddress": "bts-cloud://pubsub:22019",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://127.0.0.1:12345",
"ServerAddress": "bts-cloud://pubsub:22019",
"TagRegExp": ".*"
}
]
@@ -57,8 +66,18 @@
]
}
},
"GarbageCollector": {
"ActivationThreshold": 0,
"Interval": "00:00:01"
"DataStreamPublisher": {
"PublisherNodes": [
{
"Name": "test_data",
"Streams": [
{
"Name": "test_literal",
"Expression": "literal(\"x\", \"420.0\")",
"IsOnWebsocket": true
}
]
}
]
}
}
@@ -0,0 +1,15 @@
Logging {
value {
console {
default_level: "Info"
}
syslog {
default_level: "Info"
}
file {
default_level: "Info"
}
initial_queue_size: 120001
max_queue_growth: 4
}
}
+6
View File
@@ -40,6 +40,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.DataStreamWebApp.Server
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_websocket", "..\client\spark_websocket\spark_websocket.csproj", "{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{5C528A65-E43C-48A2-8CA9-ED2D0E356A37}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -125,5 +127,9 @@ Global
{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Debug|Any CPU.Build.0 = Debug|x64
{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Release|Any CPU.ActiveCfg = Release|x64
{09B0FF33-D34F-4BB4-BE96-B7D35960BB3B}.Release|Any CPU.Build.0 = Release|x64
{5C528A65-E43C-48A2-8CA9-ED2D0E356A37}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5C528A65-E43C-48A2-8CA9-ED2D0E356A37}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5C528A65-E43C-48A2-8CA9-ED2D0E356A37}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5C528A65-E43C-48A2-8CA9-ED2D0E356A37}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal
+4
View File
@@ -0,0 +1,4 @@
FROM bluetradingsystems/linux_dotnet7
COPY --chown=bts:bts "build/Release/bin/net7.0/linux-x64/publish" "bin"
WORKDIR "/home/bts/bin"
@@ -1,61 +0,0 @@
{
"user": "bts",
"pubsub": {
"milliseconds_between_sub_info_updates": 1000,
"milliseconds_between_time_slice_updates": 5000,
"milliseconds_between_no_buffer_space_retries": 100,
"retries_on_no_buffer_space": 1,
"do_it_all_seconds_between_logs": 0,
"heartbeat_interval": 15000,
"missing_heartbeats_before_gc": 50,
"use_managed_sockets": true,
"state_update_period": 5000,
"statistics_update_period": 5000,
"disposed_connection_check_period": 1000,
"message_processing_warning_threshold": 1000,
"server_port": 12345,
"publisher_routes": [
{
"name": "default",
"server_address": "bts-cloud://192.168.1.80:12345",
"tag": ".*"
}
],
"subscriber_routes": [
{
"name": "default",
"server_address": "bts-cloud://192.168.1.80:12345",
"tag": ".*"
}
],
"max_packet_size": 500000000,
"blue_comm_must_die": true,
"crushable_tags": [],
"tag_type_to_assembly_map": [
{
"handler_class": "BCG.PubSubServerTagHandler",
"assembly": "DataStreamCore.dll"
}
],
"server_queue": "PubSubQueue",
"port_range": {
"first": 30400,
"last": 31700
},
"bind_to_random_port": false,
"debug_tags": "^NOTAGS$",
"latency_report_threshold_ms": 1500,
"use_substations": false,
"use_dynamic_clients": false,
"is_substation": false,
"internally_publish_sub_info": true,
"use_managed_clients_in_native_code": false,
"compressed_tags": "^NOTAGS$",
"tags_to_report_compression_ratio": "^NOTAGS$",
"disable_msmq": false,
"use_prepublished_server_state": false,
"ip_translation": null,
"gc_activation_threshold": 0,
"wait_period_after_gc": 1000
}
}
@@ -0,0 +1,8 @@
FROM bluetradingsystems/linux_dotnet7
ARG SOURCE_DIR="build/Release/bin/net7.0/linux-x64/publish"
ARG CONF_FILE="src/DataStreamSuite/PubSubServer/spark.conf"
COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
COPY --chown=bts:bts "${CONF_FILE}" ".local/share/DataStream/conf/spark.conf"
WORKDIR "/home/bts/bin"
+2 -11
View File
@@ -1,3 +1,4 @@
using Bts.Extensions.Hosting;
using Bts.PubSub.Services;
using Microsoft.Extensions.Configuration;
@@ -8,8 +9,7 @@ using Spark.Core.Services;
using System;
var host = Host.CreateDefaultBuilder(args)
.ConfigureAppConfiguration(ConfigureApp)
var host = HostBuilderExtensions.CreateBtsBuilder(args)
.ConfigureServices(ConfigureServices)
.Build();
try {
@@ -22,15 +22,6 @@ try {
}
return 0;
static void ConfigureApp(HostBuilderContext context, IConfigurationBuilder configBuilder) {
configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
var cfgRoot = configBuilder.Build();
var cfg = cfgRoot["config"];
if (cfg is not null) {
configBuilder.AddJsonFile(cfg, false);
}
}
static void ConfigureServices(HostBuilderContext context, IServiceCollection services) {
var echoServerCfg = context.Configuration.GetSection("SimpleEchoServer");
if (echoServerCfg.Get<EchoServerConfig>() is not null) {
@@ -12,11 +12,17 @@
<ItemGroup>
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
<ProjectReference Include="..\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj" />
<ProjectReference Include="..\PubSub\PubSub.csproj" />
</ItemGroup>
<ItemGroup>
<ConfigurationProfile Include="Configurations\*.json" />
<ConfigurationProfile Include="appsettings.Development.json" />
<ConfigurationProfile Include="appsettings.EchoServer.json" />
<ConfigurationProfile Include="appsettings.Production.json" />
<ConfigurationProfile Include="appsettings.ProductionAws.json" />
<ConfigurationProfile Include="appsettings.Staging.json" />
<ConfigurationProfile Include="appsettings.json" />
</ItemGroup>
</Project>
@@ -0,0 +1,20 @@
{
"PubSub": {
"Routing": {
"PublisherRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://localhost:22019",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://localhost:22019",
"TagRegExp": ".*"
}
]
}
}
}
@@ -0,0 +1 @@
{}
@@ -0,0 +1,20 @@
{
"PubSub": {
"Routing": {
"PublisherRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://34.237.179.93:22019",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://34.237.179.93:22019",
"TagRegExp": ".*"
}
]
}
}
}
@@ -0,0 +1 @@
{}
@@ -12,14 +12,14 @@
"PublisherRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://34.237.179.93:22019",
"ServerAddress": "bts-cloud://pubsub:22019",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://34.237.179.93:22019",
"ServerAddress": "bts-cloud://pubsub:22019",
"TagRegExp": ".*"
}
]
@@ -0,0 +1,15 @@
Logging {
value {
console {
default_level: "Info"
}
syslog {
default_level: "Info"
}
file {
default_level: "Info"
}
initial_queue_size: 120001
max_queue_growth: 4
}
}
@@ -0,0 +1 @@
ASPNETCORE_ENVIRONMENT=Staging
+3
View File
@@ -91,6 +91,9 @@
<ItemDefinitionGroup>
<ConfigurationProfile>
<OutDir>$(OutputPath)\Configurations\$(MSBuildProjectName)</OutDir>
<OutFile>$(OutputPath)\$(MSBuildProjectName).%(FileName)%(Extension)</OutFile>
<PublishOutDir>$(OutputPath)\publish\Configurations\$(MSBuildProjectName)</PublishOutDir>
<PublishOutFile>$(OutputPath)\publish\$(MSBuildProjectName).%(FileName)%(Extension)</PublishOutFile>
</ConfigurationProfile>
</ItemDefinitionGroup>
+10 -7
View File
@@ -43,6 +43,7 @@
Inputs="%(ConfigurationProfile.Identity)"
Outputs="%(ConfigurationProfile.OutDir)%(ConfigurationProfile.Identity)">
<Copy SourceFiles="%(ConfigurationProfile.FullPath)" DestinationFolder="%(ConfigurationProfile.OutDir)" />
<Copy SourceFiles="%(ConfigurationProfile.FullPath)" DestinationFiles="%(ConfigurationProfile.OutFile)" />
</Target>
<Target
@@ -52,6 +53,7 @@
Inputs="%(ConfigurationProfile.Identity)"
Outputs="%(ConfigurationProfile.PublishOutDir)%(ConfigurationProfile.Identity)">
<Copy SourceFiles="%(ConfigurationProfile.FullPath)" DestinationFolder="%(ConfigurationProfile.PublishOutDir)" />
<Copy SourceFiles="%(ConfigurationProfile.FullPath)" DestinationFiles="%(ConfigurationProfile.PublishOutFile)" />
</Target>
<!--
@@ -110,13 +112,13 @@
Name="GenerateScanners"
BeforeTargets="BeforeBuild"
Condition="'@(Scanner)'!=''"
Outputs="$([MSBuild]::MakeRelative($(MSBuildProjectDirectory), $(ScannerOutputPath)\%(Scanner.RelativeDir)))\%(Scanner.FileName).Scanner.g.cs">
Outputs="%(Scanner.RelativeOutputDir)\%(Scanner.FileName).Scanner.g.cs">
<Message Text="Generating Scanner %(Scanner.Identity)" Importance="High" />
<ItemGroup>
<LexFile Include="%(Scanner.Identity)">
<OutputFile>$([MSBuild]::MakeRelative($(MSBuildProjectDirectory), $(ScannerOutputPath)\%(Scanner.RelativeDir)))\%(Scanner.FileName).Scanner.g.cs</OutputFile>
<OutputFile>$([MSBuild]::MakeRelative(%(Scanner.RootDir)%(Scanner.Directory), $(ScannerOutputPath)))\%(Scanner.FileName).Scanner.g.cs</OutputFile>
</LexFile>
<Compile Include="$([MSBuild]::MakeRelative($(MSBuildProjectDirectory), $(ScannerOutputPath)\%(Scanner.RelativeDir)))\%(Scanner.FileName).Scanner.g.cs">
<Compile Include="$([MSBuild]::MakeRelative(%(Scanner.RootDir)%(Scanner.Directory), $(ScannerOutputPath)))\%(Scanner.FileName).Scanner.g.cs">
<AutoGen>True</AutoGen>
<DesignTime>True</DesignTime>
<DependentUpon>%(Scanner.Identity)</DependentUpon>
@@ -127,18 +129,19 @@
<Target
Name="GenerateParsers"
BeforeTargets="BeforeBuild"
AfterTargets="GenerateScanners"
DependsOnTargets="GenerateScanners"
Condition="'@(Parser)'!=''"
Outputs="$([MSBuild]::MakeRelative($(MSBuildProjectDirectory), $(ParserOutputPath)\%(Parser.RelativeDir)))\%(Parser.FileName).Parser.g.cs">
Outputs="%(Parser.RelativeOutputDir)\%(Parser.FileName).Parser.g.cs">
<Message
Text="Generating Parser %(Parser.Identity)"
Importance="High" />
<ItemGroup>
<YaccFile Include="%(Parser.Identity)">
<OutputFile>$([MSBuild]::MakeRelative($(MSBuildProjectDirectory), $(ParserOutputPath)\%(Parser.RelativeDir)))\%(Parser.FileName).Parser.g.cs</OutputFile>
<OutputFile>$([MSBuild]::MakeRelative(%(Parser.RootDir)%(Parser.Directory),
$(ParserOutputPath)))\%(Parser.FileName).Parser.g.cs</OutputFile>
<Arguments>/gplex /nolines</Arguments>
</YaccFile>
<Compile Include="$([MSBuild]::MakeRelative($(MSBuildProjectDirectory), $(ParserOutputPath)\%(Parser.RelativeDir)))\%(Parser.FileName).Parser.g.cs">
<Compile Include="$([MSBuild]::MakeRelative(%(Parser.RootDir)%(Parser.Directory), $(ParserOutputPath)))\%(Parser.FileName).Parser.g.cs">
<AutoGen>True</AutoGen>
<DesignTime>True</DesignTime>
<DependentUpon>%(Parser.Identity)</DependentUpon>
+13
View File
@@ -0,0 +1,13 @@
FROM bluetradingsystems/linux_dotnet7
ARG SOURCE_DIR="install/Release/bin"
ARG CONF_FILE="src/server/sparkd/spark.conf"
USER "root"
RUN apt-get update \
&& apt-get install -y liblttng-ust-dev
USER "bts"
COPY --chown=bts:bts "${SOURCE_DIR}" "bin"
COPY --chown=bts:bts "${CONF_FILE}" "sparkdata/conf/spark.conf"
WORKDIR "/home/bts/bin"
+16
View File
@@ -0,0 +1,16 @@
MarketDataConfig {
value {
binance_us {
num_depth_levels: 5
url_rest_base: "https://api.binance.us"
url_ws_base: "wss://stream.binance.us:9443"
security_criteria {
clauses {
security_types: SECURITY_TYPE_SPOT
symbols: "BTC"
symbols: "ETH"
}
}
}
}
}