refactor: Add DataStream service modules for Dependency Injection

Namely, in DataStreamPublisher and PubSubServer. This is part of a broader
paradigm shift towards Dependency Injection and Factory Pattern.

https://learn.microsoft.com/en-us/dotnet/core/extensions/dependency-injection-guidelines

* Use interfaces to decouple services:
  - Decouple ManagedSubscriber from ISubscriberCore impl
  - Decouple PubTableBroker from IPublisherCore impl
  - Decouple RemoteQueryManager from IManagedSubscriber, IPubTableBroker impl

* Rename Publishers -> PublisherNodes in DataStreamPublisherConfig
* Model config profiles after dotnet appsettings
  - Rename Configurations/<name>.json -> appsettings.<name>.json
  - Make json fields match corresponding csharp props verbatim
    * The Dependency Injection framework doesn't support JsonPropertyName
      attributes. Config class props must match json fields verbatim.

* Move reusable startup code into new Bts.Extensions.Hosting proj
* Add pubsub and md-publisher services to compose.yaml
This commit is contained in:
John Colvin committed 2026-01-04 21:08:11 +00:00
1 parent 716bc92394
commit 76fa35382e
102 files changed
+2948 -1169

No files matched your search

@@ -1,62 +1,106 @@
{
"user": "bts",
"sparkd": "http://127.0.0.1:8000/config",
"pubsub": {
"milliseconds_between_sub_info_updates": 1000,
"milliseconds_between_time_slice_updates": 5000,
"milliseconds_between_no_buffer_space_retries": 100,
"retries_on_no_buffer_space": 1,
"do_it_all_seconds_between_logs": 0,
"heartbeat_interval": 15000,
"missing_heartbeats_before_gc": 50,
"use_managed_sockets": true,
"state_update_period": 5000,
"statistics_update_period": 5000,
"disposed_connection_check_period": 1000,
"message_processing_warning_threshold": 1000,
"server_port": 12345,
"publisher_routes": [
{
"name": "default",
"server_address": "bts-cloud://127.0.0.1:12345",
"tag": ".*"
}
],
"subscriber_routes": [
{
"name": "default",
"server_address": "bts-cloud://127.0.0.1:12345",
"tag": ".*"
}
],
"max_packet_size": 500000000,
"blue_comm_must_die": true,
"crushable_tags": [],
"tag_type_to_assembly_map": [
{
"handler_class": "BCG.PubSubServerTagHandler",
"assembly": "DataStreamCore.dll"
}
],
"server_queue": "PubSubQueue",
"port_range": {
"first": 30400,
"last": 31700
"MasterWorker": {
"Interval": "00:00:00.1"
},
"ViewStreamDatabase": {
"Interval": "00:00:00.1"
},
"Spark": {
"ConfigServer": "http://127.0.0.1:8000/config"
},
"SparkApi": {
"Credentials": {
"User": "bts",
"Password": ""
},
"bind_to_random_port": false,
"debug_tags": "^NOTAGS$",
"latency_report_threshold_ms": 1500,
"use_substations": false,
"use_dynamic_clients": false,
"is_substation": false,
"internally_publish_sub_info": true,
"use_managed_clients_in_native_code": false,
"compressed_tags": "^NOTAGS$",
"tags_to_report_compression_ratio": "^NOTAGS$",
"disable_msmq": false,
"use_prepublished_server_state": false,
"ip_translation": null,
"gc_activation_threshold": 0,
"wait_period_after_gc": 1000
"EnableAdmin": true,
"EnableOrders": true,
"EnablePosition": true
},
"PubSub": {
"MillisecondsBetweenSubInfoUpdates": 1000,
"MillisecondsBetweenTimeSliceUpdates": 5000,
"MillisecondsBetweenNoBufferSpaceRetries": 100,
"RetriesOnNoBufferSpace": 1,
"DoItAllSecondsBetweenLogs": 0,
"HeartbeatInterval": 15000,
"MissingHeartbeatsBeforeGc": 50,
"UseManagedSockets": true,
"StateUpdatePeriod": 5000,
"StatisticsUpdatePeriod": 5000,
"DisposedConnectionCheckPeriod": 1000,
"MessageProcessingWarningThreshold": 1000,
"ServerPort": 12345,
"Routing": {
"PublisherRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://127.0.0.1:12345",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://127.0.0.1:12345",
"TagRegExp": ".*"
}
]
},
"MaxPacketSize": 500000000,
"BlueCommMustDie": true,
"CrushableTags": [],
"TagTypeToAssemblyMap": {
"Entries": [
{
"HandlerClass": "BCG.PubSubServerTagHandler",
"Assembly": "DataStreamCore.dll"
}
]
},
"ServerQueue": "PubSubQueue",
"PortRange": {
"First": 30400,
"Last": 31700
},
"BindToRandomPort": false,
"DebugTags": "^NOTAGS$",
"LatencyReportThresholdMs": 1500,
"UseSubstations": false,
"UseDynamicClients": false,
"IsSubstation": false,
"InternallyPublishSubInfo": true,
"UseManagedClientsInNativeCode": false,
"CompressedTags": "^NOTAGS$",
"TagsToReportCompressionRatio": "^NOTAGS$",
"DisableMsmq": false,
"UsePrepublishedServerState": false,
"IpTranslation": null,
"GcActivationThreshold": 0,
"WaitPeriodAfterGc": 1000,
"TagHandlerFactory": {
"Handlers": [
{
"TagType": "DataStream",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerStreamTagHandler"
},
{
"TagType": "System_Sub",
"Assembly": "PubSub",
"Class": "BCG.SystemSubscriptionMapPublisher"
},
{
"TagType": "System_Stats",
"Assembly": "PubSub",
"Class": "BCG.SystemStatsPublisher"
},
{
"TagType": ".*",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerGenericDataHandler"
}
]
}
}
}
@@ -1,29 +1,17 @@
using bts;
using bts.utils;
using mbts.config;
using mbts.spark;
using Bts.DataStreamExpressions;
using Bts.PubSub;
using System;
using System.Drawing;
using System.Collections;
using System.Collections.Generic;
using System.Collections;
using System.ComponentModel;
using System.Windows.Forms;
using System.Text.RegularExpressions;
using System.IO;
using System.Diagnostics;
using BCG.Utilities;
// using BCG.Utilities.BlueConfiguration;
using BCG.Utilities.General;
using Spark.DataStream;
using Bts.DataStreamExpressions;
// using BCG.Evaluator.DataStreamExpression;
// using BCG.Utilities.Win32;
using System.Threading.Tasks;
using System.Drawing;
using System.IO;
using System.Text.RegularExpressions;
using System.Windows.Forms;
namespace BCG
{
@@ -79,7 +67,7 @@ namespace BCG
{
// TODO: Port PublishedTagsStream from edge (then we can uncomment impl below)
//ViewStream vs = new PubSubServerStateStream();
ViewStream vs = new PubSubServerStateStream();
ViewStream vs = new PubSubServerStateStream(new SubscriberCoreFactory(), ViewStreamDatabase.instance);
return new FilterStream(vs, "tag~='DS_'");
//return new LiteralStream("tag/String", "DS_table1;DS_table2");
}
@@ -589,7 +577,7 @@ namespace BCG
myLoadAssemblies();
ViewStreamDatabase.instance = new ViewStreamDatabase();
// ViewStreamDatabase.instance = new ViewStreamDatabase();
myAllTagsStream = myGetAllTagsStream();
myAllTagsStream.onChange += new BCG.DataStream.ChangeDelegate(myHandleTableListChange);
myTabControl.SelectedIndex = 3;
@@ -614,7 +602,7 @@ namespace BCG
// SitesCore.setMySiteName (mySite);
ViewStreamDatabase.instance = new ViewStreamDatabase();
// ViewStreamDatabase.instance = new ViewStreamDatabase();
myAllTagsStream = myGetAllTagsStream();
myAllTagsStream.onChange += new BCG.DataStream.ChangeDelegate(myHandleTableListChange);