refactor: Add DataStream service modules for Dependency Injection

Namely, in DataStreamPublisher and PubSubServer. This is part of a broader
paradigm shift towards Dependency Injection and Factory Pattern.

https://learn.microsoft.com/en-us/dotnet/core/extensions/dependency-injection-guidelines

* Use interfaces to decouple services:
  - Decouple ManagedSubscriber from ISubscriberCore impl
  - Decouple PubTableBroker from IPublisherCore impl
  - Decouple RemoteQueryManager from IManagedSubscriber, IPubTableBroker impl

* Rename Publishers -> PublisherNodes in DataStreamPublisherConfig
* Model config profiles after dotnet appsettings
  - Rename Configurations/<name>.json -> appsettings.<name>.json
  - Make json fields match corresponding csharp props verbatim
    * The Dependency Injection framework doesn't support JsonPropertyName
      attributes. Config class props must match json fields verbatim.

* Move reusable startup code into new Bts.Extensions.Hosting proj
* Add pubsub and md-publisher services to compose.yaml
This commit is contained in:
John Colvin committed 2026-01-04 21:08:11 +00:00
1 parent 716bc92394
commit 76fa35382e
102 files changed
+2948 -1169

No files matched your search

@@ -0,0 +1,8 @@
namespace Bts.DataStreamCore;
using BCG;
public interface IPubTableBroker {
void create_table(IBlueRowCollection table);
void make_batch_changes(IBlueRowCollection new_rows, IBlueRowCollection del_rows, IBlueRowCollection added_rows);
}
@@ -8,12 +8,14 @@ using BCG.Utilities;
using bts;
using Bts.DataStreamCore;
namespace BCG {
/// <summary>
/// This class will be used by the clients to publish tabular data to the pubsub
/// </summary>
public class PubTableBroker {
protected PublisherCore _publisher_core;
public class PubTableBroker : IPubTableBroker {
protected IPublisherCore _publisher_core;
protected IDictionary<string, SinglePubTableBroker> _publisher_map =
new Dictionary<string, SinglePubTableBroker>();
@@ -22,7 +24,7 @@ namespace BCG {
public PubTableBroker()
: this(new PublisherCore()) { }
public PubTableBroker(PublisherCore publisher) {
public PubTableBroker(IPublisherCore publisher) {
_publisher_core = publisher;
Log.Gen.info("PubTableBroker constructed");
}
@@ -14,9 +14,9 @@ namespace BCG {
public class SinglePubTableBroker {
private readonly string _tag;
protected IStreamIOMsgSerializer _serializer;
protected PublisherCore _publisher_core;
protected IPublisherCore _publisher_core;
public SinglePubTableBroker(string tag, PublisherCore publisher_core) {
public SinglePubTableBroker(string tag, IPublisherCore publisher_core) {
bts.Log.Gen.info($"SinglePubTableBroker.cons created for tag: {tag}");
_tag = tag;
_publisher_core = publisher_core;
@@ -51,4 +51,4 @@ namespace BCG {
}
}
}
}
@@ -0,0 +1,19 @@
namespace Bts.DataStreamCore.Services;
using BCG;
using BCG.Utilities.General;
using Bts.PubSub;
using Microsoft.Extensions.DependencyInjection;
public static class DataStreamCoreServices {
public static IServiceCollection AddPubTableBroker(this IServiceCollection services) =>
services.AddSingleton<IPubTableBroker, PubTableBroker>();
public static IServiceCollection AddViewStreamDatabase(this IServiceCollection services) =>
services.AddSingleton<IWorker, ViewStreamDatabase>(sp => {
ViewStreamDatabase.SetGlobalInstance(new ViewStreamDatabase(sp.GetRequiredService<ISubscriberCoreFactory>()));
return ViewStreamDatabase.instance;
});
}
@@ -1,16 +1,12 @@
using System;
using System.Collections;
using System.IO;
using BCG.Utilities;
using BCG.Utilities.General;
//using BCG.Utilities.BlueConfiguration;
using System.Collections.Generic;
using System.Threading;
using mbts;
using System.Linq;
using System.Diagnostics;
using Bts.PubSub;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading;
namespace BCG {
public class ViewStreamDatabase : IWorker, IWorkScheduler, IDisposable {
internal delegate void TagFireDelegate(string tag);
@@ -38,11 +34,14 @@ namespace BCG {
private static readonly object myInstanceLockObject = new object();
private readonly object myStateLock = new object();
public ViewStreamDatabase()
: this(new SubscriberCore()) {
}
public ViewStreamDatabase(ISubscriberCoreFactory subscriberCoreFactory)
: this(subscriberCoreFactory.Create()) { }
public ViewStreamDatabase(ISubscriberCore subscriberCore) {
// This ctr is private to ensure the ISubscriberCore is created by ISubscriberCoreFactory
// (we'll temporarily continue to use it internally to create ViewStreamDatabase.instance,
// but new code should use the factory). The factory itself is created via dependency
// injection so it has the correct runtime config (for pubsub routing, etc.).
ViewStreamDatabase(ISubscriberCore subscriberCore) {
mySubscriberCore = subscriberCore;
// if (DataStreamLoadFileConfig.instance.profileStreams)
// new DataStreamPerfRecorder(this);
@@ -66,7 +65,7 @@ namespace BCG {
if (mySelf == null || mySelf.myIsDisposed)
lock (myInstanceLockObject) {
if (mySelf == null || mySelf.myIsDisposed) {
mySelf = new ViewStreamDatabase();
mySelf = new ViewStreamDatabase(new SubscriberCore());
instance_hashcode = mySelf.GetHashCode();
}
}
@@ -235,6 +234,11 @@ namespace BCG {
}
}
internal static void SetGlobalInstance(ViewStreamDatabase vsd) {
Log.isnull(mySelf);
mySelf = vsd;
}
private List<PendingAddRemoveWorker> _pending_worker_ops = new List<PendingAddRemoveWorker>();
private bool _is_worker_call_in_progress = false;
class PendingAddRemoveWorker {