From b61bf16a42b596f0555ce7652fb9df7355b847bb Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 11 Jul 2024 10:15:40 -0400 Subject: [PATCH] Add PubSub service modules designed for Dependency Injection --- .../Primitives/Misc/PubsubEchoTestStream.cs | 9 +- .../DataStream/Primitives/Misc/TestStream.cs | 3 +- .../DataStream/Publish/CircularPublisher.cs | 13 +- .../DataStreamCore/DataStreamCore.csproj | 1 + .../Services/DataStreamCoreServices.cs | 21 +++ .../DataStreamCore/ViewStreamDatabase.cs | 6 + .../Client/IPubSubServerTagHandlerFactory.cs | 7 + .../Client/PubSubTagHandlerFactoryConfig.cs | 42 +++++ .../PubSub/Client/PubsubTagHandlerFactory.cs | 130 +++++++++------ .../PubSub/Client/Routing/PubSubRoute.cs | 11 +- .../PubSub/Client/Routing/PubSubRouter.cs | 33 ++-- .../Client/Routing/PubSubRouterConfig.cs | 40 +++++ .../PubSub/Config/PubSubConfig.cs | 24 +-- .../PubSub/Misc/IParameterizedWorker.cs | 10 ++ .../PubSub/Misc/IWorkScheduler.cs | 10 ++ .../PubSub/Misc/MasterWorker.cs | 10 +- .../PubSub/Misc/WorkSchedulerExtensions.cs | 25 +++ src/DataStreamSuite/PubSub/PubSub.csproj | 4 + src/DataStreamSuite/PubSub/PublishedCache.cs | 9 +- src/DataStreamSuite/PubSub/PublisherCore.cs | 22 +-- .../PubSub/Server/IInternalPublisher.cs | 7 + .../PubSub/Server/IInternalPublisherQueue.cs | 16 ++ .../PubSub/Server/IPubSubState.cs | 33 ++++ .../PubSub/Server/InternalPublisher.cs | 25 +++ .../PubSub/Server/InternalPublisherQueue.cs | 21 +++ .../SystemHeartbeatPublisher.cs | 26 +-- .../SystemStatsPublisher.cs | 25 ++- .../SystemSubscriptionMapPublisher.cs | 27 ++-- .../PubSub/Server/PubSubMessageReactor.cs | 28 ++-- .../PubSub/Server/PubSubState.cs | 17 +- .../PubSub/Server/SingleTagData.cs | 24 +-- .../PubSub/Services/MasterWorkerService.cs | 43 +++++ .../PubSub/Services/PubSubServerService.cs | 112 +++++++++++++ .../PubSub/Services/PubSubServices.cs | 148 ++++++++++++++++++ .../PubSub/Services/WorkerConfig.cs | 7 + .../PubSub/Services/WorkerServiceConfig.cs | 7 + src/DataStreamSuite/PubSub/SubscriberCore.cs | 30 ++-- src/DataStreamSuite/PubSubServer/Program.cs | 102 ++++-------- 38 files changed, 851 insertions(+), 277 deletions(-) create mode 100644 src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs create mode 100644 src/DataStreamSuite/PubSub/Client/IPubSubServerTagHandlerFactory.cs create mode 100644 src/DataStreamSuite/PubSub/Client/PubSubTagHandlerFactoryConfig.cs create mode 100644 src/DataStreamSuite/PubSub/Client/Routing/PubSubRouterConfig.cs create mode 100644 src/DataStreamSuite/PubSub/Misc/IParameterizedWorker.cs create mode 100644 src/DataStreamSuite/PubSub/Misc/IWorkScheduler.cs create mode 100644 src/DataStreamSuite/PubSub/Misc/WorkSchedulerExtensions.cs create mode 100644 src/DataStreamSuite/PubSub/Server/IInternalPublisher.cs create mode 100644 src/DataStreamSuite/PubSub/Server/IInternalPublisherQueue.cs create mode 100644 src/DataStreamSuite/PubSub/Server/IPubSubState.cs create mode 100644 src/DataStreamSuite/PubSub/Server/InternalPublisher.cs create mode 100644 src/DataStreamSuite/PubSub/Server/InternalPublisherQueue.cs create mode 100644 src/DataStreamSuite/PubSub/Services/MasterWorkerService.cs create mode 100644 src/DataStreamSuite/PubSub/Services/PubSubServerService.cs create mode 100644 src/DataStreamSuite/PubSub/Services/PubSubServices.cs create mode 100644 src/DataStreamSuite/PubSub/Services/WorkerConfig.cs create mode 100644 src/DataStreamSuite/PubSub/Services/WorkerServiceConfig.cs diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs index 35ad2fbc6..a4e6820ba 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs @@ -8,6 +8,7 @@ using System.IO; using System.Linq; using System.Runtime.CompilerServices; using bts.api; +using Bts.PubSub.Routing; namespace bts { [DataStreamUserRespresentation("pss_echo")] @@ -87,8 +88,10 @@ namespace bts { IOState _state = IOState.SEND; public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd) { - var route = new PubSubRoute(server_url, ".*"); - _router = new PubSubRouter(route); + _router = new PubSubRouter(new PubSubRouterConfig() { + PublisherRoutes = new() { new(server_url, ".*") }, + SubscriberRoutes = new() { new(server_url, ".*") }, + }); _vsd = vsd; _payload_sz = payload_size; var schema = BlueDataUtility.form_schema("PubSubTest", @@ -196,4 +199,4 @@ namespace bts { } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs index b6c6d6628..f3b5efeed 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs @@ -617,6 +617,7 @@ namespace BCG { private static TestResult _run_single_tag_data_test_clean() { + // var tagHandlerFactory // create message var test_name = MethodBase.GetCurrentMethod()?.Name; var schema = create_schema_with_all_types(); @@ -624,7 +625,7 @@ namespace BCG { BlueTable table = new BlueTable(schema); IStreamIOMsgSerializer serializer = new StreamIOSerializationController(); IStreamMsgDeserilizer deser = new StreamIODeserializationController(); - bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType); + bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType, new PubsubServerStreamTagHandler()); // create with 100 rows int n_rows = 100; bool is_succ = true; diff --git a/src/DataStreamSuite/DataStream/Publish/CircularPublisher.cs b/src/DataStreamSuite/DataStream/Publish/CircularPublisher.cs index 464e30dcc..9f7964ba6 100644 --- a/src/DataStreamSuite/DataStream/Publish/CircularPublisher.cs +++ b/src/DataStreamSuite/DataStream/Publish/CircularPublisher.cs @@ -1,3 +1,5 @@ +using Bts.DataStreamCore; + using System; namespace BCG { @@ -7,14 +9,15 @@ namespace BCG { /// public class CircularPublisher { private CircularRowCollection myBuffer; - private PubTableBroker myBroker; + private readonly IPubTableBroker myBroker; private int myBufferSize; - public CircularPublisher(int bufferSize) { + public CircularPublisher(int bufferSize, IPubTableBroker broker) { myBufferSize = bufferSize; + myBroker = broker; } - public CircularPublisher(int bufferSize, IBlueRowCollection rows) - : this(bufferSize) { + public CircularPublisher(int bufferSize, IPubTableBroker broker, IBlueRowCollection rows) + : this(bufferSize, broker) { publishRows(rows); } @@ -23,7 +26,6 @@ namespace BCG { myBuffer = new CircularRowCollection( new BlueTableSchema(nr.schema.tableName, nr.schema.columnInfo), myBufferSize); - myBroker = new PubTableBroker(); myBroker.create_table(myBuffer); } IBlueRow br = myBuffer.addRowWithReturn(nr); @@ -38,7 +40,6 @@ namespace BCG { new BlueTableSchema(BlueDataUtility.extractRow(nrs, 0).schema.tableName, BlueDataUtility.extractRow(nrs, 0).schema.columnInfo), myBufferSize); - myBroker = new PubTableBroker(); myBroker.create_table(myBuffer); } BlueTable deletes = new BlueTable(myBuffer.schema); diff --git a/src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj b/src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj index 95d4024d2..39f1f470f 100644 --- a/src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj +++ b/src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj @@ -8,6 +8,7 @@ + diff --git a/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs b/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs new file mode 100644 index 000000000..b53509263 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Services/DataStreamCoreServices.cs @@ -0,0 +1,21 @@ +namespace Bts.DataStreamCore.Services; + +using BCG; +using BCG.Utilities.General; + +using Bts.PubSub.Services; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +public static class DataStreamCoreServices { + public static IServiceCollection AddPubTableBroker(this IServiceCollection services) => + services.AddSingleton(); + + public static IServiceCollection AddViewStreamDatabase(this IServiceCollection services, IConfiguration configurationSection) => + services.Configure(configurationSection) + .AddSingleton(sp => { + ViewStreamDatabase.SetGlobalInstance(new ViewStreamDatabase(sp.GetRequiredService())); + return ViewStreamDatabase.instance; + }); +} diff --git a/src/DataStreamSuite/DataStreamCore/ViewStreamDatabase.cs b/src/DataStreamSuite/DataStreamCore/ViewStreamDatabase.cs index a314b308b..2bcc5040c 100644 --- a/src/DataStreamSuite/DataStreamCore/ViewStreamDatabase.cs +++ b/src/DataStreamSuite/DataStreamCore/ViewStreamDatabase.cs @@ -9,6 +9,7 @@ using System.Threading; using mbts; using System.Linq; using System.Diagnostics; +using bts; namespace BCG { public class ViewStreamDatabase : IWorker, IDisposable { @@ -230,6 +231,11 @@ namespace BCG { } } + internal static void SetGlobalInstance(ViewStreamDatabase vsd) { + Log.isnull(mySelf); + mySelf = vsd; + } + private List _pending_worker_ops = new List(); private bool _is_worker_call_in_progress = false; class PendingAddRemoveWorker { diff --git a/src/DataStreamSuite/PubSub/Client/IPubSubServerTagHandlerFactory.cs b/src/DataStreamSuite/PubSub/Client/IPubSubServerTagHandlerFactory.cs new file mode 100644 index 000000000..a50279455 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/IPubSubServerTagHandlerFactory.cs @@ -0,0 +1,7 @@ +namespace Bts.PubSub; + +using BCG; + +public interface IPubSubTagHandlerFactory { + IPubSubServerTagHandler? GetHandler(string tagType); +} diff --git a/src/DataStreamSuite/PubSub/Client/PubSubTagHandlerFactoryConfig.cs b/src/DataStreamSuite/PubSub/Client/PubSubTagHandlerFactoryConfig.cs new file mode 100644 index 000000000..cbb288403 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/PubSubTagHandlerFactoryConfig.cs @@ -0,0 +1,42 @@ +namespace Bts.PubSub; + +using BCG; + +using System.Collections.Generic; + +public class PubSubTagTypeHandlerConfig { + public string? TagType { get; set; } + public string? Assembly { get; set; } + public string? Class { get; set; } +} + +public class PubSubTagHandlerFactoryConfig { + public List? Handlers { get; set; } + + public static PubSubTagHandlerFactoryConfig Default() { + return new() { + Handlers = new() { + new() { + TagType = string.Empty, + Assembly = "DataStreamCore", + Class = "bts.PubsubServerStreamTagHandler", + }, + new() { + TagType = DSTagManager.DataStreamTagType, + Assembly = "DataStreamCore", + Class = "bts.PubsubServerStreamTagHandler", + }, + new() { + TagType = "System_Sub", + Assembly = "PubSub", + Class = "BCG.SystemSubscriptionMapPublisher", + }, + new() { + TagType = "System_Stats", + Assembly = "PubSub", + Class = "BCG.SystemStatsPublisher", + } + } + }; + } +} diff --git a/src/DataStreamSuite/PubSub/Client/PubsubTagHandlerFactory.cs b/src/DataStreamSuite/PubSub/Client/PubsubTagHandlerFactory.cs index 5da719b42..4f2e4c0fd 100644 --- a/src/DataStreamSuite/PubSub/Client/PubsubTagHandlerFactory.cs +++ b/src/DataStreamSuite/PubSub/Client/PubsubTagHandlerFactory.cs @@ -1,76 +1,106 @@ using System; -using System.Collections; using System.Reflection; -using BCG.Utilities; using System.Collections.Generic; using BCG; -using System.Runtime.InteropServices; using System.IO; - - +using System.Linq; +using Bts.PubSub; +using Microsoft.Extensions.Options; +using System.Text.RegularExpressions; +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; namespace bts { - public static class PubsubTagHandlerFactory { - // each tag type is represented by this - public class TagHandlerEntry { - public readonly string tag_type; - public readonly string assembly_name; - public readonly string handler_class_name; - public TagHandlerEntry(string tag_type_, string assembly_name_, string handler_class_name_) { - tag_type = tag_type_; - assembly_name = assembly_name_; - handler_class_name = handler_class_name_; - } + public class PubsubTagHandlerFactory : IPubSubTagHandlerFactory { + readonly List handlers; + readonly IServiceProvider serviceProvider; + + public PubsubTagHandlerFactory(IServiceProvider serviceProvider, IOptions config) + : this(serviceProvider, config.Value) { } + + public PubsubTagHandlerFactory(IServiceProvider serviceProvider, PubSubTagHandlerFactoryConfig config) { + this.serviceProvider = serviceProvider; + Log.notnull(config.Handlers); + handlers = config.Handlers.Select(h => new TagTypeHandlerData(h)).ToList(); } - private static Dictionary _config_map = new Dictionary() - { - {String.Empty, new TagHandlerEntry(DSTagManager.DataStreamTagType , "DataStreamCore", "bts.PubsubServerStreamTagHandler")}, - {DSTagManager.DataStreamTagType, new TagHandlerEntry(DSTagManager.DataStreamTagType , "DataStreamCore", "bts.PubsubServerStreamTagHandler")}, - {"System_Sub", new TagHandlerEntry("System_Sub", "PubSub", "BCG.SystemSubscriptionMapPublisher")}, - {"System_Stats", new TagHandlerEntry("System_Stats", "PubSub", "BCG.SystemStatsPublisher")} + + public IPubSubServerTagHandler? GetHandler(string tag_type) { + if (_handler_map.TryGetValue(tag_type, out var handler)) { + return handler; + } + return get_handler(tag_type, true); + } + + private static PubSubTagTypeHandlerConfig _wildcard_handler = new() { + TagType = "*", + Assembly = "DataStreamCore", + Class = "bts.PubsubServerGenericDataHandler", }; - private static TagHandlerEntry _wildcard_handler = new TagHandlerEntry("*", "DataStreamCore", "bts.PubsubServerGenericDataHandler"); - private static Dictionary _handler_map = new Dictionary(); + private Dictionary _handler_map = new Dictionary(); - public static string assembly_location => Assembly.GetExecutingAssembly().Location; + public string assembly_location => Assembly.GetExecutingAssembly().Location; - public static IPubSubServerTagHandler get_handler(string tag_type) { - TagHandlerEntry handler_entry = _find_config_entry(tag_type); - Log.assert(handler_entry != null); - return _find_or_build_handler(handler_entry); + public BCG.IPubSubServerTagHandler get_handler(string tag_type, bool on_pubsub_server) { + var handler_entry = _find_config_entry(tag_type); + return _find_or_build_handler(handler_entry.Config); } - private static TagHandlerEntry _find_config_entry(string tag_type) { - TagHandlerEntry rv = null; - if (_config_map.TryGetValue(tag_type, out rv)) - return rv; - else - return _wildcard_handler; - } - - private static BCG.IPubSubServerTagHandler _find_or_build_handler(TagHandlerEntry entry) { - IPubSubServerTagHandler rv = null; - if (_handler_map.TryGetValue(entry.tag_type, out rv)) { - return rv; + private TagTypeHandlerData _find_config_entry(string tag_type) { + // Look for an exact match. + foreach (var handler in handlers) { + if (handler.TagType == tag_type) { + return handler; + } } - { + // If an exact match wasn't found, look for a regex match + foreach (var handler in handlers) { + if (handler.TagTypeRegex.IsMatch(tag_type)) { + return handler; + } + } + Log.assert(false); + throw new InvalidOperationException($"Failed to handler config for tag type '{tag_type}'"); + } + + private BCG.IPubSubServerTagHandler _find_or_build_handler(PubSubTagTypeHandlerConfig entry) { + ArgumentNullException.ThrowIfNull(entry.TagType); + lock (_handler_map) { + if (_handler_map.TryGetValue(entry.TagType, out var rv)) { + return rv; + } rv = _build_handler(entry); - _handler_map.Add(entry.tag_type, rv); + _handler_map.Add(entry.TagType, rv); + return rv; } - return rv; } - private static BCG.IPubSubServerTagHandler _build_handler(TagHandlerEntry entry) { + private BCG.IPubSubServerTagHandler _build_handler(PubSubTagTypeHandlerConfig entry) { + Log.notnull(entry.Assembly); + Log.notnull(entry.Class); var ass_loc = assembly_location; bts.Log.Gen.info($"PubsubTagHandlerFactory.load_handler entry: {entry} assembly_location: {ass_loc}"); - var assem = Assembly.LoadFrom(Path.ChangeExtension(entry.assembly_name, "dll")) + var assem = Assembly.LoadFrom(Path.ChangeExtension(entry.Assembly, "dll")) ?? throw new ArgumentException("Failed to instantiate TagHandlerEntry"); bts.Log.Gen.info($"PubsubTagHandlerFactory.load_handler successfully loaded assembly: {assem}"); - Type t = assem.GetType(entry.handler_class_name, true, true); - object obj = Activator.CreateInstance(t); - BCG.IPubSubServerTagHandler tag_handler = obj as BCG.IPubSubServerTagHandler; - bts.Log.Gen.info($"PubsubTagHandlerFactory.load_handler successfully created obj: {obj} of type: {t} from assem: {assem}"); + var t = assem.GetType(entry.Class, true, true); + Log.notnull(t); + var obj = ActivatorUtilities.GetServiceOrCreateInstance(serviceProvider, t); + var tag_handler = Log.notnull(obj as IPubSubServerTagHandler); + Log.Gen.info($"PubsubTagHandlerFactory created handler. {JsonSerializer.Serialize(entry)}"); return tag_handler; } + + readonly struct TagTypeHandlerData { + public TagTypeHandlerData(PubSubTagTypeHandlerConfig config) { + Config = config; + Log.notnull(config.TagType); + TagType = config.TagType; + TagTypeRegex = new(TagType); + } + + public PubSubTagTypeHandlerConfig Config { get; } + public string TagType { get; } + public Regex TagTypeRegex { get; } + } } } diff --git a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs index 61333116b..6c1e5be88 100644 --- a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs +++ b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs @@ -1,3 +1,5 @@ +using bts; + using mbts.utils; using System; @@ -8,7 +10,7 @@ using System.Text.RegularExpressions; namespace BCG; -public class PubSubRoute : IEquatable, IComparable { +public class PubSubRoute : IEquatable, IComparable, ICloneable { [JsonPropertyName("server_address")] public string? ServerAddress { get; set; } @@ -42,6 +44,13 @@ public class PubSubRoute : IEquatable, IComparable { return myRegex.IsMatch(tag); } + public PubSubRoute Clone() => + new() { + ServerAddress = ServerAddress, + TagRegExp = TagRegExp, + Name = Name, + }; + public int CompareTo(PubSubRoute? other) => other is null ? -1 : comparer.Compare(this, other); diff --git a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs index 0304c1411..f6d69702f 100644 --- a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs +++ b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs @@ -1,14 +1,15 @@ -using System; -using System.Text.RegularExpressions; -using System.Collections; -using System.Collections.Generic; - -using BCG; -using Bts.PubSub; using bts; + +using Bts.PubSub; +using Bts.PubSub.Routing; + +using mbts.utils; + +using Microsoft.Extensions.Options; + +using System; +using System.Collections.Generic; using System.Linq; -//using BCG.Utilities; -//using bts; namespace BCG { public class PubSubRouter : IPubSubRouter { @@ -16,13 +17,15 @@ namespace BCG { private PubSubRoute[] _pub_routes; private PubSubRoute[] _sub_routes; - public PubSubRouter() : this( - PubSubConfig.Instance.SubscriberRoutes?.ToArray() - ?? throw new InvalidOperationException("Subscriber routes are not configured"), - PubSubConfig.Instance.PublisherRoutes?.ToArray() - ?? throw new InvalidOperationException("Publisher routes are not configured")) { } + public PubSubRouter() + : this(Log.notnull(PubSubConfig.Instance.Routing)) { } - public PubSubRouter(params PubSubRoute[] routes) : this(routes, routes) { } + public PubSubRouter(IOptions config) + : this(config.Value) { } + + public PubSubRouter(PubSubRouterConfig config) : this( + ExceptionAssert.Argument.NotNullOrEmpty(config.SubscriberRoutes?.ToArray()), + ExceptionAssert.Argument.NotNullOrEmpty(config.PublisherRoutes?.ToArray())) { } public PubSubRouter(PubSubRoute[] sub_routes, PubSubRoute[] pub_routes) { string log_msg = String.Empty; diff --git a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouterConfig.cs b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouterConfig.cs new file mode 100644 index 000000000..165279123 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouterConfig.cs @@ -0,0 +1,40 @@ +namespace Bts.PubSub.Routing; + +using BCG; + +using bts; + +using System.Collections.Generic; +using System.Linq; + +public class PubSubRouterConfig : ICloneable { + public List? PublisherRoutes { get; set; } + + public List? SubscriberRoutes { get; set; } + + public void MergeWith(PubSubRouterConfig other) { + PublisherRoutes = MergeRoutes(PublisherRoutes, other.PublisherRoutes); + SubscriberRoutes = MergeRoutes(SubscriberRoutes, other.SubscriberRoutes); + + List? MergeRoutes(List? routes, List? otherRoutes) { + if (otherRoutes is null) { + return routes; + } + if (routes is null) { + return otherRoutes; + } + foreach (var route in otherRoutes) { + if (!routes.Any(r => r.TagRegExp == route.TagRegExp || r.Name == route.Name)) { + routes.Add(route); + } + } + return routes; + } + } + + public PubSubRouterConfig Clone() => + new() { + PublisherRoutes = PublisherRoutes?.Select(r => r.Clone()).ToList(), + SubscriberRoutes = SubscriberRoutes?.Select(r => r.Clone()).ToList(), + }; +} diff --git a/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs b/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs index 605a8b823..cce67c879 100644 --- a/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs +++ b/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs @@ -4,6 +4,8 @@ using BCG; using bts; +using Bts.PubSub.Routing; + using System; using System.Collections.Generic; using System.Text.Json.Serialization; @@ -71,6 +73,8 @@ public class PubSubConfig { [JsonPropertyName("tag_type_to_assembly_map")] public TagTypeToAssembly? TagTypeToAssemblyMap { get; set; } + public PubSubTagHandlerFactoryConfig? TagHandlerFactory { get; set; } + // unused [JsonPropertyName("latency_report_threshold_ms")] public int? LatencyReportThresholdMs { get; set; } @@ -89,11 +93,10 @@ public class PubSubConfig { [JsonPropertyName("wait_period_after_gc")] public int? WaitPeriodAfterGc { get; set; } - [JsonPropertyName("publisher_routes")] - public List? PublisherRoutes { get; set; } + public PubSubRouterConfig? Routing { get; set; } - [JsonPropertyName("subscriber_routes")] - public List? SubscriberRoutes { get; set; } + public List? PublisherRoutes => Routing?.PublisherRoutes; + public List? SubscriberRoutes => Routing?.SubscriberRoutes; public static PubSubConfig Default() { const string NOTAGS = "^NOTAGS$"; @@ -110,17 +113,20 @@ public class PubSubConfig { DisposedConnectionCheckPeriod = 1000, ServerPort = 12345, TagTypeToAssemblyMap = new(), + TagHandlerFactory = PubSubTagHandlerFactoryConfig.Default(), LatencyReportThresholdMs = 1500, InternallyPublishSubInfo = true, CompressedTags = NOTAGS, GcActivationThreshold = 0, WaitPeriodAfterGc = 1000, SimpleEchoServer = false, - PublisherRoutes = new() { - new("bts-cloud://127.0.0.1:12345", ".*") - }, - SubscriberRoutes = new() { - new("bts-cloud://127.0.0.1:12345", ".*") + Routing = new() { + PublisherRoutes = new() { + new("bts-cloud://127.0.0.1:12345", ".*") + }, + SubscriberRoutes = new() { + new("bts-cloud://127.0.0.1:12345", ".*") + } } }; return rv; diff --git a/src/DataStreamSuite/PubSub/Misc/IParameterizedWorker.cs b/src/DataStreamSuite/PubSub/Misc/IParameterizedWorker.cs new file mode 100644 index 000000000..2a255ebb8 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/IParameterizedWorker.cs @@ -0,0 +1,10 @@ +namespace Bts.PubSub; + +using BCG.Utilities.General; + +using System; + +public interface IParameterizedWorker : IWorker { + string? Group => null; + TimeSpan Interval => TimeSpan.Zero; +} diff --git a/src/DataStreamSuite/PubSub/Misc/IWorkScheduler.cs b/src/DataStreamSuite/PubSub/Misc/IWorkScheduler.cs new file mode 100644 index 000000000..da8a09601 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/IWorkScheduler.cs @@ -0,0 +1,10 @@ +namespace Bts.PubSub; + +using BCG.Utilities.General; + +using System; + +public interface IWorkScheduler { + void Add(IWorker worker, TimeSpan interval); + void Remove(IWorker worker); +} diff --git a/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs b/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs index 9f91a6f1e..6f19b0df5 100644 --- a/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs +++ b/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs @@ -2,6 +2,8 @@ using BCG.Utilities.General; using bts; +using Bts.PubSub; + using mbts; using System; @@ -9,7 +11,7 @@ using System.Collections.Generic; using System.Diagnostics; namespace BCG { - public class MasterWorker : IWorker { + public class MasterWorker : IWorker, IWorkScheduler { private DateTime myLastDoWorkTime = DateTime.MinValue, myLastLogTime = DateTime.MinValue; @@ -86,6 +88,12 @@ namespace BCG { } + void IWorkScheduler.Add(IWorker worker, TimeSpan interval) => + addWorker(worker, interval); + + void IWorkScheduler.Remove(IWorker worker) => + removeWorker(worker); + protected virtual void myLogEntries() { if (DateTime.UtcNow - myLastLogTime > myLogPeriod) { myLastLogTime = DateTime.UtcNow; diff --git a/src/DataStreamSuite/PubSub/Misc/WorkSchedulerExtensions.cs b/src/DataStreamSuite/PubSub/Misc/WorkSchedulerExtensions.cs new file mode 100644 index 000000000..f7d4a272e --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/WorkSchedulerExtensions.cs @@ -0,0 +1,25 @@ +namespace Bts.PubSub; + +using BCG.Utilities.General; + +using Spark.Core.Disposables; + +using System; + +public static class WorkSchedulerExtensions { + public static void Add(this IWorkScheduler scheduler, IWorker worker) => + scheduler.Add(worker, (worker as IParameterizedWorker)?.Interval ?? TimeSpan.Zero); + + public static IDisposable Subscribe(this IWorkScheduler scheduler, IWorker worker, TimeSpan? interval = null) { + scheduler.Add(worker, interval ?? TimeSpan.Zero); + return Disposable.Create(Dispose); + void Dispose() => + scheduler.Remove(worker); + } + + public static void Add(this IWorkScheduler scheduler, Action work) => + scheduler.Add(new AnonymousWorker(work), TimeSpan.Zero); + + public static void Add(this IWorkScheduler scheduler, Action work, TimeSpan interval) => + scheduler.Add(new AnonymousWorker(work), interval); +} diff --git a/src/DataStreamSuite/PubSub/PubSub.csproj b/src/DataStreamSuite/PubSub/PubSub.csproj index 772f6fb9a..89f5e0d0f 100644 --- a/src/DataStreamSuite/PubSub/PubSub.csproj +++ b/src/DataStreamSuite/PubSub/PubSub.csproj @@ -1,5 +1,9 @@ + + + + diff --git a/src/DataStreamSuite/PubSub/PublishedCache.cs b/src/DataStreamSuite/PubSub/PublishedCache.cs index 899396a02..c9d090bba 100644 --- a/src/DataStreamSuite/PubSub/PublishedCache.cs +++ b/src/DataStreamSuite/PubSub/PublishedCache.cs @@ -1,3 +1,5 @@ +using Bts.PubSub; + using System; using System.Collections.Generic; @@ -8,14 +10,19 @@ namespace bts { // PublisherCore calls get_content to get the data to re-publish, to // re-seed the PubSubServer public class PublishedCache { + readonly IPubSubTagHandlerFactory tagHandlerFactory; private Dictionary _data_map = new Dictionary(); + public PublishedCache(IPubSubTagHandlerFactory tagHandlerFactory) => + this.tagHandlerFactory = tagHandlerFactory; + public void publish(PublishMsg msg) { Log.Gen.info($"PublishedCache.publish msg: {msg}"); ArgumentException.ThrowIfNullOrEmpty(msg.tag); if (!_data_map.TryGetValue(msg.tag, out var tag_data)) { ArgumentException.ThrowIfNullOrEmpty(msg.tag_type); - tag_data = new SingleTagData(msg.tag, msg.tag_type); + var handler = Log.notnull(tagHandlerFactory.GetHandler(msg.tag_type)); + tag_data = new SingleTagData(msg.tag, msg.tag_type, handler); _data_map.Add(msg.tag, tag_data); } tag_data.handle_publish(msg); diff --git a/src/DataStreamSuite/PubSub/PublisherCore.cs b/src/DataStreamSuite/PubSub/PublisherCore.cs index 7e1b57b7a..f921578ea 100644 --- a/src/DataStreamSuite/PubSub/PublisherCore.cs +++ b/src/DataStreamSuite/PubSub/PublisherCore.cs @@ -3,6 +3,8 @@ using System.IO; using System.Collections.Generic; using System.Threading; using bts; +using Bts.PubSub; +using Bts.PubSub.Services; namespace BCG { @@ -21,7 +23,7 @@ namespace BCG { // all the data published through this instance of PublisherCore // is kept there, in a form that can be used repopulate a pubsub server // in the event of a conection re-establishment to the PubSubServer - protected PublishedCache _state = new PublishedCache(); + protected PublishedCache _state; // every PubSubServer is communicated with a adifferent IPubSubServerConnection object. @@ -41,15 +43,11 @@ namespace BCG { #region constructors - public PublisherCore() - // TODOds - : this(new PubSubRouter()) { } - - public PublisherCore(string server_url) - : this(new PubSubRouter(new PubSubRoute(server_url, ".*"))) { } - - public PublisherCore(IPubSubRouter r) { - _router = r; + public PublisherCore(IPubSubRouter? router = null, IPubSubTagHandlerFactory? tagHandlerFactory = null) { + router ??= new PubSubRouter(); + tagHandlerFactory ??= new PubsubTagHandlerFactory(PubSubServices.ServiceProvider, PubSubTagHandlerFactoryConfig.Default()); + _router = router; + _state = new(tagHandlerFactory); Thread pump_thread = new Thread(new ThreadStart(_message_pump)); pump_thread.IsBackground = true; pump_thread.Start(); @@ -114,6 +112,10 @@ namespace BCG { #endregion + // static void setGlobalInstance() { + + // } + #region Message Pump Thread - this is where messages are dequeued and the action performed // all the state manahement happens here, the public interface methods merely enqueue the operation private void _message_pump() { diff --git a/src/DataStreamSuite/PubSub/Server/IInternalPublisher.cs b/src/DataStreamSuite/PubSub/Server/IInternalPublisher.cs new file mode 100644 index 000000000..bd0533eff --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/IInternalPublisher.cs @@ -0,0 +1,7 @@ +namespace Bts.PubSub.Server; + +using bts; + +interface IInternalPublisher { + void Publish(PublishMsg message); +} diff --git a/src/DataStreamSuite/PubSub/Server/IInternalPublisherQueue.cs b/src/DataStreamSuite/PubSub/Server/IInternalPublisherQueue.cs new file mode 100644 index 000000000..0d1ed8b8f --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/IInternalPublisherQueue.cs @@ -0,0 +1,16 @@ +namespace Bts.PubSub.Server; + +using bts; + +using System; +using System.Collections.Generic; + +/// +/// An instance of this interface is passed to the () so it can enqueue +/// messages to be published. It is also passed to the , so it +/// can dequeue these messages and send them over the wire. +/// +interface IInternalPublisherQueue { + void Enqueue(PublishMsg message); + List DequeueMultiple(TimeSpan ts, int maxElements); +} diff --git a/src/DataStreamSuite/PubSub/Server/IPubSubState.cs b/src/DataStreamSuite/PubSub/Server/IPubSubState.cs new file mode 100644 index 000000000..ecad176ec --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/IPubSubState.cs @@ -0,0 +1,33 @@ +namespace Bts.PubSub.Server; + +using BCG; + +using mbts; + +using System.Collections.Generic; + +public delegate void PubSubStateSingleTagChangeDelegate(PubSubSingleTagStateEvent theEvent); +public delegate void PubSubStateChangeDelegate(string[] tags); +public delegate void PubSubStatsDelegate(string tag, int size); + +public interface IPubSubState { + event PubSubStateChangeDelegate? onTagMapChanged; + event PubSubStatsDelegate? onPublishStats; + + ICollection all_tags { get; } + + byte[] get_content(string tag); + string getTagType(string tag); + void get_all_connections( + string tag, + out Dictionary subscribers, + out Dictionary activePublishers, + out ICollection> deadPublishers); + + ICollection get_subscriber_connections_for_tag(string tag); + + void remove_connection(string connection); + void update_pub_data(bts.PublishMsg pub_msg); + void handle_sub(bts.SubscribeMsg sub_msg); + void handle_unsub(bts.UnsubscribeMsg unsub_msg); +} diff --git a/src/DataStreamSuite/PubSub/Server/InternalPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalPublisher.cs new file mode 100644 index 000000000..04e7cd40c --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/InternalPublisher.cs @@ -0,0 +1,25 @@ +namespace Bts.PubSub.Server; + +using bts; + +using System.Threading; + +/// +/// A simple queue based implementation of . +/// +class InternalPublisher : IInternalPublisher { + static int instanceCounter = 0; + + readonly IInternalPublisherQueue queue; + + public InternalPublisher(IInternalPublisherQueue queue) { + Log.assert(Interlocked.Increment(ref instanceCounter) == 1); + this.queue = queue; + } + + public void Publish(PublishMsg message) { + Log.Gen.diag($"InternalPublisher.Publish message: {message}"); + Log.assert(!string.IsNullOrEmpty(message.tag)); + queue.Enqueue(message); + } +} diff --git a/src/DataStreamSuite/PubSub/Server/InternalPublisherQueue.cs b/src/DataStreamSuite/PubSub/Server/InternalPublisherQueue.cs new file mode 100644 index 000000000..54c6b8cda --- /dev/null +++ b/src/DataStreamSuite/PubSub/Server/InternalPublisherQueue.cs @@ -0,0 +1,21 @@ +namespace Bts.PubSub.Server; + +using BCG; + +using bts; + +using System; +using System.Collections.Generic; + +class InternalPublisherQueue : IInternalPublisherQueue { + readonly SyncQueue queue = new(); + + public void Enqueue(PublishMsg message) { + Log.Gen.diag($"PubsubMessageReactor.internal_publish msg: {message}"); + Log.Gen.debug_assert(!string.IsNullOrEmpty(message.tag)); + queue.enqueue(message); + } + + public List DequeueMultiple(TimeSpan ts, int maxElements) => + queue.dequeueMultipleElements(ts, maxElements); +} diff --git a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs index 7867661c7..59d6b2401 100644 --- a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs +++ b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs @@ -4,6 +4,8 @@ using System.Text; using BCG.Utilities.General; using System.IO; using bts; +using Bts.PubSub; +using Bts.PubSub.Server; namespace BCG { class Pubsubheartbeat { @@ -32,27 +34,15 @@ namespace BCG { /// A subscriber can subscribe to it, and find out of a PubSub is alive or /// not. Infact more so, if the subsriptions are working or not. /// - internal class SystemHeartbeatPublisher : IWorker { + internal class SystemHeartbeatPublisher : IWorker, IParameterizedWorker { public static string tag_type = "System_Heartbeat"; public static string tag_name = "System_Heartbeat"; + readonly IInternalPublisher internalPublisher; - private static SystemHeartbeatPublisher? _self; + public SystemHeartbeatPublisher(IInternalPublisher internalPublisher) => + this.internalPublisher = internalPublisher; - private SystemHeartbeatPublisher() { - - ///this is using a multithreaded MasterWorker since the - ///regular one is called within the MessageReactor, which - ///waits 2500 msecs before calling doWork. This 2500 msecs - ///is only interceded by an external event (publication or - ///subscription) - MasterWorker.getInstance(true).addWorker(this, - TimeSpan.FromMilliseconds(500)); - } - - public static void init() { - if (_self == null) - _self = new SystemHeartbeatPublisher(); - } + TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(500); #region IWorker Members @@ -67,7 +57,7 @@ namespace BCG { publish_time_epoch_ms = hb.hb_time_epoch, data_to_publish = hb.serialize() }; - PubSubMessageReactor.internal_publish(msg); + internalPublisher.Publish(msg); } #endregion diff --git a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs index 8e7b0fbac..f0fe3271a 100644 --- a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs +++ b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs @@ -4,6 +4,8 @@ using System.Text; using System.IO; using BCG.Utilities.General; using bts; +using Bts.PubSub; +using Bts.PubSub.Server; namespace BCG { /// @@ -13,29 +15,22 @@ namespace BCG { /// but returns the state in get_content (though it could have perhaps used update instead ofth spcialized event, onPublishStats) to /// keep track of publish sizes. /// - internal class SystemStatsPublisher : IPubSubServerTagHandler, IWorker { + internal class SystemStatsPublisher : IPubSubServerTagHandler, IParameterizedWorker { public static string tag_type = "System_Stats"; public static string tag_name = "System_AllStatistics"; // these tags have been touched, need to publish in the next doWork private HashSet _dirty_tags = new HashSet(); private Dictionary _tag_stats_map = new Dictionary(); - private MasterWorker _master_worker = MasterWorker.getInstance(false); - private static SystemStatsPublisher? _self; + readonly IInternalPublisher internalPublisher; - private SystemStatsPublisher(PubSubState state) { + public SystemStatsPublisher(IPubSubState state, IInternalPublisher internalPublisher) { bts.Log.Gen.info("SystemStatsPublisher constructed"); - state.onPublishStats += new PubSubState.PubSubStatsDelegate(_handle_publish); - _master_worker.addWorker(this, TimeSpan.FromMilliseconds(250)); + this.internalPublisher = internalPublisher; + state.onPublishStats += _handle_publish; } - public static void init(PubSubState state) { - bts.Log.Gen.info("SystemStatsPublisher.init"); - _self = new SystemStatsPublisher(state); - } - - public static IPubSubServerTagHandler? get_instance() { - return _self; - } + public string TagType => tag_type; + TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(250); void _handle_publish(string tag, int size) { bts.Log.Gen.info($"SystemStatsPublisher._handle_publish tag: {tag} size: {size}"); @@ -70,7 +65,7 @@ namespace BCG { publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), data_to_publish = bytes }; - PubSubMessageReactor.internal_publish(pub_msg); + internalPublisher.Publish(pub_msg); } } diff --git a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs index 55c462d6a..9af49d33c 100644 --- a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs +++ b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs @@ -7,6 +7,8 @@ using System.Threading; using mbts; using System.Threading.Tasks; using bts; +using Bts.PubSub; +using Bts.PubSub.Server; namespace BCG { /// @@ -20,30 +22,25 @@ namespace BCG { /// It does not have much state except for the dirty state. It builds the state to publish /// as well as return in get_content, from calling into PububState /// - class SystemSubscriptionMapPublisher : IPubSubServerTagHandler, IWorker { - private PubSubState _state; + class SystemSubscriptionMapPublisher : IPubSubServerTagHandler, IParameterizedWorker { + private IPubSubState _state; private HashSet _dirty_tags = new HashSet(); private readonly string _ip = SysInfoManager.getIP(); public static string tag_type = "System_Sub"; public static string tag_name = "System_AllSubscriptions"; - private MasterWorker _master_worker = MasterWorker.getInstance(false); - private static IPubSubServerTagHandler _self; + private static SystemSubscriptionMapPublisher _self; + readonly IInternalPublisher internalPublisher; - private SystemSubscriptionMapPublisher(PubSubState state) { + public SystemSubscriptionMapPublisher(IPubSubState state, IInternalPublisher internalPublisher) { bts.Log.Gen.info("SystemSubscriptionMapPublisher.constructor"); + this.internalPublisher = internalPublisher; _state = state; - _state.onTagMapChanged += new PubSubState.PubSubStateChangeDelegate(_on_tag_state_changes); - _master_worker.addWorker(this, TimeSpan.FromMilliseconds(500)); + _state.onTagMapChanged += _on_tag_state_changes; } - public static void init(PubSubState state) { - bts.Log.Gen.info("SystemSubscriptionMapPublisher.init"); - _self = new SystemSubscriptionMapPublisher(state); - } + public string TagType => tag_type; - public static IPubSubServerTagHandler getInstance() { - return _self; - } + public TimeSpan Interval => TimeSpan.FromMilliseconds(500); private void _on_tag_state_changes(string[] tags) { Func tags_to_string = () => { @@ -79,7 +76,7 @@ namespace BCG { is_compressed = false, data_to_publish = ser_bytes }; - PubSubMessageReactor.internal_publish(pub_msg); + internalPublisher.Publish(pub_msg); } } diff --git a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs index a043649ae..35b4dea86 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs @@ -9,6 +9,8 @@ using Bts.PubSub; using Spark.Core; using System.Text; using System.Diagnostics; +using Bts.PubSub.Server; +using Microsoft.Extensions.Options; namespace BCG { // This is the main PubSub Server class where it all starts (called by Program.cs). Its constructed and start is called and thats it. @@ -20,23 +22,22 @@ namespace BCG { // private Dictionary myOperationToHandler = new Dictionary(); private Thread _msg_polling_thread; // all the data,list of subscribers etc are in the state - private PubSubState _state = new PubSubState(); - private MasterWorker _mainloop_master_worker = MasterWorker.getInstance(false); - private static SyncQueue _internal_pub_queue = new SyncQueue(); + private IPubSubState _state; + private IInternalPublisherQueue _internal_pub_queue; private Stopwatch _stop_watch = new Stopwatch(); - public PubSubMessageReactor(int port_no) { + internal PubSubMessageReactor(IPubSubState state, IInternalPublisherQueue internalPublisherQueue, int port_no, TimeSpan heartbeatInterval, int maxMissedHeartbeats, TimeSpan collectDisposedConnectionsInterval) { bts.Log.Gen.info("PubSubMessageReactor.cons System Initialized"); + _state = state; + _internal_pub_queue = internalPublisherQueue; _msg_polling_thread = new Thread(new ThreadStart(_polling_thread_func)); _msg_polling_thread.Priority = ThreadPriority.AboveNormal; - // TODO, fix it - //var port_no = 12345; _msg_hub = new ManagedTCPMessageHub( port_no, - TimeSpan.FromMilliseconds(PubSubConfig.Instance.HeartbeatInterval.assert_value()), - PubSubConfig.Instance.MissingHeartbeatsBeforeGc.assert_value(), - TimeSpan.FromMilliseconds(PubSubConfig.Instance.DisposedConnectionCheckPeriod.assert_value())); + heartbeatInterval, + maxMissedHeartbeats, + collectDisposedConnectionsInterval); // this is the only event this class gets from hub _msg_hub.register_connection_callback(new PubSubCallbacks.PubSubServerConnectionStatusChange(_handle_client_connection)); @@ -67,7 +68,7 @@ namespace BCG { _msg_hub.activate(); Action handle_internal_pubs = () => { Log.Gen.info($"PubSubMessageReactor._polling_thread_func handle_internal_pubs"); - var internal_pubs = _internal_pub_queue.dequeueMultipleElements(TimeSpan.FromSeconds(1), 64000); + var internal_pubs = _internal_pub_queue.DequeueMultiple(TimeSpan.FromSeconds(1), 64000); //Log.Gen.info($"PubSubMessageReactor._polling_thread_func internal_pubs count: {internal_pubs.Count}"); foreach (var msg in internal_pubs) { try { @@ -95,7 +96,6 @@ namespace BCG { _handle_client_msg(kv.first, kv.second); } handle_internal_pubs(); - _mainloop_master_worker.doWork(); } catch (Exception ex) { bts.Log.Gen.error($"PubSubMessageReactor._polling_thread_func ex: {ex}"); if (ex is OutOfMemoryException) { @@ -211,11 +211,5 @@ namespace BCG { // TODO: check ubnsub_msg.connection _state.handle_unsub(ubnsub_msg); } - - public static void internal_publish(PublishMsg msg) { - bts.Log.Gen.diag($"PubsubMessageReactor.internal_publish msg: {msg}"); - bts.Log.Gen.debug_assert(!String.IsNullOrEmpty(msg.tag)); - _internal_pub_queue.enqueue(msg); - } } } diff --git a/src/DataStreamSuite/PubSub/Server/PubSubState.cs b/src/DataStreamSuite/PubSub/Server/PubSubState.cs index aa456c595..8ddc44ae1 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubState.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubState.cs @@ -10,6 +10,8 @@ using System.Linq.Expressions; using bts; using mbts.utils; using bts.collections; +using Bts.PubSub; +using Bts.PubSub.Server; namespace BCG { /// @@ -24,22 +26,18 @@ namespace BCG { /// are subsribing or publishing the tag) /// /// - public class PubSubState { + public class PubSubState : IPubSubState { - public delegate void PubSubStateSingleTagChangeDelegate(PubSubSingleTagStateEvent theEvent); private event PubSubStateSingleTagChangeDelegate? onSingleTagChange; - public delegate void PubSubStateChangeDelegate(string[] tags); public event PubSubStateChangeDelegate? onTagMapChanged; - public delegate void PubSubStatsDelegate(string tag, int size); public event PubSubStatsDelegate? onPublishStats; + readonly IPubSubTagHandlerFactory tagHandlerFactory; private Dictionary _tag_to_data_map = new Dictionary(); private Dictionary _tag_to_connection_map = new Dictionary(); - public PubSubState() { - SystemSubscriptionMapPublisher.init(this); - SystemStatsPublisher.init(this); - SystemHeartbeatPublisher.init(); + public PubSubState(IPubSubTagHandlerFactory tagHandlerFactory) { + this.tagHandlerFactory = tagHandlerFactory; } #region subscription based state changes @@ -112,7 +110,8 @@ namespace BCG { ///this is the only place where SingleTagData should be created if (!_tag_to_data_map.ContainsKey(pub_msg.tag)) { ///first time data is published for this tag - _tag_to_data_map.Add(pub_msg.tag, new SingleTagData(pub_msg.tag, pub_msg.tag_type)); + var handler = tagHandlerFactory.GetHandler(pub_msg.tag_type); + _tag_to_data_map.Add(pub_msg.tag, new SingleTagData(pub_msg.tag, pub_msg.tag_type, handler)); } _tag_to_data_map[pub_msg.tag].handle_publish(pub_msg); diff --git a/src/DataStreamSuite/PubSub/Server/SingleTagData.cs b/src/DataStreamSuite/PubSub/Server/SingleTagData.cs index 1d751c9c1..02f06636d 100644 --- a/src/DataStreamSuite/PubSub/Server/SingleTagData.cs +++ b/src/DataStreamSuite/PubSub/Server/SingleTagData.cs @@ -20,24 +20,15 @@ namespace bts { public class SingleTagData { private readonly IPubSubServerTagHandler? _tag_handler; - public SingleTagData(string tag, string tag_type) { + public SingleTagData(string tag, string tag_type, IPubSubServerTagHandler tag_handler) { bts.Log.Gen.info($"SingleTagData constructed for tag: {tag} tag_type: {tag_type}"); this.tag = tag; this.tag_type = tag_type; - _tag_handler = get_handler(tag_type); + _tag_handler = tag_handler; Log.notnull(_tag_handler); Log.Gen.info($"SingleTagData.cons tag: {tag} tag_type: {tag_type} _tag_handler: {_tag_handler}"); } - public IPubSubServerTagHandler? get_handler(string tag_type) { - if (tag_type == SystemSubscriptionMapPublisher.tag_type) - return SystemSubscriptionMapPublisher.getInstance(); - else if (tag_type == SystemStatsPublisher.tag_type) - return SystemStatsPublisher.get_instance(); - return PubsubTagHandlerFactory.get_handler(tag_type); - - } - public string tag { get; } public string tag_type { get; } @@ -66,16 +57,5 @@ namespace bts { Log.Gen.info($"SingleTagData.get_current_content tag: {tag} rv (n bytes): {rv.Length}"); return rv; } - - private void _on_handler_internal_publish(ICollection> cbItems) { - foreach (var p in cbItems) { - PublishMsg pub_msg = new PublishMsg() { - tag = p.first, - tag_type = p.second, - data_to_publish = p.third - }; - PubSubMessageReactor.internal_publish(pub_msg); - } - } } } diff --git a/src/DataStreamSuite/PubSub/Services/MasterWorkerService.cs b/src/DataStreamSuite/PubSub/Services/MasterWorkerService.cs new file mode 100644 index 000000000..bf0d887de --- /dev/null +++ b/src/DataStreamSuite/PubSub/Services/MasterWorkerService.cs @@ -0,0 +1,43 @@ +namespace Bts.PubSub.Services; + +using BCG; +using BCG.Utilities.General; + +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +/// +/// An that imports a collection of s and +/// calls each in a loop. +/// +public class MasterWorkerService : BackgroundService { + readonly IWorker masterWorker; + readonly WorkerServiceConfig config; + + public MasterWorkerService(MasterWorker masterWorker, IEnumerable workers, IOptions config) + : this(masterWorker, workers, config.Value) { } + + public MasterWorkerService(MasterWorker masterWorker, IEnumerable workers, WorkerServiceConfig config) { + this.masterWorker = masterWorker; + this.config = config; + foreach (var worker in workers) { + masterWorker.addWorker(worker, (worker as IParameterizedWorker)?.Interval ?? TimeSpan.Zero); + } + } + + protected override Task ExecuteAsync(CancellationToken cancellationToken) { + return Task.Run(Run, cancellationToken); + + void Run() { + while (!cancellationToken.IsCancellationRequested) { + masterWorker.doWork(); + Thread.Sleep(config.Interval); + } + } + } +} diff --git a/src/DataStreamSuite/PubSub/Services/PubSubServerService.cs b/src/DataStreamSuite/PubSub/Services/PubSubServerService.cs new file mode 100644 index 000000000..0c6ccf961 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Services/PubSubServerService.cs @@ -0,0 +1,112 @@ +namespace Bts.PubSub.Services; + +using BCG; + +using bts; + +using Bts.PubSub.Server; + +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; + +using Spark.Core; + +using System; +using System.Threading; +using System.Threading.Tasks; + +class PubSubServerService : IHostedService { + readonly PubSubConfig config; + readonly TaskCompletionSource taskCompletionSource = new(); + readonly CancellationTokenSource cancellationTokenSource = new(); + readonly PubSubMessageReactor pubSubMessageReactor; + + public PubSubServerService(IPubSubState pubSubState, IInternalPublisherQueue internalPublisherQueue, IOptions config) + : this(pubSubState, internalPublisherQueue, config.Value) { } + + public PubSubServerService(IPubSubState pubSubState, IInternalPublisherQueue internalPublisherQueue, PubSubConfig config) { + this.config = config; + pubSubMessageReactor = new( + pubSubState, + internalPublisherQueue, + config.ServerPort.assert_value(), + TimeSpan.FromMilliseconds(config.HeartbeatInterval.assert_value()), + config.MissingHeartbeatsBeforeGc.assert_value(), + TimeSpan.FromMilliseconds(config.DisposedConnectionCheckPeriod.assert_value())); + } + + public Task StartAsync(CancellationToken cancellationToken) { + if (config.SimpleEchoServer ?? false) { + Log.Gen.info("Starting PubSubServerService in echo mode"); + Task.Factory.StartNew( + RunEchoServer, + cancellationToken, + TaskCreationOptions.LongRunning, + TaskScheduler.Default); + } else { + Log.Gen.info("Starting PubSubServerService"); + Task.Factory.StartNew( + RunPubSubServer, + cancellationToken, + TaskCreationOptions.LongRunning, + TaskScheduler.Default); + pubSubMessageReactor.start(); + } + + return Task.CompletedTask; + + void RunPubSubServer() { + long currentMemory = 0; + int defaultWaitTime = 100; + bool gced = false; + Log.Gen.info("PubSubServerService STARTED"); + var innerCancellationToken = cancellationTokenSource.Token; + Log.Gen.info($"Starting PubSubServerService execution loop"); + while (!innerCancellationToken.IsCancellationRequested) { + gced = false; + currentMemory = GC.GetTotalMemory(false); + if (config.GcActivationThreshold > 0 && currentMemory > config.GcActivationThreshold) { + Console.WriteLine($"GCing memory, mem before GC = {currentMemory}"); + GC.Collect(); + gced = true; + } + Thread.Sleep(gced && config.WaitPeriodAfterGc.HasValue + ? config.WaitPeriodAfterGc.Value + : defaultWaitTime); + } + Log.Gen.info("PubSubServerService STOPPED"); + taskCompletionSource.SetResult(); + } + + void RunEchoServer() { + Log.Gen.info("PubSubServerService STARTED in echo mode"); + var innerCancellationToken = cancellationTokenSource.Token; + var port = config.ServerPort.assert_value(); + var echo = false; + string log_msg = $"Running as TCP Echo server, port: {port} echo: {echo} Hit enter to continue:"; + Console.WriteLine(log_msg); + Console.ReadLine(); + var ready_msg = $"ready to listen to port: {port}"; + Log.Gen.info(ready_msg); + Console.WriteLine(ready_msg); + var sts = new SimpleTCPServer(port, echo); + sts.init(); + Console.CancelKeyPress += OnCancelKeyPress; + while (!innerCancellationToken.IsCancellationRequested) { + Thread.Sleep(TimeSpan.FromSeconds(3)); + } + Log.Gen.info("PubSubServerService STOPPED in echo mode"); + + void OnCancelKeyPress(object? sender, ConsoleCancelEventArgs e) { + Log.Gen.error("Cancel key press. Disposing echo server"); + sts.Dispose(); + } + } + } + + public Task StopAsync(CancellationToken cancellationToken) { + Log.Gen.info("Stopping PubSubMessageReactorService"); + cancellationTokenSource.Cancel(); + return taskCompletionSource.Task; + } +} diff --git a/src/DataStreamSuite/PubSub/Services/PubSubServices.cs b/src/DataStreamSuite/PubSub/Services/PubSubServices.cs new file mode 100644 index 000000000..90431ac37 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Services/PubSubServices.cs @@ -0,0 +1,148 @@ +namespace Bts.PubSub.Services; + +using BCG; +using BCG.Utilities.General; + +using bts; + +using Bts.PubSub.Routing; +using Bts.PubSub.Server; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +public static class PubSubServices { + // TODO: get rid of this static singleton. For now, we need it to support + // many classes that have historically relied on static singletons. Rather than + // accessing such instances via 'Foo.getInstance' or something like that, these + // classes should call "PubSubServices.ServiceProvider.GetRequiredService" + + static IServiceProvider? serviceProvider; + + public static IServiceProvider ServiceProvider => + Log.notnull( + serviceProvider, + $"Global ServiceProvider is null. Initialize it via '{nameof(PubSubServices)}.{nameof(AddGlobalServiceProvider)}'"); + + public static IServiceCollection AddGlobalServiceProvider(this IServiceCollection services) { + if (!services.Any(s => s.ServiceType == typeof(GlobalServiceProvider))) { + services.AddHostedService(); + } + return services; + } + + internal static IServiceCollection AddGlobalPubSubConfig(this IServiceCollection services) { + if (!services.Any(s => s.ServiceType == typeof(GlobalServiceProvider))) { + services.AddHostedService(); + } + return services; + } + + public static IServiceCollection AddMasterWorker(this IServiceCollection services, IConfigurationSection configurationSection) => + services + .Configure(configurationSection) + .AddSingleton(sp => MasterWorker.getInstance(false)) + .AddSingleton(sp => sp.GetRequiredService()) + .AddHostedService(); + + public static IServiceCollection AddManagedSubscriber(this IServiceCollection services) => + services.AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()) + .AddSingleton(sp => sp.GetRequiredService()); + + public static IServiceCollection AddHeartbeatPublisher(this IServiceCollection services) => + services.AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()); + + public static IServiceCollection AddPublisherCore(this IServiceCollection services) => + services.AddSingleton(); + + public static IServiceCollection AddSubscriberCore(this IServiceCollection services) => + services.AddSingleton(); + + public static IServiceCollection AddPubSubState(this IServiceCollection services) => + services.AddSingleton(); + + public static IServiceCollection AddSubscriptionMapPublisher(this IServiceCollection services) => + services.AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()) + .AddSingleton(sp => sp.GetRequiredService()); + + public static IServiceCollection AddSystemStatsPublisher(this IServiceCollection services) => + services.AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()) + .AddSingleton(sp => sp.GetRequiredService()); + + public static IServiceCollection AddTagHandlerFactory(this IServiceCollection services, IConfigurationSection configurationSection) => + services.Configure(configurationSection) + .AddGlobalServiceProvider() + .AddSingleton(); + + public static IServiceCollection AddPubSubRouter(this IServiceCollection services, IConfiguration configurationSection) => + services.Configure(configurationSection) + .AddSingleton(); + + public static IServiceCollection AddPubSubServer(this IServiceCollection services, IConfiguration configurationSection) => + services.Configure(configurationSection) + .AddGlobalServiceProvider() + .AddTagHandlerFactory(configurationSection.GetSection(nameof(PubSubConfig.TagHandlerFactory))) + .AddSystemStatsPublisher() + .AddHeartbeatPublisher() + .AddSubscriptionMapPublisher() + .AddPubSubState() + .AddInternalPublisher() + .AddHostedService(); + + internal static IServiceCollection AddInternalPublisher(this IServiceCollection services) => + services.AddSingleton() + .AddSingleton(); + + public static IServiceCollection AddPubSubClient(this IServiceCollection services, IConfiguration configurationSection) { + services.Configure(configurationSection); + var cfg = configurationSection.Get(); + PubSubConfig.SetGlobalConfig(cfg ?? PubSubConfig.Default()); + return services.AddGlobalServiceProvider() + .AddPubSubRouter(configurationSection.GetSection(nameof(PubSubConfig.Routing))) + .AddPublisherCore() + .AddSubscriberCore() + .AddManagedSubscriber(); + } + + internal static void SetGlobalServiceProvider(IServiceProvider serviceProvider) { + Log.isnull(PubSubServices.serviceProvider); + PubSubServices.serviceProvider = serviceProvider; + } + + class GlobalServiceProvider : IHostedService { + public GlobalServiceProvider(IServiceProvider serviceProvider) { + if (PubSubServices.serviceProvider is not null) { + Log.assert(serviceProvider == PubSubServices.serviceProvider); + return; + } + SetGlobalServiceProvider(serviceProvider); + } + + public Task StartAsync(CancellationToken cancellationToken) => + Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken) => + Task.CompletedTask; + } + + class AnonymousTagHandlerFactory : IPubSubTagHandlerFactory { + readonly Func getHandler; + + public AnonymousTagHandlerFactory(Func getHandler) { + this.getHandler = getHandler; + } + + public IPubSubServerTagHandler GetHandler(string tagType) => + getHandler(tagType); + } +} diff --git a/src/DataStreamSuite/PubSub/Services/WorkerConfig.cs b/src/DataStreamSuite/PubSub/Services/WorkerConfig.cs new file mode 100644 index 000000000..718141572 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Services/WorkerConfig.cs @@ -0,0 +1,7 @@ +namespace Bts.PubSub.Services; + +using System; + +public class WorkerConfig { + public TimeSpan Interval { get; set; } +} diff --git a/src/DataStreamSuite/PubSub/Services/WorkerServiceConfig.cs b/src/DataStreamSuite/PubSub/Services/WorkerServiceConfig.cs new file mode 100644 index 000000000..9e06bd494 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Services/WorkerServiceConfig.cs @@ -0,0 +1,7 @@ +namespace Bts.PubSub.Services; + +using System; + +public class WorkerServiceConfig { + public TimeSpan Interval { get; set; } +} diff --git a/src/DataStreamSuite/PubSub/SubscriberCore.cs b/src/DataStreamSuite/PubSub/SubscriberCore.cs index eb102e03f..bd5b1daea 100644 --- a/src/DataStreamSuite/PubSub/SubscriberCore.cs +++ b/src/DataStreamSuite/PubSub/SubscriberCore.cs @@ -1,6 +1,7 @@ using bts; using Bts.PubSub; +using Bts.PubSub.Routing; using System; using System.Collections.Generic; @@ -26,20 +27,11 @@ namespace BCG { #region constructors - public SubscriberCore() { - //this defaults to a basic pubsubrouter if the config key is not filled out - _tag_router = new PubSubRouter(Log.notnull(PubSubConfig.Instance.SubscriberRoutes).ToArray()); - } - public SubscriberCore(string site_name) { - _tag_router = new PubSubRouter(PubSubConfig.Instance.SubscriberRoutesFor(site_name).ToArray()); - } + public SubscriberCore() + : this((string?)null) { } - private IPubSubRouter _get_router() => - new PubSubRouter(Log.notnull(PubSubConfig.Instance.SubscriberRoutes).ToArray()); - - public SubscriberCore(PubSubRoute[] routes) { - _tag_router = new PubSubRouter(routes); - } + public SubscriberCore(string? siteName) + : this(new PubSubRouter(routerConfig(siteName))) { } public SubscriberCore(IPubSubRouter router) { _tag_router = router; @@ -66,6 +58,18 @@ namespace BCG { } } + static PubSubRouterConfig routerConfig(string? siteName = null) { + var pubSubCfg = PubSubConfig.Instance; + if (siteName is null) { + return Log.notnull(pubSubCfg.Routing); + } + var routes = PubSubConfig.Instance.SubscriberRoutesFor(siteName); + return new() { + PublisherRoutes = routes, + SubscriberRoutes = routes, + }; + } + private ISingleServerSubscriberCore _find_subscribercore(RemoteConnectionInfo server_rci) { if (!_server_route_to_subcore_map.TryGetValue(server_rci, out var rv)) { rv = new SingleServerSubscriberCore(server_rci, _connection_factory); diff --git a/src/DataStreamSuite/PubSubServer/Program.cs b/src/DataStreamSuite/PubSubServer/Program.cs index bbb0b46c7..5ce5d0f3e 100644 --- a/src/DataStreamSuite/PubSubServer/Program.cs +++ b/src/DataStreamSuite/PubSubServer/Program.cs @@ -1,81 +1,41 @@ -using BCG; +using Bts.PubSub.Services; -using bts; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; -using Bts.PubSub; -using Bts.PubSubServer; - -using CommandLine; - -using mbts.config; - -using Spark.Core; +using Spark.Core.Services; using System; -using System.Collections.Specialized; -using System.Threading; -using System.Threading.Tasks; -Parser.Default.ParseArguments(args) - .MapResult(Run, errs => 1); +var host = Host.CreateDefaultBuilder(args) + .ConfigureAppConfiguration(ConfigureApp) + .ConfigureServices(ConfigureServices) + .Build(); +try { + Console.Write($"Starting Application"); + host.Run(); + Console.Write($"Application exited gracefully."); +} catch (Exception ex) { + Console.Error.Write($"Application crashed: {ex}"); + return 1; +} +return 0; -int Run(PubSubServerConfig cfg) { - try { - cfg.MergeWithConfigFile(); - cfg.MergeWith(PubSubServerConfig.Default()); - PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); - Configuration.initialize(cfg.Sparkd); - Log.Gen.info("Starting PubSubServer"); - return cfg.SimpleEchoServer ?? false ? - RunEchoServer(cfg.PubSub.ServerPort.assert_value(), false) : - RunPubSubServer(); - } catch (Exception ex) { - // Write the error to console first, in case the logging system is compromised. - var error_msg = $"PubSubServer threw exception:\n{ex}"; - Console.Error.WriteLine(error_msg); - Console.Error.Flush(); - Log.Gen.error(error_msg); - return 1; +static void ConfigureApp(HostBuilderContext context, IConfigurationBuilder configBuilder) { + configBuilder.AddCommandLine(Environment.GetCommandLineArgs()); + var cfgRoot = configBuilder.Build(); + var cfg = cfgRoot["config"]; + if (cfg is not null) { + configBuilder.AddJsonFile(cfg, false); } } -int RunPubSubServer() { - var config = PubSubConfig.Instance; - long currentMemory = 0; - int defaultWaitTime = 100; - bool gced = false; - Task.Run(() => { - while (true) { - gced = false; - currentMemory = GC.GetTotalMemory(false); - if (config.GcActivationThreshold > 0 && currentMemory > config.GcActivationThreshold) { - Console.WriteLine($"GCing memory, mem before GC = {currentMemory}"); - GC.Collect(); - gced = true; - } - Thread.Sleep(gced ? config.WaitPeriodAfterGc.assert_value() : defaultWaitTime); - } - }); - new PubSubMessageReactor(config.ServerPort.assert_value()).start(); - return 0; -} - -int RunEchoServer(short port, bool echo) { - string log_msg = $"Running as TCP Echo server, port: {port} echo: {echo} Hit enter to continue:"; - Console.WriteLine(log_msg); - Console.ReadLine(); - var ready_msg = $"ready to listen to port: {port}"; - Log.Gen.info(ready_msg); - Console.WriteLine(ready_msg); - var sts = new SimpleTCPServer(port, echo); - sts.init(); - Console.CancelKeyPress += OnCancelKeyPress; - while (true) { - Thread.Sleep(TimeSpan.FromSeconds(3)); - } - - void OnCancelKeyPress(object? sender, ConsoleCancelEventArgs e) { - Log.Gen.error("Cancel key press. Disposing echo server"); - sts.Dispose(); - } +static void ConfigureServices(HostBuilderContext context, IServiceCollection services) { + /// Bind configuration json fields to strong types. This allows us to add services with + /// constructors that accept these types as arguments. + services + .AddSpark(context.Configuration.GetSection("Spark")) + .AddMasterWorker(context.Configuration.GetSection("MasterWorker")) + .AddPubSubServer(context.Configuration.GetSection("PubSub")); }