From 554ceea2eefd3e2191edac29171cadde10380a2b Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sun, 7 Jul 2024 22:19:25 +0000 Subject: [PATCH] check pt --- .../Misc/TCPConnectionTestStream.cs | 36 +++-- .../Primitives/Time/TimeUnitStream.cs | 5 + .../DataStreamCoreExtensions.cs | 8 ++ .../DataStreamCore/ViewStream.cs | 2 +- .../Configurations/MarketDataPublisher.json | 58 +++------ .../Services/PublisherService.cs | 19 ++- .../PubSub/Client/ThreadDrivenMasterWorker.cs | 68 ++++++++-- .../AsyncTCPConnectionAcceptor_APM.cs | 78 ----------- .../PubSub/Misc/MasterWorker.cs | 22 +++- .../PubSub/Misc/WorkerGroups.cs | 6 + .../SystemHeartbeatPublisher.cs | 3 +- .../SystemStatsPublisher.cs | 4 +- .../PubSub/Server/PubSubMessageReactor.cs | 2 - .../Services/BackgroundHostedService.cs | 70 ++++++++++ .../PubSub/Services/MasterWorkerConfig.cs | 3 +- .../PubSub/Services/MasterWorkerService.cs | 98 +++++++++----- .../PubSub/Services/PubSubServices.cs | 1 + .../PubSub/Services/WorkSchedulerConfig.cs | 8 ++ .../PubSub/Services/WorkSchedulerProvider.cs | 22 ++++ .../PubSub/Services/WorkSchedulerService.cs | 123 ++++++++++++++++++ src/DataStreamSuite/PubSub/WorkActivator.cs | 115 ++++++++++++++++ .../Configurations/PubSubServer.json | 2 +- 22 files changed, 566 insertions(+), 187 deletions(-) create mode 100644 src/DataStreamSuite/DataStreamCore/DataStreamCoreExtensions.cs delete mode 100644 src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs create mode 100644 src/DataStreamSuite/PubSub/Misc/WorkerGroups.cs create mode 100644 src/DataStreamSuite/PubSub/Services/BackgroundHostedService.cs create mode 100644 src/DataStreamSuite/PubSub/Services/WorkSchedulerConfig.cs create mode 100644 src/DataStreamSuite/PubSub/Services/WorkSchedulerProvider.cs create mode 100644 src/DataStreamSuite/PubSub/Services/WorkSchedulerService.cs create mode 100644 src/DataStreamSuite/PubSub/WorkActivator.cs diff --git a/src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs b/src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs index b361e7740..e518934d6 100644 --- a/src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs +++ b/src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs @@ -1,12 +1,16 @@ using BCG.Utilities.General; +using Bts.PubSub; + +using Microsoft.Extensions.DependencyInjection; + using System; namespace BCG { - public class TCPConnectionTestStream : ViewStream, IWorker, ITCPConnectionHandler, ITCPMessageHandler { + public class TCPConnectionTestStream : ViewStream, IParameterizedWorker, ITCPConnectionHandler, ITCPMessageHandler { class State { - public int n_sends = 0; - public int n_recvs = 0; + public int n_sends = 0; + public int n_recvs = 0; } private readonly string _ip; private readonly int _port; @@ -17,7 +21,8 @@ namespace BCG { private TimeSpan _send_period; private ITCPConnection _connection = null; private byte[] _buff; - + WorkActivator workActivator; + public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) { _ip = ip; _port = port; @@ -31,16 +36,21 @@ namespace BCG { for (int i = 0; i < _buff.Length; i++) { _buff[i] = (byte)i; } + workActivator = new(this); + } + + TimeSpan IParameterizedWorker.Interval => _send_period; + + protected override void OnServiceProvider(IServiceProvider serviceProvider) { + workActivator.OnWorkScheduler(serviceProvider.GetRequiredService()); } protected override void myOnFirstSubscriber() { - MasterWorker.getInstance(false).addWorker(this, _send_period); - var server_rci = new RemoteConnectionInfo(_ip, _port); - bts.AsyncTCPConnector_APM.connect_to(server_rci, this); + workActivator.Start(); } protected override void myOnLastSubscriber() { - MasterWorker.getInstance(false).removeWorker(this); + workActivator.Stop(); } public void doWork() { @@ -56,18 +66,18 @@ namespace BCG { } public void onConnection(ITCPConnection connection) { - _connection = connection; - _connection.register(this); + _connection = connection; + _connection.register(this); } public bool isDisposed { get; } public void onData(RemoteMessage msg) { - bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}"); - _state.n_recvs++; + bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}"); + _state.n_recvs++; } public void Dispose() { - + workActivator.Stop(); } diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/TimeUnitStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/TimeUnitStream.cs index bef419e69..0b65fd7b8 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Time/TimeUnitStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Time/TimeUnitStream.cs @@ -2,6 +2,8 @@ using System; using BCG.Utilities.General; +using Bts.DataStreamCore; + namespace BCG { public abstract class TimeUnitStream : ViewStream, IWorker { protected TimeSpan myOffset; @@ -31,6 +33,9 @@ namespace BCG { #endregion protected virtual void myInit() { + if (!this.HasSubscribers()) { + return; + } int periods = (int) (DateTime.Now - myConstructionTime).TotalMilliseconds / (int)myOffset.TotalMilliseconds; diff --git a/src/DataStreamSuite/DataStreamCore/DataStreamCoreExtensions.cs b/src/DataStreamSuite/DataStreamCore/DataStreamCoreExtensions.cs new file mode 100644 index 000000000..2472d547f --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/DataStreamCoreExtensions.cs @@ -0,0 +1,8 @@ +namespace Bts.DataStreamCore; + +using BCG; + +public static class DataStreamCoreExtensions { + public static bool HasSubscribers(this DataStream dataStream) => + dataStream.numberOfSubscribers > 0; +} diff --git a/src/DataStreamSuite/DataStreamCore/ViewStream.cs b/src/DataStreamSuite/DataStreamCore/ViewStream.cs index a3baa5caa..a192d2ab3 100644 --- a/src/DataStreamSuite/DataStreamCore/ViewStream.cs +++ b/src/DataStreamSuite/DataStreamCore/ViewStream.cs @@ -156,7 +156,7 @@ namespace BCG protected virtual void OnViewStreamDatabase(IWorkScheduler viewStreamDatabase) { var w = this as IWorker; if (w is not null) { - worker = viewStreamDatabase.Subscribe(w, (this as IParameterizedWorker)?.Interval); + worker = viewStreamDatabase.Subscribe(w); } } diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json index 8a404f0ac..c6b876151 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json +++ b/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json @@ -7,35 +7,10 @@ "Password": "" }, "MasterWorker": { - "Interval": "00:00:00.1" + "Interval": "00:00:01" }, "Publisher": { "Publishers": [ - { - "Name": "market_data", - "Streams": [ - { - "Name": "bbo", - "Expression": "bbo()", - "IsOnWebsocket": true - }, - { - "Name": "instruments", - "Expression": "instruments()", - "IsOnWebsocket": true - }, - { - "Name": "timesales", - "Expression": "timesales(10000)", - "IsOnWebsocket": true - }, - { - "Name": "books", - "Expression": "books()", - "IsOnWebsocket": true - } - ] - }, { "Name": "test_data", "Streams": [ @@ -43,16 +18,6 @@ "Name": "seconds_1_10", "Expression": "second(1,10)", "IsOnWebsocket": true - }, - { - "Name": "test_literal", - "Expression": "literal(\"x\", \"420.0\")", - "IsOnWebsocket": true - }, - { - "Name": "test_not_on_websocket", - "Expression": "literal(\"x\", \"840.0\")", - "IsOnWebsocket": false } ] } @@ -118,7 +83,26 @@ "UsePrepublishedServerState": false, "IpTranslation": null, "GcActivationThreshold": 0, - "WaitPeriodAfterGc": 1000 + "WaitPeriodAfterGc": 1000, + "TagHandlerFactory": { + "Handlers": [ + { + "TagType": "DataStream", + "Assembly": "DataStreamCore", + "Class": "bts.PubsubServerStreamTagHandler" + }, + { + "TagType": "System_Sub", + "Assembly": "PubSub", + "Class": "BCG.SystemSubscriptionMapPublisher" + }, + { + "TagType": "System_Stats", + "Assembly": "PubSub", + "Class": "BCG.SystemStatsPublisher" + } + ] + } }, "ViewStreamDatabase": { "Interval": "00:00:01" diff --git a/src/DataStreamSuite/DataStreamPublisher/Services/PublisherService.cs b/src/DataStreamSuite/DataStreamPublisher/Services/PublisherService.cs index c3a92a3c1..f879bc88a 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Services/PublisherService.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Services/PublisherService.cs @@ -8,7 +8,9 @@ using bts.api.core; using Bts.DataStreamCore; using Bts.DataStreamExpressions; +using Bts.PubSub; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Options; @@ -29,17 +31,20 @@ public class PublisherService : IHostedService { readonly IPubTableBroker pubTableBroker; readonly PrimitiveManager primitiveManager; readonly IServiceProvider serviceProvider; + readonly IWorkScheduler workScheduler; public PublisherService( IPubTableBroker pubTableBroker, PrimitiveManager primitiveManager, IServiceProvider serviceProvider, - IOptions config + IOptions config, + [FromKeyedServices(WorkerGroups.Main)]IWorkScheduler workScheduler ) { this.pubTableBroker = pubTableBroker; this.primitiveManager = primitiveManager; this.serviceProvider = serviceProvider; this.config = config.Value; + this.workScheduler = workScheduler; } public Task StartAsync(CancellationToken cancellationToken) { @@ -93,7 +98,9 @@ public class PublisherService : IHostedService { foreach (var publisher in publishers) { if (Directory.Exists(publisher.CsvDir)) { - csvPublishers.Add(new(publisher.CsvDir, pubTableBroker)); + var csvPublisher = new CSVPublisher(publisher.CsvDir, pubTableBroker); + csvPublishers.Add(csvPublisher); + workScheduler.Add(csvPublisher); } } @@ -122,7 +129,7 @@ public class PublisherService : IHostedService { void on_change(object obj) => Log.Gen.info($"DSPublisher on_change dp: {obj}"); - public class CSVPublisher : IWorker { + public class CSVPublisher : IParameterizedWorker { private IDictionary _pubs = new Dictionary(); private readonly string _csv_directory; private IPubTableBroker _pub_table_broker; @@ -130,8 +137,12 @@ public class PublisherService : IHostedService { public CSVPublisher(string csv_directory, IPubTableBroker pub_table_broker) { _csv_directory = csv_directory; _pub_table_broker = pub_table_broker; - MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); } + + string IParameterizedWorker.Group => WorkerGroups.Main; + + TimeSpan IParameterizedWorker.Interval => TimeSpan.FromSeconds(1); + public void doWork() { var file_names = Directory.GetFiles(_csv_directory); foreach (var fn in file_names) { diff --git a/src/DataStreamSuite/PubSub/Client/ThreadDrivenMasterWorker.cs b/src/DataStreamSuite/PubSub/Client/ThreadDrivenMasterWorker.cs index 2a4a2125b..4792d78ef 100644 --- a/src/DataStreamSuite/PubSub/Client/ThreadDrivenMasterWorker.cs +++ b/src/DataStreamSuite/PubSub/Client/ThreadDrivenMasterWorker.cs @@ -2,8 +2,14 @@ using System; using System.Collections.Generic; using System.Text; using System.Threading; +using System.Threading.Tasks; + // using BCG.Utilities; - using BCG.Utilities.General; +using BCG.Utilities.General; + +using Bts.PubSub; +using Bts.PubSub.Services; + using mbts; namespace BCG @@ -16,15 +22,58 @@ namespace BCG /// public class ThreadDrivenMasterWorker : MasterWorker { + readonly object syncRoot = new(); + readonly string? name; + readonly CancellationToken cancellationToken; + readonly TaskCompletionSource taskCompletionSource = new(); + Thread? thread; + internal ThreadDrivenMasterWorker() - : base() - { } + : this(new WorkSchedulerConfig() { Interval = TimeSpan.FromMilliseconds(100)}) { } + + internal ThreadDrivenMasterWorker(string? name) + : this(new WorkSchedulerConfig() { Name = name, Interval = TimeSpan.FromMilliseconds(100) }) { } + + internal ThreadDrivenMasterWorker(WorkSchedulerConfig config) : this(config, CancellationToken.None) { } + + internal ThreadDrivenMasterWorker(WorkSchedulerConfig config, CancellationToken cancellationToken) : base(config.Interval) { + name = config.Name; + this.cancellationToken = cancellationToken; + } + + public static ThreadDrivenMasterWorker StartNew(string? name) { + var rv = new ThreadDrivenMasterWorker(name); + rv.Start(); + return rv; + } + + public static ThreadDrivenMasterWorker StartNew(IWorker worker, TimeSpan interval) { + var rv = new ThreadDrivenMasterWorker(); + rv.addWorker(worker, interval); + rv.Start(); + return rv; + } + + public Task StartAsync() { + Start(); + return taskCompletionSource.Task; + } protected override void myInit() { - Thread doWorkThread = new Thread(new ThreadStart(myDoWork)); - doWorkThread.IsBackground = true; - doWorkThread.Start(); + if (thread is not null) { + // Already started + return; + } + lock (syncRoot) { + if (thread is null) { + thread = new(myDoWork); + if (name is not null) { + thread.Name = name; + } + thread.Start(); + } + } } public override void addWorker(IWorker worker, TimeSpan period) @@ -50,7 +99,7 @@ namespace BCG private void myDoWork() { TimeSpan sleepTime = new TimeSpan(myPeriod.Ticks / 2); - while (true) + while (!cancellationToken.IsCancellationRequested) { try { @@ -76,11 +125,10 @@ namespace BCG } catch (Exception ex) { - // BlueEventLog.logEvent(ex); - if (ex is OutOfMemoryException) - throw ex; + taskCompletionSource.SetException(ex); } } + taskCompletionSource.SetResult(); } } diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs deleted file mode 100644 index ab5f4dea4..000000000 --- a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs +++ /dev/null @@ -1,78 +0,0 @@ -using System.Net.Sockets; -using System.Net; - -using BCG; -using System; -using System.Collections.Generic; - - -namespace bts { - - // opens a TCP port and starts listening to connections - public class AsyncTCPConnectionAcceptor_APM { - private readonly int _port; - private Socket _accept_socket; - private ITCPConnectionHandler _connection_handler; - private HashSet _key_map = new HashSet(); - public AsyncTCPConnectionAcceptor_APM(int port, ITCPConnectionHandler connection_handler) { - Log.Gen.info($"AsyncTCPConnectionAcceptor_APM. cons started on port: {port}"); - _port = port; - _connection_handler = connection_handler; - } - - public void init() { - _init(); - _begin_accepting(); - } - - private void _init() { - Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._init started on port: {_port}"); - _accept_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); - TCPSocketUtils.set_server_socket_options(ref _accept_socket); - IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, _port); - _accept_socket.Bind(endpoint); - _accept_socket.Listen(250); - } - - private void _begin_accepting() { - var accept_key = Guid.NewGuid().ToString(); - Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._begin_accepting port: {_port} accept_key: {accept_key}"); - var res = _accept_socket.BeginAccept(new AsyncCallback(_on_accept), accept_key); - if (res != null && res.IsCompleted && res.CompletedSynchronously) - _on_accept(res); - } - - private void _on_accept(IAsyncResult res) { - var log_msg = $"AsyncTCPConnectionAcceptor_APM._on_accept state: {res.AsyncState}"; - Log.Gen.info(log_msg); - Console.WriteLine(log_msg); - string key = res.AsyncState.ToString(); - if (_key_map.Contains(key)) { - return; - } - _key_map.Add(key); - Socket new_sock; - try { - new_sock = _accept_socket.EndAccept(res); - } catch (Exception ex) { - bts.Log.Gen.info("AsyncTCPConnectionAcceptor_APM._on_accept exception thrown by EndAccept ex: {ex}"); - System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(250)); - _begin_accepting(); - return; - } - new_sock.LingerState = new LingerOption(false, 0); - new_sock.Blocking = false; - ITCPConnection newConn = new AsyncTCPConnection_APM(new_sock, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Server); - _connection_handler.onConnection(newConn); - // go back to accepting more conns - _begin_accepting(); - var peer_rci = newConn.get_peer_address(); - Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._on_accept peer_rci: {peer_rci}"); - } - - - - - } - -} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs b/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs index 181b18e34..db24e51ac 100644 --- a/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs +++ b/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs @@ -15,8 +15,8 @@ namespace BCG { private DateTime myLastDoWorkTime = DateTime.MinValue, myLastLogTime = DateTime.MinValue; - protected TimeSpan myPeriod = new TimeSpan(0, 0, 0, 0, 100), - myLogPeriod = new TimeSpan(0, 1, 0); + protected readonly TimeSpan myPeriod; + protected readonly TimeSpan myLogPeriod = TimeSpan.FromMinutes(1); protected Dictionary> myWorkers = new Dictionary>(), myJustAdded = new Dictionary>(), myJustRemoved = new Dictionary>(); @@ -24,7 +24,18 @@ namespace BCG { private static MasterWorker myThreadDriven = null, myDoWorkDriven = null; - internal MasterWorker() { + internal MasterWorker() : this(TimeSpan.FromMilliseconds(100)) { } + internal MasterWorker(TimeSpan interval) { + myPeriod = interval; + } + + public static MasterWorker StartNew() { + var rv = new MasterWorker(); + rv.Start(); + return rv; + } + + public void Start() { myInit(); } @@ -34,11 +45,10 @@ namespace BCG { lock (typeof(MasterWorker)) { if (threadDriven) { // TODO - myThreadDriven = myThreadDriven ?? new ThreadDrivenMasterWorker(); + myThreadDriven ??= ThreadDrivenMasterWorker.StartNew(); return myThreadDriven; - return null; } else { - myDoWorkDriven = myDoWorkDriven ?? new MasterWorker(); + myDoWorkDriven ??= StartNew(); return myDoWorkDriven; } } diff --git a/src/DataStreamSuite/PubSub/Misc/WorkerGroups.cs b/src/DataStreamSuite/PubSub/Misc/WorkerGroups.cs new file mode 100644 index 000000000..52e306b13 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/WorkerGroups.cs @@ -0,0 +1,6 @@ +namespace Bts.PubSub; + +public static class WorkerGroups { + public const string Main = "Main"; + public const string Background = "Background"; +} diff --git a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs index f14a8a8ff..8fdfcd591 100644 --- a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs +++ b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs @@ -48,7 +48,8 @@ namespace BCG { this.publisher = publisher; } - public TimeSpan Interval => TimeSpan.FromMilliseconds(500); + string IParameterizedWorker.Group => WorkerGroups.Background; + TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(500); #region IWorker Members diff --git a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs index 7cf8a1d2d..4d05af81d 100644 --- a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs +++ b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs @@ -39,8 +39,8 @@ namespace BCG { public string TagType => tag_type; public string Tag => tag_name; - TimeSpan IParameterizedWorker.Interval => - TimeSpan.FromMilliseconds(250); + string IParameterizedWorker.Group => WorkerGroups.Main; + TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(250); internal static SystemStatsPublisher get_instance() { return Log.notnull(instance); diff --git a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs index 50cf7feb5..6d005288a 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs @@ -23,7 +23,6 @@ namespace BCG { private Thread _msg_polling_thread; // all the data,list of subscribers etc are in the state private IPubSubState _state; - private MasterWorker _mainloop_master_worker = MasterWorker.getInstance(false); // private static SyncQueue _internal_pub_queue = new SyncQueue(); private static IInternalPublisherQueue? _internal_pub_queue; static int instanceCount = 0; @@ -109,7 +108,6 @@ namespace BCG { _handle_client_msg(kv.first, kv.second); } handle_internal_pubs(); - _mainloop_master_worker.doWork(); } catch (Exception ex) { bts.Log.Gen.error($"PubSubMessageReactor._polling_thread_func ex: {ex}"); if (ex is OutOfMemoryException) { diff --git a/src/DataStreamSuite/PubSub/Services/BackgroundHostedService.cs b/src/DataStreamSuite/PubSub/Services/BackgroundHostedService.cs new file mode 100644 index 000000000..8948e1bf4 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Services/BackgroundHostedService.cs @@ -0,0 +1,70 @@ +namespace Bts.PubSub.Services; + +using bts; + +using Microsoft.Extensions.Hosting; + +using System; +using System.Threading; +using System.Threading.Tasks; + +/// +/// An that runs on a background thread. +/// +public abstract class BackgroundHostedService : IHostedService { + readonly TaskCompletionSource tcs = new(); + readonly CancellationTokenSource cancellationTokenSource = new(); + readonly object syncRoot = new(); + Thread? thread; + State state; + + protected BackgroundHostedService() { } + + enum State { + Idle, + Starting, + Running, + Stopping, + } + + string name => GetType().Name; + + public Task StartAsync(CancellationToken cancellationToken) { + Log.isnull(thread); + lock (syncRoot) { + Log.assert(state == State.Idle); + state = State.Starting; + } + Log.Gen.info($"Starting {name}"); + thread = new Thread(run) { + Name = name + }; + thread.Start(); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) { + Log.Gen.info($"Stopping {name}..."); + cancellationTokenSource.Cancel(); + return tcs.Task; + } + + protected abstract void Run(CancellationToken cancellationToken); + + void run() { + Log.Gen.info($"{GetType()} STARTED."); + lock (syncRoot) { + state = State.Running; + } + try { + Run(cancellationTokenSource.Token); + Log.Gen.error($"{name} '{nameof(Run)}' finished"); + } catch (Exception ex) { + Log.Gen.error($"{name} '{nameof(Run)}' threw an exception: {ex}"); + } + lock (syncRoot) { + state = State.Idle; + } + tcs.SetResult(); + } +} diff --git a/src/DataStreamSuite/PubSub/Services/MasterWorkerConfig.cs b/src/DataStreamSuite/PubSub/Services/MasterWorkerConfig.cs index 7d855b4ee..377f2bc71 100644 --- a/src/DataStreamSuite/PubSub/Services/MasterWorkerConfig.cs +++ b/src/DataStreamSuite/PubSub/Services/MasterWorkerConfig.cs @@ -1,7 +1,8 @@ namespace Bts.PubSub.Services; using System; +using System.Collections.Generic; public class MasterWorkerConfig { - public TimeSpan Interval { get; set; } = TimeSpan.FromMilliseconds(100); + public List? WorkSchedulers { get; set; } } diff --git a/src/DataStreamSuite/PubSub/Services/MasterWorkerService.cs b/src/DataStreamSuite/PubSub/Services/MasterWorkerService.cs index 051497a03..7eb125162 100644 --- a/src/DataStreamSuite/PubSub/Services/MasterWorkerService.cs +++ b/src/DataStreamSuite/PubSub/Services/MasterWorkerService.cs @@ -9,7 +9,9 @@ using bts.collections; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Options; +using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.Linq; using System.Text.Json; using System.Text.Json.Nodes; @@ -20,54 +22,57 @@ using System.Threading.Tasks; /// An that imports a collection of s and /// calls each in a loop. /// -public class MasterWorkerService : IHostedService { +public class MasterWorkerService : BackgroundHostedService, IWorkScheduler { readonly TaskCompletionSource tcs = new(); readonly CancellationTokenSource cancellationTokenSource = new(); readonly List workers; readonly MasterWorkerConfig config; - MasterWorker? masterWorker; - Dictionary? backgroundMasterWorkers; + readonly object syncRoot = new(); + readonly Dictionary backgroundWorkers = new(); public MasterWorkerService(IEnumerable workers, IOptions config) { this.workers = workers.ToList(); this.config = config.Value; + Log.notnull(this.config.WorkSchedulers); + Log.assert(this.config.WorkSchedulers.Count == 2); + Log.assert(this.config.WorkSchedulers[0].Name == WorkerGroups.Main); + Log.assert(this.config.WorkSchedulers[1].Name == WorkerGroups.Background); + foreach (var w in this.config.WorkSchedulers) { + Log.notnull(w.Name); + backgroundWorkers.Add(w.Name, new(w.Name)); + } } - public Task StartAsync(CancellationToken cancellationToken) { + enum State { + Idle, + Working + } + + public void Add(IWorker worker, TimeSpan interval) { + var group = getWorkerGroup(worker); + var scheduler = getOrAddScheduler(group); + scheduler.Add(worker, interval); + } + + public void Remove(IWorker worker) { + var group = getWorkerGroup(worker); + var scheduler = getScheduler(group); + Log.notnull(scheduler).Remove(worker); + } + + protected override void Run(CancellationToken cancellationToken) { var workersDesc = JsonSerializer.Serialize(workersToJson(workers)); Log.Gen.info($"Starting MasterWorkerService. {workersDesc}"); - foreach (var w in workers) { - GetWorkScheduler((w as IParameterizedWorker)?.Group).Add(w); - } - new Thread(workLoop).Start(); - return Task.CompletedTask; - - IWorkScheduler GetWorkScheduler(string? key) => - key is null - ? (masterWorker ??= new()) - : (backgroundMasterWorkers ??= new()).GetOrAdd(key, _ => new ThreadDrivenMasterWorker()); - } - - public Task StopAsync(CancellationToken cancellationToken) { - Log.Gen.info("Stopping MasterWorkerService..."); - cancellationTokenSource.Cancel(); - return tcs.Task; - } - - void workLoop() { - Log.Gen.info("MasterWorkerService STARTED."); - if (masterWorker is null) { - Log.Gen.warn("MasterWorkerService STOPPED early because there is no work to do."); - return; - } - while (!cancellationTokenSource.IsCancellationRequested) { - masterWorker?.doWork(); - Thread.Sleep(config.Interval); + foreach (var workScheduler in backgroundWorkers.Values) { + workScheduler.Start(); } Log.Gen.info("MasterWorkerService STOPPED."); tcs.SetResult(); } + static string getWorkerGroup(IWorker worker) => + (worker as IParameterizedWorker)?.Group ?? WorkerGroups.Main; + static JsonObject workersToJson(IEnumerable workers) { var rootJson = new JsonObject(); foreach (var wg in workers.GroupBy(w => (w as IParameterizedWorker)?.Group ?? "Main")) { @@ -77,4 +82,35 @@ public class MasterWorkerService : IHostedService { } return rootJson; } + + IWorkScheduler? getScheduler(string group) { + lock (syncRoot) { + if (backgroundWorkers.TryGetValue(group, out var bgw)) { + return bgw; + } + } + return null; + } + + IWorkScheduler getOrAddScheduler(IWorker worker) => + getOrAddScheduler((worker as IParameterizedWorker)?.Group ?? WorkerGroups.Main); + + IWorkScheduler getOrAddScheduler(string group) { + var (bgWorker, isNew) = getOrAddBackgroundWorker(group); + if (isNew) { + bgWorker.Start(); + } + return bgWorker; + } + + (ThreadDrivenMasterWorker worker, bool isNew) getOrAddBackgroundWorker(string group) { + lock (syncRoot) { + if (backgroundWorkers.TryGetValue(group, out var bgWorker)) { + return (bgWorker, false); + } + bgWorker = new(group); + backgroundWorkers.Add(group, bgWorker); + return (bgWorker, true); + } + } } diff --git a/src/DataStreamSuite/PubSub/Services/PubSubServices.cs b/src/DataStreamSuite/PubSub/Services/PubSubServices.cs index 604098431..e69480ff1 100644 --- a/src/DataStreamSuite/PubSub/Services/PubSubServices.cs +++ b/src/DataStreamSuite/PubSub/Services/PubSubServices.cs @@ -65,6 +65,7 @@ public static class PubSubServices { services .Configure(configurationSection) .AddInternalPublisher() + .AddPubSubTagHandlerFactory(configurationSection.GetSection(nameof(PubSubConfig.TagHandlerFactory))) .AddSingleton() .AddSingleton() .AddPublisherCore() diff --git a/src/DataStreamSuite/PubSub/Services/WorkSchedulerConfig.cs b/src/DataStreamSuite/PubSub/Services/WorkSchedulerConfig.cs new file mode 100644 index 000000000..1dd532399 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Services/WorkSchedulerConfig.cs @@ -0,0 +1,8 @@ +namespace Bts.PubSub.Services; + +using System; + +public class WorkSchedulerConfig { + public string? Name { get; set; } + public TimeSpan Interval { get; set; } = TimeSpan.FromMilliseconds(100); +} diff --git a/src/DataStreamSuite/PubSub/Services/WorkSchedulerProvider.cs b/src/DataStreamSuite/PubSub/Services/WorkSchedulerProvider.cs new file mode 100644 index 000000000..0299c43bd --- /dev/null +++ b/src/DataStreamSuite/PubSub/Services/WorkSchedulerProvider.cs @@ -0,0 +1,22 @@ +namespace Bts.PubSub.Services; + +using BCG; + + +using System.Collections.Generic; + +public class WorkSchedulerProvider { + readonly Dictionary workSchedulers = new(); + readonly object syncRoot = new(); + + public IWorkScheduler GetScheduler(string? group) { + group ??= string.Empty; + lock (syncRoot) { + return workSchedulers.GetOrAdd( + group, + g => string.IsNullOrEmpty(g) + ? new MasterWorker() + : new ThreadDrivenMasterWorker()); + } + } +} diff --git a/src/DataStreamSuite/PubSub/Services/WorkSchedulerService.cs b/src/DataStreamSuite/PubSub/Services/WorkSchedulerService.cs new file mode 100644 index 000000000..1dcb9eac9 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Services/WorkSchedulerService.cs @@ -0,0 +1,123 @@ +namespace Bts.PubSub; + +using BCG; +using BCG.Utilities.General; + +using bts; +using bts.collections; + +using Bts.PubSub.Services; + +using Microsoft.Extensions.Options; + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; + +public class WorkSchedulerService : BackgroundHostedService, IWorkScheduler { + readonly List workers; + readonly MasterWorkerConfig config; + readonly IWorkScheduler mainWorker = new MasterWorker(); + readonly object syncRoot = new(); + ImmutableDictionary backgroundWorkers = ImmutableDictionary.Empty; + + public WorkSchedulerService(IEnumerable workers, IOptions config) { + this.workers = workers.ToList(); + this.config = config.Value; + } + + public void Add(IWorker worker, TimeSpan interval) { + // getOrAddScheduler(worker).Add(worker, interval); + var group = (worker as IParameterizedWorker)?.Group; + if (string.IsNullOrEmpty(group)) { + lock (syncRoot) { + mainWorker.Add(worker, interval); + } + return; + } + if (backgroundWorkers.TryGetValue(group, out var bgw)) { + bgw.Add(worker, interval); + } + bgw = ThreadDrivenMasterWorker.StartNew(worker, interval); + lock (syncRoot) { + backgroundWorkers = backgroundWorkers.Add(group, bgw); + } + } + + public void Remove(IWorker worker) { + var group = (worker as IParameterizedWorker)?.Group; + if (string.IsNullOrEmpty(group)) { + lock (syncRoot) { + mainWorker.Remove(worker); + } + return; + } + if (backgroundWorkers.TryGetValue(group, out var bgw)) { + bgw.Remove(worker); + } + var scheduler = getScheduler(worker); + Log.notnull(scheduler).Remove(worker); + } + + protected override void OnStart(CancellationToken cancellationToken) { + var workersDesc = JsonSerializer.Serialize(workersToJson(workers)); + Log.Gen.info($"Starting MasterWorkerService. {workersDesc}"); + foreach (var w in workers) { + getOrAddScheduler(w).Add(w); + } + } + + static JsonObject workersToJson(IEnumerable workers) { + var rootJson = new JsonObject(); + foreach (var wg in workers.GroupBy(w => (w as IParameterizedWorker)?.Group ?? "Main")) { + var wgJson = new JsonArray(); + wgJson.AddRange(wg.Select(w => JsonValue.Create(w.ToString()))); + rootJson.Add(wg.Key, wgJson); + } + return rootJson; + } + + IWorkScheduler? getScheduler(IWorker worker) => + getScheduler((worker as IParameterizedWorker)?.Group); + + IWorkScheduler? getScheduler(string? group) { + if (string.IsNullOrEmpty(group)) { + return mainWorker; + } + lock (syncRoot) { + if (backgroundWorkers.TryGetValue(group, out var bgw)) { + return bgw; + } + } + return null; + } + + IWorkScheduler getOrAddScheduler(IWorker worker) => + getOrAddScheduler((worker as IParameterizedWorker)?.Group); + + IWorkScheduler getOrAddScheduler(string? group) { + if (string.IsNullOrEmpty(group)) { + return mainWorker; + } + lock (syncRoot) { + if (backgroundWorkers.TryGetValue(group, out var bgw)) { + return bgw; + } + bgw = new(); + backgroundWorkers = backgroundWorkers.Add(group, bgw); + bgw.Start(); + return bgw; + } + } + + protected override void Run(CancellationToken cancellationToken) { + while (!cancellationToken.IsCancellationRequested) { + mainWorker.doWork(); + Thread.Sleep(config.Interval); + } + } +} diff --git a/src/DataStreamSuite/PubSub/WorkActivator.cs b/src/DataStreamSuite/PubSub/WorkActivator.cs new file mode 100644 index 000000000..aa1289a63 --- /dev/null +++ b/src/DataStreamSuite/PubSub/WorkActivator.cs @@ -0,0 +1,115 @@ +#nullable enable +namespace Bts.PubSub; + +using BCG.Utilities.General; + +using bts; + +using System; + +public class WorkActivator : IDisposable { + readonly object syncRoot = new(); + readonly IWorker worker; + IWorkScheduler? workScheduler; + + State state = State.Idle; + + public WorkActivator(IWorker worker) { + this.worker = worker; + } + + enum State { + Idle, + WaitingToWork, + Working + } + + public void OnWorkScheduler(IWorkScheduler workScheduler) { + lock (syncRoot) { + if (workScheduler == this.workScheduler) { + return; + } + switch (state) { + case State.Idle: + this.workScheduler = workScheduler; + break; + case State.WaitingToWork: + /// We only transition when is + /// called and is null. + Log.isnull(this.workScheduler); + this.workScheduler = workScheduler; + if (workScheduler is not null) { + state = State.Working; + workScheduler.Add(worker); + } + break; + case State.Working: + Log.notnull(this.workScheduler).Remove(worker); + this.workScheduler = workScheduler; + if (workScheduler is not null) { + workScheduler.Add(worker); + } else { + state = State.WaitingToWork; + } + break; + } + } + } + + public void Start() { + lock (syncRoot) { + switch (state) { + case State.Idle: + if (workScheduler is null) { + state = State.WaitingToWork; + } else { + state = State.Working; + workScheduler.Add(worker); + } + break; + case State.WaitingToWork: + // Sanity check - "WaitingToWork" means we are waiting for a non-null IWorkScheduler. + // It follows that if we're "WaitingToWork", then our IWorkScheduler should be null. + Log.isnull(workScheduler); + break; + case State.Working: + Log.notnull(workScheduler); + break; + } + } + } + + public void Stop() { + lock (syncRoot) { + switch (state) { + case State.Idle: + break; + case State.WaitingToWork: + state = State.Idle; + break; + case State.Working: + Log.notnull(workScheduler).Remove(worker); + state = State.Idle; + break; + } + } + } + + public void Dispose() { + lock (syncRoot) { + stopImpl(); + workScheduler = null; + } + } + + void stopImpl() { + if (workScheduler is not null) { + switch (state) { + case State.Working: + workScheduler.Remove(worker); + break; + } + } + state = State.Idle; + } +} diff --git a/src/DataStreamSuite/PubSubServer/Configurations/PubSubServer.json b/src/DataStreamSuite/PubSubServer/Configurations/PubSubServer.json index 02fd6f10e..06053914a 100644 --- a/src/DataStreamSuite/PubSubServer/Configurations/PubSubServer.json +++ b/src/DataStreamSuite/PubSubServer/Configurations/PubSubServer.json @@ -1,6 +1,6 @@ { "MasterWorker": { - "Interval": "00:00:00.1" + "Interval": "00:00:01" }, "SparkLogging": { "ConfigServer": "http://localhost:8000/config"