This commit is contained in:
John Colvin committed 2024-07-07 22:19:25 +00:00
1 parent d54b0437b0
commit 554ceea2ee
22 files changed
+566 -187

No files matched your search

@@ -1,12 +1,16 @@
using BCG.Utilities.General;
using Bts.PubSub;
using Microsoft.Extensions.DependencyInjection;
using System;
namespace BCG {
public class TCPConnectionTestStream : ViewStream, IWorker, ITCPConnectionHandler, ITCPMessageHandler {
public class TCPConnectionTestStream : ViewStream, IParameterizedWorker, ITCPConnectionHandler, ITCPMessageHandler {
class State {
public int n_sends = 0;
public int n_recvs = 0;
public int n_sends = 0;
public int n_recvs = 0;
}
private readonly string _ip;
private readonly int _port;
@@ -17,7 +21,8 @@ namespace BCG {
private TimeSpan _send_period;
private ITCPConnection _connection = null;
private byte[] _buff;
WorkActivator workActivator;
public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) {
_ip = ip;
_port = port;
@@ -31,16 +36,21 @@ namespace BCG {
for (int i = 0; i < _buff.Length; i++) {
_buff[i] = (byte)i;
}
workActivator = new(this);
}
TimeSpan IParameterizedWorker.Interval => _send_period;
protected override void OnServiceProvider(IServiceProvider serviceProvider) {
workActivator.OnWorkScheduler(serviceProvider.GetRequiredService<IWorkScheduler>());
}
protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, _send_period);
var server_rci = new RemoteConnectionInfo(_ip, _port);
bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
workActivator.Start();
}
protected override void myOnLastSubscriber() {
MasterWorker.getInstance(false).removeWorker(this);
workActivator.Stop();
}
public void doWork() {
@@ -56,18 +66,18 @@ namespace BCG {
}
public void onConnection(ITCPConnection connection) {
_connection = connection;
_connection.register(this);
_connection = connection;
_connection.register(this);
}
public bool isDisposed { get; }
public void onData(RemoteMessage msg) {
bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
_state.n_recvs++;
bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
_state.n_recvs++;
}
public void Dispose() {
workActivator.Stop();
}
@@ -2,6 +2,8 @@ using System;
using BCG.Utilities.General;
using Bts.DataStreamCore;
namespace BCG {
public abstract class TimeUnitStream : ViewStream, IWorker {
protected TimeSpan myOffset;
@@ -31,6 +33,9 @@ namespace BCG {
#endregion
protected virtual void myInit() {
if (!this.HasSubscribers()) {
return;
}
int periods = (int)
(DateTime.Now - myConstructionTime).TotalMilliseconds / (int)myOffset.TotalMilliseconds;
@@ -0,0 +1,8 @@
namespace Bts.DataStreamCore;
using BCG;
public static class DataStreamCoreExtensions {
public static bool HasSubscribers(this DataStream dataStream) =>
dataStream.numberOfSubscribers > 0;
}
@@ -156,7 +156,7 @@ namespace BCG
protected virtual void OnViewStreamDatabase(IWorkScheduler viewStreamDatabase) {
var w = this as IWorker;
if (w is not null) {
worker = viewStreamDatabase.Subscribe(w, (this as IParameterizedWorker)?.Interval);
worker = viewStreamDatabase.Subscribe(w);
}
}
@@ -7,35 +7,10 @@
"Password": ""
},
"MasterWorker": {
"Interval": "00:00:00.1"
"Interval": "00:00:01"
},
"Publisher": {
"Publishers": [
{
"Name": "market_data",
"Streams": [
{
"Name": "bbo",
"Expression": "bbo()",
"IsOnWebsocket": true
},
{
"Name": "instruments",
"Expression": "instruments()",
"IsOnWebsocket": true
},
{
"Name": "timesales",
"Expression": "timesales(10000)",
"IsOnWebsocket": true
},
{
"Name": "books",
"Expression": "books()",
"IsOnWebsocket": true
}
]
},
{
"Name": "test_data",
"Streams": [
@@ -43,16 +18,6 @@
"Name": "seconds_1_10",
"Expression": "second(1,10)",
"IsOnWebsocket": true
},
{
"Name": "test_literal",
"Expression": "literal(\"x\", \"420.0\")",
"IsOnWebsocket": true
},
{
"Name": "test_not_on_websocket",
"Expression": "literal(\"x\", \"840.0\")",
"IsOnWebsocket": false
}
]
}
@@ -118,7 +83,26 @@
"UsePrepublishedServerState": false,
"IpTranslation": null,
"GcActivationThreshold": 0,
"WaitPeriodAfterGc": 1000
"WaitPeriodAfterGc": 1000,
"TagHandlerFactory": {
"Handlers": [
{
"TagType": "DataStream",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerStreamTagHandler"
},
{
"TagType": "System_Sub",
"Assembly": "PubSub",
"Class": "BCG.SystemSubscriptionMapPublisher"
},
{
"TagType": "System_Stats",
"Assembly": "PubSub",
"Class": "BCG.SystemStatsPublisher"
}
]
}
},
"ViewStreamDatabase": {
"Interval": "00:00:01"
@@ -8,7 +8,9 @@ using bts.api.core;
using Bts.DataStreamCore;
using Bts.DataStreamExpressions;
using Bts.PubSub;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Options;
@@ -29,17 +31,20 @@ public class PublisherService : IHostedService {
readonly IPubTableBroker pubTableBroker;
readonly PrimitiveManager primitiveManager;
readonly IServiceProvider serviceProvider;
readonly IWorkScheduler workScheduler;
public PublisherService(
IPubTableBroker pubTableBroker,
PrimitiveManager primitiveManager,
IServiceProvider serviceProvider,
IOptions<PublisherServiceConfig> config
IOptions<PublisherServiceConfig> config,
[FromKeyedServices(WorkerGroups.Main)]IWorkScheduler workScheduler
) {
this.pubTableBroker = pubTableBroker;
this.primitiveManager = primitiveManager;
this.serviceProvider = serviceProvider;
this.config = config.Value;
this.workScheduler = workScheduler;
}
public Task StartAsync(CancellationToken cancellationToken) {
@@ -93,7 +98,9 @@ public class PublisherService : IHostedService {
foreach (var publisher in publishers) {
if (Directory.Exists(publisher.CsvDir)) {
csvPublishers.Add(new(publisher.CsvDir, pubTableBroker));
var csvPublisher = new CSVPublisher(publisher.CsvDir, pubTableBroker);
csvPublishers.Add(csvPublisher);
workScheduler.Add(csvPublisher);
}
}
@@ -122,7 +129,7 @@ public class PublisherService : IHostedService {
void on_change(object obj) =>
Log.Gen.info($"DSPublisher on_change dp: {obj}");
public class CSVPublisher : IWorker {
public class CSVPublisher : IParameterizedWorker {
private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
private readonly string _csv_directory;
private IPubTableBroker _pub_table_broker;
@@ -130,8 +137,12 @@ public class PublisherService : IHostedService {
public CSVPublisher(string csv_directory, IPubTableBroker pub_table_broker) {
_csv_directory = csv_directory;
_pub_table_broker = pub_table_broker;
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
}
string IParameterizedWorker.Group => WorkerGroups.Main;
TimeSpan IParameterizedWorker.Interval => TimeSpan.FromSeconds(1);
public void doWork() {
var file_names = Directory.GetFiles(_csv_directory);
foreach (var fn in file_names) {
@@ -2,8 +2,14 @@ using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
// using BCG.Utilities;
using BCG.Utilities.General;
using BCG.Utilities.General;
using Bts.PubSub;
using Bts.PubSub.Services;
using mbts;
namespace BCG
@@ -16,15 +22,58 @@ namespace BCG
/// </summary>
public class ThreadDrivenMasterWorker : MasterWorker
{
readonly object syncRoot = new();
readonly string? name;
readonly CancellationToken cancellationToken;
readonly TaskCompletionSource taskCompletionSource = new();
Thread? thread;
internal ThreadDrivenMasterWorker()
: base()
{ }
: this(new WorkSchedulerConfig() { Interval = TimeSpan.FromMilliseconds(100)}) { }
internal ThreadDrivenMasterWorker(string? name)
: this(new WorkSchedulerConfig() { Name = name, Interval = TimeSpan.FromMilliseconds(100) }) { }
internal ThreadDrivenMasterWorker(WorkSchedulerConfig config) : this(config, CancellationToken.None) { }
internal ThreadDrivenMasterWorker(WorkSchedulerConfig config, CancellationToken cancellationToken) : base(config.Interval) {
name = config.Name;
this.cancellationToken = cancellationToken;
}
public static ThreadDrivenMasterWorker StartNew(string? name) {
var rv = new ThreadDrivenMasterWorker(name);
rv.Start();
return rv;
}
public static ThreadDrivenMasterWorker StartNew(IWorker worker, TimeSpan interval) {
var rv = new ThreadDrivenMasterWorker();
rv.addWorker(worker, interval);
rv.Start();
return rv;
}
public Task StartAsync() {
Start();
return taskCompletionSource.Task;
}
protected override void myInit()
{
Thread doWorkThread = new Thread(new ThreadStart(myDoWork));
doWorkThread.IsBackground = true;
doWorkThread.Start();
if (thread is not null) {
// Already started
return;
}
lock (syncRoot) {
if (thread is null) {
thread = new(myDoWork);
if (name is not null) {
thread.Name = name;
}
thread.Start();
}
}
}
public override void addWorker(IWorker worker, TimeSpan period)
@@ -50,7 +99,7 @@ namespace BCG
private void myDoWork()
{
TimeSpan sleepTime = new TimeSpan(myPeriod.Ticks / 2);
while (true)
while (!cancellationToken.IsCancellationRequested)
{
try
{
@@ -76,11 +125,10 @@ namespace BCG
}
catch (Exception ex)
{
// BlueEventLog.logEvent(ex);
if (ex is OutOfMemoryException)
throw ex;
taskCompletionSource.SetException(ex);
}
}
taskCompletionSource.SetResult();
}
}
@@ -1,78 +0,0 @@
using System.Net.Sockets;
using System.Net;
using BCG;
using System;
using System.Collections.Generic;
namespace bts {
// opens a TCP port and starts listening to connections
public class AsyncTCPConnectionAcceptor_APM {
private readonly int _port;
private Socket _accept_socket;
private ITCPConnectionHandler _connection_handler;
private HashSet<string> _key_map = new HashSet<string>();
public AsyncTCPConnectionAcceptor_APM(int port, ITCPConnectionHandler connection_handler) {
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM. cons started on port: {port}");
_port = port;
_connection_handler = connection_handler;
}
public void init() {
_init();
_begin_accepting();
}
private void _init() {
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._init started on port: {_port}");
_accept_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
TCPSocketUtils.set_server_socket_options(ref _accept_socket);
IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, _port);
_accept_socket.Bind(endpoint);
_accept_socket.Listen(250);
}
private void _begin_accepting() {
var accept_key = Guid.NewGuid().ToString();
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._begin_accepting port: {_port} accept_key: {accept_key}");
var res = _accept_socket.BeginAccept(new AsyncCallback(_on_accept), accept_key);
if (res != null && res.IsCompleted && res.CompletedSynchronously)
_on_accept(res);
}
private void _on_accept(IAsyncResult res) {
var log_msg = $"AsyncTCPConnectionAcceptor_APM._on_accept state: {res.AsyncState}";
Log.Gen.info(log_msg);
Console.WriteLine(log_msg);
string key = res.AsyncState.ToString();
if (_key_map.Contains(key)) {
return;
}
_key_map.Add(key);
Socket new_sock;
try {
new_sock = _accept_socket.EndAccept(res);
} catch (Exception ex) {
bts.Log.Gen.info("AsyncTCPConnectionAcceptor_APM._on_accept exception thrown by EndAccept ex: {ex}");
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(250));
_begin_accepting();
return;
}
new_sock.LingerState = new LingerOption(false, 0);
new_sock.Blocking = false;
ITCPConnection newConn = new AsyncTCPConnection_APM(new_sock, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Server);
_connection_handler.onConnection(newConn);
// go back to accepting more conns
_begin_accepting();
var peer_rci = newConn.get_peer_address();
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._on_accept peer_rci: {peer_rci}");
}
}
}
@@ -15,8 +15,8 @@ namespace BCG {
private DateTime myLastDoWorkTime = DateTime.MinValue,
myLastLogTime = DateTime.MinValue;
protected TimeSpan myPeriod = new TimeSpan(0, 0, 0, 0, 100),
myLogPeriod = new TimeSpan(0, 1, 0);
protected readonly TimeSpan myPeriod;
protected readonly TimeSpan myLogPeriod = TimeSpan.FromMinutes(1);
protected Dictionary<IWorker, Pair<TimeSpan, DateTime>> myWorkers = new Dictionary<IWorker, Pair<TimeSpan, DateTime>>(),
myJustAdded = new Dictionary<IWorker, Pair<TimeSpan, DateTime>>(),
myJustRemoved = new Dictionary<IWorker, Pair<TimeSpan, DateTime>>();
@@ -24,7 +24,18 @@ namespace BCG {
private static MasterWorker myThreadDriven = null,
myDoWorkDriven = null;
internal MasterWorker() {
internal MasterWorker() : this(TimeSpan.FromMilliseconds(100)) { }
internal MasterWorker(TimeSpan interval) {
myPeriod = interval;
}
public static MasterWorker StartNew() {
var rv = new MasterWorker();
rv.Start();
return rv;
}
public void Start() {
myInit();
}
@@ -34,11 +45,10 @@ namespace BCG {
lock (typeof(MasterWorker)) {
if (threadDriven) {
// TODO
myThreadDriven = myThreadDriven ?? new ThreadDrivenMasterWorker();
myThreadDriven ??= ThreadDrivenMasterWorker.StartNew();
return myThreadDriven;
return null;
} else {
myDoWorkDriven = myDoWorkDriven ?? new MasterWorker();
myDoWorkDriven ??= StartNew();
return myDoWorkDriven;
}
}
@@ -0,0 +1,6 @@
namespace Bts.PubSub;
public static class WorkerGroups {
public const string Main = "Main";
public const string Background = "Background";
}
@@ -48,7 +48,8 @@ namespace BCG {
this.publisher = publisher;
}
public TimeSpan Interval => TimeSpan.FromMilliseconds(500);
string IParameterizedWorker.Group => WorkerGroups.Background;
TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(500);
#region IWorker Members
@@ -39,8 +39,8 @@ namespace BCG {
public string TagType => tag_type;
public string Tag => tag_name;
TimeSpan IParameterizedWorker.Interval =>
TimeSpan.FromMilliseconds(250);
string IParameterizedWorker.Group => WorkerGroups.Main;
TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(250);
internal static SystemStatsPublisher get_instance() {
return Log.notnull(instance);
@@ -23,7 +23,6 @@ namespace BCG {
private Thread _msg_polling_thread;
// all the data,list of subscribers etc are in the state
private IPubSubState _state;
private MasterWorker _mainloop_master_worker = MasterWorker.getInstance(false);
// private static SyncQueue<PublishMsg> _internal_pub_queue = new SyncQueue<PublishMsg>();
private static IInternalPublisherQueue? _internal_pub_queue;
static int instanceCount = 0;
@@ -109,7 +108,6 @@ namespace BCG {
_handle_client_msg(kv.first, kv.second);
}
handle_internal_pubs();
_mainloop_master_worker.doWork();
} catch (Exception ex) {
bts.Log.Gen.error($"PubSubMessageReactor._polling_thread_func ex: {ex}");
if (ex is OutOfMemoryException) {
@@ -0,0 +1,70 @@
namespace Bts.PubSub.Services;
using bts;
using Microsoft.Extensions.Hosting;
using System;
using System.Threading;
using System.Threading.Tasks;
/// <summary>
/// An <see cref="IHostedService"/> that runs on a background thread.
/// </summary>
public abstract class BackgroundHostedService : IHostedService {
readonly TaskCompletionSource tcs = new();
readonly CancellationTokenSource cancellationTokenSource = new();
readonly object syncRoot = new();
Thread? thread;
State state;
protected BackgroundHostedService() { }
enum State {
Idle,
Starting,
Running,
Stopping,
}
string name => GetType().Name;
public Task StartAsync(CancellationToken cancellationToken) {
Log.isnull(thread);
lock (syncRoot) {
Log.assert(state == State.Idle);
state = State.Starting;
}
Log.Gen.info($"Starting {name}");
thread = new Thread(run) {
Name = name
};
thread.Start();
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken cancellationToken) {
Log.Gen.info($"Stopping {name}...");
cancellationTokenSource.Cancel();
return tcs.Task;
}
protected abstract void Run(CancellationToken cancellationToken);
void run() {
Log.Gen.info($"{GetType()} STARTED.");
lock (syncRoot) {
state = State.Running;
}
try {
Run(cancellationTokenSource.Token);
Log.Gen.error($"{name} '{nameof(Run)}' finished");
} catch (Exception ex) {
Log.Gen.error($"{name} '{nameof(Run)}' threw an exception: {ex}");
}
lock (syncRoot) {
state = State.Idle;
}
tcs.SetResult();
}
}
@@ -1,7 +1,8 @@
namespace Bts.PubSub.Services;
using System;
using System.Collections.Generic;
public class MasterWorkerConfig {
public TimeSpan Interval { get; set; } = TimeSpan.FromMilliseconds(100);
public List<WorkSchedulerConfig>? WorkSchedulers { get; set; }
}
@@ -9,7 +9,9 @@ using bts.collections;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Options;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Nodes;
@@ -20,54 +22,57 @@ using System.Threading.Tasks;
/// An <see cref="IHostedService"/> that imports a collection of <see cref="IWorker"/>s and
/// calls each <see cref="IWorker.doWork"/> in a loop.
/// </summary>
public class MasterWorkerService : IHostedService {
public class MasterWorkerService : BackgroundHostedService, IWorkScheduler {
readonly TaskCompletionSource tcs = new();
readonly CancellationTokenSource cancellationTokenSource = new();
readonly List<IWorker> workers;
readonly MasterWorkerConfig config;
MasterWorker? masterWorker;
Dictionary<string, IWorkScheduler>? backgroundMasterWorkers;
readonly object syncRoot = new();
readonly Dictionary<string, ThreadDrivenMasterWorker> backgroundWorkers = new();
public MasterWorkerService(IEnumerable<IWorker> workers, IOptions<MasterWorkerConfig> config) {
this.workers = workers.ToList();
this.config = config.Value;
Log.notnull(this.config.WorkSchedulers);
Log.assert(this.config.WorkSchedulers.Count == 2);
Log.assert(this.config.WorkSchedulers[0].Name == WorkerGroups.Main);
Log.assert(this.config.WorkSchedulers[1].Name == WorkerGroups.Background);
foreach (var w in this.config.WorkSchedulers) {
Log.notnull(w.Name);
backgroundWorkers.Add(w.Name, new(w.Name));
}
}
public Task StartAsync(CancellationToken cancellationToken) {
enum State {
Idle,
Working
}
public void Add(IWorker worker, TimeSpan interval) {
var group = getWorkerGroup(worker);
var scheduler = getOrAddScheduler(group);
scheduler.Add(worker, interval);
}
public void Remove(IWorker worker) {
var group = getWorkerGroup(worker);
var scheduler = getScheduler(group);
Log.notnull(scheduler).Remove(worker);
}
protected override void Run(CancellationToken cancellationToken) {
var workersDesc = JsonSerializer.Serialize(workersToJson(workers));
Log.Gen.info($"Starting MasterWorkerService. {workersDesc}");
foreach (var w in workers) {
GetWorkScheduler((w as IParameterizedWorker)?.Group).Add(w);
}
new Thread(workLoop).Start();
return Task.CompletedTask;
IWorkScheduler GetWorkScheduler(string? key) =>
key is null
? (masterWorker ??= new())
: (backgroundMasterWorkers ??= new()).GetOrAdd(key, _ => new ThreadDrivenMasterWorker());
}
public Task StopAsync(CancellationToken cancellationToken) {
Log.Gen.info("Stopping MasterWorkerService...");
cancellationTokenSource.Cancel();
return tcs.Task;
}
void workLoop() {
Log.Gen.info("MasterWorkerService STARTED.");
if (masterWorker is null) {
Log.Gen.warn("MasterWorkerService STOPPED early because there is no work to do.");
return;
}
while (!cancellationTokenSource.IsCancellationRequested) {
masterWorker?.doWork();
Thread.Sleep(config.Interval);
foreach (var workScheduler in backgroundWorkers.Values) {
workScheduler.Start();
}
Log.Gen.info("MasterWorkerService STOPPED.");
tcs.SetResult();
}
static string getWorkerGroup(IWorker worker) =>
(worker as IParameterizedWorker)?.Group ?? WorkerGroups.Main;
static JsonObject workersToJson(IEnumerable<IWorker> workers) {
var rootJson = new JsonObject();
foreach (var wg in workers.GroupBy(w => (w as IParameterizedWorker)?.Group ?? "Main")) {
@@ -77,4 +82,35 @@ public class MasterWorkerService : IHostedService {
}
return rootJson;
}
IWorkScheduler? getScheduler(string group) {
lock (syncRoot) {
if (backgroundWorkers.TryGetValue(group, out var bgw)) {
return bgw;
}
}
return null;
}
IWorkScheduler getOrAddScheduler(IWorker worker) =>
getOrAddScheduler((worker as IParameterizedWorker)?.Group ?? WorkerGroups.Main);
IWorkScheduler getOrAddScheduler(string group) {
var (bgWorker, isNew) = getOrAddBackgroundWorker(group);
if (isNew) {
bgWorker.Start();
}
return bgWorker;
}
(ThreadDrivenMasterWorker worker, bool isNew) getOrAddBackgroundWorker(string group) {
lock (syncRoot) {
if (backgroundWorkers.TryGetValue(group, out var bgWorker)) {
return (bgWorker, false);
}
bgWorker = new(group);
backgroundWorkers.Add(group, bgWorker);
return (bgWorker, true);
}
}
}
@@ -65,6 +65,7 @@ public static class PubSubServices {
services
.Configure<PubSubConfig>(configurationSection)
.AddInternalPublisher()
.AddPubSubTagHandlerFactory(configurationSection.GetSection(nameof(PubSubConfig.TagHandlerFactory)))
.AddSingleton<IPubSubTagHandlerFactory, PubsubTagHandlerFactory>()
.AddSingleton<IPubSubRouter, PubSubRouter>()
.AddPublisherCore()
@@ -0,0 +1,8 @@
namespace Bts.PubSub.Services;
using System;
public class WorkSchedulerConfig {
public string? Name { get; set; }
public TimeSpan Interval { get; set; } = TimeSpan.FromMilliseconds(100);
}
@@ -0,0 +1,22 @@
namespace Bts.PubSub.Services;
using BCG;
using System.Collections.Generic;
public class WorkSchedulerProvider {
readonly Dictionary<string, IWorkScheduler> workSchedulers = new();
readonly object syncRoot = new();
public IWorkScheduler GetScheduler(string? group) {
group ??= string.Empty;
lock (syncRoot) {
return workSchedulers.GetOrAdd(
group,
g => string.IsNullOrEmpty(g)
? new MasterWorker()
: new ThreadDrivenMasterWorker());
}
}
}
@@ -0,0 +1,123 @@
namespace Bts.PubSub;
using BCG;
using BCG.Utilities.General;
using bts;
using bts.collections;
using Bts.PubSub.Services;
using Microsoft.Extensions.Options;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Threading;
public class WorkSchedulerService : BackgroundHostedService, IWorkScheduler {
readonly List<IWorker> workers;
readonly MasterWorkerConfig config;
readonly IWorkScheduler mainWorker = new MasterWorker();
readonly object syncRoot = new();
ImmutableDictionary<string, IWorkScheduler> backgroundWorkers = ImmutableDictionary<string, IWorkScheduler>.Empty;
public WorkSchedulerService(IEnumerable<IWorker> workers, IOptions<MasterWorkerConfig> config) {
this.workers = workers.ToList();
this.config = config.Value;
}
public void Add(IWorker worker, TimeSpan interval) {
// getOrAddScheduler(worker).Add(worker, interval);
var group = (worker as IParameterizedWorker)?.Group;
if (string.IsNullOrEmpty(group)) {
lock (syncRoot) {
mainWorker.Add(worker, interval);
}
return;
}
if (backgroundWorkers.TryGetValue(group, out var bgw)) {
bgw.Add(worker, interval);
}
bgw = ThreadDrivenMasterWorker.StartNew(worker, interval);
lock (syncRoot) {
backgroundWorkers = backgroundWorkers.Add(group, bgw);
}
}
public void Remove(IWorker worker) {
var group = (worker as IParameterizedWorker)?.Group;
if (string.IsNullOrEmpty(group)) {
lock (syncRoot) {
mainWorker.Remove(worker);
}
return;
}
if (backgroundWorkers.TryGetValue(group, out var bgw)) {
bgw.Remove(worker);
}
var scheduler = getScheduler(worker);
Log.notnull(scheduler).Remove(worker);
}
protected override void OnStart(CancellationToken cancellationToken) {
var workersDesc = JsonSerializer.Serialize(workersToJson(workers));
Log.Gen.info($"Starting MasterWorkerService. {workersDesc}");
foreach (var w in workers) {
getOrAddScheduler(w).Add(w);
}
}
static JsonObject workersToJson(IEnumerable<IWorker> workers) {
var rootJson = new JsonObject();
foreach (var wg in workers.GroupBy(w => (w as IParameterizedWorker)?.Group ?? "Main")) {
var wgJson = new JsonArray();
wgJson.AddRange(wg.Select(w => JsonValue.Create(w.ToString())));
rootJson.Add(wg.Key, wgJson);
}
return rootJson;
}
IWorkScheduler? getScheduler(IWorker worker) =>
getScheduler((worker as IParameterizedWorker)?.Group);
IWorkScheduler? getScheduler(string? group) {
if (string.IsNullOrEmpty(group)) {
return mainWorker;
}
lock (syncRoot) {
if (backgroundWorkers.TryGetValue(group, out var bgw)) {
return bgw;
}
}
return null;
}
IWorkScheduler getOrAddScheduler(IWorker worker) =>
getOrAddScheduler((worker as IParameterizedWorker)?.Group);
IWorkScheduler getOrAddScheduler(string? group) {
if (string.IsNullOrEmpty(group)) {
return mainWorker;
}
lock (syncRoot) {
if (backgroundWorkers.TryGetValue(group, out var bgw)) {
return bgw;
}
bgw = new();
backgroundWorkers = backgroundWorkers.Add(group, bgw);
bgw.Start();
return bgw;
}
}
protected override void Run(CancellationToken cancellationToken) {
while (!cancellationToken.IsCancellationRequested) {
mainWorker.doWork();
Thread.Sleep(config.Interval);
}
}
}
+115
View File
@@ -0,0 +1,115 @@
#nullable enable
namespace Bts.PubSub;
using BCG.Utilities.General;
using bts;
using System;
public class WorkActivator : IDisposable {
readonly object syncRoot = new();
readonly IWorker worker;
IWorkScheduler? workScheduler;
State state = State.Idle;
public WorkActivator(IWorker worker) {
this.worker = worker;
}
enum State {
Idle,
WaitingToWork,
Working
}
public void OnWorkScheduler(IWorkScheduler workScheduler) {
lock (syncRoot) {
if (workScheduler == this.workScheduler) {
return;
}
switch (state) {
case State.Idle:
this.workScheduler = workScheduler;
break;
case State.WaitingToWork:
/// We only transition <see cref="State.WaitingToWork"/> when <see cref="Start"/> is
/// called and <see cref="worker"/> is null.
Log.isnull(this.workScheduler);
this.workScheduler = workScheduler;
if (workScheduler is not null) {
state = State.Working;
workScheduler.Add(worker);
}
break;
case State.Working:
Log.notnull(this.workScheduler).Remove(worker);
this.workScheduler = workScheduler;
if (workScheduler is not null) {
workScheduler.Add(worker);
} else {
state = State.WaitingToWork;
}
break;
}
}
}
public void Start() {
lock (syncRoot) {
switch (state) {
case State.Idle:
if (workScheduler is null) {
state = State.WaitingToWork;
} else {
state = State.Working;
workScheduler.Add(worker);
}
break;
case State.WaitingToWork:
// Sanity check - "WaitingToWork" means we are waiting for a non-null IWorkScheduler.
// It follows that if we're "WaitingToWork", then our IWorkScheduler should be null.
Log.isnull(workScheduler);
break;
case State.Working:
Log.notnull(workScheduler);
break;
}
}
}
public void Stop() {
lock (syncRoot) {
switch (state) {
case State.Idle:
break;
case State.WaitingToWork:
state = State.Idle;
break;
case State.Working:
Log.notnull(workScheduler).Remove(worker);
state = State.Idle;
break;
}
}
}
public void Dispose() {
lock (syncRoot) {
stopImpl();
workScheduler = null;
}
}
void stopImpl() {
if (workScheduler is not null) {
switch (state) {
case State.Working:
workScheduler.Remove(worker);
break;
}
}
state = State.Idle;
}
}
@@ -1,6 +1,6 @@
{
"MasterWorker": {
"Interval": "00:00:00.1"
"Interval": "00:00:01"
},
"SparkLogging": {
"ConfigServer": "http://localhost:8000/config"