check pt
This commit is contained in:
1 parent
d54b0437b0
commit
554ceea2ee
22 files changed
+566
-187
No files matched your search
@@ -1,12 +1,16 @@
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using Bts.PubSub;
|
||||
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
using System;
|
||||
|
||||
namespace BCG {
|
||||
public class TCPConnectionTestStream : ViewStream, IWorker, ITCPConnectionHandler, ITCPMessageHandler {
|
||||
public class TCPConnectionTestStream : ViewStream, IParameterizedWorker, ITCPConnectionHandler, ITCPMessageHandler {
|
||||
class State {
|
||||
public int n_sends = 0;
|
||||
public int n_recvs = 0;
|
||||
public int n_sends = 0;
|
||||
public int n_recvs = 0;
|
||||
}
|
||||
private readonly string _ip;
|
||||
private readonly int _port;
|
||||
@@ -17,7 +21,8 @@ namespace BCG {
|
||||
private TimeSpan _send_period;
|
||||
private ITCPConnection _connection = null;
|
||||
private byte[] _buff;
|
||||
|
||||
WorkActivator workActivator;
|
||||
|
||||
public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) {
|
||||
_ip = ip;
|
||||
_port = port;
|
||||
@@ -31,16 +36,21 @@ namespace BCG {
|
||||
for (int i = 0; i < _buff.Length; i++) {
|
||||
_buff[i] = (byte)i;
|
||||
}
|
||||
workActivator = new(this);
|
||||
}
|
||||
|
||||
TimeSpan IParameterizedWorker.Interval => _send_period;
|
||||
|
||||
protected override void OnServiceProvider(IServiceProvider serviceProvider) {
|
||||
workActivator.OnWorkScheduler(serviceProvider.GetRequiredService<IWorkScheduler>());
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
MasterWorker.getInstance(false).addWorker(this, _send_period);
|
||||
var server_rci = new RemoteConnectionInfo(_ip, _port);
|
||||
bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
|
||||
workActivator.Start();
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
MasterWorker.getInstance(false).removeWorker(this);
|
||||
workActivator.Stop();
|
||||
}
|
||||
|
||||
public void doWork() {
|
||||
@@ -56,18 +66,18 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public void onConnection(ITCPConnection connection) {
|
||||
_connection = connection;
|
||||
_connection.register(this);
|
||||
_connection = connection;
|
||||
_connection.register(this);
|
||||
}
|
||||
public bool isDisposed { get; }
|
||||
|
||||
public void onData(RemoteMessage msg) {
|
||||
bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
|
||||
_state.n_recvs++;
|
||||
bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
|
||||
_state.n_recvs++;
|
||||
}
|
||||
|
||||
public void Dispose() {
|
||||
|
||||
workActivator.Stop();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@ using System;
|
||||
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
namespace BCG {
|
||||
public abstract class TimeUnitStream : ViewStream, IWorker {
|
||||
protected TimeSpan myOffset;
|
||||
@@ -31,6 +33,9 @@ namespace BCG {
|
||||
#endregion
|
||||
|
||||
protected virtual void myInit() {
|
||||
if (!this.HasSubscribers()) {
|
||||
return;
|
||||
}
|
||||
int periods = (int)
|
||||
(DateTime.Now - myConstructionTime).TotalMilliseconds / (int)myOffset.TotalMilliseconds;
|
||||
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace Bts.DataStreamCore;
|
||||
|
||||
using BCG;
|
||||
|
||||
public static class DataStreamCoreExtensions {
|
||||
public static bool HasSubscribers(this DataStream dataStream) =>
|
||||
dataStream.numberOfSubscribers > 0;
|
||||
}
|
||||
@@ -156,7 +156,7 @@ namespace BCG
|
||||
protected virtual void OnViewStreamDatabase(IWorkScheduler viewStreamDatabase) {
|
||||
var w = this as IWorker;
|
||||
if (w is not null) {
|
||||
worker = viewStreamDatabase.Subscribe(w, (this as IParameterizedWorker)?.Interval);
|
||||
worker = viewStreamDatabase.Subscribe(w);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,35 +7,10 @@
|
||||
"Password": ""
|
||||
},
|
||||
"MasterWorker": {
|
||||
"Interval": "00:00:00.1"
|
||||
"Interval": "00:00:01"
|
||||
},
|
||||
"Publisher": {
|
||||
"Publishers": [
|
||||
{
|
||||
"Name": "market_data",
|
||||
"Streams": [
|
||||
{
|
||||
"Name": "bbo",
|
||||
"Expression": "bbo()",
|
||||
"IsOnWebsocket": true
|
||||
},
|
||||
{
|
||||
"Name": "instruments",
|
||||
"Expression": "instruments()",
|
||||
"IsOnWebsocket": true
|
||||
},
|
||||
{
|
||||
"Name": "timesales",
|
||||
"Expression": "timesales(10000)",
|
||||
"IsOnWebsocket": true
|
||||
},
|
||||
{
|
||||
"Name": "books",
|
||||
"Expression": "books()",
|
||||
"IsOnWebsocket": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Name": "test_data",
|
||||
"Streams": [
|
||||
@@ -43,16 +18,6 @@
|
||||
"Name": "seconds_1_10",
|
||||
"Expression": "second(1,10)",
|
||||
"IsOnWebsocket": true
|
||||
},
|
||||
{
|
||||
"Name": "test_literal",
|
||||
"Expression": "literal(\"x\", \"420.0\")",
|
||||
"IsOnWebsocket": true
|
||||
},
|
||||
{
|
||||
"Name": "test_not_on_websocket",
|
||||
"Expression": "literal(\"x\", \"840.0\")",
|
||||
"IsOnWebsocket": false
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -118,7 +83,26 @@
|
||||
"UsePrepublishedServerState": false,
|
||||
"IpTranslation": null,
|
||||
"GcActivationThreshold": 0,
|
||||
"WaitPeriodAfterGc": 1000
|
||||
"WaitPeriodAfterGc": 1000,
|
||||
"TagHandlerFactory": {
|
||||
"Handlers": [
|
||||
{
|
||||
"TagType": "DataStream",
|
||||
"Assembly": "DataStreamCore",
|
||||
"Class": "bts.PubsubServerStreamTagHandler"
|
||||
},
|
||||
{
|
||||
"TagType": "System_Sub",
|
||||
"Assembly": "PubSub",
|
||||
"Class": "BCG.SystemSubscriptionMapPublisher"
|
||||
},
|
||||
{
|
||||
"TagType": "System_Stats",
|
||||
"Assembly": "PubSub",
|
||||
"Class": "BCG.SystemStatsPublisher"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"ViewStreamDatabase": {
|
||||
"Interval": "00:00:01"
|
||||
|
||||
@@ -8,7 +8,9 @@ using bts.api.core;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
using Bts.DataStreamExpressions;
|
||||
using Bts.PubSub;
|
||||
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
@@ -29,17 +31,20 @@ public class PublisherService : IHostedService {
|
||||
readonly IPubTableBroker pubTableBroker;
|
||||
readonly PrimitiveManager primitiveManager;
|
||||
readonly IServiceProvider serviceProvider;
|
||||
readonly IWorkScheduler workScheduler;
|
||||
|
||||
public PublisherService(
|
||||
IPubTableBroker pubTableBroker,
|
||||
PrimitiveManager primitiveManager,
|
||||
IServiceProvider serviceProvider,
|
||||
IOptions<PublisherServiceConfig> config
|
||||
IOptions<PublisherServiceConfig> config,
|
||||
[FromKeyedServices(WorkerGroups.Main)]IWorkScheduler workScheduler
|
||||
) {
|
||||
this.pubTableBroker = pubTableBroker;
|
||||
this.primitiveManager = primitiveManager;
|
||||
this.serviceProvider = serviceProvider;
|
||||
this.config = config.Value;
|
||||
this.workScheduler = workScheduler;
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken) {
|
||||
@@ -93,7 +98,9 @@ public class PublisherService : IHostedService {
|
||||
|
||||
foreach (var publisher in publishers) {
|
||||
if (Directory.Exists(publisher.CsvDir)) {
|
||||
csvPublishers.Add(new(publisher.CsvDir, pubTableBroker));
|
||||
var csvPublisher = new CSVPublisher(publisher.CsvDir, pubTableBroker);
|
||||
csvPublishers.Add(csvPublisher);
|
||||
workScheduler.Add(csvPublisher);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -122,7 +129,7 @@ public class PublisherService : IHostedService {
|
||||
void on_change(object obj) =>
|
||||
Log.Gen.info($"DSPublisher on_change dp: {obj}");
|
||||
|
||||
public class CSVPublisher : IWorker {
|
||||
public class CSVPublisher : IParameterizedWorker {
|
||||
private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
|
||||
private readonly string _csv_directory;
|
||||
private IPubTableBroker _pub_table_broker;
|
||||
@@ -130,8 +137,12 @@ public class PublisherService : IHostedService {
|
||||
public CSVPublisher(string csv_directory, IPubTableBroker pub_table_broker) {
|
||||
_csv_directory = csv_directory;
|
||||
_pub_table_broker = pub_table_broker;
|
||||
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
|
||||
string IParameterizedWorker.Group => WorkerGroups.Main;
|
||||
|
||||
TimeSpan IParameterizedWorker.Interval => TimeSpan.FromSeconds(1);
|
||||
|
||||
public void doWork() {
|
||||
var file_names = Directory.GetFiles(_csv_directory);
|
||||
foreach (var fn in file_names) {
|
||||
|
||||
@@ -2,8 +2,14 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
// using BCG.Utilities;
|
||||
using BCG.Utilities.General;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using Bts.PubSub;
|
||||
using Bts.PubSub.Services;
|
||||
|
||||
using mbts;
|
||||
|
||||
namespace BCG
|
||||
@@ -16,15 +22,58 @@ namespace BCG
|
||||
/// </summary>
|
||||
public class ThreadDrivenMasterWorker : MasterWorker
|
||||
{
|
||||
readonly object syncRoot = new();
|
||||
readonly string? name;
|
||||
readonly CancellationToken cancellationToken;
|
||||
readonly TaskCompletionSource taskCompletionSource = new();
|
||||
Thread? thread;
|
||||
|
||||
internal ThreadDrivenMasterWorker()
|
||||
: base()
|
||||
{ }
|
||||
: this(new WorkSchedulerConfig() { Interval = TimeSpan.FromMilliseconds(100)}) { }
|
||||
|
||||
internal ThreadDrivenMasterWorker(string? name)
|
||||
: this(new WorkSchedulerConfig() { Name = name, Interval = TimeSpan.FromMilliseconds(100) }) { }
|
||||
|
||||
internal ThreadDrivenMasterWorker(WorkSchedulerConfig config) : this(config, CancellationToken.None) { }
|
||||
|
||||
internal ThreadDrivenMasterWorker(WorkSchedulerConfig config, CancellationToken cancellationToken) : base(config.Interval) {
|
||||
name = config.Name;
|
||||
this.cancellationToken = cancellationToken;
|
||||
}
|
||||
|
||||
public static ThreadDrivenMasterWorker StartNew(string? name) {
|
||||
var rv = new ThreadDrivenMasterWorker(name);
|
||||
rv.Start();
|
||||
return rv;
|
||||
}
|
||||
|
||||
public static ThreadDrivenMasterWorker StartNew(IWorker worker, TimeSpan interval) {
|
||||
var rv = new ThreadDrivenMasterWorker();
|
||||
rv.addWorker(worker, interval);
|
||||
rv.Start();
|
||||
return rv;
|
||||
}
|
||||
|
||||
public Task StartAsync() {
|
||||
Start();
|
||||
return taskCompletionSource.Task;
|
||||
}
|
||||
|
||||
protected override void myInit()
|
||||
{
|
||||
Thread doWorkThread = new Thread(new ThreadStart(myDoWork));
|
||||
doWorkThread.IsBackground = true;
|
||||
doWorkThread.Start();
|
||||
if (thread is not null) {
|
||||
// Already started
|
||||
return;
|
||||
}
|
||||
lock (syncRoot) {
|
||||
if (thread is null) {
|
||||
thread = new(myDoWork);
|
||||
if (name is not null) {
|
||||
thread.Name = name;
|
||||
}
|
||||
thread.Start();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public override void addWorker(IWorker worker, TimeSpan period)
|
||||
@@ -50,7 +99,7 @@ namespace BCG
|
||||
private void myDoWork()
|
||||
{
|
||||
TimeSpan sleepTime = new TimeSpan(myPeriod.Ticks / 2);
|
||||
while (true)
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -76,11 +125,10 @@ namespace BCG
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// BlueEventLog.logEvent(ex);
|
||||
if (ex is OutOfMemoryException)
|
||||
throw ex;
|
||||
taskCompletionSource.SetException(ex);
|
||||
}
|
||||
}
|
||||
taskCompletionSource.SetResult();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
using System.Net.Sockets;
|
||||
using System.Net;
|
||||
|
||||
using BCG;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
|
||||
namespace bts {
|
||||
|
||||
// opens a TCP port and starts listening to connections
|
||||
public class AsyncTCPConnectionAcceptor_APM {
|
||||
private readonly int _port;
|
||||
private Socket _accept_socket;
|
||||
private ITCPConnectionHandler _connection_handler;
|
||||
private HashSet<string> _key_map = new HashSet<string>();
|
||||
public AsyncTCPConnectionAcceptor_APM(int port, ITCPConnectionHandler connection_handler) {
|
||||
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM. cons started on port: {port}");
|
||||
_port = port;
|
||||
_connection_handler = connection_handler;
|
||||
}
|
||||
|
||||
public void init() {
|
||||
_init();
|
||||
_begin_accepting();
|
||||
}
|
||||
|
||||
private void _init() {
|
||||
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._init started on port: {_port}");
|
||||
_accept_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
TCPSocketUtils.set_server_socket_options(ref _accept_socket);
|
||||
IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, _port);
|
||||
_accept_socket.Bind(endpoint);
|
||||
_accept_socket.Listen(250);
|
||||
}
|
||||
|
||||
private void _begin_accepting() {
|
||||
var accept_key = Guid.NewGuid().ToString();
|
||||
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._begin_accepting port: {_port} accept_key: {accept_key}");
|
||||
var res = _accept_socket.BeginAccept(new AsyncCallback(_on_accept), accept_key);
|
||||
if (res != null && res.IsCompleted && res.CompletedSynchronously)
|
||||
_on_accept(res);
|
||||
}
|
||||
|
||||
private void _on_accept(IAsyncResult res) {
|
||||
var log_msg = $"AsyncTCPConnectionAcceptor_APM._on_accept state: {res.AsyncState}";
|
||||
Log.Gen.info(log_msg);
|
||||
Console.WriteLine(log_msg);
|
||||
string key = res.AsyncState.ToString();
|
||||
if (_key_map.Contains(key)) {
|
||||
return;
|
||||
}
|
||||
_key_map.Add(key);
|
||||
Socket new_sock;
|
||||
try {
|
||||
new_sock = _accept_socket.EndAccept(res);
|
||||
} catch (Exception ex) {
|
||||
bts.Log.Gen.info("AsyncTCPConnectionAcceptor_APM._on_accept exception thrown by EndAccept ex: {ex}");
|
||||
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(250));
|
||||
_begin_accepting();
|
||||
return;
|
||||
}
|
||||
new_sock.LingerState = new LingerOption(false, 0);
|
||||
new_sock.Blocking = false;
|
||||
ITCPConnection newConn = new AsyncTCPConnection_APM(new_sock, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Server);
|
||||
_connection_handler.onConnection(newConn);
|
||||
// go back to accepting more conns
|
||||
_begin_accepting();
|
||||
var peer_rci = newConn.get_peer_address();
|
||||
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._on_accept peer_rci: {peer_rci}");
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -15,8 +15,8 @@ namespace BCG {
|
||||
|
||||
private DateTime myLastDoWorkTime = DateTime.MinValue,
|
||||
myLastLogTime = DateTime.MinValue;
|
||||
protected TimeSpan myPeriod = new TimeSpan(0, 0, 0, 0, 100),
|
||||
myLogPeriod = new TimeSpan(0, 1, 0);
|
||||
protected readonly TimeSpan myPeriod;
|
||||
protected readonly TimeSpan myLogPeriod = TimeSpan.FromMinutes(1);
|
||||
protected Dictionary<IWorker, Pair<TimeSpan, DateTime>> myWorkers = new Dictionary<IWorker, Pair<TimeSpan, DateTime>>(),
|
||||
myJustAdded = new Dictionary<IWorker, Pair<TimeSpan, DateTime>>(),
|
||||
myJustRemoved = new Dictionary<IWorker, Pair<TimeSpan, DateTime>>();
|
||||
@@ -24,7 +24,18 @@ namespace BCG {
|
||||
private static MasterWorker myThreadDriven = null,
|
||||
myDoWorkDriven = null;
|
||||
|
||||
internal MasterWorker() {
|
||||
internal MasterWorker() : this(TimeSpan.FromMilliseconds(100)) { }
|
||||
internal MasterWorker(TimeSpan interval) {
|
||||
myPeriod = interval;
|
||||
}
|
||||
|
||||
public static MasterWorker StartNew() {
|
||||
var rv = new MasterWorker();
|
||||
rv.Start();
|
||||
return rv;
|
||||
}
|
||||
|
||||
public void Start() {
|
||||
myInit();
|
||||
}
|
||||
|
||||
@@ -34,11 +45,10 @@ namespace BCG {
|
||||
lock (typeof(MasterWorker)) {
|
||||
if (threadDriven) {
|
||||
// TODO
|
||||
myThreadDriven = myThreadDriven ?? new ThreadDrivenMasterWorker();
|
||||
myThreadDriven ??= ThreadDrivenMasterWorker.StartNew();
|
||||
return myThreadDriven;
|
||||
return null;
|
||||
} else {
|
||||
myDoWorkDriven = myDoWorkDriven ?? new MasterWorker();
|
||||
myDoWorkDriven ??= StartNew();
|
||||
return myDoWorkDriven;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace Bts.PubSub;
|
||||
|
||||
public static class WorkerGroups {
|
||||
public const string Main = "Main";
|
||||
public const string Background = "Background";
|
||||
}
|
||||
@@ -48,7 +48,8 @@ namespace BCG {
|
||||
this.publisher = publisher;
|
||||
}
|
||||
|
||||
public TimeSpan Interval => TimeSpan.FromMilliseconds(500);
|
||||
string IParameterizedWorker.Group => WorkerGroups.Background;
|
||||
TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(500);
|
||||
|
||||
#region IWorker Members
|
||||
|
||||
|
||||
@@ -39,8 +39,8 @@ namespace BCG {
|
||||
public string TagType => tag_type;
|
||||
public string Tag => tag_name;
|
||||
|
||||
TimeSpan IParameterizedWorker.Interval =>
|
||||
TimeSpan.FromMilliseconds(250);
|
||||
string IParameterizedWorker.Group => WorkerGroups.Main;
|
||||
TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(250);
|
||||
|
||||
internal static SystemStatsPublisher get_instance() {
|
||||
return Log.notnull(instance);
|
||||
|
||||
@@ -23,7 +23,6 @@ namespace BCG {
|
||||
private Thread _msg_polling_thread;
|
||||
// all the data,list of subscribers etc are in the state
|
||||
private IPubSubState _state;
|
||||
private MasterWorker _mainloop_master_worker = MasterWorker.getInstance(false);
|
||||
// private static SyncQueue<PublishMsg> _internal_pub_queue = new SyncQueue<PublishMsg>();
|
||||
private static IInternalPublisherQueue? _internal_pub_queue;
|
||||
static int instanceCount = 0;
|
||||
@@ -109,7 +108,6 @@ namespace BCG {
|
||||
_handle_client_msg(kv.first, kv.second);
|
||||
}
|
||||
handle_internal_pubs();
|
||||
_mainloop_master_worker.doWork();
|
||||
} catch (Exception ex) {
|
||||
bts.Log.Gen.error($"PubSubMessageReactor._polling_thread_func ex: {ex}");
|
||||
if (ex is OutOfMemoryException) {
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
namespace Bts.PubSub.Services;
|
||||
|
||||
using bts;
|
||||
|
||||
using Microsoft.Extensions.Hosting;
|
||||
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
/// <summary>
|
||||
/// An <see cref="IHostedService"/> that runs on a background thread.
|
||||
/// </summary>
|
||||
public abstract class BackgroundHostedService : IHostedService {
|
||||
readonly TaskCompletionSource tcs = new();
|
||||
readonly CancellationTokenSource cancellationTokenSource = new();
|
||||
readonly object syncRoot = new();
|
||||
Thread? thread;
|
||||
State state;
|
||||
|
||||
protected BackgroundHostedService() { }
|
||||
|
||||
enum State {
|
||||
Idle,
|
||||
Starting,
|
||||
Running,
|
||||
Stopping,
|
||||
}
|
||||
|
||||
string name => GetType().Name;
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken) {
|
||||
Log.isnull(thread);
|
||||
lock (syncRoot) {
|
||||
Log.assert(state == State.Idle);
|
||||
state = State.Starting;
|
||||
}
|
||||
Log.Gen.info($"Starting {name}");
|
||||
thread = new Thread(run) {
|
||||
Name = name
|
||||
};
|
||||
thread.Start();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken) {
|
||||
Log.Gen.info($"Stopping {name}...");
|
||||
cancellationTokenSource.Cancel();
|
||||
return tcs.Task;
|
||||
}
|
||||
|
||||
protected abstract void Run(CancellationToken cancellationToken);
|
||||
|
||||
void run() {
|
||||
Log.Gen.info($"{GetType()} STARTED.");
|
||||
lock (syncRoot) {
|
||||
state = State.Running;
|
||||
}
|
||||
try {
|
||||
Run(cancellationTokenSource.Token);
|
||||
Log.Gen.error($"{name} '{nameof(Run)}' finished");
|
||||
} catch (Exception ex) {
|
||||
Log.Gen.error($"{name} '{nameof(Run)}' threw an exception: {ex}");
|
||||
}
|
||||
lock (syncRoot) {
|
||||
state = State.Idle;
|
||||
}
|
||||
tcs.SetResult();
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,8 @@
|
||||
namespace Bts.PubSub.Services;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
public class MasterWorkerConfig {
|
||||
public TimeSpan Interval { get; set; } = TimeSpan.FromMilliseconds(100);
|
||||
public List<WorkSchedulerConfig>? WorkSchedulers { get; set; }
|
||||
}
|
||||
@@ -9,7 +9,9 @@ using bts.collections;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
@@ -20,54 +22,57 @@ using System.Threading.Tasks;
|
||||
/// An <see cref="IHostedService"/> that imports a collection of <see cref="IWorker"/>s and
|
||||
/// calls each <see cref="IWorker.doWork"/> in a loop.
|
||||
/// </summary>
|
||||
public class MasterWorkerService : IHostedService {
|
||||
public class MasterWorkerService : BackgroundHostedService, IWorkScheduler {
|
||||
readonly TaskCompletionSource tcs = new();
|
||||
readonly CancellationTokenSource cancellationTokenSource = new();
|
||||
readonly List<IWorker> workers;
|
||||
readonly MasterWorkerConfig config;
|
||||
MasterWorker? masterWorker;
|
||||
Dictionary<string, IWorkScheduler>? backgroundMasterWorkers;
|
||||
readonly object syncRoot = new();
|
||||
readonly Dictionary<string, ThreadDrivenMasterWorker> backgroundWorkers = new();
|
||||
|
||||
public MasterWorkerService(IEnumerable<IWorker> workers, IOptions<MasterWorkerConfig> config) {
|
||||
this.workers = workers.ToList();
|
||||
this.config = config.Value;
|
||||
Log.notnull(this.config.WorkSchedulers);
|
||||
Log.assert(this.config.WorkSchedulers.Count == 2);
|
||||
Log.assert(this.config.WorkSchedulers[0].Name == WorkerGroups.Main);
|
||||
Log.assert(this.config.WorkSchedulers[1].Name == WorkerGroups.Background);
|
||||
foreach (var w in this.config.WorkSchedulers) {
|
||||
Log.notnull(w.Name);
|
||||
backgroundWorkers.Add(w.Name, new(w.Name));
|
||||
}
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken) {
|
||||
enum State {
|
||||
Idle,
|
||||
Working
|
||||
}
|
||||
|
||||
public void Add(IWorker worker, TimeSpan interval) {
|
||||
var group = getWorkerGroup(worker);
|
||||
var scheduler = getOrAddScheduler(group);
|
||||
scheduler.Add(worker, interval);
|
||||
}
|
||||
|
||||
public void Remove(IWorker worker) {
|
||||
var group = getWorkerGroup(worker);
|
||||
var scheduler = getScheduler(group);
|
||||
Log.notnull(scheduler).Remove(worker);
|
||||
}
|
||||
|
||||
protected override void Run(CancellationToken cancellationToken) {
|
||||
var workersDesc = JsonSerializer.Serialize(workersToJson(workers));
|
||||
Log.Gen.info($"Starting MasterWorkerService. {workersDesc}");
|
||||
foreach (var w in workers) {
|
||||
GetWorkScheduler((w as IParameterizedWorker)?.Group).Add(w);
|
||||
}
|
||||
new Thread(workLoop).Start();
|
||||
return Task.CompletedTask;
|
||||
|
||||
IWorkScheduler GetWorkScheduler(string? key) =>
|
||||
key is null
|
||||
? (masterWorker ??= new())
|
||||
: (backgroundMasterWorkers ??= new()).GetOrAdd(key, _ => new ThreadDrivenMasterWorker());
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken) {
|
||||
Log.Gen.info("Stopping MasterWorkerService...");
|
||||
cancellationTokenSource.Cancel();
|
||||
return tcs.Task;
|
||||
}
|
||||
|
||||
void workLoop() {
|
||||
Log.Gen.info("MasterWorkerService STARTED.");
|
||||
if (masterWorker is null) {
|
||||
Log.Gen.warn("MasterWorkerService STOPPED early because there is no work to do.");
|
||||
return;
|
||||
}
|
||||
while (!cancellationTokenSource.IsCancellationRequested) {
|
||||
masterWorker?.doWork();
|
||||
Thread.Sleep(config.Interval);
|
||||
foreach (var workScheduler in backgroundWorkers.Values) {
|
||||
workScheduler.Start();
|
||||
}
|
||||
Log.Gen.info("MasterWorkerService STOPPED.");
|
||||
tcs.SetResult();
|
||||
}
|
||||
|
||||
static string getWorkerGroup(IWorker worker) =>
|
||||
(worker as IParameterizedWorker)?.Group ?? WorkerGroups.Main;
|
||||
|
||||
static JsonObject workersToJson(IEnumerable<IWorker> workers) {
|
||||
var rootJson = new JsonObject();
|
||||
foreach (var wg in workers.GroupBy(w => (w as IParameterizedWorker)?.Group ?? "Main")) {
|
||||
@@ -77,4 +82,35 @@ public class MasterWorkerService : IHostedService {
|
||||
}
|
||||
return rootJson;
|
||||
}
|
||||
|
||||
IWorkScheduler? getScheduler(string group) {
|
||||
lock (syncRoot) {
|
||||
if (backgroundWorkers.TryGetValue(group, out var bgw)) {
|
||||
return bgw;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
IWorkScheduler getOrAddScheduler(IWorker worker) =>
|
||||
getOrAddScheduler((worker as IParameterizedWorker)?.Group ?? WorkerGroups.Main);
|
||||
|
||||
IWorkScheduler getOrAddScheduler(string group) {
|
||||
var (bgWorker, isNew) = getOrAddBackgroundWorker(group);
|
||||
if (isNew) {
|
||||
bgWorker.Start();
|
||||
}
|
||||
return bgWorker;
|
||||
}
|
||||
|
||||
(ThreadDrivenMasterWorker worker, bool isNew) getOrAddBackgroundWorker(string group) {
|
||||
lock (syncRoot) {
|
||||
if (backgroundWorkers.TryGetValue(group, out var bgWorker)) {
|
||||
return (bgWorker, false);
|
||||
}
|
||||
bgWorker = new(group);
|
||||
backgroundWorkers.Add(group, bgWorker);
|
||||
return (bgWorker, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -65,6 +65,7 @@ public static class PubSubServices {
|
||||
services
|
||||
.Configure<PubSubConfig>(configurationSection)
|
||||
.AddInternalPublisher()
|
||||
.AddPubSubTagHandlerFactory(configurationSection.GetSection(nameof(PubSubConfig.TagHandlerFactory)))
|
||||
.AddSingleton<IPubSubTagHandlerFactory, PubsubTagHandlerFactory>()
|
||||
.AddSingleton<IPubSubRouter, PubSubRouter>()
|
||||
.AddPublisherCore()
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace Bts.PubSub.Services;
|
||||
|
||||
using System;
|
||||
|
||||
public class WorkSchedulerConfig {
|
||||
public string? Name { get; set; }
|
||||
public TimeSpan Interval { get; set; } = TimeSpan.FromMilliseconds(100);
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace Bts.PubSub.Services;
|
||||
|
||||
using BCG;
|
||||
|
||||
|
||||
using System.Collections.Generic;
|
||||
|
||||
public class WorkSchedulerProvider {
|
||||
readonly Dictionary<string, IWorkScheduler> workSchedulers = new();
|
||||
readonly object syncRoot = new();
|
||||
|
||||
public IWorkScheduler GetScheduler(string? group) {
|
||||
group ??= string.Empty;
|
||||
lock (syncRoot) {
|
||||
return workSchedulers.GetOrAdd(
|
||||
group,
|
||||
g => string.IsNullOrEmpty(g)
|
||||
? new MasterWorker()
|
||||
: new ThreadDrivenMasterWorker());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
namespace Bts.PubSub;
|
||||
|
||||
using BCG;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using bts;
|
||||
using bts.collections;
|
||||
|
||||
using Bts.PubSub.Services;
|
||||
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Threading;
|
||||
|
||||
public class WorkSchedulerService : BackgroundHostedService, IWorkScheduler {
|
||||
readonly List<IWorker> workers;
|
||||
readonly MasterWorkerConfig config;
|
||||
readonly IWorkScheduler mainWorker = new MasterWorker();
|
||||
readonly object syncRoot = new();
|
||||
ImmutableDictionary<string, IWorkScheduler> backgroundWorkers = ImmutableDictionary<string, IWorkScheduler>.Empty;
|
||||
|
||||
public WorkSchedulerService(IEnumerable<IWorker> workers, IOptions<MasterWorkerConfig> config) {
|
||||
this.workers = workers.ToList();
|
||||
this.config = config.Value;
|
||||
}
|
||||
|
||||
public void Add(IWorker worker, TimeSpan interval) {
|
||||
// getOrAddScheduler(worker).Add(worker, interval);
|
||||
var group = (worker as IParameterizedWorker)?.Group;
|
||||
if (string.IsNullOrEmpty(group)) {
|
||||
lock (syncRoot) {
|
||||
mainWorker.Add(worker, interval);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (backgroundWorkers.TryGetValue(group, out var bgw)) {
|
||||
bgw.Add(worker, interval);
|
||||
}
|
||||
bgw = ThreadDrivenMasterWorker.StartNew(worker, interval);
|
||||
lock (syncRoot) {
|
||||
backgroundWorkers = backgroundWorkers.Add(group, bgw);
|
||||
}
|
||||
}
|
||||
|
||||
public void Remove(IWorker worker) {
|
||||
var group = (worker as IParameterizedWorker)?.Group;
|
||||
if (string.IsNullOrEmpty(group)) {
|
||||
lock (syncRoot) {
|
||||
mainWorker.Remove(worker);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (backgroundWorkers.TryGetValue(group, out var bgw)) {
|
||||
bgw.Remove(worker);
|
||||
}
|
||||
var scheduler = getScheduler(worker);
|
||||
Log.notnull(scheduler).Remove(worker);
|
||||
}
|
||||
|
||||
protected override void OnStart(CancellationToken cancellationToken) {
|
||||
var workersDesc = JsonSerializer.Serialize(workersToJson(workers));
|
||||
Log.Gen.info($"Starting MasterWorkerService. {workersDesc}");
|
||||
foreach (var w in workers) {
|
||||
getOrAddScheduler(w).Add(w);
|
||||
}
|
||||
}
|
||||
|
||||
static JsonObject workersToJson(IEnumerable<IWorker> workers) {
|
||||
var rootJson = new JsonObject();
|
||||
foreach (var wg in workers.GroupBy(w => (w as IParameterizedWorker)?.Group ?? "Main")) {
|
||||
var wgJson = new JsonArray();
|
||||
wgJson.AddRange(wg.Select(w => JsonValue.Create(w.ToString())));
|
||||
rootJson.Add(wg.Key, wgJson);
|
||||
}
|
||||
return rootJson;
|
||||
}
|
||||
|
||||
IWorkScheduler? getScheduler(IWorker worker) =>
|
||||
getScheduler((worker as IParameterizedWorker)?.Group);
|
||||
|
||||
IWorkScheduler? getScheduler(string? group) {
|
||||
if (string.IsNullOrEmpty(group)) {
|
||||
return mainWorker;
|
||||
}
|
||||
lock (syncRoot) {
|
||||
if (backgroundWorkers.TryGetValue(group, out var bgw)) {
|
||||
return bgw;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
IWorkScheduler getOrAddScheduler(IWorker worker) =>
|
||||
getOrAddScheduler((worker as IParameterizedWorker)?.Group);
|
||||
|
||||
IWorkScheduler getOrAddScheduler(string? group) {
|
||||
if (string.IsNullOrEmpty(group)) {
|
||||
return mainWorker;
|
||||
}
|
||||
lock (syncRoot) {
|
||||
if (backgroundWorkers.TryGetValue(group, out var bgw)) {
|
||||
return bgw;
|
||||
}
|
||||
bgw = new();
|
||||
backgroundWorkers = backgroundWorkers.Add(group, bgw);
|
||||
bgw.Start();
|
||||
return bgw;
|
||||
}
|
||||
}
|
||||
|
||||
protected override void Run(CancellationToken cancellationToken) {
|
||||
while (!cancellationToken.IsCancellationRequested) {
|
||||
mainWorker.doWork();
|
||||
Thread.Sleep(config.Interval);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
#nullable enable
|
||||
namespace Bts.PubSub;
|
||||
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using bts;
|
||||
|
||||
using System;
|
||||
|
||||
public class WorkActivator : IDisposable {
|
||||
readonly object syncRoot = new();
|
||||
readonly IWorker worker;
|
||||
IWorkScheduler? workScheduler;
|
||||
|
||||
State state = State.Idle;
|
||||
|
||||
public WorkActivator(IWorker worker) {
|
||||
this.worker = worker;
|
||||
}
|
||||
|
||||
enum State {
|
||||
Idle,
|
||||
WaitingToWork,
|
||||
Working
|
||||
}
|
||||
|
||||
public void OnWorkScheduler(IWorkScheduler workScheduler) {
|
||||
lock (syncRoot) {
|
||||
if (workScheduler == this.workScheduler) {
|
||||
return;
|
||||
}
|
||||
switch (state) {
|
||||
case State.Idle:
|
||||
this.workScheduler = workScheduler;
|
||||
break;
|
||||
case State.WaitingToWork:
|
||||
/// We only transition <see cref="State.WaitingToWork"/> when <see cref="Start"/> is
|
||||
/// called and <see cref="worker"/> is null.
|
||||
Log.isnull(this.workScheduler);
|
||||
this.workScheduler = workScheduler;
|
||||
if (workScheduler is not null) {
|
||||
state = State.Working;
|
||||
workScheduler.Add(worker);
|
||||
}
|
||||
break;
|
||||
case State.Working:
|
||||
Log.notnull(this.workScheduler).Remove(worker);
|
||||
this.workScheduler = workScheduler;
|
||||
if (workScheduler is not null) {
|
||||
workScheduler.Add(worker);
|
||||
} else {
|
||||
state = State.WaitingToWork;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Start() {
|
||||
lock (syncRoot) {
|
||||
switch (state) {
|
||||
case State.Idle:
|
||||
if (workScheduler is null) {
|
||||
state = State.WaitingToWork;
|
||||
} else {
|
||||
state = State.Working;
|
||||
workScheduler.Add(worker);
|
||||
}
|
||||
break;
|
||||
case State.WaitingToWork:
|
||||
// Sanity check - "WaitingToWork" means we are waiting for a non-null IWorkScheduler.
|
||||
// It follows that if we're "WaitingToWork", then our IWorkScheduler should be null.
|
||||
Log.isnull(workScheduler);
|
||||
break;
|
||||
case State.Working:
|
||||
Log.notnull(workScheduler);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Stop() {
|
||||
lock (syncRoot) {
|
||||
switch (state) {
|
||||
case State.Idle:
|
||||
break;
|
||||
case State.WaitingToWork:
|
||||
state = State.Idle;
|
||||
break;
|
||||
case State.Working:
|
||||
Log.notnull(workScheduler).Remove(worker);
|
||||
state = State.Idle;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose() {
|
||||
lock (syncRoot) {
|
||||
stopImpl();
|
||||
workScheduler = null;
|
||||
}
|
||||
}
|
||||
|
||||
void stopImpl() {
|
||||
if (workScheduler is not null) {
|
||||
switch (state) {
|
||||
case State.Working:
|
||||
workScheduler.Remove(worker);
|
||||
break;
|
||||
}
|
||||
}
|
||||
state = State.Idle;
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"MasterWorker": {
|
||||
"Interval": "00:00:00.1"
|
||||
"Interval": "00:00:01"
|
||||
},
|
||||
"SparkLogging": {
|
||||
"ConfigServer": "http://localhost:8000/config"
|
||||
|
||||
Reference in new issue
Block a user