diff --git a/src/DataStreamSuite/DataStream/Services/DataStreamServices.cs b/src/DataStreamSuite/DataStream/Services/DataStreamServices.cs new file mode 100644 index 000000000..d35f836da --- /dev/null +++ b/src/DataStreamSuite/DataStream/Services/DataStreamServices.cs @@ -0,0 +1,11 @@ +namespace Bts.DataStream.Services; + +using BCG; +using BCG.Utilities.General; + +using Microsoft.Extensions.DependencyInjection; + +public static class DataStreamServices { + public static IServiceCollection AddRemoteQueryManager(this IServiceCollection services) => + services.AddSingleton(); +} diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json index 2f7a35544..69af34479 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json +++ b/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json @@ -1,129 +1,155 @@ -{ - "user": "bts", - "sparkd": "http://192.168.1.80:8000/config", - "pubsub": { - "milliseconds_between_sub_info_updates": 1000, - "milliseconds_between_time_slice_updates": 5000, - "milliseconds_between_no_buffer_space_retries": 100, - "retries_on_no_buffer_space": 1, - "do_it_all_seconds_between_logs": 0, - "heartbeat_interval": 15000, - "missing_heartbeats_before_gc": 50, - "use_managed_sockets": true, - "state_update_period": 5000, - "statistics_update_period": 5000, - "disposed_connection_check_period": 1000, - "message_processing_warning_threshold": 1000, - "server_port": 12345, - "publisher_routes": [ - { - "name": "default", - "server_address": "bts-cloud://127.0.0.1:12345", - "tag": ".*" - } - ], - "subscriber_routes": [ - { - "name": "default", - "server_address": "bts-cloud://127.0.0.1:12345", - "tag": ".*" - } - ], - "max_packet_size": 500000000, - "blue_comm_must_die": true, - "crushable_tags": [], - "tag_type_to_assembly_map": [ - { - "handler_class": "BCG.PubSubServerTagHandler", - "assembly": "DataStreamCore.dll" - } - ], - "server_queue": "PubSubQueue", - "port_range": { - "first": 30400, - "last": 31700 - }, - "bind_to_random_port": false, - "debug_tags": "^NOTAGS$", - "latency_report_threshold_ms": 1500, - "use_substations": false, - "use_dynamic_clients": false, - "is_substation": false, - "internally_publish_sub_info": true, - "use_managed_clients_in_native_code": false, - "compressed_tags": "^NOTAGS$", - "tags_to_report_compression_ratio": "^NOTAGS$", - "disable_msmq": false, - "use_prepublished_server_state": false, - "ip_translation": null, - "gc_activation_threshold": 0, - "wait_period_after_gc": 1000 - }, - "publishers": [ - { - "name": "market_data", - "streams": [ - { - "name": "bbo", - "expression": "bbo()", - "is_on_websocket": true - }, - { - "name": "instruments", - "expression": "instruments()", - "is_on_websocket": true - }, - { - "name": "timesales", - "expression": "timesales(10000)", - "is_on_websocket": true - }, - { - "name": "books", - "expression": "books()", - "is_on_websocket": true - } - ] - }, - { - "name": "test_data", - "streams": [ - { - "name": "milliseconds_1000_100", - "expression": "milliseconds(1000,100)", - "is_on_websocket": true - }, - { - "name": "test_literal", - "expression": "literal(\"x\", \"420.0\")", - "is_on_websocket": true - }, - { - "name": "test_not_on_websocket", - "expression": "literal(\"x\", \"840.0\")", - "is_on_websocket": false - } - ] - } - // { - // "name": "news_data", - // "streams": [ - // { - // "name": "benzinga_news", - // "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")", - // "is_on_websocket": true - // }, - // { - // "name": "benzinga_news_symbols", - // "expression": "benzingaNewsSymbols(benzinga_news)", - // "is_on_websocket": true - // }, - // { - // "name": "benzinga_news_symbol_state", - // "expression": "benzingaNewsSymbolState(benzinga_news)", - // "is_on_websocket": true - // } - // ] - // } - ] -} \ No newline at end of file +{ + "Spark": { + "ConfigServer": "http://localhost:8000/config" + }, + "SparkApi": { + "Credentials": { + "User": "bts", + "Password": "" + }, + "EnableAdmin": true, + "EnableOrders": true, + "EnablePosition": true + }, + "MasterWorker": { + "Interval": "00:00:00.1" + }, + "ViewStreamDatabase": { + "Interval": "00:00:00.1" + }, + "PubSub": { + "MillisecondsBetweenSubInfoUpdates": 1000, + "MillisecondsBetweenTimeSliceUpdates": 5000, + "MillisecondsBetweenNoBufferSpaceRetries": 100, + "RetriesOnNoBufferSpace": 1, + "DoItAllSecondsBetweenLogs": 0, + "HeartbeatInterval": 15000, + "MissingHeartbeatsBeforeGc": 50, + "UseManagedSockets": true, + "StateUpdatePeriod": 5000, + "StatisticsUpdatePeriod": 5000, + "DisposedConnectionCheckPeriod": 1000, + "MessageProcessingWarningThreshold": 1000, + "ServerPort": 12345, + "Routing": { + "PublisherRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://127.0.0.1:12345", + "TagRegExp": ".*" + } + ], + "SubscriberRoutes": [ + { + "Name": "default", + "ServerAddress": "bts-cloud://127.0.0.1:12345", + "TagRegExp": ".*" + } + ] + }, + "MaxPacketSize": 500000000, + "BlueCommMustDie": true, + "CrushableTags": [], + "TagTypeToAssemblyMap": { + "Entries": [ + { + "HandlerClass": "BCG.PubSubServerTagHandler", + "Assembly": "DataStreamCore.dll" + } + ] + }, + "TagHandlerFactory": { + "Handlers": [ + { + "TagType": "DataStream", + "Assembly": "DataStreamCore", + "Class": "bts.PubsubServerStreamTagHandler" + }, + { + "TagType": "System_Sub", + "Assembly": "PubSub", + "Class": "BCG.SystemSubscriptionMapPublisher" + }, + { + "TagType": "System_Stats", + "Assembly": "PubSub", + "Class": "BCG.SystemStatsPublisher" + }, + { + "TagType": ".*", + "Assembly": "DataStreamCore", + "Class": "bts.PubsubServerGenericDataHandler" + } + ] + }, + "ServerQueue": "PubSubQueue", + "PortRange": { + "First": 30400, + "Last": 31700 + }, + "BindToRandomPort": false, + "DebugTags": "^NOTAGS$", + "LatencyReportThresholdMs": 1500, + "UseSubstations": false, + "UseDynamicClients": false, + "IsSubstation": false, + "InternallyPublishSubInfo": true, + "UseManagedClientsInNativeCode": false, + "CompressedTags": "^NOTAGS$", + "TagsToReportCompressionRatio": "^NOTAGS$", + "DisableMsmq": false, + "UsePrepublishedServerState": false, + "IpTranslation": null, + "GcActivationThreshold": 0, + "WaitPeriodAfterGc": 1000 + }, + "DataStreamPublisher": { + "PublisherNodes": [ + { + "Name": "market_data", + "Streams": [ + { + "Name": "bbo", + "Expression": "bbo()", + "IsOnWebsocket": true + }, + { + "Name": "instruments", + "Expression": "instruments()", + "IsOnWebsocket": true + }, + { + "Name": "timesales", + "Expression": "timesales(10000)", + "IsOnWebsocket": true + }, + { + "Name": "books", + "Expression": "books()", + "IsOnWebsocket": true + } + ] + }, + { + "Name": "test_data", + "Streams": [ + { + "Name": "milliseconds_1000_100", + "Expression": "milliseconds(3000,5)", + "IsOnWebsocket": true + }, + { + "Name": "test_literal", + "Expression": "literal(\"x\", \"420.0\")", + "IsOnWebsocket": true + }, + { + "Name": "test_not_on_websocket", + "Expression": "literal(\"x\", \"840.0\")", + "IsOnWebsocket": false + } + ] + } + ] + } +} diff --git a/src/DataStreamSuite/DataStreamPublisher/CsvPublisherConfig.cs b/src/DataStreamSuite/DataStreamPublisher/CsvPublisherConfig.cs new file mode 100644 index 000000000..7fecb3321 --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/CsvPublisherConfig.cs @@ -0,0 +1,6 @@ +namespace Bts.DataStreamPublisher; + +public class CsvPublisherConfig { + public string? Name { get; set; } + public string? CsvDir { get; set; } +} diff --git a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj index 179d1c62e..015e92e14 100644 --- a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj +++ b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj @@ -9,6 +9,7 @@ + diff --git a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs index 5e6dba0e3..d4bb4ac72 100644 --- a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs +++ b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs @@ -1,7 +1,5 @@ namespace Bts.DataStreamPublisher; -using Bts.PubSub; - using CommandLine; using System; @@ -10,49 +8,13 @@ using System.IO; using System.Text.Json; using System.Text.Json.Serialization; -public class DataStreamPublisherConfig -{ - /// - /// The login user for sparkd - /// - [JsonPropertyName("user")] - [Option("user", HelpText = "The sparkd login user")] - public string? User { get; set; } - - /// - /// The login password for sparkd - /// - [JsonPropertyName("password")] - [Option("pass", HelpText = "The sparkd login password")] - public string? Password { get; set; } - - /// - /// The config server url for sparkd - /// - [JsonPropertyName("sparkd")] - [Option("sparkd", HelpText = "The sparkd config server")] - public string? Sparkd { get; set; } - +public class DataStreamPublisherConfig { /// /// The tables to publish on startup. Each key/value represents a table name/query to publish. /// - [JsonPropertyName("publishers")] - public List? Publishers { get; set; } + public List? PublisherNodes { get; set; } - [JsonPropertyName("mongodb_url")] - public string MONGO_DB_URL { get; set; } - - [JsonPropertyName("mongodb_db_name")] - public string MONGO_DB_NAME { get; set; } - - [JsonPropertyName("mongodb_db_collection")] - public string MONGO_DB_COLLECTION { get; set; } - - [JsonPropertyName("benzinga_news_ttl")] - public string BENZINGA_NEWS_TTL { get; set; } - - [JsonPropertyName("pubsub")] - public PubSubConfig? PubSub { get; set; } + public List? CsvDirs { get; set; } [Option( "config", @@ -62,18 +24,10 @@ public class DataStreamPublisherConfig public string? ConfigFile { get; set; } public static DataStreamPublisherConfig Default() => - new() - { - PubSub = PubSubConfig.Default(), - }; + new() { }; - public void MergeWith(DataStreamPublisherConfig other) - { - User ??= other.User; - Password ??= other.Password; - Sparkd ??= other.Sparkd; - Publishers ??= other.Publishers; - PubSub ??= other.PubSub; + public void MergeWith(DataStreamPublisherConfig other) { + PublisherNodes ??= other.PublisherNodes; ConfigFile ??= other.ConfigFile; } @@ -84,10 +38,8 @@ public class DataStreamPublisherConfig /// /// JsonSerializer.Deserialize returns null /// - public void MergeWithConfigFile() - { - if (ConfigFile is null) - { + public void MergeWithConfigFile() { + if (ConfigFile is null) { return; } var cfgString = File.ReadAllText(ConfigFile); diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs index 82f117b5c..555631256 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Program.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs @@ -1,166 +1,34 @@ -using BCG; -using BCG.Utilities.General; +using Bts.DataStreamPublisher.Services; -using bts; -using bts.api.core; -using bts.utils; - -using Bts.DataStreamExpressions; -using Bts.DataStreamPublisher; -using Bts.PubSub; - -using CommandLine; - -using mbts.config; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text.Json; -using System.Threading.Tasks; -await Parser.Default.ParseArguments(args) - .MapResult(RunAsync, errs => Task.FromResult(1)); +var host = Host.CreateDefaultBuilder(args) + .ConfigureAppConfiguration(ConfigureApp) + .ConfigureServices(ConfigureServices) + .Build(); -async Task RunAsync(DataStreamPublisherConfig cfg) -{ - try - { - cfg.MergeWithConfigFile(); - cfg.MergeWith(DataStreamPublisherConfig.Default()); - PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); - Configuration.initialize(cfg.Sparkd); - Log.Gen.info("Starting DataStreamPublisher"); - var service_ctx = ServiceContext.create(cfg.Sparkd); - await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty); - _publish_streams(cfg.Publishers ?? new(0)); - } - catch (Exception ex) - { - // Write the error to console first, in case the logging system is compromised. - var error_msg = $"DataStreamPublisher threw exception:\n{ex}"; - Console.Error.WriteLine(error_msg); - Console.Error.Flush(); - bts.Log.Gen.error(error_msg); - return 1; - } - return 0; +try { + Console.Write("Starting DataStreamPublisher"); + host.Run(); + Console.Write("DataStreamPublisher exited gracefully."); +} catch (Exception ex) { + Console.Error.Write($"DataStreamPublisher crashed: {ex}"); + return 1; } +return 0; -static ViewStream? build_stream(string? stream_expression) -{ - // Console.WriteLine("Hit key to continue"); - // Console.ReadLine(); - if (stream_expression is null) - { - return null; - } - try - { - bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}"); - var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); - var rv = exp.getView(new Dictionary(0)); - return rv; - - } - catch (Exception ex) - { - bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}"); - return null; +static void ConfigureApp(HostBuilderContext context, IConfigurationBuilder configBuilder) { + configBuilder.AddCommandLine(Environment.GetCommandLineArgs()); + var cfgRoot = configBuilder.Build(); + var cfg = cfgRoot["config"]; + if (cfg is not null) { + configBuilder.AddJsonFile(cfg, false); } } -void _publish_streams(IEnumerable publishers) -{ - PubTableBroker pb_broker = new PubTableBroker(); - RemoteQueryManager csm = new RemoteQueryManager(); - var csvPublishers = new List(); - - foreach (var publisher in publishers) - { - if (Directory.Exists(publisher.CsvDir)) - { - csvPublishers.Add(new(publisher.CsvDir, pb_broker)); - } - } - - var viewPublishers = publishers - .SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher)) - .ToList(); - - while (true) - { - System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1)); - ViewStreamDatabase.instance.doWork(); - csm.doWork(); - MasterWorker.getInstance(false).doWork(); - ManagedSubscriber.instance.doWork(); - } - - ViewPublisher? createViewPublisher(PublishedStream ps) - { - Log.Gen.info( - $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}"); - if (string.IsNullOrEmpty(ps.Name)) - { - Log.Gen.warn( - $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}"); - return null; - } - var vs = build_stream(ps.Expression); - if (vs is null) - { - Log.Gen.warn( - $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}"); - return null; - } - var vpub = new ViewPublisher(ps.Name, pb_broker, vs); - vs.onChange += on_change; - return vpub; - } -} - -void on_change(object obj) -{ - bts.Log.Gen.info($"DSPublisher on_change dp: {obj}"); -} - -public class CSVPublisher : IWorker -{ - private IDictionary _pubs = new Dictionary(); - private readonly string _csv_directory; - private PubTableBroker _pub_table_broker; - - public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) - { - _csv_directory = csv_directory; - _pub_table_broker = pub_table_broker; - MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); - } - public void doWork() - { - var file_names = Directory.GetFiles(_csv_directory); - foreach (var fn in file_names) - { - if (!_pubs.ContainsKey(fn)) - { - _handle_new_file(fn); - } - } - } - - private void _handle_new_file(string file_name) - { - if (file_name.EndsWith(".csv")) - { - ViewStream vs = new CSVReaderStream(file_name); - vs = new SchemaGuesserStream(vs); - var toks = file_name.Split('\\'); - string table_name = toks.Last(); - table_name = table_name.Remove(table_name.Length - ".csv".Length); - ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs); - _pubs.Add(file_name, vp); - } - } -} +static void ConfigureServices(HostBuilderContext context, IServiceCollection services) => + services.AddDataStreamPublisher(context.Configuration); diff --git a/src/DataStreamSuite/DataStreamPublisher/PublisherNode.cs b/src/DataStreamSuite/DataStreamPublisher/PublisherNode.cs index 0df5b39a9..ec9092fd7 100644 --- a/src/DataStreamSuite/DataStreamPublisher/PublisherNode.cs +++ b/src/DataStreamSuite/DataStreamPublisher/PublisherNode.cs @@ -9,7 +9,4 @@ public class PublisherNode { [JsonPropertyName("streams")] public List? Streams { get; set; } - - [JsonPropertyName("csv_dir")] - public string? CsvDir { get; set; } } diff --git a/src/DataStreamSuite/DataStreamPublisher/Services/CsvPublisher.cs b/src/DataStreamSuite/DataStreamPublisher/Services/CsvPublisher.cs new file mode 100644 index 000000000..3c04e893b --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/Services/CsvPublisher.cs @@ -0,0 +1,39 @@ +namespace Bts.DataStreamPublisher.Services; + +using BCG; +using BCG.Utilities.General; + +using bts; + +using Bts.DataStreamCore; + +using System; +using System.Collections.Generic; +using System.IO; + +public class CsvPublisher : IWorker, IDisposable { + readonly Dictionary publishers = new(); + readonly string directory; + readonly IPubTableBroker pubTableBroker; + + public CsvPublisher(string directory, IPubTableBroker pubTableBroker) { + this.directory = directory; + this.pubTableBroker = pubTableBroker; + } + + public void doWork() { + var newFiles = Directory.GetFiles(directory, "*.csv").WhereNot(publishers.ContainsKey); + foreach (var file in newFiles) { + ViewStream vs = new CSVReaderStream(file); + vs = new SchemaGuesserStream(vs); + var tableName = Path.GetFileNameWithoutExtension(file); + var vp = new ViewPublisher(tableName, pubTableBroker, vs); + publishers.Add(file, vp); + } + } + + public void Dispose() { + publishers.Values.DisposeAll(); + publishers.Clear(); + } +} diff --git a/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs new file mode 100644 index 000000000..7467cbd90 --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs @@ -0,0 +1,105 @@ +namespace Bts.DataStreamPublisher.Services; + +using BCG; + +using bts; + +using Bts.DataStreamCore; +using Bts.DataStreamExpressions; +using Bts.PubSub; + +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; + +using Spark.Core.Disposables; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +/// +/// An that publishes s to the PubSub. +/// It can be added to the application's services via +/// +public sealed class DataStreamPublisherService : IHostedService, IDisposable { + readonly DataStreamPublisherConfig config; + readonly IPubTableBroker pubTableBroker; + readonly IWorkScheduler workScheduler; + List? resources = new(); + + public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, IOptions config) + : this(pubTableBroker, workScheduler, config.Value) { } + + public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, DataStreamPublisherConfig config) { + this.pubTableBroker = pubTableBroker; + this.workScheduler = workScheduler; + this.config = config; + } + + public Task StartAsync(CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); + resources = new(); + foreach (var publisherNode in config.PublisherNodes.empty_if_null()) { + foreach (var publishedStream in publisherNode.Streams.empty_if_null()) { + if (string.IsNullOrEmpty(publishedStream.Name)) { + Log.Gen.warn("Skipping PublishedStream because it doesn't have a name (must define 'Name' in config json)."); + continue; + } + var vs = CreateViewStream(publishedStream); + if (vs is null) { + continue; + } + resources.Add(new ViewPublisher(publishedStream.Name, pubTableBroker, vs)); + } + } + var csvPublishers = config.CsvDirs.empty_if_null().Select(csvDir => new CsvPublisher(csvDir, pubTableBroker)).ToList(); + resources.AddRange(csvPublishers); + resources.Add(workScheduler.Subscribe(new CompositeWorker(csvPublishers))); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); + Dispose(); + return Task.CompletedTask; + } + + public void Dispose() { + resources?.DisposeAll(); + resources?.Clear(); + } + + static ViewStream? CreateViewStream(PublishedStream ps) { + Log.Gen.info( + $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}"); + if (string.IsNullOrEmpty(ps.Name)) { + Log.Gen.warn( + $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}"); + return null; + } + var vs = buildStream(ps.Expression); + if (vs is null) { + Log.Gen.warn( + $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}"); + return null; + } + return vs; + } + + static ViewStream? buildStream(string? streamExpression) { + if (streamExpression is null) { + return null; + } + try { + Log.Gen.info($"Building ViewStream: {streamExpression}"); + var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression); + return exp.getView(new Dictionary(0)); + } catch (Exception ex) { + Log.assert(false, $"Could not build stream. expression: {streamExpression}; ex: {ex}"); + return null; + } + } +} diff --git a/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherServices.cs b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherServices.cs new file mode 100644 index 000000000..fef611bd0 --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherServices.cs @@ -0,0 +1,24 @@ +namespace Bts.DataStreamPublisher.Services; + +using Bts.DataStream.Services; +using Bts.DataStreamCore.Services; +using Bts.PubSub.Services; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +using Spark.Api.Services; +using Spark.Core.Services; + +public static class DataStreamPublisherServices { + public static IServiceCollection AddDataStreamPublisher(this IServiceCollection services, IConfiguration configurationSection) => + services.AddSpark(configurationSection.GetSection("Spark")) + .AddSparkApi(configurationSection.GetSection("SparkApi")) + .AddMasterWorker(configurationSection.GetSection("MasterWorker")) + .AddViewStreamDatabase(configurationSection.GetSection("ViewStreamDatabase")) + .AddPubSubClient(configurationSection.GetSection("PubSub")) + .AddPubTableBroker() + .AddRemoteQueryManager() + .Configure(configurationSection.GetSection("DataStreamPublisher")) + .AddHostedService(); +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompositeWorker.cs b/src/DataStreamSuite/PubSub/Misc/CompositeWorker.cs new file mode 100644 index 000000000..dae20933e --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompositeWorker.cs @@ -0,0 +1,18 @@ +namespace Bts.PubSub; + +using BCG.Utilities.General; + +using System.Collections.Generic; + +public class CompositeWorker : IWorker { + readonly List workers; + + public CompositeWorker(IEnumerable workers) => + this.workers = new(workers); + + public void doWork() { + foreach (var worker in workers) { + worker.doWork(); + } + } +}