diff --git a/src/DataStreamSuite/DataStream/Services/DataStreamServices.cs b/src/DataStreamSuite/DataStream/Services/DataStreamServices.cs new file mode 100644 index 000000000..d35f836da --- /dev/null +++ b/src/DataStreamSuite/DataStream/Services/DataStreamServices.cs @@ -0,0 +1,11 @@ +namespace Bts.DataStream.Services; + +using BCG; +using BCG.Utilities.General; + +using Microsoft.Extensions.DependencyInjection; + +public static class DataStreamServices { + public static IServiceCollection AddRemoteQueryManager(this IServiceCollection services) => + services.AddSingleton(); +} diff --git a/src/DataStreamSuite/DataStreamPublisher/CsvPublisherConfig.cs b/src/DataStreamSuite/DataStreamPublisher/CsvPublisherConfig.cs new file mode 100644 index 000000000..7fecb3321 --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/CsvPublisherConfig.cs @@ -0,0 +1,6 @@ +namespace Bts.DataStreamPublisher; + +public class CsvPublisherConfig { + public string? Name { get; set; } + public string? CsvDir { get; set; } +} diff --git a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj index 22308aee5..189dfa0fb 100644 --- a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj +++ b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisher.csproj @@ -9,6 +9,7 @@ + diff --git a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs index 2f4f52d45..64646f810 100644 --- a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs +++ b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs @@ -1,7 +1,5 @@ namespace Bts.DataStreamPublisher; -using Bts.PubSub; - using CommandLine; using System; @@ -11,35 +9,12 @@ using System.Text.Json; using System.Text.Json.Serialization; public class DataStreamPublisherConfig { - /// - /// The login user for sparkd - /// - [JsonPropertyName("user")] - [Option("user", HelpText = "The sparkd login user")] - public string? User { get; set; } - - /// - /// The login password for sparkd - /// - [JsonPropertyName("password")] - [Option("pass", HelpText = "The sparkd login password")] - public string? Password { get; set; } - - /// - /// The config server url for sparkd - /// - [JsonPropertyName("sparkd")] - [Option("sparkd", HelpText = "The sparkd config server")] - public string? Sparkd { get; set; } - /// /// The tables to publish on startup. Each key/value represents a table name/query to publish. /// - [JsonPropertyName("publishers")] public List? Publishers { get; set; } - [JsonPropertyName("pubsub")] - public PubSubConfig? PubSub { get; set; } + public List? CsvDirs { get; set; } [Option( "config", @@ -49,16 +24,10 @@ public class DataStreamPublisherConfig { public string? ConfigFile { get; set; } public static DataStreamPublisherConfig Default() => - new() { - PubSub = PubSubConfig.Default(), - }; + new() { }; public void MergeWith(DataStreamPublisherConfig other) { - User ??= other.User; - Password ??= other.Password; - Sparkd ??= other.Sparkd; Publishers ??= other.Publishers; - PubSub ??= other.PubSub; ConfigFile ??= other.ConfigFile; } diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs index 11aab26c1..555631256 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Program.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs @@ -1,140 +1,34 @@ -using BCG; -using BCG.Utilities.General; +using Bts.DataStreamPublisher.Services; -using bts; -using bts.api.core; -using bts.utils; - -using Bts.DataStreamExpressions; -using Bts.DataStreamPublisher; -using Bts.PubSub; - -using CommandLine; - -using mbts.config; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text.Json; -using System.Threading.Tasks; -await Parser.Default.ParseArguments(args) - .MapResult(RunAsync, errs => Task.FromResult(1)); +var host = Host.CreateDefaultBuilder(args) + .ConfigureAppConfiguration(ConfigureApp) + .ConfigureServices(ConfigureServices) + .Build(); -async Task RunAsync(DataStreamPublisherConfig cfg) { - try { - cfg.MergeWithConfigFile(); - cfg.MergeWith(DataStreamPublisherConfig.Default()); - PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); - Configuration.initialize(cfg.Sparkd); - Log.Gen.info("Starting DataStreamPublisher"); - var service_ctx = ServiceContext.create(cfg.Sparkd); - await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty); - _publish_streams(cfg.Publishers ?? new(0)); - } catch (Exception ex) { - // Write the error to console first, in case the logging system is compromised. - var error_msg = $"DataStreamPublisher threw exception:\n{ex}"; - Console.Error.WriteLine(error_msg); - Console.Error.Flush(); - bts.Log.Gen.error(error_msg); - return 1; - } - return 0; +try { + Console.Write("Starting DataStreamPublisher"); + host.Run(); + Console.Write("DataStreamPublisher exited gracefully."); +} catch (Exception ex) { + Console.Error.Write($"DataStreamPublisher crashed: {ex}"); + return 1; } +return 0; -static ViewStream? build_stream(string? stream_expression) { - if (stream_expression is null) { - return null; - } - try { - bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}"); - var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); - var rv = exp.getView(new Dictionary(0)); - return rv; - - } catch (Exception ex) { - bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}"); - return null; +static void ConfigureApp(HostBuilderContext context, IConfigurationBuilder configBuilder) { + configBuilder.AddCommandLine(Environment.GetCommandLineArgs()); + var cfgRoot = configBuilder.Build(); + var cfg = cfgRoot["config"]; + if (cfg is not null) { + configBuilder.AddJsonFile(cfg, false); } } -void _publish_streams(IEnumerable publishers) { - PubTableBroker pb_broker = new PubTableBroker(); - RemoteQueryManager csm = new RemoteQueryManager(); - var csvPublishers = new List(); - - foreach (var publisher in publishers) { - if (Directory.Exists(publisher.CsvDir)) { - csvPublishers.Add(new(publisher.CsvDir, pb_broker)); - } - } - - var viewPublishers = publishers - .SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher)) - .ToList(); - - while (true) { - System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1)); - ViewStreamDatabase.instance.doWork(); - csm.doWork(); - MasterWorker.getInstance(false).doWork(); - ManagedSubscriber.instance.doWork(); - } - - ViewPublisher? createViewPublisher(PublishedStream ps) { - Log.Gen.info( - $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}"); - if (string.IsNullOrEmpty(ps.Name)) { - Log.Gen.warn( - $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}"); - return null; - } - var vs = build_stream(ps.Expression); - if (vs is null) { - Log.Gen.warn( - $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}"); - return null; - } - var vpub = new ViewPublisher(ps.Name, pb_broker, vs); - vs.onChange += on_change; - return vpub; - } -} - -void on_change(object obj) { - bts.Log.Gen.info($"DSPublisher on_change dp: {obj}"); -} - -public class CSVPublisher : IWorker { - private IDictionary _pubs = new Dictionary(); - private readonly string _csv_directory; - private PubTableBroker _pub_table_broker; - - public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) { - _csv_directory = csv_directory; - _pub_table_broker = pub_table_broker; - MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); - } - public void doWork() { - var file_names = Directory.GetFiles(_csv_directory); - foreach (var fn in file_names) { - if (!_pubs.ContainsKey(fn)) { - _handle_new_file(fn); - } - } - } - - private void _handle_new_file(string file_name) { - if (file_name.EndsWith(".csv")) { - ViewStream vs = new CSVReaderStream(file_name); - vs = new SchemaGuesserStream(vs); - var toks = file_name.Split('\\'); - string table_name = toks.Last(); - table_name = table_name.Remove(table_name.Length - ".csv".Length); - ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs); - _pubs.Add(file_name, vp); - } - } -} +static void ConfigureServices(HostBuilderContext context, IServiceCollection services) => + services.AddDataStreamPublisher(context.Configuration); diff --git a/src/DataStreamSuite/DataStreamPublisher/PublisherNode.cs b/src/DataStreamSuite/DataStreamPublisher/PublisherNode.cs index 0df5b39a9..ec9092fd7 100644 --- a/src/DataStreamSuite/DataStreamPublisher/PublisherNode.cs +++ b/src/DataStreamSuite/DataStreamPublisher/PublisherNode.cs @@ -9,7 +9,4 @@ public class PublisherNode { [JsonPropertyName("streams")] public List? Streams { get; set; } - - [JsonPropertyName("csv_dir")] - public string? CsvDir { get; set; } } diff --git a/src/DataStreamSuite/DataStreamPublisher/Services/CsvPublisher.cs b/src/DataStreamSuite/DataStreamPublisher/Services/CsvPublisher.cs new file mode 100644 index 000000000..3c04e893b --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/Services/CsvPublisher.cs @@ -0,0 +1,39 @@ +namespace Bts.DataStreamPublisher.Services; + +using BCG; +using BCG.Utilities.General; + +using bts; + +using Bts.DataStreamCore; + +using System; +using System.Collections.Generic; +using System.IO; + +public class CsvPublisher : IWorker, IDisposable { + readonly Dictionary publishers = new(); + readonly string directory; + readonly IPubTableBroker pubTableBroker; + + public CsvPublisher(string directory, IPubTableBroker pubTableBroker) { + this.directory = directory; + this.pubTableBroker = pubTableBroker; + } + + public void doWork() { + var newFiles = Directory.GetFiles(directory, "*.csv").WhereNot(publishers.ContainsKey); + foreach (var file in newFiles) { + ViewStream vs = new CSVReaderStream(file); + vs = new SchemaGuesserStream(vs); + var tableName = Path.GetFileNameWithoutExtension(file); + var vp = new ViewPublisher(tableName, pubTableBroker, vs); + publishers.Add(file, vp); + } + } + + public void Dispose() { + publishers.Values.DisposeAll(); + publishers.Clear(); + } +} diff --git a/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs new file mode 100644 index 000000000..557dd6e28 --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs @@ -0,0 +1,105 @@ +namespace Bts.DataStreamPublisher.Services; + +using BCG; + +using bts; + +using Bts.DataStreamCore; +using Bts.DataStreamExpressions; +using Bts.PubSub; + +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; + +using Spark.Core.Disposables; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +/// +/// An that publishes s to the PubSub. +/// It can be added to the application's services via +/// +public class DataStreamPublisherService : IHostedService, IDisposable { + readonly DataStreamPublisherConfig config; + readonly IPubTableBroker pubTableBroker; + readonly IWorkScheduler workScheduler; + List? resources = new(); + + public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, IOptions config) + : this(pubTableBroker, workScheduler, config.Value) { } + + public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, DataStreamPublisherConfig config) { + this.pubTableBroker = pubTableBroker; + this.workScheduler = workScheduler; + this.config = config; + } + + public Task StartAsync(CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); + resources = new(); + foreach (var publisherNode in config.Publishers.empty_if_null()) { + foreach (var publishedStream in publisherNode.Streams.empty_if_null()) { + if (string.IsNullOrEmpty(publishedStream.Name)) { + Log.Gen.warn("Skipping PublishedStream because it doesn't have a name (must define 'Name' in config json)."); + continue; + } + var vs = CreateViewStream(publishedStream); + if (vs is null) { + continue; + } + resources.Add(new ViewPublisher(publishedStream.Name, pubTableBroker, vs)); + } + } + var csvPublishers = config.CsvDirs.empty_if_null().Select(csvDir => new CsvPublisher(csvDir, pubTableBroker)).ToList(); + resources.AddRange(csvPublishers); + resources.Add(workScheduler.Subscribe(new CompositeWorker(csvPublishers))); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); + Dispose(); + return Task.CompletedTask; + } + + public void Dispose() { + resources?.DisposeAll(); + resources?.Clear(); + } + + static ViewStream? CreateViewStream(PublishedStream ps) { + Log.Gen.info( + $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}"); + if (string.IsNullOrEmpty(ps.Name)) { + Log.Gen.warn( + $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}"); + return null; + } + var vs = buildStream(ps.Expression); + if (vs is null) { + Log.Gen.warn( + $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}"); + return null; + } + return vs; + } + + static ViewStream? buildStream(string? streamExpression) { + if (streamExpression is null) { + return null; + } + try { + Log.Gen.info($"Building ViewStream: {streamExpression}"); + var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression); + return exp.getView(new Dictionary(0)); + } catch (Exception ex) { + Log.assert(false, $"Could not build stream. expression: {streamExpression}; ex: {ex}"); + return null; + } + } +} diff --git a/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherServices.cs b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherServices.cs new file mode 100644 index 000000000..8c98d54f0 --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherServices.cs @@ -0,0 +1,62 @@ +namespace Bts.DataStreamPublisher.Services; + +using BCG; +using BCG.Utilities.General; + +using bts; + +using Bts.DataStream.Services; +using Bts.DataStreamCore; +using Bts.DataStreamCore.Services; +using Bts.DataStreamExpressions; +using Bts.PubSub.Services; + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +using Spark.Api.Services; +using Spark.Core.Services; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; + +public static class DataStreamPublisherServices { + public static IServiceCollection AddDataStreamPublisher(this IServiceCollection services, IConfigurationSection configurationSection) => + services.Configure(configurationSection) + .AddHostedService(); + + public static IServiceCollection AddCsvPublishers(this IServiceCollection services, IConfigurationSection configurationSection) { + foreach (var csvDir in configurationSection.Get>().empty_if_null()) { + services.AddSingleton(sp => new(csvDir, sp.GetRequiredService())); + } + return services; + } + + public static IServiceCollection AddDataStreamPublisher(this IServiceCollection services, IConfiguration configurationSection) { + services.AddMasterWorker(configurationSection.GetSection("MasterWorker")) + .AddViewStreamDatabase(configurationSection.GetSection("ViewStreamDatabase")) + .AddPubSubClient(configurationSection.GetSection("PubSub")) + .AddPubTableBroker() + .AddRemoteQueryManager() + .AddDataStreamPublisher(configurationSection.GetSection("DataStreamPublisher")) + .AddSpark(configurationSection.GetSection("Spark")) + .AddSparkd(configurationSection.GetSection("Sparkd")); + return services; + } + + static ViewStream? buildStream(string? streamExpression) { + if (streamExpression is null) { + return null; + } + try { + Log.Gen.info($"build_stream stream_expression: {streamExpression}"); + var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression); + return exp.getView(new Dictionary(0)); + } catch (Exception ex) { + Log.assert(false, $"Could not build stream. expression: {streamExpression}; ex: {ex}"); + return null; + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompositeWorker.cs b/src/DataStreamSuite/PubSub/Misc/CompositeWorker.cs new file mode 100644 index 000000000..dae20933e --- /dev/null +++ b/src/DataStreamSuite/PubSub/Misc/CompositeWorker.cs @@ -0,0 +1,18 @@ +namespace Bts.PubSub; + +using BCG.Utilities.General; + +using System.Collections.Generic; + +public class CompositeWorker : IWorker { + readonly List workers; + + public CompositeWorker(IEnumerable workers) => + this.workers = new(workers); + + public void doWork() { + foreach (var worker in workers) { + worker.doWork(); + } + } +} diff --git a/src/DataStreamSuite/PubSub/Misc/WorkSchedulerExtensions.cs b/src/DataStreamSuite/PubSub/Misc/WorkSchedulerExtensions.cs index f7d4a272e..583b6d727 100644 --- a/src/DataStreamSuite/PubSub/Misc/WorkSchedulerExtensions.cs +++ b/src/DataStreamSuite/PubSub/Misc/WorkSchedulerExtensions.cs @@ -16,10 +16,4 @@ public static class WorkSchedulerExtensions { void Dispose() => scheduler.Remove(worker); } - - public static void Add(this IWorkScheduler scheduler, Action work) => - scheduler.Add(new AnonymousWorker(work), TimeSpan.Zero); - - public static void Add(this IWorkScheduler scheduler, Action work, TimeSpan interval) => - scheduler.Add(new AnonymousWorker(work), interval); }