Add DataStreamPublisher service modules designed for Dependency Injection

This commit is contained in:
John Colvin committed 2024-07-11 10:37:28 -04:00
1 parent 8127c9c9fe
commit db18aebf92
11 files changed
+268 -172

No files matched your search

@@ -0,0 +1,11 @@
namespace Bts.DataStream.Services;
using BCG;
using BCG.Utilities.General;
using Microsoft.Extensions.DependencyInjection;
public static class DataStreamServices {
public static IServiceCollection AddRemoteQueryManager(this IServiceCollection services) =>
services.AddSingleton<IWorker, RemoteQueryManager>();
}
@@ -0,0 +1,6 @@
namespace Bts.DataStreamPublisher;
public class CsvPublisherConfig {
public string? Name { get; set; }
public string? CsvDir { get; set; }
}
@@ -9,6 +9,7 @@
<ItemGroup>
<PackageReference Include="CommandLineParser" Version="2.9.1" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
</ItemGroup>
<ItemGroup>
@@ -1,7 +1,5 @@
namespace Bts.DataStreamPublisher;
using Bts.PubSub;
using CommandLine;
using System;
@@ -11,35 +9,12 @@ using System.Text.Json;
using System.Text.Json.Serialization;
public class DataStreamPublisherConfig {
/// <summary>
/// The login user for sparkd
/// </summary>
[JsonPropertyName("user")]
[Option("user", HelpText = "The sparkd login user")]
public string? User { get; set; }
/// <summary>
/// The login password for sparkd
/// </summary>
[JsonPropertyName("password")]
[Option("pass", HelpText = "The sparkd login password")]
public string? Password { get; set; }
/// <summary>
/// The config server url for sparkd
/// </summary>
[JsonPropertyName("sparkd")]
[Option("sparkd", HelpText = "The sparkd config server")]
public string? Sparkd { get; set; }
/// <summary>
/// The tables to publish on startup. Each key/value represents a table name/query to publish.
/// </summary>
[JsonPropertyName("publishers")]
public List<PublisherNode>? Publishers { get; set; }
[JsonPropertyName("pubsub")]
public PubSubConfig? PubSub { get; set; }
public List<string>? CsvDirs { get; set; }
[Option(
"config",
@@ -49,16 +24,10 @@ public class DataStreamPublisherConfig {
public string? ConfigFile { get; set; }
public static DataStreamPublisherConfig Default() =>
new() {
PubSub = PubSubConfig.Default(),
};
new() { };
public void MergeWith(DataStreamPublisherConfig other) {
User ??= other.User;
Password ??= other.Password;
Sparkd ??= other.Sparkd;
Publishers ??= other.Publishers;
PubSub ??= other.PubSub;
ConfigFile ??= other.ConfigFile;
}
@@ -1,140 +1,34 @@
using BCG;
using BCG.Utilities.General;
using Bts.DataStreamPublisher.Services;
using bts;
using bts.api.core;
using bts.utils;
using Bts.DataStreamExpressions;
using Bts.DataStreamPublisher;
using Bts.PubSub;
using CommandLine;
using mbts.config;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args)
.MapResult(RunAsync, errs => Task.FromResult(1));
var host = Host.CreateDefaultBuilder(args)
.ConfigureAppConfiguration(ConfigureApp)
.ConfigureServices(ConfigureServices)
.Build();
async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
try {
cfg.MergeWithConfigFile();
cfg.MergeWith(DataStreamPublisherConfig.Default());
PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub));
Configuration.initialize(cfg.Sparkd);
Log.Gen.info("Starting DataStreamPublisher");
var service_ctx = ServiceContext.create(cfg.Sparkd);
await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty);
_publish_streams(cfg.Publishers ?? new(0));
} catch (Exception ex) {
// Write the error to console first, in case the logging system is compromised.
var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
Console.Error.WriteLine(error_msg);
Console.Error.Flush();
bts.Log.Gen.error(error_msg);
return 1;
}
return 0;
try {
Console.Write("Starting DataStreamPublisher");
host.Run();
Console.Write("DataStreamPublisher exited gracefully.");
} catch (Exception ex) {
Console.Error.Write($"DataStreamPublisher crashed: {ex}");
return 1;
}
return 0;
static ViewStream? build_stream(string? stream_expression) {
if (stream_expression is null) {
return null;
}
try {
bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv;
} catch (Exception ex) {
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
return null;
static void ConfigureApp(HostBuilderContext context, IConfigurationBuilder configBuilder) {
configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
var cfgRoot = configBuilder.Build();
var cfg = cfgRoot["config"];
if (cfg is not null) {
configBuilder.AddJsonFile(cfg, false);
}
}
void _publish_streams(IEnumerable<PublisherNode> publishers) {
PubTableBroker pb_broker = new PubTableBroker();
RemoteQueryManager csm = new RemoteQueryManager();
var csvPublishers = new List<CSVPublisher>();
foreach (var publisher in publishers) {
if (Directory.Exists(publisher.CsvDir)) {
csvPublishers.Add(new(publisher.CsvDir, pb_broker));
}
}
var viewPublishers = publishers
.SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher))
.ToList();
while (true) {
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
ViewStreamDatabase.instance.doWork();
csm.doWork();
MasterWorker.getInstance(false).doWork();
ManagedSubscriber.instance.doWork();
}
ViewPublisher? createViewPublisher(PublishedStream ps) {
Log.Gen.info(
$"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
if (string.IsNullOrEmpty(ps.Name)) {
Log.Gen.warn(
$"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
return null;
}
var vs = build_stream(ps.Expression);
if (vs is null) {
Log.Gen.warn(
$"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
return null;
}
var vpub = new ViewPublisher(ps.Name, pb_broker, vs);
vs.onChange += on_change;
return vpub;
}
}
void on_change(object obj) {
bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
}
public class CSVPublisher : IWorker {
private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
private readonly string _csv_directory;
private PubTableBroker _pub_table_broker;
public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
_csv_directory = csv_directory;
_pub_table_broker = pub_table_broker;
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
}
public void doWork() {
var file_names = Directory.GetFiles(_csv_directory);
foreach (var fn in file_names) {
if (!_pubs.ContainsKey(fn)) {
_handle_new_file(fn);
}
}
}
private void _handle_new_file(string file_name) {
if (file_name.EndsWith(".csv")) {
ViewStream vs = new CSVReaderStream(file_name);
vs = new SchemaGuesserStream(vs);
var toks = file_name.Split('\\');
string table_name = toks.Last();
table_name = table_name.Remove(table_name.Length - ".csv".Length);
ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
_pubs.Add(file_name, vp);
}
}
}
static void ConfigureServices(HostBuilderContext context, IServiceCollection services) =>
services.AddDataStreamPublisher(context.Configuration);
@@ -9,7 +9,4 @@ public class PublisherNode {
[JsonPropertyName("streams")]
public List<PublishedStream>? Streams { get; set; }
[JsonPropertyName("csv_dir")]
public string? CsvDir { get; set; }
}
@@ -0,0 +1,39 @@
namespace Bts.DataStreamPublisher.Services;
using BCG;
using BCG.Utilities.General;
using bts;
using Bts.DataStreamCore;
using System;
using System.Collections.Generic;
using System.IO;
public class CsvPublisher : IWorker, IDisposable {
readonly Dictionary<string, ViewPublisher> publishers = new();
readonly string directory;
readonly IPubTableBroker pubTableBroker;
public CsvPublisher(string directory, IPubTableBroker pubTableBroker) {
this.directory = directory;
this.pubTableBroker = pubTableBroker;
}
public void doWork() {
var newFiles = Directory.GetFiles(directory, "*.csv").WhereNot(publishers.ContainsKey);
foreach (var file in newFiles) {
ViewStream vs = new CSVReaderStream(file);
vs = new SchemaGuesserStream(vs);
var tableName = Path.GetFileNameWithoutExtension(file);
var vp = new ViewPublisher(tableName, pubTableBroker, vs);
publishers.Add(file, vp);
}
}
public void Dispose() {
publishers.Values.DisposeAll();
publishers.Clear();
}
}
@@ -0,0 +1,105 @@
namespace Bts.DataStreamPublisher.Services;
using BCG;
using bts;
using Bts.DataStreamCore;
using Bts.DataStreamExpressions;
using Bts.PubSub;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Options;
using Spark.Core.Disposables;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
/// <summary>
/// An <see cref="IHostedService"/> that publishes <see cref="ViewStream"/>s to the PubSub.
/// It can be added to the application's services via <see cref="DataStreamPublisherServices.AddDataStreamPublisher"/>
/// </summary>
public class DataStreamPublisherService : IHostedService, IDisposable {
readonly DataStreamPublisherConfig config;
readonly IPubTableBroker pubTableBroker;
readonly IWorkScheduler workScheduler;
List<IDisposable>? resources = new();
public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, IOptions<DataStreamPublisherConfig> config)
: this(pubTableBroker, workScheduler, config.Value) { }
public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, DataStreamPublisherConfig config) {
this.pubTableBroker = pubTableBroker;
this.workScheduler = workScheduler;
this.config = config;
}
public Task StartAsync(CancellationToken cancellationToken) {
cancellationToken.ThrowIfCancellationRequested();
resources = new();
foreach (var publisherNode in config.Publishers.empty_if_null()) {
foreach (var publishedStream in publisherNode.Streams.empty_if_null()) {
if (string.IsNullOrEmpty(publishedStream.Name)) {
Log.Gen.warn("Skipping PublishedStream because it doesn't have a name (must define 'Name' in config json).");
continue;
}
var vs = CreateViewStream(publishedStream);
if (vs is null) {
continue;
}
resources.Add(new ViewPublisher(publishedStream.Name, pubTableBroker, vs));
}
}
var csvPublishers = config.CsvDirs.empty_if_null().Select(csvDir => new CsvPublisher(csvDir, pubTableBroker)).ToList();
resources.AddRange(csvPublishers);
resources.Add(workScheduler.Subscribe(new CompositeWorker(csvPublishers)));
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken cancellationToken) {
cancellationToken.ThrowIfCancellationRequested();
Dispose();
return Task.CompletedTask;
}
public void Dispose() {
resources?.DisposeAll();
resources?.Clear();
}
static ViewStream? CreateViewStream(PublishedStream ps) {
Log.Gen.info(
$"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
if (string.IsNullOrEmpty(ps.Name)) {
Log.Gen.warn(
$"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
return null;
}
var vs = buildStream(ps.Expression);
if (vs is null) {
Log.Gen.warn(
$"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
return null;
}
return vs;
}
static ViewStream? buildStream(string? streamExpression) {
if (streamExpression is null) {
return null;
}
try {
Log.Gen.info($"Building ViewStream: {streamExpression}");
var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression);
return exp.getView(new Dictionary<string, StatementExpression>(0));
} catch (Exception ex) {
Log.assert(false, $"Could not build stream. expression: {streamExpression}; ex: {ex}");
return null;
}
}
}
@@ -0,0 +1,62 @@
namespace Bts.DataStreamPublisher.Services;
using BCG;
using BCG.Utilities.General;
using bts;
using Bts.DataStream.Services;
using Bts.DataStreamCore;
using Bts.DataStreamCore.Services;
using Bts.DataStreamExpressions;
using Bts.PubSub.Services;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Spark.Api.Services;
using Spark.Core.Services;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
public static class DataStreamPublisherServices {
public static IServiceCollection AddDataStreamPublisher(this IServiceCollection services, IConfigurationSection configurationSection) =>
services.Configure<DataStreamPublisherConfig>(configurationSection)
.AddHostedService<DataStreamPublisherService>();
public static IServiceCollection AddCsvPublishers(this IServiceCollection services, IConfigurationSection configurationSection) {
foreach (var csvDir in configurationSection.Get<List<string>>().empty_if_null()) {
services.AddSingleton<IWorker, CsvPublisher>(sp => new(csvDir, sp.GetRequiredService<IPubTableBroker>()));
}
return services;
}
public static IServiceCollection AddDataStreamPublisher(this IServiceCollection services, IConfiguration configurationSection) {
services.AddMasterWorker(configurationSection.GetSection("MasterWorker"))
.AddViewStreamDatabase(configurationSection.GetSection("ViewStreamDatabase"))
.AddPubSubClient(configurationSection.GetSection("PubSub"))
.AddPubTableBroker()
.AddRemoteQueryManager()
.AddDataStreamPublisher(configurationSection.GetSection("DataStreamPublisher"))
.AddSpark(configurationSection.GetSection("Spark"))
.AddSparkd(configurationSection.GetSection("Sparkd"));
return services;
}
static ViewStream? buildStream(string? streamExpression) {
if (streamExpression is null) {
return null;
}
try {
Log.Gen.info($"build_stream stream_expression: {streamExpression}");
var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression);
return exp.getView(new Dictionary<string, StatementExpression>(0));
} catch (Exception ex) {
Log.assert(false, $"Could not build stream. expression: {streamExpression}; ex: {ex}");
return null;
}
}
}
@@ -0,0 +1,18 @@
namespace Bts.PubSub;
using BCG.Utilities.General;
using System.Collections.Generic;
public class CompositeWorker : IWorker {
readonly List<IWorker> workers;
public CompositeWorker(IEnumerable<IWorker> workers) =>
this.workers = new(workers);
public void doWork() {
foreach (var worker in workers) {
worker.doWork();
}
}
}
@@ -16,10 +16,4 @@ public static class WorkSchedulerExtensions {
void Dispose() =>
scheduler.Remove(worker);
}
public static void Add(this IWorkScheduler scheduler, Action work) =>
scheduler.Add(new AnonymousWorker(work), TimeSpan.Zero);
public static void Add(this IWorkScheduler scheduler, Action work, TimeSpan interval) =>
scheduler.Add(new AnonymousWorker(work), interval);
}