Files
spark/src/DataStreamSuite/DataStreamPublisher/Services/PublisherService.cs
T
John Colvin 5fa284cad3 refactor: Use DependencyInjection paradigm in DataStreamSuite
This allows us to create loosely coupled services with automatic
dependency resolution
2024-07-07 18:13:59 -04:00

157 lines
5.2 KiB
C#

namespace Bts.DataStreamPublisher;
using BCG;
using BCG.Utilities.General;
using bts;
using bts.api.core;
using Bts.DataStreamCore;
using Bts.DataStreamExpressions;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Options;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
/// <summary>
/// An IHostedService wrapper for ViewStream publishers. It provides a way for the Application to
/// start/stop the view publishers.
/// </summary>
public class PublisherService : IHostedService {
readonly PublisherServiceConfig config;
readonly IPubTableBroker pubTableBroker;
readonly PrimitiveManager primitiveManager;
readonly IServiceProvider serviceProvider;
public PublisherService(
IPubTableBroker pubTableBroker,
PrimitiveManager primitiveManager,
IServiceProvider serviceProvider,
IOptions<PublisherServiceConfig> config
) {
this.pubTableBroker = pubTableBroker;
this.primitiveManager = primitiveManager;
this.serviceProvider = serviceProvider;
this.config = config.Value;
}
public Task StartAsync(CancellationToken cancellationToken) {
try {
Log.Gen.info("Starting DataStreamPublisher");
if (config.SimpleEchoClient ?? false) {
startEchoClient();
} else {
startPublisher(config.Publishers ?? new(0));
}
} catch (Exception ex) {
// Write the error to console first, in case the logging system is compromised.
var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
Console.Error.WriteLine(error_msg);
Console.Error.Flush();
Log.Gen.error(error_msg);
throw;
}
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken cancellationToken) =>
Task.CompletedTask;
ViewStream? buildStream(string? streamExpression) {
if (streamExpression is null) {
return null;
}
try {
Log.Gen.info($"buildStream streamExpression: {streamExpression}");
var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0), primitiveManager);
rv?.Reset(serviceProvider);
return rv;
} catch (Exception ex) {
bts.Log.assert(false, $"Could not build stream: {streamExpression} ex: {ex}");
return null;
}
}
static void startEchoClient() {
Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue");
Console.ReadLine();
var client = new SimpleTCPClient("127.0.0.1", 12345, 1000, 1000, int.MaxValue);
client.start();
}
void startPublisher(IEnumerable<PublisherNode> publishers) {
var csvPublishers = new List<CSVPublisher>();
foreach (var publisher in publishers) {
if (Directory.Exists(publisher.CsvDir)) {
csvPublishers.Add(new(publisher.CsvDir, pubTableBroker));
}
}
var viewPublishers = publishers.SelectMany(p => p.Streams.empty_if_null().Select(CreateViewPublisher)).ToList();
ViewPublisher? CreateViewPublisher(PublishedStream ps) {
Log.Gen.info(
$"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
if (string.IsNullOrEmpty(ps.Name)) {
Log.Gen.warn(
$"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
return null;
}
var vs = buildStream(ps.Expression);
if (vs is null) {
Log.Gen.warn(
$"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
return null;
}
var vpub = new ViewPublisher(ps.Name, pubTableBroker, vs);
vs.onChange += on_change;
return vpub;
}
}
void on_change(object obj) =>
Log.Gen.info($"DSPublisher on_change dp: {obj}");
public class CSVPublisher : IWorker {
private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
private readonly string _csv_directory;
private IPubTableBroker _pub_table_broker;
public CSVPublisher(string csv_directory, IPubTableBroker pub_table_broker) {
_csv_directory = csv_directory;
_pub_table_broker = pub_table_broker;
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
}
public void doWork() {
var file_names = Directory.GetFiles(_csv_directory);
foreach (var fn in file_names) {
if (!_pubs.ContainsKey(fn)) {
_handle_new_file(fn);
}
}
}
private void _handle_new_file(string file_name) {
if (file_name.EndsWith(".csv")) {
ViewStream vs = new CSVReaderStream(file_name);
vs = new SchemaGuesserStream(vs);
var toks = file_name.Split('\\');
string table_name = toks.Last();
table_name = table_name.Remove(table_name.Length - ".csv".Length);
ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
_pubs.Add(file_name, vp);
}
}
}
}