Add DataStreamPublisher service modules designed for Dependency Injection
* Change DataStreamPublisher json fields to match cs class property names verbatim * Rename Publishers -> PublisherNodes in DataStreamPublisherConfig
This commit is contained in:
1 parent
2b7fa477fb
commit
c47ddf041f
12 files changed
+391
-350
No files matched your search
@@ -0,0 +1,11 @@
|
||||
namespace Bts.DataStream.Services;
|
||||
|
||||
using BCG;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
public static class DataStreamServices {
|
||||
public static IServiceCollection AddRemoteQueryManager(this IServiceCollection services) =>
|
||||
services.AddSingleton<IWorker, RemoteQueryManager>();
|
||||
}
|
||||
@@ -1,129 +1,155 @@
|
||||
{
|
||||
"user": "bts",
|
||||
"sparkd": "http://192.168.1.80:8000/config",
|
||||
"pubsub": {
|
||||
"milliseconds_between_sub_info_updates": 1000,
|
||||
"milliseconds_between_time_slice_updates": 5000,
|
||||
"milliseconds_between_no_buffer_space_retries": 100,
|
||||
"retries_on_no_buffer_space": 1,
|
||||
"do_it_all_seconds_between_logs": 0,
|
||||
"heartbeat_interval": 15000,
|
||||
"missing_heartbeats_before_gc": 50,
|
||||
"use_managed_sockets": true,
|
||||
"state_update_period": 5000,
|
||||
"statistics_update_period": 5000,
|
||||
"disposed_connection_check_period": 1000,
|
||||
"message_processing_warning_threshold": 1000,
|
||||
"server_port": 12345,
|
||||
"publisher_routes": [
|
||||
{
|
||||
"name": "default",
|
||||
"server_address": "bts-cloud://127.0.0.1:12345",
|
||||
"tag": ".*"
|
||||
}
|
||||
],
|
||||
"subscriber_routes": [
|
||||
{
|
||||
"name": "default",
|
||||
"server_address": "bts-cloud://127.0.0.1:12345",
|
||||
"tag": ".*"
|
||||
}
|
||||
],
|
||||
"max_packet_size": 500000000,
|
||||
"blue_comm_must_die": true,
|
||||
"crushable_tags": [],
|
||||
"tag_type_to_assembly_map": [
|
||||
{
|
||||
"handler_class": "BCG.PubSubServerTagHandler",
|
||||
"assembly": "DataStreamCore.dll"
|
||||
}
|
||||
],
|
||||
"server_queue": "PubSubQueue",
|
||||
"port_range": {
|
||||
"first": 30400,
|
||||
"last": 31700
|
||||
},
|
||||
"bind_to_random_port": false,
|
||||
"debug_tags": "^NOTAGS$",
|
||||
"latency_report_threshold_ms": 1500,
|
||||
"use_substations": false,
|
||||
"use_dynamic_clients": false,
|
||||
"is_substation": false,
|
||||
"internally_publish_sub_info": true,
|
||||
"use_managed_clients_in_native_code": false,
|
||||
"compressed_tags": "^NOTAGS$",
|
||||
"tags_to_report_compression_ratio": "^NOTAGS$",
|
||||
"disable_msmq": false,
|
||||
"use_prepublished_server_state": false,
|
||||
"ip_translation": null,
|
||||
"gc_activation_threshold": 0,
|
||||
"wait_period_after_gc": 1000
|
||||
},
|
||||
"publishers": [
|
||||
{
|
||||
"name": "market_data",
|
||||
"streams": [
|
||||
{
|
||||
"name": "bbo",
|
||||
"expression": "bbo()",
|
||||
"is_on_websocket": true
|
||||
},
|
||||
{
|
||||
"name": "instruments",
|
||||
"expression": "instruments()",
|
||||
"is_on_websocket": true
|
||||
},
|
||||
{
|
||||
"name": "timesales",
|
||||
"expression": "timesales(10000)",
|
||||
"is_on_websocket": true
|
||||
},
|
||||
{
|
||||
"name": "books",
|
||||
"expression": "books()",
|
||||
"is_on_websocket": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "test_data",
|
||||
"streams": [
|
||||
{
|
||||
"name": "milliseconds_1000_100",
|
||||
"expression": "milliseconds(1000,100)",
|
||||
"is_on_websocket": true
|
||||
},
|
||||
{
|
||||
"name": "test_literal",
|
||||
"expression": "literal(\"x\", \"420.0\")",
|
||||
"is_on_websocket": true
|
||||
},
|
||||
{
|
||||
"name": "test_not_on_websocket",
|
||||
"expression": "literal(\"x\", \"840.0\")",
|
||||
"is_on_websocket": false
|
||||
}
|
||||
]
|
||||
}
|
||||
// {
|
||||
// "name": "news_data",
|
||||
// "streams": [
|
||||
// {
|
||||
// "name": "benzinga_news",
|
||||
// "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")",
|
||||
// "is_on_websocket": true
|
||||
// },
|
||||
// {
|
||||
// "name": "benzinga_news_symbols",
|
||||
// "expression": "benzingaNewsSymbols(benzinga_news)",
|
||||
// "is_on_websocket": true
|
||||
// },
|
||||
// {
|
||||
// "name": "benzinga_news_symbol_state",
|
||||
// "expression": "benzingaNewsSymbolState(benzinga_news)",
|
||||
// "is_on_websocket": true
|
||||
// }
|
||||
// ]
|
||||
// }
|
||||
]
|
||||
}
|
||||
{
|
||||
"Spark": {
|
||||
"ConfigServer": "http://localhost:8000/config"
|
||||
},
|
||||
"SparkApi": {
|
||||
"Credentials": {
|
||||
"User": "bts",
|
||||
"Password": ""
|
||||
},
|
||||
"EnableAdmin": true,
|
||||
"EnableOrders": true,
|
||||
"EnablePosition": true
|
||||
},
|
||||
"MasterWorker": {
|
||||
"Interval": "00:00:00.1"
|
||||
},
|
||||
"ViewStreamDatabase": {
|
||||
"Interval": "00:00:00.1"
|
||||
},
|
||||
"PubSub": {
|
||||
"MillisecondsBetweenSubInfoUpdates": 1000,
|
||||
"MillisecondsBetweenTimeSliceUpdates": 5000,
|
||||
"MillisecondsBetweenNoBufferSpaceRetries": 100,
|
||||
"RetriesOnNoBufferSpace": 1,
|
||||
"DoItAllSecondsBetweenLogs": 0,
|
||||
"HeartbeatInterval": 15000,
|
||||
"MissingHeartbeatsBeforeGc": 50,
|
||||
"UseManagedSockets": true,
|
||||
"StateUpdatePeriod": 5000,
|
||||
"StatisticsUpdatePeriod": 5000,
|
||||
"DisposedConnectionCheckPeriod": 1000,
|
||||
"MessageProcessingWarningThreshold": 1000,
|
||||
"ServerPort": 12345,
|
||||
"Routing": {
|
||||
"PublisherRoutes": [
|
||||
{
|
||||
"Name": "default",
|
||||
"ServerAddress": "bts-cloud://127.0.0.1:12345",
|
||||
"TagRegExp": ".*"
|
||||
}
|
||||
],
|
||||
"SubscriberRoutes": [
|
||||
{
|
||||
"Name": "default",
|
||||
"ServerAddress": "bts-cloud://127.0.0.1:12345",
|
||||
"TagRegExp": ".*"
|
||||
}
|
||||
]
|
||||
},
|
||||
"MaxPacketSize": 500000000,
|
||||
"BlueCommMustDie": true,
|
||||
"CrushableTags": [],
|
||||
"TagTypeToAssemblyMap": {
|
||||
"Entries": [
|
||||
{
|
||||
"HandlerClass": "BCG.PubSubServerTagHandler",
|
||||
"Assembly": "DataStreamCore.dll"
|
||||
}
|
||||
]
|
||||
},
|
||||
"TagHandlerFactory": {
|
||||
"Handlers": [
|
||||
{
|
||||
"TagType": "DataStream",
|
||||
"Assembly": "DataStreamCore",
|
||||
"Class": "bts.PubsubServerStreamTagHandler"
|
||||
},
|
||||
{
|
||||
"TagType": "System_Sub",
|
||||
"Assembly": "PubSub",
|
||||
"Class": "BCG.SystemSubscriptionMapPublisher"
|
||||
},
|
||||
{
|
||||
"TagType": "System_Stats",
|
||||
"Assembly": "PubSub",
|
||||
"Class": "BCG.SystemStatsPublisher"
|
||||
},
|
||||
{
|
||||
"TagType": ".*",
|
||||
"Assembly": "DataStreamCore",
|
||||
"Class": "bts.PubsubServerGenericDataHandler"
|
||||
}
|
||||
]
|
||||
},
|
||||
"ServerQueue": "PubSubQueue",
|
||||
"PortRange": {
|
||||
"First": 30400,
|
||||
"Last": 31700
|
||||
},
|
||||
"BindToRandomPort": false,
|
||||
"DebugTags": "^NOTAGS$",
|
||||
"LatencyReportThresholdMs": 1500,
|
||||
"UseSubstations": false,
|
||||
"UseDynamicClients": false,
|
||||
"IsSubstation": false,
|
||||
"InternallyPublishSubInfo": true,
|
||||
"UseManagedClientsInNativeCode": false,
|
||||
"CompressedTags": "^NOTAGS$",
|
||||
"TagsToReportCompressionRatio": "^NOTAGS$",
|
||||
"DisableMsmq": false,
|
||||
"UsePrepublishedServerState": false,
|
||||
"IpTranslation": null,
|
||||
"GcActivationThreshold": 0,
|
||||
"WaitPeriodAfterGc": 1000
|
||||
},
|
||||
"DataStreamPublisher": {
|
||||
"PublisherNodes": [
|
||||
{
|
||||
"Name": "market_data",
|
||||
"Streams": [
|
||||
{
|
||||
"Name": "bbo",
|
||||
"Expression": "bbo()",
|
||||
"IsOnWebsocket": true
|
||||
},
|
||||
{
|
||||
"Name": "instruments",
|
||||
"Expression": "instruments()",
|
||||
"IsOnWebsocket": true
|
||||
},
|
||||
{
|
||||
"Name": "timesales",
|
||||
"Expression": "timesales(10000)",
|
||||
"IsOnWebsocket": true
|
||||
},
|
||||
{
|
||||
"Name": "books",
|
||||
"Expression": "books()",
|
||||
"IsOnWebsocket": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Name": "test_data",
|
||||
"Streams": [
|
||||
{
|
||||
"Name": "milliseconds_1000_100",
|
||||
"Expression": "milliseconds(3000,5)",
|
||||
"IsOnWebsocket": true
|
||||
},
|
||||
{
|
||||
"Name": "test_literal",
|
||||
"Expression": "literal(\"x\", \"420.0\")",
|
||||
"IsOnWebsocket": true
|
||||
},
|
||||
{
|
||||
"Name": "test_not_on_websocket",
|
||||
"Expression": "literal(\"x\", \"840.0\")",
|
||||
"IsOnWebsocket": false
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace Bts.DataStreamPublisher;
|
||||
|
||||
public class CsvPublisherConfig {
|
||||
public string? Name { get; set; }
|
||||
public string? CsvDir { get; set; }
|
||||
}
|
||||
@@ -9,6 +9,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="CommandLineParser" Version="2.9.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
namespace Bts.DataStreamPublisher;
|
||||
|
||||
using Bts.PubSub;
|
||||
|
||||
using CommandLine;
|
||||
|
||||
using System;
|
||||
@@ -10,49 +8,13 @@ using System.IO;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
public class DataStreamPublisherConfig
|
||||
{
|
||||
/// <summary>
|
||||
/// The login user for sparkd
|
||||
/// </summary>
|
||||
[JsonPropertyName("user")]
|
||||
[Option("user", HelpText = "The sparkd login user")]
|
||||
public string? User { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The login password for sparkd
|
||||
/// </summary>
|
||||
[JsonPropertyName("password")]
|
||||
[Option("pass", HelpText = "The sparkd login password")]
|
||||
public string? Password { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The config server url for sparkd
|
||||
/// </summary>
|
||||
[JsonPropertyName("sparkd")]
|
||||
[Option("sparkd", HelpText = "The sparkd config server")]
|
||||
public string? Sparkd { get; set; }
|
||||
|
||||
public class DataStreamPublisherConfig {
|
||||
/// <summary>
|
||||
/// The tables to publish on startup. Each key/value represents a table name/query to publish.
|
||||
/// </summary>
|
||||
[JsonPropertyName("publishers")]
|
||||
public List<PublisherNode>? Publishers { get; set; }
|
||||
public List<PublisherNode>? PublisherNodes { get; set; }
|
||||
|
||||
[JsonPropertyName("mongodb_url")]
|
||||
public string MONGO_DB_URL { get; set; }
|
||||
|
||||
[JsonPropertyName("mongodb_db_name")]
|
||||
public string MONGO_DB_NAME { get; set; }
|
||||
|
||||
[JsonPropertyName("mongodb_db_collection")]
|
||||
public string MONGO_DB_COLLECTION { get; set; }
|
||||
|
||||
[JsonPropertyName("benzinga_news_ttl")]
|
||||
public string BENZINGA_NEWS_TTL { get; set; }
|
||||
|
||||
[JsonPropertyName("pubsub")]
|
||||
public PubSubConfig? PubSub { get; set; }
|
||||
public List<string>? CsvDirs { get; set; }
|
||||
|
||||
[Option(
|
||||
"config",
|
||||
@@ -62,18 +24,10 @@ public class DataStreamPublisherConfig
|
||||
public string? ConfigFile { get; set; }
|
||||
|
||||
public static DataStreamPublisherConfig Default() =>
|
||||
new()
|
||||
{
|
||||
PubSub = PubSubConfig.Default(),
|
||||
};
|
||||
new() { };
|
||||
|
||||
public void MergeWith(DataStreamPublisherConfig other)
|
||||
{
|
||||
User ??= other.User;
|
||||
Password ??= other.Password;
|
||||
Sparkd ??= other.Sparkd;
|
||||
Publishers ??= other.Publishers;
|
||||
PubSub ??= other.PubSub;
|
||||
public void MergeWith(DataStreamPublisherConfig other) {
|
||||
PublisherNodes ??= other.PublisherNodes;
|
||||
ConfigFile ??= other.ConfigFile;
|
||||
}
|
||||
|
||||
@@ -84,10 +38,8 @@ public class DataStreamPublisherConfig
|
||||
/// <exception cref="InvalidOperationException">
|
||||
/// JsonSerializer.Deserialize returns null
|
||||
/// </exception>
|
||||
public void MergeWithConfigFile()
|
||||
{
|
||||
if (ConfigFile is null)
|
||||
{
|
||||
public void MergeWithConfigFile() {
|
||||
if (ConfigFile is null) {
|
||||
return;
|
||||
}
|
||||
var cfgString = File.ReadAllText(ConfigFile);
|
||||
|
||||
@@ -1,166 +1,34 @@
|
||||
using BCG;
|
||||
using BCG.Utilities.General;
|
||||
using Bts.DataStreamPublisher.Services;
|
||||
|
||||
using bts;
|
||||
using bts.api.core;
|
||||
using bts.utils;
|
||||
|
||||
using Bts.DataStreamExpressions;
|
||||
using Bts.DataStreamPublisher;
|
||||
using Bts.PubSub;
|
||||
|
||||
using CommandLine;
|
||||
|
||||
using mbts.config;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args)
|
||||
.MapResult(RunAsync, errs => Task.FromResult(1));
|
||||
var host = Host.CreateDefaultBuilder(args)
|
||||
.ConfigureAppConfiguration(ConfigureApp)
|
||||
.ConfigureServices(ConfigureServices)
|
||||
.Build();
|
||||
|
||||
async Task<int> RunAsync(DataStreamPublisherConfig cfg)
|
||||
{
|
||||
try
|
||||
{
|
||||
cfg.MergeWithConfigFile();
|
||||
cfg.MergeWith(DataStreamPublisherConfig.Default());
|
||||
PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub));
|
||||
Configuration.initialize(cfg.Sparkd);
|
||||
Log.Gen.info("Starting DataStreamPublisher");
|
||||
var service_ctx = ServiceContext.create(cfg.Sparkd);
|
||||
await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty);
|
||||
_publish_streams(cfg.Publishers ?? new(0));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Write the error to console first, in case the logging system is compromised.
|
||||
var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
|
||||
Console.Error.WriteLine(error_msg);
|
||||
Console.Error.Flush();
|
||||
bts.Log.Gen.error(error_msg);
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
try {
|
||||
Console.Write("Starting DataStreamPublisher");
|
||||
host.Run();
|
||||
Console.Write("DataStreamPublisher exited gracefully.");
|
||||
} catch (Exception ex) {
|
||||
Console.Error.Write($"DataStreamPublisher crashed: {ex}");
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
|
||||
static ViewStream? build_stream(string? stream_expression)
|
||||
{
|
||||
// Console.WriteLine("Hit key to continue");
|
||||
// Console.ReadLine();
|
||||
if (stream_expression is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
try
|
||||
{
|
||||
bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
|
||||
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
||||
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||
return rv;
|
||||
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
|
||||
return null;
|
||||
static void ConfigureApp(HostBuilderContext context, IConfigurationBuilder configBuilder) {
|
||||
configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
|
||||
var cfgRoot = configBuilder.Build();
|
||||
var cfg = cfgRoot["config"];
|
||||
if (cfg is not null) {
|
||||
configBuilder.AddJsonFile(cfg, false);
|
||||
}
|
||||
}
|
||||
|
||||
void _publish_streams(IEnumerable<PublisherNode> publishers)
|
||||
{
|
||||
PubTableBroker pb_broker = new PubTableBroker();
|
||||
RemoteQueryManager csm = new RemoteQueryManager();
|
||||
var csvPublishers = new List<CSVPublisher>();
|
||||
|
||||
foreach (var publisher in publishers)
|
||||
{
|
||||
if (Directory.Exists(publisher.CsvDir))
|
||||
{
|
||||
csvPublishers.Add(new(publisher.CsvDir, pb_broker));
|
||||
}
|
||||
}
|
||||
|
||||
var viewPublishers = publishers
|
||||
.SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher))
|
||||
.ToList();
|
||||
|
||||
while (true)
|
||||
{
|
||||
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
|
||||
ViewStreamDatabase.instance.doWork();
|
||||
csm.doWork();
|
||||
MasterWorker.getInstance(false).doWork();
|
||||
ManagedSubscriber.instance.doWork();
|
||||
}
|
||||
|
||||
ViewPublisher? createViewPublisher(PublishedStream ps)
|
||||
{
|
||||
Log.Gen.info(
|
||||
$"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
|
||||
if (string.IsNullOrEmpty(ps.Name))
|
||||
{
|
||||
Log.Gen.warn(
|
||||
$"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
|
||||
return null;
|
||||
}
|
||||
var vs = build_stream(ps.Expression);
|
||||
if (vs is null)
|
||||
{
|
||||
Log.Gen.warn(
|
||||
$"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
|
||||
return null;
|
||||
}
|
||||
var vpub = new ViewPublisher(ps.Name, pb_broker, vs);
|
||||
vs.onChange += on_change;
|
||||
return vpub;
|
||||
}
|
||||
}
|
||||
|
||||
void on_change(object obj)
|
||||
{
|
||||
bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
|
||||
}
|
||||
|
||||
public class CSVPublisher : IWorker
|
||||
{
|
||||
private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
|
||||
private readonly string _csv_directory;
|
||||
private PubTableBroker _pub_table_broker;
|
||||
|
||||
public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker)
|
||||
{
|
||||
_csv_directory = csv_directory;
|
||||
_pub_table_broker = pub_table_broker;
|
||||
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
public void doWork()
|
||||
{
|
||||
var file_names = Directory.GetFiles(_csv_directory);
|
||||
foreach (var fn in file_names)
|
||||
{
|
||||
if (!_pubs.ContainsKey(fn))
|
||||
{
|
||||
_handle_new_file(fn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void _handle_new_file(string file_name)
|
||||
{
|
||||
if (file_name.EndsWith(".csv"))
|
||||
{
|
||||
ViewStream vs = new CSVReaderStream(file_name);
|
||||
vs = new SchemaGuesserStream(vs);
|
||||
var toks = file_name.Split('\\');
|
||||
string table_name = toks.Last();
|
||||
table_name = table_name.Remove(table_name.Length - ".csv".Length);
|
||||
ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
|
||||
_pubs.Add(file_name, vp);
|
||||
}
|
||||
}
|
||||
}
|
||||
static void ConfigureServices(HostBuilderContext context, IServiceCollection services) =>
|
||||
services.AddDataStreamPublisher(context.Configuration);
|
||||
@@ -9,7 +9,4 @@ public class PublisherNode {
|
||||
|
||||
[JsonPropertyName("streams")]
|
||||
public List<PublishedStream>? Streams { get; set; }
|
||||
|
||||
[JsonPropertyName("csv_dir")]
|
||||
public string? CsvDir { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
namespace Bts.DataStreamPublisher.Services;
|
||||
|
||||
using BCG;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using bts;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
|
||||
public class CsvPublisher : IWorker, IDisposable {
|
||||
readonly Dictionary<string, ViewPublisher> publishers = new();
|
||||
readonly string directory;
|
||||
readonly IPubTableBroker pubTableBroker;
|
||||
|
||||
public CsvPublisher(string directory, IPubTableBroker pubTableBroker) {
|
||||
this.directory = directory;
|
||||
this.pubTableBroker = pubTableBroker;
|
||||
}
|
||||
|
||||
public void doWork() {
|
||||
var newFiles = Directory.GetFiles(directory, "*.csv").WhereNot(publishers.ContainsKey);
|
||||
foreach (var file in newFiles) {
|
||||
ViewStream vs = new CSVReaderStream(file);
|
||||
vs = new SchemaGuesserStream(vs);
|
||||
var tableName = Path.GetFileNameWithoutExtension(file);
|
||||
var vp = new ViewPublisher(tableName, pubTableBroker, vs);
|
||||
publishers.Add(file, vp);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose() {
|
||||
publishers.Values.DisposeAll();
|
||||
publishers.Clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
namespace Bts.DataStreamPublisher.Services;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
using Bts.DataStreamExpressions;
|
||||
using Bts.PubSub;
|
||||
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
using Spark.Core.Disposables;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
/// <summary>
|
||||
/// An <see cref="IHostedService"/> that publishes <see cref="ViewStream"/>s to the PubSub.
|
||||
/// It can be added to the application's services via <see cref="DataStreamPublisherServices.AddDataStreamPublisher"/>
|
||||
/// </summary>
|
||||
public class DataStreamPublisherService : IHostedService, IDisposable {
|
||||
readonly DataStreamPublisherConfig config;
|
||||
readonly IPubTableBroker pubTableBroker;
|
||||
readonly IWorkScheduler workScheduler;
|
||||
List<IDisposable>? resources = new();
|
||||
|
||||
public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, IOptions<DataStreamPublisherConfig> config)
|
||||
: this(pubTableBroker, workScheduler, config.Value) { }
|
||||
|
||||
public DataStreamPublisherService(IPubTableBroker pubTableBroker, IWorkScheduler workScheduler, DataStreamPublisherConfig config) {
|
||||
this.pubTableBroker = pubTableBroker;
|
||||
this.workScheduler = workScheduler;
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken) {
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
resources = new();
|
||||
foreach (var publisherNode in config.PublisherNodes.empty_if_null()) {
|
||||
foreach (var publishedStream in publisherNode.Streams.empty_if_null()) {
|
||||
if (string.IsNullOrEmpty(publishedStream.Name)) {
|
||||
Log.Gen.warn("Skipping PublishedStream because it doesn't have a name (must define 'Name' in config json).");
|
||||
continue;
|
||||
}
|
||||
var vs = CreateViewStream(publishedStream);
|
||||
if (vs is null) {
|
||||
continue;
|
||||
}
|
||||
resources.Add(new ViewPublisher(publishedStream.Name, pubTableBroker, vs));
|
||||
}
|
||||
}
|
||||
var csvPublishers = config.CsvDirs.empty_if_null().Select(csvDir => new CsvPublisher(csvDir, pubTableBroker)).ToList();
|
||||
resources.AddRange(csvPublishers);
|
||||
resources.Add(workScheduler.Subscribe(new CompositeWorker(csvPublishers)));
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken) {
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public void Dispose() {
|
||||
resources?.DisposeAll();
|
||||
resources?.Clear();
|
||||
}
|
||||
|
||||
static ViewStream? CreateViewStream(PublishedStream ps) {
|
||||
Log.Gen.info(
|
||||
$"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
|
||||
if (string.IsNullOrEmpty(ps.Name)) {
|
||||
Log.Gen.warn(
|
||||
$"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
|
||||
return null;
|
||||
}
|
||||
var vs = buildStream(ps.Expression);
|
||||
if (vs is null) {
|
||||
Log.Gen.warn(
|
||||
$"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
|
||||
return null;
|
||||
}
|
||||
return vs;
|
||||
}
|
||||
|
||||
static ViewStream? buildStream(string? streamExpression) {
|
||||
if (streamExpression is null) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
Log.Gen.info($"Building ViewStream: {streamExpression}");
|
||||
var exp = DataStreamExpressionsInterpreter.Evaluate(streamExpression);
|
||||
return exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||
} catch (Exception ex) {
|
||||
Log.assert(false, $"Could not build stream. expression: {streamExpression}; ex: {ex}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
namespace Bts.DataStreamPublisher.Services;
|
||||
|
||||
using Bts.DataStream.Services;
|
||||
using Bts.DataStreamCore.Services;
|
||||
using Bts.PubSub.Services;
|
||||
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
using Spark.Api.Services;
|
||||
using Spark.Core.Services;
|
||||
|
||||
public static class DataStreamPublisherServices {
|
||||
public static IServiceCollection AddDataStreamPublisher(this IServiceCollection services, IConfiguration configurationSection) =>
|
||||
services.AddSpark(configurationSection.GetSection("Spark"))
|
||||
.AddSparkApi(configurationSection.GetSection("SparkApi"))
|
||||
.AddMasterWorker(configurationSection.GetSection("MasterWorker"))
|
||||
.AddViewStreamDatabase(configurationSection.GetSection("ViewStreamDatabase"))
|
||||
.AddPubSubClient(configurationSection.GetSection("PubSub"))
|
||||
.AddPubTableBroker()
|
||||
.AddRemoteQueryManager()
|
||||
.Configure<DataStreamPublisherConfig>(configurationSection.GetSection("DataStreamPublisher"))
|
||||
.AddHostedService<DataStreamPublisherService>();
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
namespace Bts.PubSub;
|
||||
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using System.Collections.Generic;
|
||||
|
||||
public class CompositeWorker : IWorker {
|
||||
readonly List<IWorker> workers;
|
||||
|
||||
public CompositeWorker(IEnumerable<IWorker> workers) =>
|
||||
this.workers = new(workers);
|
||||
|
||||
public void doWork() {
|
||||
foreach (var worker in workers) {
|
||||
worker.doWork();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,10 +16,4 @@ public static class WorkSchedulerExtensions {
|
||||
void Dispose() =>
|
||||
scheduler.Remove(worker);
|
||||
}
|
||||
|
||||
public static void Add(this IWorkScheduler scheduler, Action work) =>
|
||||
scheduler.Add(new AnonymousWorker(work), TimeSpan.Zero);
|
||||
|
||||
public static void Add(this IWorkScheduler scheduler, Action work, TimeSpan interval) =>
|
||||
scheduler.Add(new AnonymousWorker(work), interval);
|
||||
}
|
||||
Reference in new issue
Block a user