From 7ee693378941a4bfe247536357f313d8d996086b Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 23 May 2024 12:24:26 -0500 Subject: [PATCH] refactor: Add formal config classes for prepublish streams They can be passed to command line via --config --- .../Configurations/MarketDataPublisher.json | 65 ++++++++++++++++--- .../Configurations/TestDataPublisher.json | 25 +++++++ .../DataStreamPublisherConfig.cs | 6 +- .../DataStreamPublisher/Program.cs | 43 ++++++++---- .../DataStreamPublisher/PublishedStream.cs | 14 ++++ .../DataStreamPublisher/PublisherNode.cs | 15 +++++ 6 files changed, 144 insertions(+), 24 deletions(-) create mode 100644 src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json create mode 100644 src/DataStreamSuite/DataStreamPublisher/PublishedStream.cs create mode 100644 src/DataStreamSuite/DataStreamPublisher/PublisherNode.cs diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json index 42cf76548..59c36cfeb 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json +++ b/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json @@ -1,12 +1,59 @@ { "user": "bts", - "prepublish_tables": { - "bbo": "bbo()", - "instruments": "instruments()", - "timesales": "timesales(10000)", - "books": "books()", - "seconds_1_10": "second(1,10)", - "literal_420": "literal(\"x\", \"420.0\")", - "mongodb_testcollection": "mongodb(\"mongodb+srv://johnc-dev:DMhDhP!g-TKeK.5@atlas-online-archive-5e18ab3c77c74d1cffa64c03-m7jkr.a.query.mongodb.net/?ssl=true&authSource=admin\",\"admin\",\"test_collection\")" - } + "publishers": [ + { + "name": "market_data", + "streams": [ + { + "name": "bbo", + "expression": "bbo()", + "is_on_websocket": true + }, + { + "name": "instruments", + "expression": "instruments()", + "is_on_websocket": true + }, + { + "name": "timesales", + "expression": "timesales(10000)", + "is_on_websocket": true + }, + { + "name": "books", + "expression": "books()", + "is_on_websocket": true + } + ] + }, + { + "name": "mongo", + "streams": [ + { + "name": "test_mongo", + "expression": "mongodb(\"mongodb+srv://johnc-dev:DMhDhP!g-TKeK.5@atlas-online-archive-5e18ab3c77c74d1cffa64c03-m7jkr.a.query.mongodb.net/?ssl=true&authSource=admin\",\"admin\",\"test_collection\")" + } + ] + }, + { + "name": "test_data", + "streams": [ + { + "name": "seconds_1_10", + "expression": "second(1,10)", + "is_on_websocket": true + }, + { + "name": "test_literal", + "expression": "literal(\"x\", \"420.0\")", + "is_on_websocket": true + }, + { + "name": "test_not_on_websocket", + "expression": "literal(\"x\", \"840.0\")", + "is_on_websocket": false + } + ] + } + ] } diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json new file mode 100644 index 000000000..04ebfef32 --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json @@ -0,0 +1,25 @@ +{ + "user": "bts", + "publishers": [ + { + "name": "test_data", + "streams": [ + { + "name": "seconds_1_10", + "expression": "second(1,10)", + "is_on_websocket": true + }, + { + "name": "test_literal", + "expression": "literal(\"x\", \"420.0\")", + "is_on_websocket": true + }, + { + "name": "test_not_on_websocket", + "expression": "literal(\"x\", \"840.0\")", + "is_on_websocket": false + } + ] + } + ] +} diff --git a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs index 598889899..e556e8acf 100644 --- a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs +++ b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs @@ -33,8 +33,8 @@ public class DataStreamPublisherConfig { /// /// The tables to publish on startup. Each key/value represents a table name/query to publish. /// - [JsonPropertyName("prepublish_tables")] - public Dictionary? PrepublishTables { get; set; } + [JsonPropertyName("publishers")] + public List? Publishers { get; set; } [JsonPropertyName("simple_echo_client")] [Option("simple-echo-client", HelpText = "Run a simple echo client (won't publish tables or connect to sparkd)")] @@ -51,7 +51,7 @@ public class DataStreamPublisherConfig { User ??= other.User; Password ??= other.Password; Sparkd ??= other.Sparkd; - PrepublishTables ??= other.PrepublishTables; + Publishers ??= other.Publishers; SimpleEchoClient ??= other.SimpleEchoClient; ConfigFile ??= other.ConfigFile; } diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs index 55609bbf0..fcadda583 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Program.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text.Json; using System.Threading.Tasks; await Parser.Default.ParseArguments(args) @@ -31,7 +32,7 @@ async Task RunAsync(DataStreamPublisherConfig cfg) { if (cfg.SimpleEchoClient ?? false) { start_echo_client(); } else { - _publish_streams(cfg.PrepublishTables ?? new(0)); + _publish_streams(cfg.Publishers ?? new(0)); } } catch (Exception ex) { // Write the error to console first, in case the logging system is compromised. @@ -44,7 +45,10 @@ async Task RunAsync(DataStreamPublisherConfig cfg) { return 0; } -static ViewStream build_stream(string stream_expression) { +static ViewStream? build_stream(string? stream_expression) { + if (stream_expression is null) { + return null; + } try { bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}"); var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); @@ -69,28 +73,43 @@ static void start_echo_client() { // Task.Delay(1000000000).Wait(); } -void _publish_streams(IEnumerable> prepublish_streams) { +void _publish_streams(IEnumerable publishers) { PubTableBroker pb_broker = new PubTableBroker(); CloudStreamManager csm = new CloudStreamManager(); string csv_folder = @"C:\Temp\csvs_topublish"; CSVPublisher csv_publisher = new CSVPublisher(csv_folder, pb_broker); - foreach (var kv in prepublish_streams) { - var name = kv.Key; - var exp = kv.Value; - BCG.ViewStream vs = build_stream(exp); - ViewPublisher vpub = new ViewPublisher(name, pb_broker, vs); - vs.onChange += on_change; - } + + var viewPublishers = publishers + .SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher)) + .ToList(); + while (true) { System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1)); ViewStreamDatabase.instance.doWork(); csm.doWork(); MasterWorker.getInstance(false).doWork(); } + + ViewPublisher? createViewPublisher(PublishedStream ps) { + Log.Gen.info( + $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}"); + if (string.IsNullOrEmpty(ps.Name)) { + Log.Gen.warn( + $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}"); + return null; + } + var vs = build_stream(ps.Expression); + if (vs is null) { + Log.Gen.warn( + $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}"); + return null; + } + var vpub = new ViewPublisher(ps.Name, pb_broker, vs); + vs.onChange += on_change; + return vpub; + } } - - void on_change(object obj) { bts.Log.Gen.info($"DSPublisher on_change dp: {obj}"); } diff --git a/src/DataStreamSuite/DataStreamPublisher/PublishedStream.cs b/src/DataStreamSuite/DataStreamPublisher/PublishedStream.cs new file mode 100644 index 000000000..a819da966 --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/PublishedStream.cs @@ -0,0 +1,14 @@ +namespace Bts.DataStreamPublisher; + +using System.Text.Json.Serialization; + +public class PublishedStream { + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("expression")] + public string? Expression { get; set; } + + [JsonPropertyName("is_on_websocket")] + public bool? IsOnWebsocket { get; set; } +} diff --git a/src/DataStreamSuite/DataStreamPublisher/PublisherNode.cs b/src/DataStreamSuite/DataStreamPublisher/PublisherNode.cs new file mode 100644 index 000000000..0df5b39a9 --- /dev/null +++ b/src/DataStreamSuite/DataStreamPublisher/PublisherNode.cs @@ -0,0 +1,15 @@ +namespace Bts.DataStreamPublisher; + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +public class PublisherNode { + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("streams")] + public List? Streams { get; set; } + + [JsonPropertyName("csv_dir")] + public string? CsvDir { get; set; } +}