diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json
index 42cf76548..59c36cfeb 100644
--- a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json
+++ b/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json
@@ -1,12 +1,59 @@
{
"user": "bts",
- "prepublish_tables": {
- "bbo": "bbo()",
- "instruments": "instruments()",
- "timesales": "timesales(10000)",
- "books": "books()",
- "seconds_1_10": "second(1,10)",
- "literal_420": "literal(\"x\", \"420.0\")",
- "mongodb_testcollection": "mongodb(\"mongodb+srv://johnc-dev:DMhDhP!g-TKeK.5@atlas-online-archive-5e18ab3c77c74d1cffa64c03-m7jkr.a.query.mongodb.net/?ssl=true&authSource=admin\",\"admin\",\"test_collection\")"
- }
+ "publishers": [
+ {
+ "name": "market_data",
+ "streams": [
+ {
+ "name": "bbo",
+ "expression": "bbo()",
+ "is_on_websocket": true
+ },
+ {
+ "name": "instruments",
+ "expression": "instruments()",
+ "is_on_websocket": true
+ },
+ {
+ "name": "timesales",
+ "expression": "timesales(10000)",
+ "is_on_websocket": true
+ },
+ {
+ "name": "books",
+ "expression": "books()",
+ "is_on_websocket": true
+ }
+ ]
+ },
+ {
+ "name": "mongo",
+ "streams": [
+ {
+ "name": "test_mongo",
+ "expression": "mongodb(\"mongodb+srv://johnc-dev:DMhDhP!g-TKeK.5@atlas-online-archive-5e18ab3c77c74d1cffa64c03-m7jkr.a.query.mongodb.net/?ssl=true&authSource=admin\",\"admin\",\"test_collection\")"
+ }
+ ]
+ },
+ {
+ "name": "test_data",
+ "streams": [
+ {
+ "name": "seconds_1_10",
+ "expression": "second(1,10)",
+ "is_on_websocket": true
+ },
+ {
+ "name": "test_literal",
+ "expression": "literal(\"x\", \"420.0\")",
+ "is_on_websocket": true
+ },
+ {
+ "name": "test_not_on_websocket",
+ "expression": "literal(\"x\", \"840.0\")",
+ "is_on_websocket": false
+ }
+ ]
+ }
+ ]
}
diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json
new file mode 100644
index 000000000..04ebfef32
--- /dev/null
+++ b/src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json
@@ -0,0 +1,25 @@
+{
+ "user": "bts",
+ "publishers": [
+ {
+ "name": "test_data",
+ "streams": [
+ {
+ "name": "seconds_1_10",
+ "expression": "second(1,10)",
+ "is_on_websocket": true
+ },
+ {
+ "name": "test_literal",
+ "expression": "literal(\"x\", \"420.0\")",
+ "is_on_websocket": true
+ },
+ {
+ "name": "test_not_on_websocket",
+ "expression": "literal(\"x\", \"840.0\")",
+ "is_on_websocket": false
+ }
+ ]
+ }
+ ]
+}
diff --git a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs
index 598889899..e556e8acf 100644
--- a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs
+++ b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs
@@ -33,8 +33,8 @@ public class DataStreamPublisherConfig {
///
/// The tables to publish on startup. Each key/value represents a table name/query to publish.
///
- [JsonPropertyName("prepublish_tables")]
- public Dictionary? PrepublishTables { get; set; }
+ [JsonPropertyName("publishers")]
+ public List? Publishers { get; set; }
[JsonPropertyName("simple_echo_client")]
[Option("simple-echo-client", HelpText = "Run a simple echo client (won't publish tables or connect to sparkd)")]
@@ -51,7 +51,7 @@ public class DataStreamPublisherConfig {
User ??= other.User;
Password ??= other.Password;
Sparkd ??= other.Sparkd;
- PrepublishTables ??= other.PrepublishTables;
+ Publishers ??= other.Publishers;
SimpleEchoClient ??= other.SimpleEchoClient;
ConfigFile ??= other.ConfigFile;
}
diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs
index 55609bbf0..fcadda583 100644
--- a/src/DataStreamSuite/DataStreamPublisher/Program.cs
+++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs
@@ -16,6 +16,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
+using System.Text.Json;
using System.Threading.Tasks;
await Parser.Default.ParseArguments(args)
@@ -31,7 +32,7 @@ async Task RunAsync(DataStreamPublisherConfig cfg) {
if (cfg.SimpleEchoClient ?? false) {
start_echo_client();
} else {
- _publish_streams(cfg.PrepublishTables ?? new(0));
+ _publish_streams(cfg.Publishers ?? new(0));
}
} catch (Exception ex) {
// Write the error to console first, in case the logging system is compromised.
@@ -44,7 +45,10 @@ async Task RunAsync(DataStreamPublisherConfig cfg) {
return 0;
}
-static ViewStream build_stream(string stream_expression) {
+static ViewStream? build_stream(string? stream_expression) {
+ if (stream_expression is null) {
+ return null;
+ }
try {
bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
@@ -69,28 +73,43 @@ static void start_echo_client() {
// Task.Delay(1000000000).Wait();
}
-void _publish_streams(IEnumerable> prepublish_streams) {
+void _publish_streams(IEnumerable publishers) {
PubTableBroker pb_broker = new PubTableBroker();
CloudStreamManager csm = new CloudStreamManager();
string csv_folder = @"C:\Temp\csvs_topublish";
CSVPublisher csv_publisher = new CSVPublisher(csv_folder, pb_broker);
- foreach (var kv in prepublish_streams) {
- var name = kv.Key;
- var exp = kv.Value;
- BCG.ViewStream vs = build_stream(exp);
- ViewPublisher vpub = new ViewPublisher(name, pb_broker, vs);
- vs.onChange += on_change;
- }
+
+ var viewPublishers = publishers
+ .SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher))
+ .ToList();
+
while (true) {
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
ViewStreamDatabase.instance.doWork();
csm.doWork();
MasterWorker.getInstance(false).doWork();
}
+
+ ViewPublisher? createViewPublisher(PublishedStream ps) {
+ Log.Gen.info(
+ $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
+ if (string.IsNullOrEmpty(ps.Name)) {
+ Log.Gen.warn(
+ $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
+ return null;
+ }
+ var vs = build_stream(ps.Expression);
+ if (vs is null) {
+ Log.Gen.warn(
+ $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
+ return null;
+ }
+ var vpub = new ViewPublisher(ps.Name, pb_broker, vs);
+ vs.onChange += on_change;
+ return vpub;
+ }
}
-
-
void on_change(object obj) {
bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
}
diff --git a/src/DataStreamSuite/DataStreamPublisher/PublishedStream.cs b/src/DataStreamSuite/DataStreamPublisher/PublishedStream.cs
new file mode 100644
index 000000000..a819da966
--- /dev/null
+++ b/src/DataStreamSuite/DataStreamPublisher/PublishedStream.cs
@@ -0,0 +1,14 @@
+namespace Bts.DataStreamPublisher;
+
+using System.Text.Json.Serialization;
+
+public class PublishedStream {
+ [JsonPropertyName("name")]
+ public string? Name { get; set; }
+
+ [JsonPropertyName("expression")]
+ public string? Expression { get; set; }
+
+ [JsonPropertyName("is_on_websocket")]
+ public bool? IsOnWebsocket { get; set; }
+}
diff --git a/src/DataStreamSuite/DataStreamPublisher/PublisherNode.cs b/src/DataStreamSuite/DataStreamPublisher/PublisherNode.cs
new file mode 100644
index 000000000..0df5b39a9
--- /dev/null
+++ b/src/DataStreamSuite/DataStreamPublisher/PublisherNode.cs
@@ -0,0 +1,15 @@
+namespace Bts.DataStreamPublisher;
+
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+
+public class PublisherNode {
+ [JsonPropertyName("name")]
+ public string? Name { get; set; }
+
+ [JsonPropertyName("streams")]
+ public List? Streams { get; set; }
+
+ [JsonPropertyName("csv_dir")]
+ public string? CsvDir { get; set; }
+}