diff --git a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json b/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json index cc714d354..76bae980e 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json +++ b/src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json @@ -1,5 +1,63 @@ { "user": "bts", + "pubsub": { + "milliseconds_between_sub_info_updates": 1000, + "milliseconds_between_time_slice_updates": 5000, + "milliseconds_between_no_buffer_space_retries": 100, + "retries_on_no_buffer_space": 1, + "do_it_all_seconds_between_logs": 0, + "heartbeat_interval": 15000, + "missing_heartbeats_before_gc": 50, + "use_managed_sockets": true, + "state_update_period": 5000, + "statistics_update_period": 5000, + "disposed_connection_check_period": 1000, + "message_processing_warning_threshold": 1000, + "server_port": 12345, + "publisher_routes": [ + { + "name": "default", + "server_address": "bluto://localhost:12345", + "tag": ".*" + } + ], + "subscriber_routes": [ + { + "name": "default", + "server_address": "bluto://localhost:12345", + "tag": ".*" + } + ], + "max_packet_size": 500000000, + "blue_comm_must_die": true, + "crushable_tags": [], + "tag_type_to_assembly_map": [ + { + "handler_class": "BCG.PubSubServerTagHandler", + "assembly": "DataStreamCore.dll" + } + ], + "server_queue": "PubSubQueue", + "port_range": { + "first": 30400, + "last": 31700 + }, + "bind_to_random_port": false, + "debug_tags": "^NOTAGS$", + "latency_report_threshold_ms": 1500, + "use_substations": false, + "use_dynamic_clients": false, + "is_substation": false, + "internally_publish_sub_info": true, + "use_managed_clients_in_native_code": false, + "compressed_tags": "^NOTAGS$", + "tags_to_report_compression_ratio": "^NOTAGS$", + "disable_msmq": false, + "use_prepublished_server_state": false, + "ip_translation": null, + "gc_activation_threshold": 0, + "wait_period_after_gc": 1000 + }, "publishers": [ { "name": "market_data", diff --git a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs index e556e8acf..5567f2f67 100644 --- a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs +++ b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs @@ -1,5 +1,7 @@ namespace Bts.DataStreamPublisher; +using Bts.PubSub; + using CommandLine; using System; @@ -40,6 +42,9 @@ public class DataStreamPublisherConfig { [Option("simple-echo-client", HelpText = "Run a simple echo client (won't publish tables or connect to sparkd)")] public bool? SimpleEchoClient { get; set; } + [JsonPropertyName("pubsub")] + public PubSubConfig? PubSub { get; set; } + [Option( "config", MetaValue = "PATH", @@ -47,11 +52,21 @@ public class DataStreamPublisherConfig { [JsonIgnore] public string? ConfigFile { get; set; } + public static DataStreamPublisherConfig Default() => + new() { + User = "bts", + Sparkd = "http://localhost:8000/config", + SimpleEchoClient = false, + PubSub = PubSubConfig.Default(), + ConfigFile = null, + }; + public void MergeWith(DataStreamPublisherConfig other) { User ??= other.User; Password ??= other.Password; Sparkd ??= other.Sparkd; Publishers ??= other.Publishers; + PubSub ??= other.PubSub; SimpleEchoClient ??= other.SimpleEchoClient; ConfigFile ??= other.ConfigFile; } diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs index 9e195d8c0..26f1934d4 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Program.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs @@ -26,7 +26,8 @@ await Parser.Default.ParseArguments(args) async Task RunAsync(DataStreamPublisherConfig cfg) { try { cfg.MergeWithConfigFile(); - PubSubConfig.SetGlobalConfig(PubSubConfig.Default()); + cfg.MergeWith(DataStreamPublisherConfig.Default()); + PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); Configuration.initialize(cfg.Sparkd); Log.Gen.info("Starting DataStreamPublisher"); var service_ctx = ServiceContext.create(cfg.Sparkd); diff --git a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs index abba241c1..64e2d0712 100644 --- a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs +++ b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs @@ -2,8 +2,12 @@ using System; using BCG; //using BCG.Utilities; using System.Text.RegularExpressions; +using System.Text.Json.Serialization; +using System.Text.Json; +using System.Text.Json.Nodes; namespace BCG { + [JsonConverter(typeof(PubSubRouteJsonConverter))] public class PubSubRoute : IComparable { public string serverAddress; public string tagRegExp; @@ -60,9 +64,42 @@ namespace BCG { return new PubSubRoute("bluto://127.0.0.0.1:12345", ".*"); } } - + public static class PubSubRouteExtensions { public static bool IsMatch(this PubSubRoute route, string tag) => route.getRegex().IsMatch(tag); } + + public class PubSubRouteJsonConverter : JsonConverter { + public override PubSubRoute? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + var jsonObj = JsonSerializer.Deserialize(ref reader); + if (jsonObj is null) { + return null; + } + var rv = new PubSubRoute(); + ReadString("name", ref rv.name); + ReadString("server_address", ref rv.serverAddress); + ReadString("tag", ref rv.tagRegExp); + return rv; + + void ReadString(string key, ref string valueOut) { + if (jsonObj.TryGetPropertyValue(key, out var valueJson)) { + var valueStr = valueJson?.GetValue(); + // Only (over)write valueOut if non-null + if (valueStr is not null) { + valueOut = valueStr; + } + } + } + } + + public override void Write(Utf8JsonWriter writer, PubSubRoute value, JsonSerializerOptions options) { + var jsonObj = new JsonObject { + ["name"] = value.name, + ["server_address"] = value.serverAddress, + ["tag"] = value.tagRegExp, + }; + JsonSerializer.Serialize(writer, jsonObj); + } + } } diff --git a/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs b/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs index bf3132f30..c1f671557 100644 --- a/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs +++ b/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs @@ -24,8 +24,8 @@ public class PubSubConfig { /// /// The config server url for sparkd /// - [JsonPropertyName("port")] - public short? Port { get; set; } + [JsonPropertyName("server_port")] + public short? ServerPort { get; set; } [JsonPropertyName("simple_echo_server")] public bool? SimpleEchoServer { get; set; } @@ -156,7 +156,7 @@ public class PubSubConfig { StatisticsUpdatePeriod = 5000, DisposedConnectionCheckPeriod = 1000, MessageProcessingWarningThreshold = 1000, - Port = 12345, + ServerPort = 12345, MaxPacketSize = 500000000, BlueCommMustDie = true, CrushableTags = new(0), @@ -201,7 +201,7 @@ public class PubSubConfig { StatisticsUpdatePeriod ??= other.StatisticsUpdatePeriod; DisposedConnectionCheckPeriod ??= other.DisposedConnectionCheckPeriod; MessageProcessingWarningThreshold ??= other.MessageProcessingWarningThreshold; - Port ??= other.Port; + ServerPort ??= other.ServerPort; MaxPacketSize ??= other.MaxPacketSize; BlueCommMustDie ??= other.BlueCommMustDie; CrushableTags ??= other.CrushableTags; diff --git a/src/DataStreamSuite/PubSubServer/Configurations/PubSubServer.json b/src/DataStreamSuite/PubSubServer/Configurations/PubSubServer.json index 250c49a0f..371346b3a 100644 --- a/src/DataStreamSuite/PubSubServer/Configurations/PubSubServer.json +++ b/src/DataStreamSuite/PubSubServer/Configurations/PubSubServer.json @@ -1,46 +1,62 @@ { "user": "bts", "sparkd": "http://localhost:8000/config", - "milliseconds_between_sub_info_updates": 1000, - "milliseconds_between_time_slice_updates": 5000, - "milliseconds_between_no_buffer_space_retries": 100, - "retries_on_no_buffer_space": 1, - "do_it_all_seconds_between_logs": 0, - "heartbeat_interval": 15000, - "missing_heartbeats_before_gc": 50, - "use_managed_sockets": true, - "state_update_period": 5000, - "statistics_update_period": 5000, - "disposed_connection_check_period": 1000, - "message_processing_warning_threshold": 1000, - "server_port": 12345, - "max_packet_size": 500000000, - "blue_comm_must_die": true, - "crushable_tags": [], - "tag_type_to_assembly_map": [ - { - "handler_class": "BCG.PubSubServerTagHandler", - "assembly": "DataStreamCore.dll" - } - ], - "server_queue": "PubSubQueue", - "port_range": { - "first": 30400, - "last": 31700 - }, - "bind_to_random_port": false, - "debug_tags": "^NOTAGS$", - "latency_report_threshold_ms": 1500, - "use_substations": false, - "use_dynamic_clients": false, - "is_substation": false, - "internally_publish_sub_info": true, - "use_managed_clients_in_native_code": false, - "compressed_tags": "^NOTAGS$", - "tags_to_report_compression_ratio": "^NOTAGS$", - "disable_msmq": false, - "use_prepublished_server_state": false, - "ip_translation": null, - "gc_activation_threshold": 0, - "wait_period_after_gc": 1000 + "pubsub": { + "milliseconds_between_sub_info_updates": 1000, + "milliseconds_between_time_slice_updates": 5000, + "milliseconds_between_no_buffer_space_retries": 100, + "retries_on_no_buffer_space": 1, + "do_it_all_seconds_between_logs": 0, + "heartbeat_interval": 15000, + "missing_heartbeats_before_gc": 50, + "use_managed_sockets": true, + "state_update_period": 5000, + "statistics_update_period": 5000, + "disposed_connection_check_period": 1000, + "message_processing_warning_threshold": 1000, + "server_port": 12345, + "publisher_routes": [ + { + "name": "default", + "server_address": "bluto://localhost:12345", + "tag": ".*" + } + ], + "subscriber_routes": [ + { + "name": "default", + "server_address": "bluto://localhost:12345", + "tag": ".*" + } + ], + "max_packet_size": 500000000, + "blue_comm_must_die": true, + "crushable_tags": [], + "tag_type_to_assembly_map": [ + { + "handler_class": "BCG.PubSubServerTagHandler", + "assembly": "DataStreamCore.dll" + } + ], + "server_queue": "PubSubQueue", + "port_range": { + "first": 30400, + "last": 31700 + }, + "bind_to_random_port": false, + "debug_tags": "^NOTAGS$", + "latency_report_threshold_ms": 1500, + "use_substations": false, + "use_dynamic_clients": false, + "is_substation": false, + "internally_publish_sub_info": true, + "use_managed_clients_in_native_code": false, + "compressed_tags": "^NOTAGS$", + "tags_to_report_compression_ratio": "^NOTAGS$", + "disable_msmq": false, + "use_prepublished_server_state": false, + "ip_translation": null, + "gc_activation_threshold": 0, + "wait_period_after_gc": 1000 + } } diff --git a/src/DataStreamSuite/PubSubServer/Program.cs b/src/DataStreamSuite/PubSubServer/Program.cs index fb25e2fd5..b86003386 100644 --- a/src/DataStreamSuite/PubSubServer/Program.cs +++ b/src/DataStreamSuite/PubSubServer/Program.cs @@ -25,7 +25,7 @@ int Run(PubSubServerConfig cfg) { PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); Configuration.initialize(); return cfg.SimpleEchoServer ?? false ? - RunEchoServer(cfg.PubSub.Port.assert_value()) : + RunEchoServer(cfg.PubSub.ServerPort.assert_value()) : RunPubSubServer(); } catch (Exception ex) { // Write the error to console first, in case the logging system is compromised. @@ -54,7 +54,7 @@ int RunPubSubServer() { Thread.Sleep(gced ? config.WaitPeriodAfterGc.assert_value() : defaultWaitTime); } }); - new PubSubMessageReactor(config.Port.assert_value()).start(); + new PubSubMessageReactor(config.ServerPort.assert_value()).start(); return 0; }