From 4b76442a8f4eaa0c8d74c85ec270646d3549f67a Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Mon, 1 Jul 2024 03:16:28 +0000 Subject: [PATCH] Cleaning Config 1.0 (not completely done) --- .../Primitives/Time/MillisecondStream.cs | 48 +++++++++ .../BlueData/CircularBufferTable.cs | 6 -- .../DataStreamPublisherConfig.cs | 6 -- .../DataStreamPublisher/Program.cs | 91 +++++------------ .../PubSub/Config/PubSubConfig.cs | 99 +++---------------- .../PS3/Streams/TablesMetaDataStream.cs | 98 ++++++++++++++++++ 6 files changed, 181 insertions(+), 167 deletions(-) create mode 100644 src/DataStreamSuite/SharedStreams/System/PS3/Streams/TablesMetaDataStream.cs diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs index fcb8214cc..e03507ed7 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Time/MillisecondStream.cs @@ -1,6 +1,54 @@ using System; +using BCG.Utilities.General; + +using bts; + namespace BCG { + [DataStreamUserRespresentation("milliseconds")] + public class MillisecondStream : ViewStream, IWorker { + private DateTime _last_fire_time = DateTime.MinValue; + private readonly TimeSpan _period; + private readonly int _size; + private readonly ViewStreamDatabase _vsd; + private IBlueTableSchema _schema; + private bts.CircularBufferTable _table; + + public MillisecondStream(int interval_msecs, int size) + : this(ViewStreamDatabase.instance, interval_msecs, size) { + } + + public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) { + _vsd = vsd; + _period = TimeSpan.FromMilliseconds(interval_msecs); + _size = size; + _schema = BlueDataUtility.form_schema("Milliseconds", + Tuple.Create("id", TypeCode.Int32), + Tuple.Create("time", TypeCode.DateTime), + Tuple.Create("display_time", TypeCode.String)); + _table = new CircularBufferTable(_schema, _size); + } + + protected override void myOnFirstSubscriber() { + _vsd.addWorker(this); + } + + protected override void myOnLastSubscriber() { + _vsd.removeWorker(this); + } + + private int _id = 0; + public void doWork() { + if (DateTime.Now - _last_fire_time >= _period) { + _last_fire_time = DateTime.Now; + IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time, + _last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff")); + _table.add_row(nr); + var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket); + myFireEvent(dp); + } + } + } public class MSecondStream : TimeUnitStream { public MSecondStream(int intervalInms, int size) : base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs b/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs index ef67a2264..c1322be68 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/CircularBufferTable.cs @@ -16,12 +16,6 @@ namespace bts { _buffer_sz = buffer_sz; } - // public OrderedBlueTable(List rows, int buffer_sz) { - // _buffer_sz = buffer_sz; - // foreach (IBlueRow r in rows) - // add_row(r); - // } - // returns the row that might need to be deleted to keep // the circular buffer intact public IBlueRow add_row(IBlueRow row) { diff --git a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs index 1367d0ab6..2f4f52d45 100644 --- a/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs +++ b/src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs @@ -38,10 +38,6 @@ public class DataStreamPublisherConfig { [JsonPropertyName("publishers")] public List? Publishers { get; set; } - [JsonPropertyName("simple_echo_client")] - [Option("simple-echo-client", HelpText = "Run a simple echo client (won't publish tables or connect to sparkd)")] - public bool? SimpleEchoClient { get; set; } - [JsonPropertyName("pubsub")] public PubSubConfig? PubSub { get; set; } @@ -54,7 +50,6 @@ public class DataStreamPublisherConfig { public static DataStreamPublisherConfig Default() => new() { - SimpleEchoClient = false, PubSub = PubSubConfig.Default(), }; @@ -64,7 +59,6 @@ public class DataStreamPublisherConfig { Sparkd ??= other.Sparkd; Publishers ??= other.Publishers; PubSub ??= other.PubSub; - SimpleEchoClient ??= other.SimpleEchoClient; ConfigFile ??= other.ConfigFile; } diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs index 398c6bf27..11aab26c1 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Program.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs @@ -23,10 +23,8 @@ using System.Threading.Tasks; await Parser.Default.ParseArguments(args) .MapResult(RunAsync, errs => Task.FromResult(1)); -async Task RunAsync(DataStreamPublisherConfig cfg) -{ - try - { +async Task RunAsync(DataStreamPublisherConfig cfg) { + try { cfg.MergeWithConfigFile(); cfg.MergeWith(DataStreamPublisherConfig.Default()); PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); @@ -34,17 +32,8 @@ async Task RunAsync(DataStreamPublisherConfig cfg) Log.Gen.info("Starting DataStreamPublisher"); var service_ctx = ServiceContext.create(cfg.Sparkd); await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty); - if (cfg.SimpleEchoClient ?? false) - { - start_echo_client(); - } - else - { - _publish_streams(cfg.Publishers ?? new(0)); - } - } - catch (Exception ex) - { + _publish_streams(cfg.Publishers ?? new(0)); + } catch (Exception ex) { // Write the error to console first, in case the logging system is compromised. var error_msg = $"DataStreamPublisher threw exception:\n{ex}"; Console.Error.WriteLine(error_msg); @@ -55,51 +44,29 @@ async Task RunAsync(DataStreamPublisherConfig cfg) return 0; } -static ViewStream? build_stream(string? stream_expression) -{ - if (stream_expression is null) - { +static ViewStream? build_stream(string? stream_expression) { + if (stream_expression is null) { return null; } - try - { + try { bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}"); var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); var rv = exp.getView(new Dictionary(0)); return rv; - } - catch (Exception ex) - { + } catch (Exception ex) { bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}"); return null; } } -static void start_echo_client() -{ - Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue"); - Console.ReadLine(); - SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345, 1000, 1000, Int32.MaxValue); - client.start(); - while (true) - { - System.Threading.Thread.Sleep(1000); - } - // await SimpleTCPClient.start("127.0.0.1", 12345); - // Task.Delay(1000000000).Wait(); -} - -void _publish_streams(IEnumerable publishers) -{ +void _publish_streams(IEnumerable publishers) { PubTableBroker pb_broker = new PubTableBroker(); RemoteQueryManager csm = new RemoteQueryManager(); var csvPublishers = new List(); - foreach (var publisher in publishers) - { - if (Directory.Exists(publisher.CsvDir)) - { + foreach (var publisher in publishers) { + if (Directory.Exists(publisher.CsvDir)) { csvPublishers.Add(new(publisher.CsvDir, pb_broker)); } } @@ -108,8 +75,7 @@ void _publish_streams(IEnumerable publishers) .SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher)) .ToList(); - while (true) - { + while (true) { System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1)); ViewStreamDatabase.instance.doWork(); csm.doWork(); @@ -117,19 +83,16 @@ void _publish_streams(IEnumerable publishers) ManagedSubscriber.instance.doWork(); } - ViewPublisher? createViewPublisher(PublishedStream ps) - { + ViewPublisher? createViewPublisher(PublishedStream ps) { Log.Gen.info( $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}"); - if (string.IsNullOrEmpty(ps.Name)) - { + if (string.IsNullOrEmpty(ps.Name)) { Log.Gen.warn( $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}"); return null; } var vs = build_stream(ps.Expression); - if (vs is null) - { + if (vs is null) { Log.Gen.warn( $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}"); return null; @@ -140,39 +103,31 @@ void _publish_streams(IEnumerable publishers) } } -void on_change(object obj) -{ +void on_change(object obj) { bts.Log.Gen.info($"DSPublisher on_change dp: {obj}"); } -public class CSVPublisher : IWorker -{ +public class CSVPublisher : IWorker { private IDictionary _pubs = new Dictionary(); private readonly string _csv_directory; private PubTableBroker _pub_table_broker; - public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) - { + public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) { _csv_directory = csv_directory; _pub_table_broker = pub_table_broker; MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); } - public void doWork() - { + public void doWork() { var file_names = Directory.GetFiles(_csv_directory); - foreach (var fn in file_names) - { - if (!_pubs.ContainsKey(fn)) - { + foreach (var fn in file_names) { + if (!_pubs.ContainsKey(fn)) { _handle_new_file(fn); } } } - private void _handle_new_file(string file_name) - { - if (file_name.EndsWith(".csv")) - { + private void _handle_new_file(string file_name) { + if (file_name.EndsWith(".csv")) { ViewStream vs = new CSVReaderStream(file_name); vs = new SchemaGuesserStream(vs); var toks = file_name.Split('\\'); diff --git a/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs b/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs index 300e0d1ce..605a8b823 100644 --- a/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs +++ b/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs @@ -27,35 +27,39 @@ public class PubSubConfig { [JsonPropertyName("simple_echo_server")] public bool? SimpleEchoServer { get; set; } + // unused [JsonPropertyName("milliseconds_between_sub_info_updates")] public int? MillisecondsBetweenSubInfoUpdates { get; set; } = 1000; + // unused [JsonPropertyName("milliseconds_between_time_slice_updates")] public int? MillisecondsBetweenTimeSliceUpdates { get; set; } = 5000; + // unused [JsonPropertyName("milliseconds_between_no_buffer_space_retries")] public int? MillisecondsBetweenNoBufferRetries { get; set; } = 100; + // unused [JsonPropertyName("retries_on_no_buffer_space")] public int? RetriesOnNoBufferSpace { get; set; } = 1; + // unused [JsonPropertyName("do_it_all_seconds_between_log")] public int? DoItAllSecondsBetweenLog { get; set; } = 0; - // Milliseconds + // Used by the PubSubMessageReactor to pass to the ManagedTCPMessageHb (but unsed within it) [JsonPropertyName("heartbeat_interval")] public int? HeartbeatInterval { get; set; } = 15000; [JsonPropertyName("missing_heartbeats_before_gc")] public int? MissingHeartbeatsBeforeGc { get; set; } = 50; - [JsonPropertyName("use_managed_sockets")] - public bool? UseManagedSockets { get; set; } = true; - + // unused // Milliseconds [JsonPropertyName("state_update_period")] public int? StateUpdatePeriod { get; set; } = 5000; + // unused // Milliseconds [JsonPropertyName("statistics_update_period")] public int? StatisticsUpdatePeriod { get; set; } = 5000; @@ -64,67 +68,20 @@ public class PubSubConfig { [JsonPropertyName("disposed_connection_check_period")] public int? DisposedConnectionCheckPeriod { get; set; } = 1000; - // Milliseconds - [JsonPropertyName("message_processing_warning_threshold")] - public int? MessageProcessingWarningThreshold { get; set; } = 1000; - - [JsonPropertyName("max_packet_size")] - public int? MaxPacketSize { get; set; } = 500000000; - - [JsonPropertyName("blue_comm_must_die")] - public bool? BlueCommMustDie { get; set; } = true; - - [JsonPropertyName("crushable_tags")] - public List? CrushableTags { get; set; } - [JsonPropertyName("tag_type_to_assembly_map")] public TagTypeToAssembly? TagTypeToAssemblyMap { get; set; } - [JsonPropertyName("server_queue")] - public string? ServerQueue { get; set; } = "PubSubQueue"; - - [JsonPropertyName("port_range")] - public PubSubPortRange? PortRange { get; set; } - - [JsonPropertyName("bind_to_random_port")] - public bool? BindToRandomPort { get; set; } = false; - - [JsonPropertyName("debug_tags")] - public string? DebugTags { get; set; } - + // unused [JsonPropertyName("latency_report_threshold_ms")] public int? LatencyReportThresholdMs { get; set; } - [JsonPropertyName("use_substations")] - public bool? UseSubstations { get; set; } - - [JsonPropertyName("use_dynamic_clients")] - public bool? UseDynamicClients { get; set; } - - [JsonPropertyName("is_substation")] - public bool? IsSubstation { get; set; } - + // unused [JsonPropertyName("internally_publish_sub_info")] public bool? InternallyPublishSubInfo { get; set; } - [JsonPropertyName("use_managed_clients_in_native_code")] - public bool? UseManagedClientsInNativeCode { get; set; } - [JsonPropertyName("compressed_tags")] public string? CompressedTags { get; set; } - [JsonPropertyName("tags_to_report_compression_ratio")] - public string? TagsToReportCompressionRatio { get; set; } - - [JsonPropertyName("disable_msmq")] - public bool? DisableMsmq { get; set; } - - [JsonPropertyName("use_prepublished_server_state")] - public bool? UsePrepublishedServerState { get; set; } - - [JsonPropertyName("ip_translation")] - public string? IpTranslation { get; set; } - [JsonPropertyName("gc_activation_threshold")] public int? GcActivationThreshold { get; set; } @@ -148,38 +105,22 @@ public class PubSubConfig { DoItAllSecondsBetweenLog = 0, HeartbeatInterval = 15000, MissingHeartbeatsBeforeGc = 50, - UseManagedSockets = true, StateUpdatePeriod = 5000, StatisticsUpdatePeriod = 5000, DisposedConnectionCheckPeriod = 1000, - MessageProcessingWarningThreshold = 1000, ServerPort = 12345, - MaxPacketSize = 500000000, - BlueCommMustDie = true, - CrushableTags = new(0), TagTypeToAssemblyMap = new(), - ServerQueue = "PubSubQueue", - PortRange = new(30400, 31700), - BindToRandomPort = false, - DebugTags = NOTAGS, LatencyReportThresholdMs = 1500, - UseSubstations = false, - IsSubstation = false, InternallyPublishSubInfo = true, - UseManagedClientsInNativeCode = false, CompressedTags = NOTAGS, - TagsToReportCompressionRatio = NOTAGS, - DisableMsmq = false, - UsePrepublishedServerState = false, - IpTranslation = null, GcActivationThreshold = 0, WaitPeriodAfterGc = 1000, SimpleEchoServer = false, PublisherRoutes = new() { - new("bluto://127.0.0.1:12345", ".*") + new("bts-cloud://127.0.0.1:12345", ".*") }, SubscriberRoutes = new() { - new("bluto://127.0.0.1:12345", ".*") + new("bts-cloud://127.0.0.1:12345", ".*") } }; return rv; @@ -193,30 +134,14 @@ public class PubSubConfig { DoItAllSecondsBetweenLog ??= other.DoItAllSecondsBetweenLog; HeartbeatInterval ??= other.HeartbeatInterval; MissingHeartbeatsBeforeGc ??= other.MissingHeartbeatsBeforeGc; - UseManagedSockets ??= other.UseManagedSockets; StateUpdatePeriod ??= other.StateUpdatePeriod; StatisticsUpdatePeriod ??= other.StatisticsUpdatePeriod; DisposedConnectionCheckPeriod ??= other.DisposedConnectionCheckPeriod; - MessageProcessingWarningThreshold ??= other.MessageProcessingWarningThreshold; ServerPort ??= other.ServerPort; - MaxPacketSize ??= other.MaxPacketSize; - BlueCommMustDie ??= other.BlueCommMustDie; - CrushableTags ??= other.CrushableTags; TagTypeToAssemblyMap ??= other.TagTypeToAssemblyMap; - ServerQueue ??= other.ServerQueue; - PortRange ??= other.PortRange; - BindToRandomPort ??= other.BindToRandomPort; - DebugTags ??= other.DebugTags; LatencyReportThresholdMs ??= other.LatencyReportThresholdMs; - UseSubstations ??= other.UseSubstations; - IsSubstation ??= other.IsSubstation; InternallyPublishSubInfo ??= other.InternallyPublishSubInfo; - UseManagedClientsInNativeCode ??= other.UseManagedClientsInNativeCode; CompressedTags ??= other.CompressedTags; - TagsToReportCompressionRatio ??= other.TagsToReportCompressionRatio; - DisableMsmq ??= other.DisableMsmq; - UsePrepublishedServerState ??= other.UsePrepublishedServerState; - IpTranslation ??= other.IpTranslation; GcActivationThreshold ??= other.GcActivationThreshold; WaitPeriodAfterGc ??= other.WaitPeriodAfterGc; SimpleEchoServer ??= other.SimpleEchoServer; diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Streams/TablesMetaDataStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Streams/TablesMetaDataStream.cs new file mode 100644 index 000000000..3ca199965 --- /dev/null +++ b/src/DataStreamSuite/SharedStreams/System/PS3/Streams/TablesMetaDataStream.cs @@ -0,0 +1,98 @@ + + +namespace bts { + using BCG; + using BCG.Utilities.General; + + public class TableMetaDataStream : ViewStream, IWorker { + private class TableState { + public ViewStream table_stream; + public IBlueTableSchema schema = null; + public bool processed = false; + public TableState(ViewStream table_stream_) { + table_stream = table_stream_; + table_stream.onChange += _handle_table_update; + } + private void _handle_table_update(object dp_obj) { + DataPacket dp = dp_obj as DataPacket; + schema = dp.schema; + } + public void unsub() { + table_stream.onChange -= _handle_table_update; + } + } + private IBlueTableSchema _schema = BlueDataUtility.form_schema("TableMetaData", + Tuple.Create("id", TypeCode.Int32), + Tuple.Create("table_name", TypeCode.String), + Tuple.Create("col_idx", TypeCode.Int32), + Tuple.Create("col_type_code", TypeCode.Int32), + Tuple.Create("col_name", TypeCode.String)); + + private ViewStreamDatabase _vsd; + private ViewStream _base_stream; + private IDictionary _table_map = new Dictionary(); + private BlueTable _state; + public TableMetaDataStream() + : this(ViewStreamDatabase.instance) { } + public TableMetaDataStream(ViewStreamDatabase vsd) { + _vsd = vsd; + _base_stream = new PubSubServerStateStream(_vsd); + _state = new BlueTable(_schema); + } + protected override void myOnFirstSubscriber() { + _base_stream.onChange += _handle_base_stream; + _vsd.addWorker(this); + } + + protected override void myOnLastSubscriber() { + _base_stream.onChange -= _handle_base_stream; + _vsd.removeWorker(this); + } + + public void doWork() { + bool changed = false; + foreach (TableState ts in _table_map.Values) { + if (!ts.processed) { + if (ts.schema != null) { + foreach (ColumnInfo ci in ts.schema.columnInfo) { + int id = _state.Count; + var table_name = ts.schema.tableName; + var col_idx = ci.colNo; + var col_name = ci.colName; + TypeCode type_code = BlueDataUtility.get_typecode(ci.colType); + BlueRow nr = new BlueRow(_schema, id, table_name, col_idx, Convert.ToInt32(type_code), + col_name); + changed = true; + _state.addRow(nr); + } + } + ts.unsub(); + } + } + if (changed) { + var dp_out = BlueDataUtility.build_dp_inc(_state, myLastValue as DataPacket); + myFireEvent(dp_out); + } + } + + private void _handle_base_stream(object dp_obj) { + DataPacket dp = dp_obj as DataPacket; + foreach (IBlueRow r in dp.newRows) { + string tag = r.get("tag"); + if (tag.StartsWith("DS_")) { + _handle_table(tag); + } + } + } + + private void _handle_table(string tag_name) { + Log.assert(tag_name.StartsWith("DS_")); + string table_name = tag_name.Remove(0, 3); + TableState ts = null; + if (!_table_map.TryGetValue(table_name, out ts)) { + ts = new TableState(new TableStream(table_name, _vsd)); + _table_map.Add(table_name, ts); + } + } + } +} \ No newline at end of file