From 3b631e0d5f725417399370764798451d87f6f920 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Fri, 7 Jun 2024 14:52:00 +0000 Subject: [PATCH 1/6] Fix a couple more typos in logs --- .../Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs b/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs index 76f3f65a2..3f7a5df14 100644 --- a/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs +++ b/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs @@ -219,7 +219,7 @@ namespace BCG { _connection_map.TryGetValue(p.first, out conn); if (conn != null) { conn.send(p.second); - Log.Gen.info("$ManagedTCPMessageHub._main_message_pump sent n_bytes : {p.second.Length} to: {p.first}"); + Log.Gen.info($"ManagedTCPMessageHub._main_message_pump sent n_bytes : {p.second.Length} to: {p.first}"); } else { string error_msg = $"sending data of sz: {p.second.Count()} to {p.first} failed, this client might have " + "disconnected, in which case it will be garbage collected within 60 secs"; @@ -251,7 +251,7 @@ namespace BCG { } foreach (string url in disposed_urls) { string dispose_msg = $"Collecting disposed connections : {url} is disposed by the connection itself, " + - "connection count before deleting this: {_connection_map.Count}"; + $"connection count before deleting this: {_connection_map.Count}"; bts.Log.Gen.warn(dispose_msg); _connection_map.Remove(url); _conn_callback_queue.enqueue(new ConnectionStatusInfo(url, PubSubCallbacks.ConnectionStatus.DISCONNECTED)); @@ -293,4 +293,4 @@ namespace BCG { -} \ No newline at end of file +} From f3e33fa390e632fa551ccb8692a800885f99564d Mon Sep 17 00:00:00 2001 From: John Colvin Date: Fri, 7 Jun 2024 16:41:42 +0000 Subject: [PATCH 2/6] vscode: Add task to run PubSubServer without building --- .vscode/tasks.json | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 859ecb519..4847f18e7 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -526,6 +526,28 @@ "color": "terminal.ansiGreen", }, }, + { + "label": "Run PubSubServer (Debug | Skip Build)", + "type": "process", + "command": "", + "linux": { + "command": "PubSubServer", + }, + "windows": { + "command": "PubSubServer.exe", + }, + "args": [ + "--config", + "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json", + ], + "options": { + "cwd": "${workspaceFolder}/build/Debug/bin/net7.0" + }, + "icon": { + "id": "run", + "color": "terminal.ansiGreen", + }, + }, { "label": "Run PubSubServer (Release)", "dependsOn": "Build DataStream (Release)", @@ -549,6 +571,28 @@ "color": "terminal.ansiGreen", }, }, + { + "label": "Run PubSubServer (Release | Skip Build)", + "type": "process", + "command": "", + "linux": { + "command": "PubSubServer", + }, + "windows": { + "command": "PubSubServer.exe", + }, + "args": [ + "--config", + "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json", + ], + "options": { + "cwd": "${workspaceFolder}/build/Release/bin/net7.0" + }, + "icon": { + "id": "run", + "color": "terminal.ansiGreen", + }, + }, { "label": "Run Spark", "dependsOn": "Build Spark", From 5620f33524a1eb7cfe5e4b6ab0a0ef886ff0c2d0 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sat, 8 Jun 2024 14:07:39 +0000 Subject: [PATCH 3/6] vscode: Add task to view logs for last PubSubServer run --- .vscode/tasks.json | 30 +++++++++++++++++++++ src/DataStreamSuite/PubSubServer/Program.cs | 1 + 2 files changed, 31 insertions(+) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 4847f18e7..0ef537298 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -728,6 +728,36 @@ "id": "wrench" }, }, + { + "label": "View PubSubServer Logs (Last Run)", + "type": "process", + "command": "", + "linux": { + "command": "tools/logging/show_last_run.sh", + "args": [ + "${env:HOME}/.local/share/DataStream/data/logging.PubSubServer", + "Starting PubSubServer" + ] + }, + "windows": { + "command": "tools/logging/show_last_run.cmd", + "args": [ + "DataStream", + "PubSubServer.exe", + "Starting PubSubServer" + ] + }, + "presentation": { + "reveal": "always", + "focus": true, + "panel": "shared", + "showReuseMessage": false, + "close": true, + }, + "icon": { + "id": "wrench" + }, + }, ], "inputs": [ { diff --git a/src/DataStreamSuite/PubSubServer/Program.cs b/src/DataStreamSuite/PubSubServer/Program.cs index 9df1c2c79..fbcdc066e 100644 --- a/src/DataStreamSuite/PubSubServer/Program.cs +++ b/src/DataStreamSuite/PubSubServer/Program.cs @@ -24,6 +24,7 @@ int Run(PubSubServerConfig cfg) { cfg.MergeWith(PubSubServerConfig.Default()); PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); Configuration.initialize(cfg.Sparkd); + Log.Gen.info("Starting PubSubServer"); return cfg.SimpleEchoServer ?? false ? RunEchoServer(cfg.PubSub.ServerPort.assert_value()) : RunPubSubServer(); From b69b7b3350884e69a6b3b1b94578bcfbf8ae4170 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Sat, 8 Jun 2024 09:56:52 -0500 Subject: [PATCH 4/6] vscode: Use '\' dir separator in show_last_run.cmd This script only runs on Windows --- tools/logging/show_last_run.cmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/logging/show_last_run.cmd b/tools/logging/show_last_run.cmd index e7c8ec1e2..7f1a79b0a 100644 --- a/tools/logging/show_last_run.cmd +++ b/tools/logging/show_last_run.cmd @@ -1,6 +1,6 @@ @echo off -set log_dir="%APPDATA%/bts/%1/data/logging.%2" +set log_dir="%APPDATA%\bts\%1\data\logging.%2" set num_lines=9999 -tac "%log_dir%/log" | grep "%3" -m 1 -B %num_lines% | tac | less +tac %log_dir%\log | grep "%3" -m 1 -B %num_lines% | tac | less From 087d8ab8c9e8ffd677ee19fdde59698dc838f1ab Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 11 Jun 2024 15:22:20 +0000 Subject: [PATCH 5/6] Set text=auto in gitattributes Otherwise git can cause files to show as "permanently modified" when you apply changes from the index (e.g., by running 'git rebase', or 'git stash pop') --- .gitattributes | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitattributes b/.gitattributes index b3caa273b..48de34a3c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,3 @@ -*.cmd eol=crlf -*.cs eol=crlf -*.sh eol=lf +*.cmd text=auto eol=crlf +*.cs text=auto eol=crlf +*.sh text=auto eol=lf From 5cf7eacd82c6daa31cfb9a1aff3492ec162be4c2 Mon Sep 17 00:00:00 2001 From: Taha Date: Mon, 24 Jun 2024 00:28:07 -0400 Subject: [PATCH 6/6] Fixing the extra .0 in the default pubsub server routeplusfixing acorner cae issue of gettign a fast streaming data cakback after an unsubscribe --- src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs | 2 +- src/DataStreamSuite/PubSub/Config/PubSubConfig.cs | 4 ++-- src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs | 6 +++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs index 64e2d0712..d3057dd81 100644 --- a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs +++ b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs @@ -61,7 +61,7 @@ namespace BCG { } public static PubSubRoute get_default_route() { - return new PubSubRoute("bluto://127.0.0.0.1:12345", ".*"); + return new PubSubRoute("bluto://127.0.0.1:12345", ".*"); } } diff --git a/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs b/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs index ee0e6c3bc..300e0d1ce 100644 --- a/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs +++ b/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs @@ -176,10 +176,10 @@ public class PubSubConfig { WaitPeriodAfterGc = 1000, SimpleEchoServer = false, PublisherRoutes = new() { - new("bluto://127.0.0.0.1:12345", ".*") + new("bluto://127.0.0.1:12345", ".*") }, SubscriberRoutes = new() { - new("bluto://127.0.0.0.1:12345", ".*") + new("bluto://127.0.0.1:12345", ".*") } }; return rv; diff --git a/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs b/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs index 882f28326..72ca1ae29 100644 --- a/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs +++ b/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs @@ -47,7 +47,11 @@ namespace BCG { foreach (var m in conn_data) { var msg = new PubSubServerMessage(); msg.consruct_deserialize(m); - rv.Add(msg); + if (_tags.Contains(msg.tag)) + rv.Add(msg); + else + bts.Log.Gen.warn("SingleServerSubscriberCore.get_updates msg: {msg} is not subscribed currently"); + } return rv; }