From 87b6a2044c8bc924ebef91fb71030f0861ac572d Mon Sep 17 00:00:00 2001 From: Taha Date: Sun, 23 Jun 2024 21:13:37 -0400 Subject: [PATCH] PubSubServer's linux issues fixed --- .devcontainer/devcontainer.json | 6 +- .vscode/launch.json | 5 + .../Misc/TCPConnectionTestStream.cs | 4 +- .../Primitives/Misc/PubSubTestStream.cs | 1 + .../Primitives/Misc/SimpleTCPEchoStream.cs | 7 +- .../DataStream/Primitives/Misc/TestStream.cs | 10 +- .../Primitives/Misc/TickingTestStream.cs | 31 + .../Serialization/StreamSerializationUtils.cs | 13 + .../PubsubServerStreamTagHandler.cs | 21 +- .../DataStreamQueryAnalyzer.json | 6 +- .../DataStreamQueryAnalyzer/MainForm.cs | 259 +++++--- .../Communication/BlutoTCP/BlutoSubAddress.cs | 130 ++-- .../BlutoTCP/SmartConnectionManagedWrapper.cs | 155 ++--- .../Communication/IPubSubServerConnection.cs | 2 +- .../Communication/MSMQ/MSMQSubAddress.cs | 88 +-- .../PubSubServerManagedTCPConnection.cs | 560 +++++++++--------- .../PubSubServerConnectionFactory.cs | 51 +- .../Client/MetaData/PubSubClientMessage.cs | 7 + .../PubSub/Client/PubsubTagHandlerFactory.cs | 1 + .../PubSub/Client/Routing/IPubSubRouter.cs | 13 +- .../PubSub/Client/Routing/PubSubRoute.cs | 55 +- .../PubSub/Client/Routing/PubSubRouter.cs | 176 +++--- .../PubSub/Client/SubAddress.cs | 58 +- .../PubSub/Config/PubSubConfig.cs | 4 +- .../PubSub/ManagedTCP/AsyncTCPConnection.cs | 542 +++++++++++++++++ .../PubSub/ManagedTCP/AsyncTCPConnection2.cs | 248 ++++++++ .../AsyncTCPConnectionAcceptor_APM.cs | 11 +- ...AsyncTCPConnectionAcceptor_AsyncWrapper.cs | 52 ++ .../ManagedTCP/AsyncTCPConnection_APM.cs | 123 ++-- .../AsyncTCPConnection_AsyncWrapper.cs | 269 +++++++++ .../ManagedTCP/AsyncTCPConnector_APM.cs | 8 +- .../AsyncTCPConnector_AsyncWrapper.cs | 83 +++ .../ClientToServerManagedTCPConnector.cs | 82 +++ .../PubSub/ManagedTCP/ITCPConnection.cs | 4 +- .../PubSub/ManagedTCP/RemoteConnectionInfo.cs | 57 +- ...verToClientManagedTCPConnectionAcceptor.cs | 11 +- ...verToClientManagedTCPConnectionAcceptor.cs | 208 +++---- .../PubSub/ManagedTCP/TCPSocketUtils.cs | 107 +++- src/DataStreamSuite/PubSub/PublishedCache.cs | 14 +- src/DataStreamSuite/PubSub/PublisherCore.cs | 139 +---- .../Server/Communication/IPubsubMessageHub.cs | 5 - .../ManagedTCP/ManagedTCPMessageHub.cs | 36 +- .../PubSub/Server/PubSubMessageReactor.cs | 34 +- .../PubSub/Server/PubSubState.cs | 1 + .../PubSub/Server/SingleTagConnectionData.cs | 1 + .../PubSub/Server/SingleTagData.cs | 9 +- .../PubSub/SingleServerSubscriberCore.cs | 14 +- src/DataStreamSuite/PubSub/SubscriberCore.cs | 251 ++++---- .../PubSub/Testing/SimpleTCPClient.cs | 39 +- .../PubSub/Testing/SimpleTCPServer.cs | 20 +- .../PubSub/documentation/readme.txt | 2 +- src/DataStreamSuite/PubSubServer/Program.cs | 17 +- .../System/PS3/Subscription/AllTagsStream.cs | 100 ++-- .../PS3/Subscription/TagSynopsisStream.cs | 234 ++++---- 54 files changed, 2920 insertions(+), 1464 deletions(-) create mode 100644 src/DataStreamSuite/DataStream/Primitives/Misc/TickingTestStream.cs create mode 100644 src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection.cs create mode 100644 src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection2.cs create mode 100644 src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_AsyncWrapper.cs create mode 100644 src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs create mode 100644 src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_AsyncWrapper.cs create mode 100644 src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 421e0a2bf..7b6c0b4ee 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -25,7 +25,9 @@ "-p", "8002:8002", "-p", - "8003:8003" + "8003:8003", + "-p", + "12345:12345", ], "mounts": [ "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind" @@ -45,4 +47,4 @@ ] } } -} +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 224b1ff56..2bfc14f64 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,6 +1,11 @@ { "version": "0.2.0", "configurations": [ + { + "name": ".NET Core Attach", + "type": "coreclr", + "request": "attach" + }, { "name": "DataStreamPublisher", "type": "coreclr", diff --git a/src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs b/src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs index b275558da..3b3dc1e49 100644 --- a/src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs +++ b/src/DataStreamSuite/DataStream/Misc/TCPConnectionTestStream.cs @@ -39,8 +39,8 @@ namespace BCG { protected override void myOnFirstSubscriber() { MasterWorker.getInstance(false).addWorker(this, _send_period); - var server_address = new BlutoSubAddress(_ip, _port); - bts.AsyncTCPConnector_APM.connect_to(server_address, this); + var server_rci = new RemoteConnectionInfo(_ip, _port); + bts.AsyncTCPConnector_APM.connect_to(server_rci, this); } protected override void myOnLastSubscriber() { diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs index 958ef6f40..36afb6811 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs @@ -153,6 +153,7 @@ namespace bts { var ser_bytes = _last_sent_msg.serialize(); _n_pubs++; _pub_core.publish(String.Empty, _tag_name, ser_bytes); + _fire_dirty = true; } private void _fire_dp() { diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/SimpleTCPEchoStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/SimpleTCPEchoStream.cs index 555d9c471..4d25d5ca4 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/SimpleTCPEchoStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/SimpleTCPEchoStream.cs @@ -14,11 +14,14 @@ namespace bts { private BlueTable _table; private SyncQueue _action_queue = new SyncQueue(); - public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms) { + public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms) + : this(ip, port, msg_sz, send_wait_period_ms, Int32.MaxValue) { } + + public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms, int max_n_msgs) { _ip = ip; _port = port; _table = new BlueTable(_schema); - _tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms); + _tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms, max_n_msgs); } protected override void myOnFirstSubscriber() { diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs index f58a7b829..ebd1652ae 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs @@ -43,11 +43,11 @@ namespace BCG { Tuple.Create("exception", TypeCode.String) ); // register tests here - // _tests.Add(new Func(_run_sub_msg_test)); - // _tests.Add(new Func(_run_pub_msg_test)); - // _tests.Add(new Func(_run_pubsub_server_msg_test)); - // _tests.Add(new Func(_run_pubsub_client_msg_test)); - // _tests.Add(new Func(_run_columninfo_serialization_test)); + _tests.Add(new Func(_run_sub_msg_test)); + _tests.Add(new Func(_run_pub_msg_test)); + _tests.Add(new Func(_run_pubsub_server_msg_test)); + _tests.Add(new Func(_run_pubsub_client_msg_test)); + _tests.Add(new Func(_run_columninfo_serialization_test)); // _tests.Add(new Func(_run_schema_serialization_test)); // _tests.Add(new Func(_row_serialization_tests_no_dbnulls)); // _tests.Add(new Func(_row_serialization_tests_with_dbnulls)); diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/TickingTestStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/TickingTestStream.cs new file mode 100644 index 000000000..b5ee8873b --- /dev/null +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/TickingTestStream.cs @@ -0,0 +1,31 @@ +namespace BCG { + using System; + + public class TickingTestStream : ViewStream { + public TickingTestStream(int ticking_period_msecs) + : + this(ViewStreamDatabase.instance, TimeSpan.FromSeconds(ticking_period_msecs)) { } + + public TickingTestStream(ViewStreamDatabase vsd, TimeSpan ticking_period_msecs) { + ViewStream vs = new DisplayableMillisecondStream(Convert.ToInt32(ticking_period_msecs.TotalMilliseconds), 10); + vs = new MathStream(vs, "x=id*123 y=x*x"); + mySetRealSource(vs); + //vs.onChange += _on_str_update; + } + + // private void _on_str_update(object dp_obj) { + // DataPacket dp = dp_obj as DataPacket; + // IBlueTableSchema out_schema = BlueDataUtility.add_column(dp.schema, "time_txt", typeof(string)); + // BlueTable state = new BlueTable(out_schema); + // foreach (IBlueRow r in dp.newRows) { + // MutableBlueRow nr = new MutableBlueRow(out_schema, r); + // DateTime time = r.get_datetime("time"); + // var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff"); + // nr.setValue("time_txt", time_txt); + // state.addRow(nr); + // } + // var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue == null ? null : myLastValue as DataPacket); + // myFireEvent(dp_out); + // } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs index 7e97f13e3..d1d802112 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs @@ -212,5 +212,18 @@ namespace bts { return rv; } + public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) { + var len = end_pos - start_pos; + StringBuilder hex_builder = new StringBuilder(len * 2); + int i = 0; + + foreach (byte b in byte_arr) { + if (i >= start_pos && i < end_pos) + hex_builder.Append($"{b:X2} "); + i++; + } + return hex_builder.ToString().Trim(); + } + } } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs index 825b7506e..279ed0bad 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs @@ -7,12 +7,23 @@ namespace bts { // for non stream tags public class PubsubServerGenericDataHandler : IPubSubServerTagHandler { - byte[] _state = null; - public void update(string tag, byte[] msg) { - _state = msg; + //byte[] _state = null; + private MemoryStream _str = new MemoryStream(); + private BinaryWriter _bw; + public PubsubServerGenericDataHandler() { + _bw = new BinaryWriter(_str); } + public void update(string tag, byte[] msg) { + _str.Position = 0; + _bw.Write(msg); + Log.Gen.info($"PubsubServerGenericDataHandler.update tag: {tag} msg_len: {msg.Length}"); + } + + public byte[] get_content(string tag) { - return _state; + var rv = _str.ToArray(); + Log.Gen.info($"PubsubServerGenericDataHandler.get_content tag: {tag} rv: {rv.Length}"); + return rv; } public string[] supportedTagTypes { get; } public void register(InternalPublishDelegate callback) { } @@ -60,7 +71,7 @@ namespace bts { }; var rv = get_content_(); var rv_txt = rv == null ? "null" : rv.Length.ToString(); - Log.Gen.info($"PubsubServerStreamTagHandler.get_content rv: {rv_txt}"); + Log.Gen.info($"PubsubServerSingleStreamTagHandler.get_content rv: {rv_txt}"); return rv; } public string[] supportedTagTypes { diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json b/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json index 42ea6ce19..a5e1aa6f7 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json @@ -17,14 +17,14 @@ "publisher_routes": [ { "name": "default", - "server_address": "bluto://localhost:12345", + "server_address": "bluto://192.168.1.80:12345", "tag": ".*" } ], "subscriber_routes": [ { "name": "default", - "server_address": "bluto://localhost:12345", + "server_address": "bluto://192.168.1.80:12345", "tag": ".*" } ], @@ -58,4 +58,4 @@ "gc_activation_threshold": 0, "wait_period_after_gc": 1000 } -} +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs index a279a3fd4..f380be9f6 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs @@ -25,8 +25,10 @@ using System.Threading.Tasks; -namespace BCG { - public partial class MainForm : System.Windows.Forms.Form { +namespace BCG +{ + public partial class MainForm : System.Windows.Forms.Form + { // static async Task _run() { // var login = "bts"; // var passwd = String.Empty; @@ -48,7 +50,8 @@ namespace BCG { private string myKQ; private TextWriter? myConsoleOutput; - public MainForm() { + public MainForm() + { //Task.Run(async () => await _run()); InitializeComponent(); // myKQ = ConfigurationRequester.getApplicationQualifier(); @@ -61,24 +64,28 @@ namespace BCG { this.Load += new EventHandler(myLoad); this.Closing += new CancelEventHandler(myClosing); - myQueryTextBox.TextChanged += (o, e) => { + myQueryTextBox.TextChanged += (o, e) => + { myQueryTextBox.ForeColor = Color.White; }; } - public string query_text { + public string query_text + { get { return myQueryTextBox.Text; } set { myQueryTextBox.Text = value; } } - private ViewStream myGetAllTagsStream() { + private ViewStream myGetAllTagsStream() + { // TODO: Port PublishedTagsStream from edge (then we can uncomment impl below) //ViewStream vs = new PubSubServerStateStream(); ViewStream vs = new PubSubServerStateStream(); return new FilterStream(vs, "tag~='DS_'"); - //return new LiteralStream("tag/String", "DS_Comedians;DS_Baldies"); + //return new LiteralStream("tag/String", "DS_table1;DS_table2"); } - private void mySetText() { + private void mySetText() + { this.Text = "Data Stream Query Analyzer | Site: " + mySite + (string.IsNullOrEmpty(myKQ) ? string.Empty : " | Key Qualifier: " + myKQ); myToolStripStatusLabelTime.Text = "Ready"; myToolStripStatusLabelRows.Text = string.Empty; @@ -88,10 +95,12 @@ namespace BCG { /// Gets the data in the form of a multilevel IDictionary /// /// - private IDictionary myGetLibraryData() { + private IDictionary myGetLibraryData() + { SortedList retVal = new SortedList(); string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames(); - foreach (string pName in primitiveNames) { + foreach (string pName in primitiveNames) + { string aName = PrimitiveManager.instance.getAssemblyName(pName); ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName); if (!retVal.Contains(aName)) @@ -106,12 +115,15 @@ namespace BCG { } - private void myLoadAssemblies() { + private void myLoadAssemblies() + { myTreeView.Nodes.Clear(); IDictionary libraryData = myGetLibraryData(); - foreach (DictionaryEntry de in libraryData) { + foreach (DictionaryEntry de in libraryData) + { TreeNode assemblyNode = new TreeNode(de.Key as String); - foreach (DictionaryEntry innerDe in (de.Value as IDictionary)) { + foreach (DictionaryEntry innerDe in (de.Value as IDictionary)) + { TreeNode primitiveNode = new TreeNode(innerDe.Key as string); foreach (string con in (innerDe.Value as ICollection)) primitiveNode.Nodes.Add(con); @@ -131,23 +143,27 @@ namespace BCG { }*/ } - private void myHandleTableListChange(object o) { + private void myHandleTableListChange(object o) + { IBlueRowCollection rows = (o as DataPacket).newRows; myUpdateTableList(); } #region Persistence Keys - public int textBoxHeight { + public int textBoxHeight + { get { return myQueryTextBox.Height; } set { myQueryTextBox.Height = value; } } #endregion - private void myTryExpression(string expStr) { + private void myTryExpression(string expStr) + { myFreezeView(false); // force unfreeze the view DataStreamExpression oldExp = myExp; - try { + try + { var exp = DataStreamExpressionsInterpreter.Evaluate(expStr); if (exp == null) @@ -173,7 +189,9 @@ namespace BCG { oldExp.getView(new Hashtable()).onChange -= new DataStream.ChangeDelegate(myCallback); */ //myQueryTextBox.Text = expStr; - } catch (Exception exc) { + } + catch (Exception exc) + { Log.Ui.warn(exc.ToString()); myDataGrid.DataSource = null; @@ -187,7 +205,8 @@ namespace BCG { } } - private string myExtractFromClass(TextReader r) { + private string myExtractFromClass(TextReader r) + { string line = r.ReadLine(); while (line != null && line.IndexOf("*Expression*") == -1) line = r.ReadLine(); @@ -200,7 +219,8 @@ namespace BCG { string result = ""; line = r.ReadLine(); - while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1) { + while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1) + { if (line != "") result += line.Substring(" ".Length) + "\n"; line = r.ReadLine(); @@ -211,17 +231,24 @@ namespace BCG { return result; } - public void loadFile(string fileName) { - try { + public void loadFile(string fileName) + { + try + { TextReader r = new StreamReader(fileName); - try { + try + { myQueryTextBox.Text = myExtractFromClass(r); - } finally { + } + finally + { r.Close(); } myLastFileName = fileName; this.Text = fileName; - } catch (Exception exc) { + } + catch (Exception exc) + { Log.Ui.warn(exc.ToString()); myDataGrid.DataSource = null; @@ -232,7 +259,8 @@ namespace BCG { } - private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code) { + private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code) + { myCSharpText.Text = "using System;\n\nnamespace BCG\n{"; @@ -252,7 +280,8 @@ namespace BCG { string lastVar = null; string variableDeclarations = ""; - foreach (string var in vars) { + foreach (string var in vars) + { variableDeclarations += " ViewStream " + var + ";\n"; lastVar = var; } @@ -282,13 +311,16 @@ namespace BCG { myCSharpText.Text += "}"; } - static DataGridViewColumn _column_factory(ColumnInfo ci) { - return new DataGridViewTextBoxColumn { + static DataGridViewColumn _column_factory(ColumnInfo ci) + { + return new DataGridViewTextBoxColumn + { HeaderText = ci.colName, Width = get_col_width(ci) }; - static int get_col_width(ColumnInfo ci) { + static int get_col_width(ColumnInfo ci) + { const int CHAR_WIDTH = 10; const int MIN_WIDTH = 50; const int MIN_DATE_WIDTH = 200; @@ -299,12 +331,14 @@ namespace BCG { } // this is the stream callback - private void myCallback(object o) { + private void myCallback(object o) + { var timer = Stopwatch.StartNew(); DataPacket dp = o as DataPacket; myCrono.Stop(); Log.Ui.diag($"Received callback for {dp.schema.tableName}"); - if (o == null) { + if (o == null) + { System.Windows.Forms.MessageBox.Show(this, "Received a Null Data packet", "Invalid Data", MessageBoxButtons.OK, MessageBoxIcon.Error); return; @@ -321,7 +355,8 @@ namespace BCG { myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows); var force = false; - if (myChangeStyle && force) { + if (myChangeStyle && force) + { myChangeStyle = false; myDataGrid.Columns.Clear(); myDataGrid.Columns.AddRange(Array.ConvertAll(dp.schema.columnInfo, _column_factory)); @@ -348,7 +383,8 @@ namespace BCG { private DataStreamExpression? myExp; - private void myTimer_Tick(object sender, System.EventArgs e) { + private void myTimer_Tick(object sender, System.EventArgs e) + { // try // { ViewStreamDatabase.instance.doWork(); @@ -365,22 +401,27 @@ namespace BCG { // } } - class StringWrapper { - public StringWrapper(string s) { + class StringWrapper + { + public StringWrapper(string s) + { myTableName = s; } - public string tableName { + public string tableName + { get { return myTableName; } } private string myTableName; } - private void MainForm_Closed(object sender, System.EventArgs e) { + private void MainForm_Closed(object sender, System.EventArgs e) + { if (myExp != null) myExp.getView(new Dictionary(0)).onChange -= new DataStream.ChangeDelegate(myCallback); } - private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) { + private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) + { myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length; myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length; myQueryTextBox.SelectionColor = Color.White; @@ -388,7 +429,8 @@ namespace BCG { myTryExpression(myQueryTextBox.Text); } - private void myToolStripMenuItemExecSel_Click(object sender, EventArgs e) { + private void myToolStripMenuItemExecSel_Click(object sender, EventArgs e) + { // int selstart = myQueryTextBox.SelectionStart; // int sellength = myQueryTextBox.SelectionLength; // myQueryTextBox.SelectionStart = 0; @@ -403,7 +445,8 @@ namespace BCG { // myQueryTextBox.SelectionColor = Color.White; } - private void myMenuItemOpenClick(object sender, System.EventArgs e) { + private void myMenuItemOpenClick(object sender, System.EventArgs e) + { OpenFileDialog od = new OpenFileDialog(); od.Filter = "C# class (*.cs)|*.cs"; @@ -411,21 +454,24 @@ namespace BCG { loadFile(od.FileName); } - private void mySave(string fileName) { + private void mySave(string fileName) + { using (TextWriter w = new StreamWriter(fileName)) w.Write(myCSharpText.Text); myLastFileName = fileName; this.Text = fileName; } - private void myMenuItemSaveClick(object sender, System.EventArgs e) { + private void myMenuItemSaveClick(object sender, System.EventArgs e) + { if (myLastFileName == null) myMenuItemSaveAsClick(sender, e); else mySave(myLastFileName); } - private void myMenuItemSaveAsClick(object sender, System.EventArgs e) { + private void myMenuItemSaveAsClick(object sender, System.EventArgs e) + { SaveFileDialog sd = new SaveFileDialog(); if (myLastFileName != null) sd.FileName = myLastFileName; @@ -436,47 +482,56 @@ namespace BCG { mySave(sd.FileName); } - private void myTableListBox_DoubleClick(object sender, System.EventArgs e) { - if (myTableListBox.SelectedItem != null) { + private void myTableListBox_DoubleClick(object sender, System.EventArgs e) + { + if (myTableListBox.SelectedItem != null) + { string tableName = myTableListBox.SelectedItem.ToString(); myQueryTextBox.Text = myQueryTextBox.Text == "" ? tableName : myQueryTextBox.Text + "\n" + tableName; myTryExpression(myQueryTextBox.Text); } } - private void myDataGrid_MouseEnter(object sender, System.EventArgs e) { + private void myDataGrid_MouseEnter(object sender, System.EventArgs e) + { //myDataGrid.TableStyles[0].GridLineColor = Color.LightSteelBlue; // if (myDataGrid.TableStyles.Count > 0) // myDataGrid.TableStyles[0].HeaderBackColor = Color.LightSteelBlue; } - private void myDataGrid_MouseLeave(object sender, System.EventArgs e) { + private void myDataGrid_MouseLeave(object sender, System.EventArgs e) + { //myDataGrid.TableStyles[0].GridLineColor = Color.Gainsboro; // if (myDataGrid.TableStyles.Count > 0) // myDataGrid.TableStyles[0].HeaderBackColor = Color.Gainsboro; } - private void myCSharpText_MouseEnter(object sender, System.EventArgs e) { + private void myCSharpText_MouseEnter(object sender, System.EventArgs e) + { //myCSharpText.BackColor = Color.Gainsboro; myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Bold); } - private void myCSharpText_MouseLeave(object sender, System.EventArgs e) { + private void myCSharpText_MouseLeave(object sender, System.EventArgs e) + { //myCSharpText.BackColor = Color.White; myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Regular); } - private void myTableListBox_MouseEnter(object sender, System.EventArgs e) { + private void myTableListBox_MouseEnter(object sender, System.EventArgs e) + { //myTableListBox.BackColor = Color.Gainsboro; myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Bold); } - private void myTableListBox_MouseLeave(object sender, System.EventArgs e) { + private void myTableListBox_MouseLeave(object sender, System.EventArgs e) + { //myTableListBox.BackColor = Color.White; myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Regular); } - private void myLoad(object sender, EventArgs e) { + private void myLoad(object sender, EventArgs e) + { myFreezeView(false); myToolStripMenuItemSite.DropDownItems.Clear(); @@ -500,7 +555,9 @@ namespace BCG { { var ex = new Win32Exception(); // same as new Win32Exception(Marshal.GetLastWin32Error()) Console.WriteLine(ex.Message); - } else { + } + else + { Console.WriteLine("Set stdout worked."); } @@ -508,7 +565,9 @@ namespace BCG { { var ex = new Win32Exception(); // same as new Win32Exception(Marshal.GetLastWin32Error()) Console.WriteLine(ex.Message); - } else { + } + else + { Console.WriteLine("Set stderr worked."); } Console.WriteLine("Redirecting console ouput to this window..."); @@ -524,10 +583,12 @@ namespace BCG { //_library = new LibraryForm(this); } - private void myMenuItemSiteClick(object sender, System.EventArgs e) { + private void myMenuItemSiteClick(object sender, System.EventArgs e) + { ToolStripMenuItem mi = (ToolStripMenuItem)sender; - if (mi.Text != mySite) { + if (mi.Text != mySite) + { mySite = mi.Text; foreach (var m in mySiteItems) m.Checked = false; @@ -548,63 +609,78 @@ namespace BCG { } } - private void myClosing(object sender, CancelEventArgs e) { + private void myClosing(object sender, CancelEventArgs e) + { } - private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e) { - if (myTreeView.SelectedNode != null) { - if (myTreeView.SelectedNode.LastNode == null) { + private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e) + { + if (myTreeView.SelectedNode != null) + { + if (myTreeView.SelectedNode.LastNode == null) + { string sn = myTreeView.SelectedNode.Parent.Text + "()"; myQueryTextBox.Text = myQueryTextBox.Text == "" ? sn : myQueryTextBox.Text + "\n" + sn; } } } - private void myToolStripMenuItemExit_Click(object sender, EventArgs e) { + private void myToolStripMenuItemExit_Click(object sender, EventArgs e) + { Close(); } - private void myToolStripMenuItemFreeze_Click(object sender, EventArgs e) { + private void myToolStripMenuItemFreeze_Click(object sender, EventArgs e) + { myFreezeView(myTimer.Enabled); } - private void myFreezeView(bool freeze) { + private void myFreezeView(bool freeze) + { myTimer.Enabled = !freeze; myToolStripMenuItemFreeze.Checked = freeze; myToolStripMenuItemStep.Enabled = freeze; } - private void myToolStripMenuItemStep_Click(object sender, EventArgs e) { + private void myToolStripMenuItemStep_Click(object sender, EventArgs e) + { ViewStreamDatabase.instance.doWork(); } - private void myToolStripMenuItemLoadAss_Click(object sender, EventArgs e) { + private void myToolStripMenuItemLoadAss_Click(object sender, EventArgs e) + { OpenFileDialog ofd = new OpenFileDialog(); ofd.ShowReadOnly = true; DialogResult dr = ofd.ShowDialog(this); - if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK) { + if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK) + { PrimitiveManager.instance.loadAssembly(ofd.FileName); myLoadAssemblies(); } } - private void myToolStripMenuItemTest_Click(object sender, EventArgs e) { - if (myCurrentStream != null) { + private void myToolStripMenuItemTest_Click(object sender, EventArgs e) + { + if (myCurrentStream != null) + { myCurrentStream.onChange -= new DataStream.ChangeDelegate(myCallback); myCurrentStream.onChange += new DataStream.ChangeDelegate(myCallback); } } - private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e) { + private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e) + { myConsoleRichTextBox.Clear(); Console.WriteLine("Redirecting console ouput to this window..."); } - private void myTableFilterBox_TextChanged(object sender, EventArgs e) { + private void myTableFilterBox_TextChanged(object sender, EventArgs e) + { myUpdateTableList(); } - private void myUpdateTableList() { + private void myUpdateTableList() + { IBlueRowCollection rows = myAllTagsStream.rows; if (rows == null) return; @@ -619,12 +695,15 @@ namespace BCG { myTableListBox.Items.Clear(); SortedList sl = new SortedList(); if (string.IsNullOrEmpty(myTableFilterBox.Text)) - foreach (IBlueRow r in rows) { + foreach (IBlueRow r in rows) + { sl[r["tag"].ToString().Substring(DSTagManager.get_stream_tag("").Length)] = true; } - else { + else + { Regex tableRegex = new Regex(myTableFilterBox.Text); - foreach (IBlueRow r in rows) { + foreach (IBlueRow r in rows) + { string thisTable = r["tag"].ToString().Substring(DSTagManager.get_stream_tag("").Length); if (tableRegex.IsMatch(thisTable)) sl[thisTable] = true; @@ -642,7 +721,8 @@ namespace BCG { myTableListBox.EndUpdate(); } - private void testGridToolStripMenuItem_Click(object sender, EventArgs e) { + private void testGridToolStripMenuItem_Click(object sender, EventArgs e) + { var timer = new Stopwatch(); //while (true) //{ @@ -664,9 +744,11 @@ namespace BCG { foreach (IBlueRow r in s.rows) rows.Add(r); - for (int i = 0; i < 100; i++) { + for (int i = 0; i < 100; i++) + { timer.Restart(); - rows.Sort(delegate (IBlueRow r1, IBlueRow r2) { + rows.Sort(delegate (IBlueRow r1, IBlueRow r2) + { return _sort_dicts_index(r1, r2, 5, true); }); timer.Stop(); @@ -674,19 +756,22 @@ namespace BCG { } } - private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high) { + private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high) + { if (r1.Contains(column) && r2.Contains(column)) return compare_send_dontcare_to_last(r1[column], r2[column], low_to_high); else return 0; } - private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high) { + private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high) + { //return compare_send_dontcare_to_last(r1[col_no], r2[col_no], low_to_high); return ((IComparable)r1[col_no]).CompareTo(r2[col_no]); } - internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) { + internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) + { if (myIsDontCare(o1)) return myIsDontCare(o2) ? 0 : 1; else @@ -694,14 +779,16 @@ namespace BCG { : ((IComparable)o2).CompareTo(o1)); } - private static bool myIsDontCare(object o) { + private static bool myIsDontCare(object o) + { return o == null || o == DBNull.Value || o is string && String.IsNullOrEmpty((string)o) || o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) || o is float && (Single.IsInfinity((float)o) || Single.IsNaN((float)o)); } - private void _new_grid_form(object args) { + private void _new_grid_form(object args) + { //System.Timers.Timer t = new System.Timers.Timer(); ////t.SynchronizingObject //BCG.Utilities.BlueEventLog.logEvent(String.Format("New BTSTabularGridTestForm created on thread {0}", System.Threading.Thread.CurrentThread.ManagedThreadId), @@ -713,11 +800,13 @@ namespace BCG { private void libraryToolStripMenuItem_Click(object sender, EventArgs e) { } - private void myTableFilterLabel_Click(object sender, EventArgs e) { + private void myTableFilterLabel_Click(object sender, EventArgs e) + { } - private void MainForm_Load(object sender, EventArgs e) { + private void MainForm_Load(object sender, EventArgs e) + { } } diff --git a/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/BlutoSubAddress.cs b/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/BlutoSubAddress.cs index 71b0ca803..5ed637877 100644 --- a/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/BlutoSubAddress.cs +++ b/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/BlutoSubAddress.cs @@ -1,77 +1,77 @@ -using System; -using System.Collections.Generic; -using System.Text; -//using BCG.Utilities; -using System.Net; +// using System; +// using System.Collections.Generic; +// using System.Text; +// //using BCG.Utilities; +// using System.Net; -namespace BCG -{ - public class BlutoSubAddress : SubAddress - { - private string myIp; - private int myPort; +// namespace BCG +// { +// public class BlutoSubAddress : SubAddress +// { +// private string myIp; +// private int myPort; - public static string prefix = "bluto"; +// public static string prefix = "bluto"; - public BlutoSubAddress(string ip, int port) - { - myIp = ip; - myPort = port; - } +// public BlutoSubAddress(string ip, int port) +// { +// myIp = ip; +// myPort = port; +// } - public BlutoSubAddress(string serverAddress) - { - string[] toks = serverAddress.Split(':'); - // TODO - // BlueEventLog.logAssert(toks.Length == 3); - // BlueEventLog.logAssert(toks[0].ToLower().StartsWith(prefix)); +// public BlutoSubAddress(string serverAddress) +// { +// string[] toks = serverAddress.Split(':'); +// // TODO +// // BlueEventLog.logAssert(toks.Length == 3); +// // BlueEventLog.logAssert(toks[0].ToLower().StartsWith(prefix)); - string machineNameOrIP = toks[1].Remove(0, 2); +// string machineNameOrIP = toks[1].Remove(0, 2); - if (machineNameOrIP.Split('.').Length == 4) - myIp = toks[1].Remove(0, 2); - else - { - //get the IP address from DNS - IPAddress[] theAddressList = Dns.GetHostEntry(machineNameOrIP).AddressList; - //BlueEventLog.logAssert(theAddressList.Length == 1, "Error: machine has multiple ip addresses."); - //if we get multiple addresses, we want the one that is of the corret dotted notation - bool oneWorked = false; - foreach (IPAddress addr in theAddressList) - { - if (addr.AddressFamily != System.Net.Sockets.AddressFamily.InterNetworkV6) - { - myIp = addr.ToString(); - oneWorked = true; - - } - } +// if (machineNameOrIP.Split('.').Length == 4) +// myIp = toks[1].Remove(0, 2); +// else +// { +// //get the IP address from DNS +// IPAddress[] theAddressList = Dns.GetHostEntry(machineNameOrIP).AddressList; +// //BlueEventLog.logAssert(theAddressList.Length == 1, "Error: machine has multiple ip addresses."); +// //if we get multiple addresses, we want the one that is of the corret dotted notation +// bool oneWorked = false; +// foreach (IPAddress addr in theAddressList) +// { +// if (addr.AddressFamily != System.Net.Sockets.AddressFamily.InterNetworkV6) +// { +// myIp = addr.ToString(); +// oneWorked = true; - if (!oneWorked) - { - throw new ArgumentException("Couldn't form server address from string " + serverAddress); - } - +// } +// } - } - myPort = Convert.ToInt32(toks[2]); - } +// if (!oneWorked) +// { +// throw new ArgumentException("Couldn't form server address from string " + serverAddress); +// } - - public string ip - { - get { return myIp; } - } +// } +// myPort = Convert.ToInt32(toks[2]); +// } - public int port - { - get { return myPort; } - } - public override string ToString() - { - return prefix + "://" + ip + ":" + port; - } - } -} + +// public string ip +// { +// get { return myIp; } +// } + +// public int port +// { +// get { return myPort; } +// } + +// public override string ToString() +// { +// return prefix + "://" + ip + ":" + port; +// } +// } +// } diff --git a/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/SmartConnectionManagedWrapper.cs b/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/SmartConnectionManagedWrapper.cs index ce9c62280..ad207743b 100644 --- a/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/SmartConnectionManagedWrapper.cs +++ b/src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/SmartConnectionManagedWrapper.cs @@ -1,88 +1,89 @@ -using System; -using System.Collections.Generic; -using System.Text; -using System.Threading; -using System.Runtime.InteropServices; +// using System; +// using System.Collections.Generic; +// using System.Text; +// using System.Threading; +// using System.Runtime.InteropServices; -namespace BCG -{ - public class SmartConnectionManagedWrapper - { - public delegate void MessageDelegate(int msgType, byte[] msg); - private Dictionary myServerToDel = new Dictionary(); +// TODO: remove +// namespace BCG +// { +// public class SmartConnectionManagedWrapper +// { +// public delegate void MessageDelegate(int msgType, byte[] msg); +// private Dictionary myServerToDel = new Dictionary(); - public SmartConnectionManagedWrapper () - { - ClientToServerConnectionWrapper.init(); - Thread readThr = new Thread(new ThreadStart(myReceiveData)); - readThr.Start(); - } +// public SmartConnectionManagedWrapper () +// { +// ClientToServerConnectionWrapper.init(); +// Thread readThr = new Thread(new ThreadStart(myReceiveData)); +// readThr.Start(); +// } - public void initConnection(BlutoSubAddress subAddress) - { - ClientToServerConnectionWrapper.initServerConnection(ClientToServerConnectionWrapper.stringToBytes(subAddress.ip), - subAddress.port); - } +// public void initConnection(BlutoSubAddress subAddress) +// { +// ClientToServerConnectionWrapper.initServerConnection(ClientToServerConnectionWrapper.stringToBytes(subAddress.ip), +// subAddress.port); +// } - public void send(BlutoSubAddress address, byte[] data) - { - byte[] ip = ClientToServerConnectionWrapper.stringToBytes (address.ip); - ClientToServerConnectionWrapper.sendDataToServer(ip, address.port, data, data.Length); - } - +// public void send(BlutoSubAddress address, byte[] data) +// { +// byte[] ip = ClientToServerConnectionWrapper.stringToBytes (address.ip); +// ClientToServerConnectionWrapper.sendDataToServer(ip, address.port, data, data.Length); +// } - public void resgisterDataCallback(BlutoSubAddress address, MessageDelegate del) - { - lock (this) - { - myServerToDel.Add(address.ToString(), del); - } - } - public void unregisterDataCallback(BlutoSubAddress address, MessageDelegate del) - { - lock (this) - { - myServerToDel.Remove(address.ip); - } - } +// public void resgisterDataCallback(BlutoSubAddress address, MessageDelegate del) +// { +// lock (this) +// { +// myServerToDel.Add(address.ToString(), del); +// } +// } - public string getClientAsSeenByServer(string serverURL) - { - return ""; - } +// public void unregisterDataCallback(BlutoSubAddress address, MessageDelegate del) +// { +// lock (this) +// { +// myServerToDel.Remove(address.ip); +// } +// } - private unsafe void myReceiveData() - { - while (true) - { - BlutoTCPClientMessage** psClientPackets = null; - int npts = 0; - ///returns as soon as it gets the first element - ClientToServerConnectionWrapper.getServerData(1000, 1, &psClientPackets, out npts); - for (int i = 0; i < npts; i++) - { - string ip = Marshal.PtrToStringAnsi(psClientPackets[i]->ip); - int port = psClientPackets[i]->port; - string serverURL = new BlutoSubAddress(ip, port).ToString(); +// public string getClientAsSeenByServer(string serverURL) +// { +// return ""; +// } - byte[] msg = null; - if (psClientPackets[i]->msgType == 1) - { - msg = new byte[psClientPackets[i]->size]; - Marshal.Copy(psClientPackets[i]->msg, msg, 0, msg.Length); - } +// private unsafe void myReceiveData() +// { +// while (true) +// { +// BlutoTCPClientMessage** psClientPackets = null; +// int npts = 0; +// ///returns as soon as it gets the first element +// ClientToServerConnectionWrapper.getServerData(1000, 1, &psClientPackets, out npts); +// for (int i = 0; i < npts; i++) +// { +// string ip = Marshal.PtrToStringAnsi(psClientPackets[i]->ip); +// int port = psClientPackets[i]->port; +// string serverURL = new BlutoSubAddress(ip, port).ToString(); - MessageDelegate cb = null; - lock (this) - { - myServerToDel.TryGetValue(serverURL, out cb); - } - if (cb != null) - cb(psClientPackets[i]->msgType, msg); - } - } - } +// byte[] msg = null; +// if (psClientPackets[i]->msgType == 1) +// { +// msg = new byte[psClientPackets[i]->size]; +// Marshal.Copy(psClientPackets[i]->msg, msg, 0, msg.Length); +// } - } -} +// MessageDelegate cb = null; +// lock (this) +// { +// myServerToDel.TryGetValue(serverURL, out cb); +// } +// if (cb != null) +// cb(psClientPackets[i]->msgType, msg); +// } +// } +// } + +// } +// } diff --git a/src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs b/src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs index ccff39522..7235b4c13 100644 --- a/src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs +++ b/src/DataStreamSuite/PubSub/Client/Communication/IPubSubServerConnection.cs @@ -72,6 +72,6 @@ namespace BCG { /// void register_data_callback(PubSubServerDataDelegate del); - void register_send_confirmation_callback(SendConfirmationDelegate del); + //void register_send_confirmation_callback(SendConfirmationDelegate del); } } diff --git a/src/DataStreamSuite/PubSub/Client/Communication/MSMQ/MSMQSubAddress.cs b/src/DataStreamSuite/PubSub/Client/Communication/MSMQ/MSMQSubAddress.cs index 9d271bbdc..c265bf16c 100644 --- a/src/DataStreamSuite/PubSub/Client/Communication/MSMQ/MSMQSubAddress.cs +++ b/src/DataStreamSuite/PubSub/Client/Communication/MSMQ/MSMQSubAddress.cs @@ -1,52 +1,52 @@ -using System; -//using System.Messaging; +// using System; +// //using System.Messaging; -using BCG; -//using BCG.Utilities; +// using BCG; +// //using BCG.Utilities; -namespace BCG -{ - public class MSMQSubAddress : SubAddress - { - private string myIP, myQ; - public static readonly string prefix = "msmq"; +// namespace BCG +// { +// public class MSMQSubAddress : SubAddress +// { +// private string myIP, myQ; +// public static readonly string prefix = "msmq"; - public MSMQSubAddress (string adr) - { - try - { - // throw away prefix - string s = adr; - s = s.Substring(prefix.Length + "://".Length); - // grab IP - myIP = s.Substring(0,s.IndexOf("/")); - // throw away ip - s = s.Substring(myIP.Length + 1); - myQ = s; - - } - catch (Exception) - { - throw (new InvalidSubAddress(adr + " is not a valid queue address")); - } - } +// public MSMQSubAddress (string adr) +// { +// try +// { +// // throw away prefix +// string s = adr; +// s = s.Substring(prefix.Length + "://".Length); +// // grab IP +// myIP = s.Substring(0,s.IndexOf("/")); +// // throw away ip +// s = s.Substring(myIP.Length + 1); +// myQ = s; - public MSMQSubAddress (string ip, string simpleQName) - { - myIP = ip; - myQ = simpleQName; - } +// } +// catch (Exception) +// { +// throw (new InvalidSubAddress(adr + " is not a valid queue address")); +// } +// } - public string ip {get {return myIP;}} +// public MSMQSubAddress (string ip, string simpleQName) +// { +// myIP = ip; +// myQ = simpleQName; +// } - public string simpleQueueName {get {return myQ;}} +// public string ip {get {return myIP;}} - public override string ToString() { return prefix + "://" + myIP + "/" + myQ; } +// public string simpleQueueName {get {return myQ;}} - public string formatName - { - get{return "FormatName:DIRECT=TCP:" + myIP + "\\private$\\" + myQ;} - } - - } -} +// public override string ToString() { return prefix + "://" + myIP + "/" + myQ; } + +// public string formatName +// { +// get{return "FormatName:DIRECT=TCP:" + myIP + "\\private$\\" + myQ;} +// } + +// } +// } diff --git a/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs b/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs index 406a258ed..f5ed81b38 100644 --- a/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs +++ b/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs @@ -1,296 +1,264 @@ -using System; -using System.Collections.Generic; -using System.Text; -using System.Net; -using System.Net.Sockets; -using System.IO; -//using BCG.Utilities; -//using BCG.Utilities.General; -using System.Threading; -using mbts; -using bts; -using System.Linq; -using System.Reflection.Metadata.Ecma335; -using Bts.PubSub; - -namespace BCG { - /// - /// Keeps an ITCPConnection for talking to the server. It also monitors heartbeats - /// from the server, and in case of missing hbs, informs its clients, and tries to - /// restablish the connection. - /// Threads : - /// its assumed that IPubSubServerConnection functions (including doWork) will - /// be called from the same thread. ITCPMessageHandler.onData and - /// ITCPConnectionHandler.onConnection will be called on any arbitrary thread - /// - /// - public class PubSubServerManagedTCPConnection : IPubSubServerConnection,// external facing interface to communicate msgs to Pubsubserver - ITCPMessageHandler, // to get messages from ITCPCOnnection - ITCPConnectionHandler { // to get connections from AsyncTCPConnector_APM - private ITCPConnection _connection, _new_connection; - private readonly BlutoSubAddress _server_address; - // if a message(with confirm) can not be sent , its id is queued here, - // on reconnection, _on_send_onfirm_callback is fired with a negative on these ids, so the user - //can re-send it. The re-send mechanism is not in implemented in this class, but by the user of this class - private ThreadSafeQueue _negative_unsent_confs = new ThreadSafeQueue(); - // in onData, data is queued here, for polling - private SyncQueue _in_data_queue = new SyncQueue(); - private DateTime _last_hb = DateTime.Now; - private PubSubServerConnectionStatusChange _conn_callback; - private PubSubServerDataDelegate _data_callback; - private SendConfirmationDelegate _on_send_confirm_callback; - private readonly TimeSpan _connection_drop_threshold = PubSubConfig.Instance.ConnectionDropThreshold(); - - public PubSubServerManagedTCPConnection(string server_url) - : this(new BlutoSubAddress(server_url)) { } - - public PubSubServerManagedTCPConnection(BlutoSubAddress server_address) { - _server_address = server_address; - _connect_to_ps_server(); - } - - private void _connect_to_ps_server() { - bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server: {_server_address}"); - AsyncTCPConnector_APM.connect_to(_server_address, this); - } - - #region IPubSubServerConnection Members - - public void send_with_confirmation(byte[] data, ref string conf_id) { - bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} _connection : {_connection != null}"); - if (_connection != null) { - _connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) => { - if (_on_send_confirm_callback != null) - _on_send_confirm_callback(msg_id, confirmed); - }); - } else { - bts.Log.Gen.warn($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} no connection, buffering send"); - conf_id = Guid.NewGuid().ToString(); - _negative_unsent_confs.enqueue(conf_id); - } - } - - private void _send(byte[] data) { - bts.Log.Gen.info($"PubSubServerManagedTCPConnection._send server: {_server_address} data: {data.Length} _connection : {_connection != null}"); - if (_connection != null) { - bts.Log.Gen.info("PubSubServerManagedTCPConnection.send server: " + _server_address + "data: " + data.Length); - _connection.send(data); - } - } - - public void send(bts.PubSubClientMessage ps_msg) { - bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_address} ps_msg: {ps_msg}"); - var msg_bytes = ps_msg.serialize_into_bytes(); - PubSubClientMessage msg_retrv = new PubSubClientMessage(); - msg_retrv.consruct_deserialize(msg_bytes); - bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_address} msg_retrv: {msg_retrv}"); - _send(msg_bytes); - } - - // private void send(Stream str) { - // BinaryReader br = new BinaryReader(str); - // _send(br.ReadBytes(Convert.ToInt32(str.Length - str.Position))); - // } - - // private void send(PubSubMessage msg) { - // send(msg.to_stream()); - // } - - // public void send_with_confirmation(PubSubMessage msg, ref string conf_id) { - // var bytes = msg.to_bytes(); - // send_with_confirmation(bytes, ref conf_id); - // } - - public void send_with_confirmation(bts.PubSubClientMessage msg, ref string conf_id, SendConfirmationDelegate del) { - if (_connection == null) { - // quq - } - MemoryStream str = new MemoryStream(); - msg.serialize_into_str(str); - var bytes = StreamUtilityMethods.getBytesFromStream(str); - _connection.sendWithConfirmation(bytes, ref conf_id, del); - } - - public IEnumerable get_updates(TimeSpan timeout, int maxElements) { - List rv_raw = _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements); - List rv = new List(); - var i = 0; - foreach (var b in rv_raw) { - PubSubServerMessage ps_msg = new PubSubServerMessage(); - ps_msg.consruct_deserialize(b); - bts.Log.Gen.info($"PubSubServerManagedTCPConnection.get_updates downloaded: {ps_msg} msg_no: {i} / {rv_raw.Count}"); - rv.Add(ps_msg); - i++; - } - return rv; - } - - public string server_url { - get { return _server_address.ToString(); } - } - - public string client_url { - get { - string rv = ""; - lock (this) { - if (_connection == null) { - rv = ""; - } else { - if (PubSubConfig.Instance.IpTranslation.is_null_or_empty()) { - string ip; - int port; - _connection.getSelfAddress(out ip, out port); - rv = new BlutoSubAddress(ip, port).ToString(); - } else { - rv = ""; - } - } - } - return rv; - } - } - - public void register_connection_callback(PubSubServerConnectionStatusChange del) { - _conn_callback = del; - } - - public void register_data_callback(PubSubServerDataDelegate del) { - _data_callback = del; - } - - public void register_send_confirmation_callback(SendConfirmationDelegate del) { - _on_send_confirm_callback = del; - } - - #endregion - - #region IDisposable Members - - private bool _is_disposed = false; - public void Dispose() { - bts.Log.Gen.info("PubSubServerManagedTCPConnection.Dispose"); - //throw new Exception("The method or operation is not implemented."); - if (_connection != null) { - _connection.Dispose(); - - } - _is_disposed = true; - } - public bool isDisposed { - get { - return _is_disposed; - } - - } - - #endregion - - #region ITCPMessageHandler Members - - /// - /// This method is executed on an arbitrary CLR thread. To avoid blocking it - /// data is written to the queue and method returns. myLastHB being a primitive - /// (long) is safe for multi threaded operations - /// - /// - public void onData(RemoteMessage message) { - bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData message: {message}"); - // TODO check for disconenction - switch (message.messageType) { - case RemoteMessageType.Data: - if (_data_callback != null) - _data_callback(message.remoteConnectionInfo.ToString(), message.message); - else - _in_data_queue.enqueue(message.message); - break; - case RemoteMessageType.Heartbeat: - _last_hb = DateTime.Now; - break; - case RemoteMessageType.Connection: - _conn_callback(_server_address.ToString(), true); - break; - case RemoteMessageType.Disconnection: - if (_conn_callback != null) - _conn_callback(_server_address.ToString(), false); - _connection.Dispose(); - _connect_to_ps_server(); - break; - - } - } - - #endregion - - #region ITCPConnectionHandler Members - - /// will be called on on an rbitrary .Net thread - public void onConnection(ITCPConnection connection) { - string ip; - int port; - connection.getPeerAddress(out ip, out port); - BlutoSubAddress add = new BlutoSubAddress(ip, port); - bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onConnection add: {add.ToString()} connection type: {connection.GetType()}"); - if (add.ToString() == _server_address.ToString()) { - _last_hb = DateTime.Now; - _new_connection = connection; - } - } - - #endregion - - void _dispose_and_reconnect_if_missing_hb() { - if (!_connection.isDisposed) { - TimeSpan hb_interval = DateTime.Now - _last_hb; - bool hb_missed = hb_interval > _connection_drop_threshold; - if (hb_missed) { - if (_conn_callback != null) - _conn_callback(_server_address.ToString(), false); - bts.Log.Gen.info("PubSubServerManagedTCPConnection._dispose_and_reconnect_if_missing_hb disposing connecion: {myServerAddress}"); - _connection.Dispose(); - _connect_to_ps_server(); - } - } else { - // already disposed, nothing to do - } - } - - void _check_for_new_connection() { - if (_new_connection != null) { - _connection = _new_connection; - _new_connection = null; // since we are using its presence as a marker of nw connections - bts.Log.Gen.info("PubSubServerManagedTCPConnection._check_for_new_connection assigning new connection"); - _connection.register(this); - if (_conn_callback != null) - _conn_callback(_server_address.ToString(), true); - if (_on_send_confirm_callback != null) { - var msg_ids = _negative_unsent_confs.dequeueAll(); - foreach (var id in msg_ids) - _on_send_confirm_callback(id, false); - } - } - } - - #region IWorker Members - - /// - /// should be called on the same thread as the other user functions. - /// the design only allows for onConnection to be on a different - /// thread. - /// - public void doWork() { - //_dispose_and_reconnect_if_missing_hb(); - //try { - _check_for_new_connection(); - // } catch (Exception ex) { - // bts.Log.Gen.warn($"PubsSubServerManagedTCPConnection.doWork ex: {ex}"); - // } - } - - #endregion - - // private void myOnSendConfirm(string messageId, bool confirmed) { - // _negative_confs.enqueue(new Pair(messageId, confirmed)); - // } - - } -} +using System; +using System.Collections.Generic; +using System.Text; +using System.Net; +using System.Net.Sockets; +using System.IO; +//using BCG.Utilities; +//using BCG.Utilities.General; +using System.Threading; +using mbts; +using bts; +using System.Linq; +using System.Reflection.Metadata.Ecma335; +using Bts.PubSub; +using Microsoft.Win32; +using System.ComponentModel; + +namespace BCG { + /// + /// Keeps an ITCPConnection for talking to the server. It also monitors heartbeats + /// from the server, and in case of missing hbs, informs its clients, and tries to + /// restablish the connection. + /// Threads : + /// its assumed that IPubSubServerConnection functions (including doWork) will + /// be called from the same thread. ITCPMessageHandler.onData and + /// ITCPConnectionHandler.onConnection are called on an arbitrary thread. + /// _conn_callback is called from within doWork, and _data_callback is called grom + // an arbitrary data thread + /// + public class PubSubServerManagedTCPConnection : IPubSubServerConnection,// external facing interface to communicate msgs to Pubsubserver + ITCPMessageHandler, // to get messages from ITCPCOnnection + ITCPConnectionHandler { // to get connections from AsyncTCPConnector_APM + private ITCPConnection _connection; + private readonly RemoteConnectionInfo _server_rc_info; + private readonly bool _is_sub; + // if _data_callback is not subscribed to , the user of this class can poll updates, + // in which case this queue will be used + private SyncQueue _in_data_queue = new SyncQueue(); + private DateTime _last_hb = DateTime.Now; + private PubSubServerConnectionStatusChange _conn_callback; + private PubSubServerDataDelegate _data_callback; + private readonly TimeSpan _connection_drop_threshold = PubSubConfig.Instance.ConnectionDropThreshold(); + private ThreadSafeQueue _conn_cb_queue = new ThreadSafeQueue(); + + // the callbacks for connection, comes from the ITCPConnection, when a connection is broken, + // and from AsyncTCPConnector whne a connection is established. ALl that info is assembled here and + // enqueued, deuqued oin the doWork and acted on + private class ConnectionStatusChange { + public enum Status { + CONNECTED, + DISCONNECTED + } + public readonly Status connection_satus; + public readonly ITCPConnection new_connection = null; + public ConnectionStatusChange(Status connection_satus_, ITCPConnection new_connection_) { + connection_satus = connection_satus_; + new_connection = new_connection_; + } + public override string ToString() { + return connection_satus.ToString(); + } + } + + public PubSubServerManagedTCPConnection(RemoteConnectionInfo server_address, bool is_sub) { + Log.Gen.info("PubSubServerManagedTCPConnection.cons server_address: {server_address} is_sub: {is_sub}"); + _server_rc_info = server_address; + _is_sub = is_sub; + // this is where it tries to connect to the pubsub server, and gets a callback on onConnection + _connect_to_ps_server(); + } + + private void _connect_to_ps_server() { + bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server _server_rc_info: {_server_rc_info} is_sub: {_is_sub}"); + // onConnection will be called on an arbitrary thread + AsyncTCPConnector_AsyncWrapper.connect_to(_server_rc_info, this); + } + + #region IPubSubServerConnection Members + + private void _send(byte[] data) { + bts.Log.Gen.info($"PubSubServerManagedTCPConnection._send server: {_server_rc_info} data: {data.Length} is_sub: {_is_sub} _connection : {_connection != null}"); + if (_connection != null) { + _connection.send(data); + bts.Log.Gen.info("PubSubServerManagedTCPConnection.send, data is sent, server: " + _server_rc_info + "data: " + data.Length); + } else { + bts.Log.Gen.warn($"PubSubServerManagedTCPConnection._send, could not send data, server: {_server_rc_info} data: {data.Length} is_sub: {_is_sub} _connection : {_connection != null} "); + } + } + + public void send(bts.PubSubClientMessage ps_msg) { + bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_rc_info} ps_msg: {ps_msg}"); + // whys it constructed again? + var msg_bytes = ps_msg.serialize_into_bytes(); + // PubSubClientMessage msg_retrv = new PubSubClientMessage(); + // msg_retrv.consruct_deserialize(msg_bytes); + // bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_rc_info} msg_retrv: {msg_retrv}"); + _send(msg_bytes); + } + + public IEnumerable get_updates(TimeSpan timeout, int maxElements) { + List rv_raw = _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements); + List rv = new List(); + var i = 0; + foreach (var b in rv_raw) { + PubSubServerMessage ps_msg = new PubSubServerMessage(); + ps_msg.consruct_deserialize(b); + bts.Log.Gen.info($"PubSubServerManagedTCPConnection.get_updates downloaded: {ps_msg} msg_no: {i} / {rv_raw.Count}"); + rv.Add(ps_msg); + i++; + } + return rv; + } + + public string server_url { + get { return _server_rc_info.ToString(); } + } + + public string client_url { + get { + if (_connection == null) + return RemoteConnectionInfo.null_instance.ToString(); + return _connection.get_self_address().ToString(); + } + } + + public void register_connection_callback(PubSubServerConnectionStatusChange del) { + _conn_callback = del; + } + + public void register_data_callback(PubSubServerDataDelegate del) { + _data_callback = del; + } + + #endregion + + #region IDisposable Members + + private bool _is_disposed = false; + public void Dispose() { + if (isDisposed) + return; + bts.Log.Gen.info("PubSubServerManagedTCPConnection.Dispose"); + if (_connection != null) { + _connection.Dispose(); + } + _is_disposed = true; + } + public bool isDisposed { + get { + return _is_disposed; + } + } + + #endregion + + #region ITCPMessageHandler Members + + /// + /// This method is executed on an arbitrary CLR thread. + /// + /// + public void onData(RemoteMessage message) { + bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData message: {message}"); + // TODO check for disconenction + switch (message.messageType) { + case RemoteMessageType.Data: + if (_data_callback != null) + _data_callback(message.remoteConnectionInfo.ToString(), message.message); + else + _in_data_queue.enqueue(message.message); + break; + case RemoteMessageType.Heartbeat: + _last_hb = DateTime.Now; + break; + case RemoteMessageType.Connection: + // since this is called from the connection, this condition will never hit, ssince + //new connections are detected only by AsyncTCPConnector + Log.assert(false); + break; + case RemoteMessageType.Disconnection: + bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData case RemoteMessageType.Disconnection: message: {message}"); + _conn_cb_queue.enqueue(new ConnectionStatusChange(ConnectionStatusChange.Status.DISCONNECTED, null)); + break; + + } + } + + #endregion + + #region ITCPConnectionHandler Members + + /// will be called on on an arbitrary .Net thread + public void onConnection(ITCPConnection connection) { + var server_rci = connection.get_peer_address(); + bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onConnection connection: {_connection_txt(connection)}"); + if (_server_rc_info.Equals(server_rci)) { + _conn_cb_queue.enqueue(new ConnectionStatusChange(ConnectionStatusChange.Status.CONNECTED, connection)); + // _last_hb = DateTime.Now; + // _new_connection = connection; + } + } + + #endregion + + void _check_missing_hb() { + if (!_connection.isDisposed) { + TimeSpan hb_interval = DateTime.Now - _last_hb; + bool hb_missed = hb_interval > _connection_drop_threshold; + if (hb_missed) { + bts.Log.Gen.warn("PubSubServerManagedTCPConnection._dispose_and_reconnect_if_missing_hb disposing connecion: {myServerAddress}"); + ConnectionStatusChange ci = new ConnectionStatusChange(ConnectionStatusChange.Status.DISCONNECTED, null); + _conn_cb_queue.enqueue(ci); + } + } else { + // already disposed, nothing to do + } + } + + private static string _connection_txt(ITCPConnection conn) { + if (conn != null) { + return $"conn_type: {conn.GetType()} server: {conn.get_peer_address()} self: {conn.get_self_address()}"; + } + return "NULL"; + } + + #region IWorker Members + + /// + /// should be called on the same thread as the other user functions. + /// the design only allows for onConnection to be on a different + /// thread. + /// + public void doWork() { + //_dispose_and_reconnect_if_missing_hb(); + if (_conn_cb_queue.getCount() > 0) { + var css = _conn_cb_queue.dequeueAll(); + foreach (var c in css) { + Log.Gen.info($"PubSubServerManagedTCPConnection.doWork c: {c}"); + switch (c.connection_satus) { + case ConnectionStatusChange.Status.CONNECTED: { + Log.assert(c.new_connection != null); + c.new_connection.register(this); + _connection = c.new_connection; + _last_hb = DateTime.Now; + if (_conn_callback != null) + _conn_callback(_server_rc_info.ToString(), true); + break; + } + case ConnectionStatusChange.Status.DISCONNECTED: { + _connection.Dispose(); + _connect_to_ps_server(); + if (_conn_callback != null) + _conn_callback(_server_rc_info.ToString(), false); + break; + } + } + } + } + } + + #endregion + + } +} diff --git a/src/DataStreamSuite/PubSub/Client/Communication/PubSubServerConnectionFactory.cs b/src/DataStreamSuite/PubSub/Client/Communication/PubSubServerConnectionFactory.cs index 313683bb4..ef030af16 100644 --- a/src/DataStreamSuite/PubSub/Client/Communication/PubSubServerConnectionFactory.cs +++ b/src/DataStreamSuite/PubSub/Client/Communication/PubSubServerConnectionFactory.cs @@ -3,55 +3,38 @@ using System.Collections.Generic; using System.Text; //using BCG.Utilities; -namespace BCG -{ +namespace BCG { /// /// Every instance of PublisherCore and SubscriberCore will have its /// factory. Rather than have once singelton factory, this approach /// is preffered since multiple pub/sub core can connect to the same /// connection and make it difficultt o manage connections. /// - public class PubSubServerConnectionFactory : IDisposable - { - private bool myIsSub; - private Dictionary myConnections = new Dictionary(); + public class PubSubServerConnectionFactory : IDisposable { + private Dictionary _conn_map = + new Dictionary(); + private readonly bool _is_sub; - internal PubSubServerConnectionFactory (bool isSub) - { - myIsSub = isSub; + internal PubSubServerConnectionFactory(bool is_sub) { + _is_sub = is_sub; } - public IPubSubServerConnection getConnection(string url) - { - if (!myConnections.ContainsKey(url)) - myConnections.Add(url, myBuildConnection(url)); - return myConnections[url]; + public IPubSubServerConnection get_connection(RemoteConnectionInfo server_rci) { + IPubSubServerConnection rv = null; + if (!_conn_map.TryGetValue(server_rci, out rv)) { + rv = _build_connection(server_rci); + _conn_map.Add(server_rci, rv); + } + return rv; } - private IPubSubServerConnection myBuildConnection(string url) - { - if (url.StartsWith(MSMQSubAddress.prefix)) - { - // TODO, no more MSMQ - //Diagnostics.assert(false); - return null; - // MSMQSubAddress serverAdd = new MSMQSubAddress(url); - // return new PubSubServerMSMQConnection(serverAdd.ToString(), myIsSub); - } - else if (url.StartsWith(BlutoSubAddress.prefix)) - { - return new PubSubServerManagedTCPConnection(url); - } - else - { - // BlueEventLog.logAssert(false, "Unknown transport, only msmq and blutoTcp allowed"); - return null; - } + private IPubSubServerConnection _build_connection(RemoteConnectionInfo server_rci) { + return new PubSubServerManagedTCPConnection(server_rci, _is_sub); } #region IDisposable Members - public void Dispose() {} + public void Dispose() { } #endregion } diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs b/src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs index 0aa0c6a02..9f26588c5 100644 --- a/src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs +++ b/src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs @@ -124,7 +124,14 @@ namespace bts { public long publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); public byte[] data_to_publish; + public void sanity_check() { + Log.assert(!String.IsNullOrEmpty(tag_type)); + Log.assert(!String.IsNullOrEmpty(tag)); + Log.assert(!String.IsNullOrEmpty(connection)); + } + public void serialize_into_str(Stream str) { + sanity_check(); BinaryWriter bw = new BinaryWriter(str); bw.write_string_cpp(connection); bw.write_string_cpp(publishing_app); diff --git a/src/DataStreamSuite/PubSub/Client/PubsubTagHandlerFactory.cs b/src/DataStreamSuite/PubSub/Client/PubsubTagHandlerFactory.cs index 901810923..bdfdd41ab 100644 --- a/src/DataStreamSuite/PubSub/Client/PubsubTagHandlerFactory.cs +++ b/src/DataStreamSuite/PubSub/Client/PubsubTagHandlerFactory.cs @@ -24,6 +24,7 @@ namespace bts { } private static Dictionary _config_map = new Dictionary() { + {String.Empty, new TagHandlerEntry(DSTagManager.DataStreamTagType , "DataStreamCore", "bts.PubsubServerStreamTagHandler")}, {DSTagManager.DataStreamTagType, new TagHandlerEntry(DSTagManager.DataStreamTagType , "DataStreamCore", "bts.PubsubServerStreamTagHandler")}, {"System_Sub", new TagHandlerEntry("System_Sub", "PubSub", "BCG.SystemSubscriptionMapPublisher")}, {"System_Stats", new TagHandlerEntry("System_Stats", "PubSub", "BCG.SystemStatsPublisher")} diff --git a/src/DataStreamSuite/PubSub/Client/Routing/IPubSubRouter.cs b/src/DataStreamSuite/PubSub/Client/Routing/IPubSubRouter.cs index ae11a0a0d..4c8fdbd55 100644 --- a/src/DataStreamSuite/PubSub/Client/Routing/IPubSubRouter.cs +++ b/src/DataStreamSuite/PubSub/Client/Routing/IPubSubRouter.cs @@ -8,11 +8,12 @@ namespace BCG { public interface IPubSubRouter { - event PubSubTagURLChangeDelegate onSubURLTagRouteChange; - event PubSubTagURLChangeDelegate onPubURLTagRouteChange; - List get_all_pubsub_sub_urls(); - List get_all_pubsub_pub_urlss(); - string get_sub_url_for_tag(string tag); - string get_pub_url_for_tag(string tag); + // event PubSubTagURLChangeDelegate onSubURLTagRouteChange; + // event PubSubTagURLChangeDelegate onPubURLTagRouteChange; + // List get_all_pubsub_sub_urls(); + // List get_all_pubsub_pub_urlss(); + + RemoteConnectionInfo get_sub_route_for_tag(string tag); + RemoteConnectionInfo get_pub_route_for_tag(string tag); } } diff --git a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs index 64e2d0712..f7d860617 100644 --- a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs +++ b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRoute.cs @@ -5,43 +5,38 @@ using System.Text.RegularExpressions; using System.Text.Json.Serialization; using System.Text.Json; using System.Text.Json.Nodes; +using System.Runtime.InteropServices; namespace BCG { [JsonConverter(typeof(PubSubRouteJsonConverter))] public class PubSubRoute : IComparable { - public string serverAddress; - public string tagRegExp; + public string server_address; + public string tag_reg_exp; //route names are used in intermediary code (i.e. while substations are being deployed) //so that MC Hammer can advertise them. public string name; - private Regex myRegex = null; + private Regex _regex = null; public PubSubRoute() { } - public PubSubRoute(string theServerAddress, string theTagRegExp) { - serverAddress = theServerAddress; - tagRegExp = theTagRegExp; + public PubSubRoute(string the_server_address, string the_tag_reg_exp) { + server_address = the_server_address; + tag_reg_exp = the_tag_reg_exp; } - // public PubSubRoute() - // { - // // serverAddress = string.Format("bluto://{0}:{1}", cfg.serverAddress, cfg.serverPort); - // // tagRegExp = cfg.tagsRegex; - // } - public string[] getAllPubSubs() { - return serverAddress.Split(';'); + return server_address.Split(';'); } - public Regex getRegex() { - return myRegex ?? new Regex(tagRegExp); + public Regex get_regex() { + return _regex ?? new Regex(tag_reg_exp); } #region IComparable Members public int CompareTo(PubSubRoute other) { - return serverAddress == other.serverAddress ? tagRegExp.CompareTo(other.tagRegExp) - : serverAddress.CompareTo(other.serverAddress); + return server_address == other.server_address ? tag_reg_exp.CompareTo(other.tag_reg_exp) + : server_address.CompareTo(other.server_address); } #endregion @@ -51,8 +46,8 @@ namespace BCG { return false; else { PubSubRoute other = obj as PubSubRoute; - return serverAddress.Equals(other.serverAddress) && - tagRegExp.Equals(other.tagRegExp); + return server_address.Equals(other.server_address) && + tag_reg_exp.Equals(other.tag_reg_exp); } } @@ -61,29 +56,33 @@ namespace BCG { } public static PubSubRoute get_default_route() { - return new PubSubRoute("bluto://127.0.0.0.1:12345", ".*"); + return new PubSubRoute("bluto://127.0.0.1:12345", ".*"); + } + + public override string ToString() { + return $"tag_reg_exp: {tag_reg_exp} server_address: {server_address}"; } } public static class PubSubRouteExtensions { public static bool IsMatch(this PubSubRoute route, string tag) => - route.getRegex().IsMatch(tag); + route.get_regex().IsMatch(tag); } public class PubSubRouteJsonConverter : JsonConverter { public override PubSubRoute? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - var jsonObj = JsonSerializer.Deserialize(ref reader); - if (jsonObj is null) { + var json_obj = JsonSerializer.Deserialize(ref reader); + if (json_obj is null) { return null; } var rv = new PubSubRoute(); ReadString("name", ref rv.name); - ReadString("server_address", ref rv.serverAddress); - ReadString("tag", ref rv.tagRegExp); + ReadString("server_address", ref rv.server_address); + ReadString("tag", ref rv.tag_reg_exp); return rv; void ReadString(string key, ref string valueOut) { - if (jsonObj.TryGetPropertyValue(key, out var valueJson)) { + if (json_obj.TryGetPropertyValue(key, out var valueJson)) { var valueStr = valueJson?.GetValue(); // Only (over)write valueOut if non-null if (valueStr is not null) { @@ -96,8 +95,8 @@ namespace BCG { public override void Write(Utf8JsonWriter writer, PubSubRoute value, JsonSerializerOptions options) { var jsonObj = new JsonObject { ["name"] = value.name, - ["server_address"] = value.serverAddress, - ["tag"] = value.tagRegExp, + ["server_address"] = value.server_address, + ["tag"] = value.tag_reg_exp, }; JsonSerializer.Serialize(writer, jsonObj); } diff --git a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs index 1eb647d3f..71552effb 100644 --- a/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs +++ b/src/DataStreamSuite/PubSub/Client/Routing/PubSubRouter.cs @@ -1,97 +1,79 @@ -using System; -using System.Text.RegularExpressions; -using System.Collections; -using System.Collections.Generic; - -using BCG; -using Bts.PubSub; -using bts; -using System.Linq; -//using BCG.Utilities; -//using bts; - -namespace BCG { - public class PubSubRouter : IPubSubRouter { - private Regex[] myPubRouteRegExps; - private Regex[] mySubRouteRegExps; - private PubSubRoute[] myPubRoutes; - private PubSubRoute[] mySubRoutes; - - public PubSubRouter() : this( - PubSubConfig.Instance.SubscriberRoutes?.ToArray() - ?? throw new InvalidOperationException("Subscriber routes are not configured"), - PubSubConfig.Instance.PublisherRoutes?.ToArray() - ?? throw new InvalidOperationException("Publisher routes are not configured")) { } - - public PubSubRouter(params PubSubRoute[] routes) : this(routes, routes) { } - - public PubSubRouter(PubSubRoute[] subRoutes, PubSubRoute[] pubRoutes) { - mySubRouteRegExps = new Regex[subRoutes.Length]; - myPubRouteRegExps = new Regex[pubRoutes.Length]; - - mySubRoutes = subRoutes; - myPubRoutes = pubRoutes; - - for (int i = 0; i < subRoutes.Length; ++i) - mySubRouteRegExps[i] = new Regex(subRoutes[i].tagRegExp); - for (int i = 0; i < pubRoutes.Length; ++i) - myPubRouteRegExps[i] = new Regex(pubRoutes[i].tagRegExp); - } - - private int myGetRouteIndex(string forTag, PubSubRoute[] routes, Regex[] regexes) { - for (int i = 0; i < routes.Length; i++) - if (regexes[i].IsMatch(forTag)) - return i; - - throw new PubSubRouteNotFound(String.Format("No route found for {0}", forTag)); - } - - #region IPubSubRouter methods - - private string myPickAServer(string tag, string[] urls) { - if (urls.Length == 1) - return urls[0]; - else { - uint hc = (uint)tag.GetHashCode(); - return urls[hc % urls.Length]; - } - } - - public string get_pub_url_for_tag(string tag) { - if (!myPubRoutes.TryGetFirst(r => r?.IsMatch(tag) ?? false, out var route)) { - throw new ArgumentException($"Failed to find route for tag '{tag}'", nameof(tag)); - } - Log.Gen.info($"PubSubRouter.get_sub_url_for_tag. tag: {tag} route: {route}"); - return route.serverAddress; - } - - public string get_sub_url_for_tag(string tag) { - if (!mySubRoutes.TryGetFirst(r => r?.IsMatch(tag) ?? false, out var route)) { - throw new ArgumentException($"Failed to find route for tag '{tag}'", nameof(tag)); - } - Log.Gen.info($"PubSubRouter.get_sub_url_for_tag. tag: {tag} route: {route}"); - return route.serverAddress; - } - - public List get_all_pubsub_sub_urls() => - mySubRoutes.Select(r => r.serverAddress).ToList(); - - public List get_all_pubsub_pub_urlss() => - myPubRoutes.Select(r => r.serverAddress).ToList(); - - public event PubSubTagURLChangeDelegate onSubURLTagRouteChange; - public event PubSubTagURLChangeDelegate onPubURLTagRouteChange; - - #endregion - - //this function is put here so that the events are used, in order to prevent build errors. - private void myNeverCallThisFunction() { - if (onPubURLTagRouteChange == null || onSubURLTagRouteChange == null) { - int aVar = 2; - aVar = aVar + 2; - } - } - - - } -} +using System; +using System.Text.RegularExpressions; +using System.Collections; +using System.Collections.Generic; + +using BCG; +using Bts.PubSub; +using bts; +using System.Linq; +//using BCG.Utilities; +//using bts; + +namespace BCG { + public class PubSubRouter : IPubSubRouter { + // private Regex[] mySubRouteRegExps; + private PubSubRoute[] _pub_routes; + private PubSubRoute[] _sub_routes; + + public PubSubRouter() : this( + PubSubConfig.Instance.SubscriberRoutes?.ToArray() + ?? throw new InvalidOperationException("Subscriber routes are not configured"), + PubSubConfig.Instance.PublisherRoutes?.ToArray() + ?? throw new InvalidOperationException("Publisher routes are not configured")) { } + + public PubSubRouter(params PubSubRoute[] routes) : this(routes, routes) { } + + public PubSubRouter(PubSubRoute[] sub_routes, PubSubRoute[] pub_routes) { + string log_msg = String.Empty; + foreach (var route in sub_routes) { + log_msg += $" sub_route: {route}"; + } + foreach (var route in pub_routes) { + log_msg += $" pub_route: {route}"; + } + Log.Gen.info($"PubSubRouter routes: {log_msg}"); + _sub_routes = sub_routes; + _pub_routes = pub_routes; + } + + #region IPubSubRouter methods + + public RemoteConnectionInfo get_pub_route_for_tag(string tag) { + if (!_pub_routes.TryGetFirst(r => r?.IsMatch(tag) ?? false, out var route)) { + throw new ArgumentException($"Failed to find route for tag '{tag}'", nameof(tag)); + } + Log.Gen.info($"PubSubRouter.get_sub_url_for_tag. tag: {tag} route: {route}"); + return RemoteConnectionInfo.parse_url(route.server_address); + } + + public RemoteConnectionInfo get_sub_route_for_tag(string tag) { + if (!_sub_routes.TryGetFirst(r => r?.IsMatch(tag) ?? false, out var route)) { + throw new ArgumentException($"Failed to find route for tag '{tag}'", nameof(tag)); + } + Log.Gen.info($"PubSubRouter.get_sub_url_for_tag. tag: {tag} route: {route}"); + return RemoteConnectionInfo.parse_url(route.server_address); + } + + public List get_all_pubsub_sub_urls() => + _sub_routes.Select(r => r.server_address).ToList(); + + public List get_all_pubsub_pub_urlss() => + _pub_routes.Select(r => r.server_address).ToList(); + + public event PubSubTagURLChangeDelegate onSubURLTagRouteChange; + public event PubSubTagURLChangeDelegate onPubURLTagRouteChange; + + #endregion + + //this function is put here so that the events are used, in order to prevent build errors. + private void myNeverCallThisFunction() { + if (onPubURLTagRouteChange == null || onSubURLTagRouteChange == null) { + int aVar = 2; + aVar = aVar + 2; + } + } + + + } +} diff --git a/src/DataStreamSuite/PubSub/Client/SubAddress.cs b/src/DataStreamSuite/PubSub/Client/SubAddress.cs index 7f83d1a0f..2dc937881 100644 --- a/src/DataStreamSuite/PubSub/Client/SubAddress.cs +++ b/src/DataStreamSuite/PubSub/Client/SubAddress.cs @@ -1,31 +1,31 @@ -using System; -using BCG; -//using BCG.Utilities; +// using System; +// using BCG; +// //using BCG.Utilities; -namespace BCG -{ - public class InvalidSubAddress : Exception - { - public InvalidSubAddress(string msg) : base (msg){} - } +// namespace BCG +// { +// public class InvalidSubAddress : Exception +// { +// public InvalidSubAddress(string msg) : base (msg){} +// } - public abstract class SubAddress - { - public static SubAddress decode (string s) - { - //if (s.StartsWith(TcpSubAddress.prefix)) - // return new TcpSubAddress(s); - //else if (s.StartsWith(SuperSocketAddress.prefix)) - // return new SuperSocketAddress(s); - if (s.StartsWith(MSMQSubAddress.prefix)) - return new MSMQSubAddress(s); - else if (s.StartsWith(BlutoSubAddress.prefix)) - return new BlutoSubAddress(s); - else - { - //BlueEventLog.logAssert(false, "Unknown pubsub transport for : " + s); - return null; - } - } - } -} +// public abstract class SubAddress +// { +// public static SubAddress decode (string s) +// { +// //if (s.StartsWith(TcpSubAddress.prefix)) +// // return new TcpSubAddress(s); +// //else if (s.StartsWith(SuperSocketAddress.prefix)) +// // return new SuperSocketAddress(s); +// if (s.StartsWith(MSMQSubAddress.prefix)) +// return new MSMQSubAddress(s); +// else if (s.StartsWith(BlutoSubAddress.prefix)) +// return new BlutoSubAddress(s); +// else +// { +// //BlueEventLog.logAssert(false, "Unknown pubsub transport for : " + s); +// return null; +// } +// } +// } +// } diff --git a/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs b/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs index ee0e6c3bc..300e0d1ce 100644 --- a/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs +++ b/src/DataStreamSuite/PubSub/Config/PubSubConfig.cs @@ -176,10 +176,10 @@ public class PubSubConfig { WaitPeriodAfterGc = 1000, SimpleEchoServer = false, PublisherRoutes = new() { - new("bluto://127.0.0.0.1:12345", ".*") + new("bluto://127.0.0.1:12345", ".*") }, SubscriberRoutes = new() { - new("bluto://127.0.0.0.1:12345", ".*") + new("bluto://127.0.0.1:12345", ".*") } }; return rv; diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection.cs new file mode 100644 index 000000000..0d5fbf05f --- /dev/null +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection.cs @@ -0,0 +1,542 @@ +// using System; +// using System.Collections.Generic; +// using System.Text; +// using System.Net; +// using System.Net.Sockets; +// using System.IO; +// //using BCG.Utilities; +// using BCG.Utilities.General; +// using System.Runtime.InteropServices; + +// namespace BCG +// { +// /// +// /// This class, mantains a connection to a peer socket, using +// /// the .Net async TCP library. It sends heartbeats at regular +// /// intervals (config). It is in active states when messages +// /// are sent and received without an error state. It assumed +// /// dead state when either of the following happens +// /// BeginReceive returns a SocketError +// /// BeginSend returns a SocketError +// /// EndReceive returns a SocketError +// /// EndSend returns a SocketError +// /// As soon as it assumes dead state it stops sending or +// /// receiving any messages (including heartbeats). +// /// The user of this class can either detect that by +// /// missing heartbeats or checking the isConnected property. +// /// Missing should be the more general way, since if the +// /// peer dies in a 'silent' way, where none of the above +// /// methods indicate an error, missing heartbeats is guaranteed +// /// to detect it. +// /// +// public class AsyncTCPConnection : ITCPConnection, IWorker, IDisposable +// { +// /// +// /// This is used as a state for the Async send. Reason being +// /// that if onSend indicates that it could not send the full +// /// data (ie EndSend does not return the full size), the offset +// /// is used to resend the data +// /// +// public class AsyncMessage +// { +// public byte[] buffer; +// public int offset; +// public string messageId; +// public SendConfirmationDelegate onSendDel; + +// public AsyncMessage(byte[] theBuffer, int theOffset) +// : this(theBuffer, theOffset, String.Empty, null) { } + +// public AsyncMessage(byte[] theBuffer, int theOffset, string theMessageId, SendConfirmationDelegate theOnSendDel) +// { +// buffer = theBuffer; +// offset = theOffset; +// messageId = theMessageId; +// onSendDel = theOnSendDel; +// } + + +// public int length +// { +// get +// { +// return buffer.Length - offset; +// } +// } +// } + +// enum ReadState +// { +// readingSize, +// readingData +// }; + +// #region work around for .Net socket's LocalEndPoint bug + +// [StructLayout(LayoutKind.Sequential)] +// private struct in_addr +// { +// [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)] +// public byte[] sin_addr; +// } + +// [StructLayout(LayoutKind.Sequential)] +// private struct sockaddr +// { +// public short sin_family; +// public ushort sin_port; +// public in_addr sin_addr; +// [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)] +// public byte[] sin_zero; +// } + +// [DllImport("wsock32.dll")] +// private static extern int getsockname(IntPtr s, ref sockaddr name, ref int namelen); + +// [DllImport("wsock32.dll")] +// private static extern int getpeername(IntPtr s, ref sockaddr name, ref int namelen); + +// #endregion + +// private readonly int myMaxMessageSize = PubSub3Config.instance.PS3_MaxPacketSize; +// private RemoteConnectionInfo myPeer, mySelf; +// private Socket mySock; +// private ITCPMessageHandler myMessageHandler; + +// private byte[] myCurrentReadBuffer; +// private int myCurrentReadBufferOffset; +// private ReadState myCurrentReadState; + +// private bool myAlive = true; + +// private byte[] myHBMessage; +// private TimeSpan myHBTimeSpan; +// private MasterWorker myMasterWorker = MasterWorker.getInstance(true); +// private bool myHBReceived = false, myDataReceived = false, myHBSent = false; +// private DateTime myLastHBReceivedTime = new DateTime(1900, 1, 1); + +// public AsyncTCPConnection(Socket sock, TimeSpan heartBeatTimeSpan) +// { +// myHBTimeSpan = heartBeatTimeSpan; +// mySock = sock; +// mySock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true); +// string ip; +// int port; +// getPeerAddress(out ip, out port); +// myPeer = new RemoteConnectionInfo(ip, port); +// getSelfAddress(out ip, out port); +// mySelf = new RemoteConnectionInfo(ip, port); +// // BlueEventLog.logEvent(String.Format("AsyncTCPConnection created : self = {0} , peer = {1}", +// // mySelf.ToString(), myPeer.ToString()), PS3LogLevels.connection); +// } + +// public override string ToString() +// { +// return String.Format("peer = {0}, self = {1}", myPeer.ToString(), mySelf.ToString()); +// } + +// #region ITCPConnection Members + +// public void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del) +// { +// confId = Guid.NewGuid().ToString(); +// byte[] msg = myBuildDataMessage(data); +// myBeginOrResumeSending(new AsyncMessage(msg, 0, confId, del)); +// } + +// public void send(byte[] data) +// { +// byte[] msg = myBuildDataMessage(data); +// myBeginOrResumeSending(new AsyncMessage(msg, 0)); +// } + +// public void register(ITCPMessageHandler messageHandler) +// { +// // BlueEventLog.logAssert(myMessageHandler == null, "Message Handler is already assigned"); +// // BlueEventLog.logEvent(String.Format("AsyncTCPConnection for peer = {0} being registered", myPeer.ToString()), +// // PS3LogLevels.connection); +// myMessageHandler = messageHandler; +// myMasterWorker.addWorker(this, myHBTimeSpan); +// myReadSize(); +// } + +// public void getPeerAddress(out string ip, out int port) +// { +// ip = ""; +// port = 0; +// if (myAlive) +// { +// IPEndPoint ipEp = mySock.RemoteEndPoint as IPEndPoint; +// ip = ipEp.Address.ToString(); +// port = ipEp.Port; +// } +// ///.Net has some problems getting the port, so using +// ///native code for that +// //sockaddr addr = new sockaddr(); +// //int namelen = 16; +// //getsockname(mySock.Handle, ref addr, ref namelen); +// //port = addr.sin_port; +// } + +// public void getSelfAddress(out string ip, out int port) +// { +// ip = ""; +// port = 0; +// if (myAlive) +// { +// IPEndPoint ipEp = mySock.LocalEndPoint as IPEndPoint; +// ip = ipEp.Address.ToString(); +// port = ipEp.Port; +// } +// ///due to this bug in .Net socket, we have to resort to +// ///native code +// //sockaddr addr = new sockaddr(); +// //int namelen = 16; +// //getsockname(mySock.Handle, ref addr, ref namelen); +// //port = addr.sin_port; +// } + +// public bool isDisposed +// { +// get { return !myAlive; } +// } + +// #endregion + +// #region IWorker Members + +// bool _last_connected = false; +// public void doWork() +// { +// if (mySock != null) +// //if (mySock.Connected != _last_connected) +// { +// Console.WriteLine($"AsyncTCPConnection.doWork connected: {mySock.Connected}"); +// _last_connected = mySock.Connected; +// } +// if (!myHBSent) +// { +// // BlueEventLog.logEvent(String.Format("Sending heartbeat to peer {0} for the first time", myPeer.ToString()), +// // PS3LogLevels.connection); +// myHBSent = true; +// } +// myBeginOrResumeSending(new AsyncMessage(myBuildHeartbeatMessage(), 0)); +// } + +// #endregion + +// #region sending data + +// private void myBeginOrResumeSending(AsyncMessage message) +// { +// if (myAlive) +// { +// try +// { +// //Console.WriteLine($"AsyncTCPConnection.myBeginOrResumeSending len: {message.length} connected: {mySock.Connected}"); +// SocketError beginSendErr; +// mySock.BeginSend(message.buffer, message.offset, message.length, SocketFlags.None, +// out beginSendErr, new AsyncCallback(myOnSend), message); +// Console.WriteLine($"AsyncTCPConnection.myBeginOrResumeSending len: {message.length} socker_error: {beginSendErr} connected: {mySock.Connected}"); +// if (beginSendErr != SocketError.Success) +// myHandleError(beginSendErr); +// } +// catch (Exception ex) +// { +// myHandleError(ex); +// } +// } +// } + +// private void myOnSend(IAsyncResult res) +// { +// if (myAlive) +// { +// try +// { +// SocketError writeErr; +// int bytesSent = mySock.EndSend(res, out writeErr); +// Console.WriteLine($"AsyncTCPConnection.myOnSend bytes: {bytesSent} writeErr: {writeErr} connected: {mySock.Connected}"); +// AsyncMessage message = res.AsyncState as AsyncMessage; +// message.offset += bytesSent; +// if (writeErr == SocketError.Success) +// { +// if (message.offset == message.buffer.Length) +// { +// // if confirmation needs to be sent +// if (String.Empty != message.messageId) +// { +// message.onSendDel(message.messageId, true); +// } +// } +// else +// { +// ///resume writing rest of the message +// // BlueEventLog.logEvent("could not send the whole message all at once , messageSize = " + +// // message.buffer.Length + " offset = " + message.offset, PS3LogLevels.info); +// myBeginOrResumeSending(message); +// } +// } +// else +// { +// myHandleError(writeErr); +// } +// } +// catch (Exception ex) +// { +// myHandleError(ex); +// } +// } +// } + +// /// +// /// Assembles the heartbeat message, and uses it for +// /// future references +// /// +// /// +// private byte[] myBuildHeartbeatMessage() +// { +// if (myHBMessage == null) +// { +// BinaryWriter bw = new BinaryWriter(new MemoryStream()); +// int sz = 4; +// int msgType = 1; +// bw.Write(sz); +// bw.Write(msgType); +// myHBMessage = (bw.BaseStream as MemoryStream).ToArray(); +// } +// return myHBMessage; +// } + +// /// +// /// Assembles the data message. This is necessary +// /// for the other side to understand the message. This can also +// /// be achieved by not forming the message and use the +// /// BeginSend (List<>) function rather than BeginSend (byte[]) +// /// +// /// +// /// +// private byte[] myBuildDataMessage(byte[] data) +// { +// BinaryWriter bw = new BinaryWriter(new MemoryStream()); +// bw.Write(data.Length + 8); +// bw.Write(0); +// bw.Write(0); ///mid for backward compatibility +// bw.Write(data); +// return (bw.BaseStream as MemoryStream).ToArray(); +// } + +// #endregion + +// #region receiving data + +// private void myReadSize() +// { +// myCurrentReadBuffer = new byte[4]; +// myCurrentReadBufferOffset = 0; +// myCurrentReadState = ReadState.readingSize; +// myBeginOrResumeReading(); +// } + +// private void myReadData(int dataSize) +// { +// myCurrentReadBuffer = new byte[dataSize]; +// myCurrentReadBufferOffset = 0; +// myCurrentReadState = ReadState.readingData; +// myBeginOrResumeReading(); +// } + +// private void myBeginOrResumeReading() +// { +// if (myAlive) +// { +// try +// { +// int beginReceiveAttempts = 0; +// while (true) +// { +// beginReceiveAttempts++; +// SocketError readSockErr; +// mySock.BeginReceive(myCurrentReadBuffer, myCurrentReadBufferOffset, +// myCurrentReadBuffer.Length - myCurrentReadBufferOffset, +// SocketFlags.None, out readSockErr, +// new AsyncCallback(myOnRead), +// myCurrentReadBuffer); +// Console.WriteLine($"AsyncTCPConnection.myBeginOrResumeReading socket_error: {readSockErr} connected: {mySock.Connected}"); +// if (readSockErr == SocketError.NoBufferSpaceAvailable) +// { +// string msg = String.Format("Not enough buffer space to receive {0} bytes. Attempt: {1} of {2}.", +// myCurrentReadBuffer.Length - myCurrentReadBufferOffset, +// beginReceiveAttempts, PubSub3Config.instance.PS3_retriesOnNoBufferSpace + 1); +// //BlueEventLog.logEvent(msg, PS3LogLevels.disconnection); + +// if (beginReceiveAttempts < (PubSub3Config.instance.PS3_retriesOnNoBufferSpace + 1)) +// System.Threading.Thread.Sleep(PubSub3Config.instance.PS3_millisecondsBetweenNoBufferSpaceRetries); +// else +// { +// //BlueEventLog.logEvent("Could not recover from NoBufferSpace condition. Breaking connection.", PS3LogLevels.disconnection); +// myHandleError(readSockErr); +// } +// } +// else if (readSockErr == SocketError.Success) +// { +// if (beginReceiveAttempts > 1) +// // BlueEventLog.logEvent(String.Format("Recovered from NoBufferSpace after {0} tries",beginReceiveAttempts-1) , +// // PS3LogLevels.disconnection); +// break; +// } +// else +// myHandleError(readSockErr); +// } +// } +// catch (Exception ex) +// { +// myHandleError(ex); +// } +// } +// } + +// private void myOnRead(IAsyncResult res) +// { +// if (myAlive) +// { +// try +// { +// SocketError readErr; +// Console.WriteLine($"AsyncTCPConnection.myOnRead pre connected: {mySock.Connected}"); +// int bytesRead = mySock.EndReceive(res, out readErr); +// Console.WriteLine($"AsyncTCPConnection.myOnRead bytes: {bytesRead} readErr: {readErr} connected: {mySock.Connected}"); +// myCurrentReadBufferOffset += bytesRead; +// if (readErr == SocketError.Success) +// { +// if (myCurrentReadBufferOffset == myCurrentReadBuffer.Length) +// { +// ///read the whole of the buffer +// if (myCurrentReadState == ReadState.readingSize) +// { +// ///the size has already been read, now convert +// ///it into an int, and start reading the data +// int dataSize = new BinaryReader(new MemoryStream(myCurrentReadBuffer)).ReadInt32(); +// if (dataSize > myMaxMessageSize) +// { +// string errMessage = "Encountered malformed message, size indicated = " + dataSize; +// //BlueEventLog.logEvent(errMessage, "Error"); +// myHandleError(new ApplicationException(errMessage)); +// } +// else +// myReadData(dataSize); +// } +// else +// { +// ///the whole data has been finally read, time to +// ///tell the subscriber about it +// BinaryReader br = new BinaryReader(new MemoryStream(myCurrentReadBuffer)); +// int msgType = br.ReadInt32(); +// if (msgType == 0) +// { +// ///skips the message ID +// br.ReadInt32(); +// byte[] data = br.ReadBytes(myCurrentReadBuffer.Length - 8); +// ///callbacks can be made through thread pool to avoid this +// ///thread from being blocked, by this callback +// if (!myDataReceived) +// { +// myDataReceived = true; +// Console.WriteLine($"AsyncTCPConnection Data received for the first time from peer {myPeer.ToString()}"); +// } +// myMessageHandler.onData(new RemoteMessage(data, RemoteMessageType.Data, myPeer)); + +// } +// else if (msgType == 1) +// { +// if (!myHBReceived) +// { +// myHBReceived = true; +// Console.WriteLine($"AsyncTCPConnection HB received for the first time from peer {myPeer.ToString()}"); +// } +// myLastHBReceivedTime = DateTime.Now; +// myMessageHandler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, myPeer)); +// } +// myReadSize(); +// } +// } +// else +// { +// ///still part of the buffer left to read +// ///continue reading +// myBeginOrResumeReading(); +// } +// } +// else +// { +// myHandleError(readErr); +// } +// } +// catch (Exception ex) +// { +// myHandleError(ex); +// } +// } +// } + +// #endregion + +// #region IDisposable Members + +// public void Dispose() +// { +// Console.WriteLine("AsyncTCPConnection.Dispose()"); +// lock (this) +// { +// if (myAlive) +// { +// myMasterWorker.removeWorker(this); +// myAlive = false; +// // BlueEventLog.logEvent(String.Format("AsycnTCPConnection with peer {0} being disposed, last HB was received at {1}" , +// // myPeer.ToString(), myLastHBReceivedTime.ToLongTimeString()), +// // PS3LogLevels.disconnection); +// try +// { +// mySock.Close(); +// } +// catch (Exception ex) +// { +// String.Format("Exception of type {0} thrown while trying to close socket connected to {1} , with" + +// " the following exception msg {2}", ex.GetType().ToString(), myPeer.ToString(), ex.Message); +// //BlueEventLog.logEvent(ex); +// if (ex is OutOfMemoryException) +// throw ex; +// } +// } +// } +// } + +// #endregion + +// private void myHandleError(object err) +// { +// lock (this) +// { +// string errMsg = ""; +// if (err is Exception) +// { +// errMsg = String.Format("AsyncTCPConnection.Dispose Encountered an exception of type '{0}' on the socket connected to ip = {1} , port = {2} ; " + +// "Exception message = {3} , if the socket has not been disposed, it will be disposed now", err.GetType(), myPeer.ip, myPeer.port, (err as Exception).Message); +// if (err is OutOfMemoryException) +// throw (err as OutOfMemoryException); +// } +// else if (err is SocketError) +// { +// errMsg = String.Format("AsyncTCPConnection.Dispose Encountered a SocketError of type '{0}' on the socket connected to ip = {1} , port = {2} ; " + +// "if the socket has not been disposed, it will be disposed now", err.ToString(), myPeer.ip, myPeer.port); +// Console.WriteLine($"AsyncTCPConnection.myHandleError errMsg: {errMsg}"); +// } +// //BlueEventLog.logEvent(errMsg, PS3LogLevels.disconnection); +// if (myAlive) +// { +// Dispose(); +// } +// } +// } +// } +// } \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection2.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection2.cs new file mode 100644 index 000000000..48e5d7d96 --- /dev/null +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection2.cs @@ -0,0 +1,248 @@ +// using BCG; + +// using Google.Protobuf; + +// using System; +// using System.IO; +// using System.Linq; +// using System.Net; +// using System.Net.Sockets; +// using System.Runtime.CompilerServices; +// using System.Text; +// using System.Threading.Tasks; +// using BCG; +// using System.Xml.Serialization; +// using System.Reflection; +// using BCG.Utilities.General; +// using System.Net.WebSockets; + +// namespace BCG { + +// public class AsyncTCPConnection2 : ITCPConnection, IWorker, IDisposable { +// private Socket _socket; +// // tied to the socket above +// private NetworkStream _nw_stream; +// private readonly ConnectionSide _side; +// ITCPMessageHandler _message_handler; +// private int _msg_id = 0; +// RemoteConnectionInfo _peer_address; +// RemoteConnectionInfo _self_address; +// private TimeSpan _hb_period = TimeSpan.FromSeconds(1); +// private DateTime _last_hb_sent = DateTime.MinValue; +// private DateTime _last_hb_received = DateTime.MaxValue; +// private static byte _msg_type_data = 1; +// private static byte _msg_type_hb = 0; +// private bool _is_disposed = false; +// SyncQueue _data_send_queue = new SyncQueue(); + +// public class QueuedMessage { +// public string id; +// public SendConfirmationDelegate del; +// public byte[] data; + +// public QueuedMessage(string id, SendConfirmationDelegate del, byte[] data) { +// id = id; +// del = del; +// data = data; +// } + +// public QueuedMessage(byte[] data) +// : this(String.Empty, null, data) { } + +// } + +// public enum ConnectionSide { +// Client, +// Server +// } + +// public AsyncTCPConnection2(Socket socket, ConnectionSide side) { +// _socket = socket; +// _nw_stream = new NetworkStream(socket); +// _side = side; +// IPEndPoint r_ep = socket.RemoteEndPoint as IPEndPoint; +// _peer_address = new RemoteConnectionInfo(r_ep.Address.ToString(), r_ep.Port); + +// var loc_ep = socket.LocalEndPoint as IPEndPoint; +// _self_address = new RemoteConnectionInfo(loc_ep.Address.ToString(), loc_ep.Port); +// } + +// public void register(ITCPMessageHandler message_handler) { +// _message_handler = message_handler; +// Task.Run(async () => _keep_reading()); +// Task.Run(async () => await _keep_sending()); +// //_keep_reading(); +// // if add a _keep_sending threadis added then we can maek send a non async +// // methid making it much easier to integrate with the rest +// } + +// // this thread pulls +// private async Task _keep_sending() { +// bts.Log.Gen.info($"TCPConnection._keep_sending starting"); +// while (!_is_disposed) { +// try { +// QueuedMessage m = _data_send_queue.dequeue(100); +// if (m == null) { +// continue; +// } +// await send_async(m.data); +// if (String.IsNullOrEmpty(m.id)) { +// m.del(m.id, true); +// } +// } catch (Exception ex) { +// bts.Log.Gen.error($"TCPConnection._keep_sending ex: {ex.Message}"); +// await Task.Delay(100); +// _handle_error(ex); +// } +// } +// } + +// private async Task _keep_reading() { +// bts.Log.Gen.info($"TCPConnection._keep_reading starting"); +// while (!_is_disposed) { +// try { +// byte[] msg = await TCPSocketUtils.read_bytes_async(_nw_stream); +// bts.Log.Gen.info($"AsyncTCPConnection2._keep_reading side: {_side} self: {get_self_address()} peer: {get_peer_address()} msg len: {msg.Length}"); +// await _handle_msg(msg); +// } catch (Exception ex) { +// bts.Log.Gen.error($"TCPConnection._keep_reading ex: {ex.Message}"); +// await Task.Delay(100); +// _handle_error(ex); +// } +// } +// } + +// private async Task _handle_msg(byte[] msg) { +// // read the first byte to read the msg type and deal with it accordingly +// BinaryReader br = new BinaryReader(new MemoryStream(msg)); +// byte msg_type = br.ReadByte(); +// if (msg_type == _msg_type_data) { +// byte[] data = br.ReadBytes(msg.Length - 1); +// RemoteMessage m = new RemoteMessage(data, RemoteMessageType.Data, _peer_address); +// _message_handler.onData(m); +// } else { +// RemoteMessage m = new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_address); +// _message_handler.onData(m); +// } +// } + +// // not very efficient +// private static byte[] _prefix_msg_type(byte[] data) { +// MemoryStream ms = new MemoryStream(); +// BinaryWriter bw = new BinaryWriter(ms); +// bw.Write(_msg_type_data); +// bw.Write(data); +// ms.Position = 0; +// return ms.ToArray(); +// } + +// private async void _handle_error(Exception ex) { +// bts.Log.Gen.error($"TCPConnection._handle_error ex: {ex}"); +// try { +// var socket = _nw_stream.Socket; +// socket.Close(); +// socket.Dispose(); +// } catch (Exception ex2) { +// bts.Log.Gen.error($"TCPConnection._handle_error got exception when closing Socket ex: {ex}"); +// } +// _is_disposed = true; +// RemoteMessage rm = new RemoteMessage(new byte[] { }, RemoteMessageType.Disconnection, _peer_address); +// _message_handler.onData(rm); +// } + +// public void send_with_confirmation(byte[] data, string confId, SendConfirmationDelegate del) { } + +// public async Task send_with_confirmation_async(byte[] data, string conf_id, SendConfirmationDelegate del) { +// try { +// conf_id = (++_msg_id).ToString(); +// await TCPSocketUtils.send_bytes_async(_nw_stream, data); +// del(conf_id, true); +// } catch (Exception ex) { +// _handle_error(ex); +// } +// } + +// public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del) { +// conf_id = (++_msg_id).ToString(); +// send(data); +// del(conf_id, true); +// } + +// public void send(byte[] data) { +// Task.Run(async () => await send_async(data)); +// bts.Log.Gen.info($"AsyncTCPConnection2.send self: {get_self_address()} peer: {get_peer_address()} msg len: {data.Length} sent successfully"); +// //_data_send_queue.enqueue(new QueuedMessage(data)); +// } + +// public async Task send_async(byte[] data) { +// // prefix 4 bytes for type (data vs hb) +// MemoryStream str = new MemoryStream(); +// BinaryWriter bw = new BinaryWriter(str); +// bw.Write(_msg_type_data); +// bw.Write(data); +// try { +// await TCPSocketUtils.send_bytes_async(_nw_stream, str.ToArray()); +// } catch (Exception ex) { +// bts.Log.Gen.error($"AsyncTCPConnection2.send_async ex: {ex}"); +// _handle_error(ex); +// } +// } + +// public void getPeerAddress(out string ip, out int port) { +// ip = _peer_address.ip; +// port = _peer_address.port; +// ///.Net has some problems getting the port, so using +// ///native code for that +// //sockaddr addr = new sockaddr(); +// //int namelen = 16; +// //getsockname(mySock.Handle, ref addr, ref namelen); +// //port = addr.sin_port; +// } + +// public RemoteConnectionInfo get_peer_address() { +// string ip; int port; +// getPeerAddress(out ip, out port); +// return new RemoteConnectionInfo(ip, port); +// } + +// public RemoteConnectionInfo get_self_address() { +// string ip; int port; +// getSelfAddress(out ip, out port); +// return new RemoteConnectionInfo(ip, port); +// } + + +// public void getSelfAddress(out string ip, out int port) { +// ip = _self_address.ip; +// port = _self_address.port; +// } + +// private async void _send_hb() { +// if (_last_hb_sent - DateTime.Now > _hb_period) { +// MemoryStream ms = new MemoryStream(); +// BinaryWriter bw = new BinaryWriter(ms); +// bw.Write(_msg_type_hb); +// ms.Position = 0; +// byte[] hb_msg = ms.ToArray(); +// await TCPSocketUtils.send_bytes_async(_nw_stream, hb_msg); +// } +// } + +// public void doWork() { +// _send_hb(); +// } + +// public void Dispose() { +// _nw_stream.Close(); +// _nw_stream.Dispose(); +// _is_disposed = true; +// } + +// public bool isDisposed { +// get { +// return _is_disposed; +// } +// } + +// } +// } \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs index 48bc7db0b..ab5f4dea4 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs @@ -15,6 +15,7 @@ namespace bts { private ITCPConnectionHandler _connection_handler; private HashSet _key_map = new HashSet(); public AsyncTCPConnectionAcceptor_APM(int port, ITCPConnectionHandler connection_handler) { + Log.Gen.info($"AsyncTCPConnectionAcceptor_APM. cons started on port: {port}"); _port = port; _connection_handler = connection_handler; } @@ -25,7 +26,9 @@ namespace bts { } private void _init() { + Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._init started on port: {_port}"); _accept_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + TCPSocketUtils.set_server_socket_options(ref _accept_socket); IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, _port); _accept_socket.Bind(endpoint); _accept_socket.Listen(250); @@ -33,6 +36,7 @@ namespace bts { private void _begin_accepting() { var accept_key = Guid.NewGuid().ToString(); + Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._begin_accepting port: {_port} accept_key: {accept_key}"); var res = _accept_socket.BeginAccept(new AsyncCallback(_on_accept), accept_key); if (res != null && res.IsCompleted && res.CompletedSynchronously) _on_accept(res); @@ -62,11 +66,8 @@ namespace bts { _connection_handler.onConnection(newConn); // go back to accepting more conns _begin_accepting(); - string ip; int port; - newConn.getPeerAddress(out ip, out port); - string connMsg = String.Format("Connection to port {2} made from {0}:{1}", ip, port, _port); - // BlueEventLog.logEvent(connMsg, PS3LogLevels.connection); - //Console.WriteLine(connMsg); + var peer_rci = newConn.get_peer_address(); + Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._on_accept peer_rci: {peer_rci}"); } diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_AsyncWrapper.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_AsyncWrapper.cs new file mode 100644 index 000000000..571a7b15d --- /dev/null +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_AsyncWrapper.cs @@ -0,0 +1,52 @@ +using System.Net.Sockets; +using System.Net; + +using BCG; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + + +namespace bts { + + // opens a TCP port and starts listening to connections. Unlike AsyncTCPConnectionAcceptor_APM, + // it uses TcpListener, instead of raw sockets. + public class AsyncTCPConnectionAcceptor_AsyncWrapper { + private readonly int _accept_port; + private Socket _accept_socket; + private ITCPConnectionHandler _connection_handler; + private TcpListener _server; + private Action _cbs = (c) => { }; + + public AsyncTCPConnectionAcceptor_AsyncWrapper(int accept_port, ITCPConnectionHandler connection_handler) { + Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper.cons started on port: {accept_port}"); + _accept_port = accept_port; + _connection_handler = connection_handler; + } + + public void resgiter_callback(Action cb) { + _cbs += cb; + } + + public void start() { + Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._init started on port: {_accept_port}"); + _server = new TcpListener(IPAddress.Any, _accept_port); + _server.Start(); + Task.Run(() => _accept_connections_forever()); + } + + private void _accept_connections_forever() { + Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port}"); + while (true) { + // this is a blocking call + Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port} starting blocking call AcceptTcpClient"); + TcpClient tcp_client = _server.AcceptTcpClient(); + ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client); + Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port}, new connection, self: {conn.get_self_address()} peer: {conn.get_peer_address()} calling onConnection now"); + _connection_handler.onConnection(conn); + } + //_cbs(conn); + } + } + +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs index afc506d11..044d63851 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs @@ -153,41 +153,14 @@ namespace bts { return $"ErrorMessage exception: {exception} context: {context}"; return $"socket_error: {socket_error} context: {context}"; } - - } - #region work around for .Net socket's LocalEndPoint bug - - // [StructLayout(LayoutKind.Sequential)] - // private struct in_addr { - // [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)] - // public byte[] sin_addr; - // } - - // [StructLayout(LayoutKind.Sequential)] - // private struct sockaddr { - // public short sin_family; - // public ushort sin_port; - // public in_addr sin_addr; - // [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)] - // public byte[] sin_zero; - // } - - // [DllImport("wsock32.dll")] - // private static extern int getsockname(IntPtr s, ref sockaddr name, ref int namelen); - - // [DllImport("wsock32.dll")] - // private static extern int getpeername(IntPtr s, ref sockaddr name, ref int namelen); - - #endregion - private const int DATA_MSG_SIGNAL = 0; private const int HB_MSG_SIGNAL = 1; private int _send_key = 0, _receive_key = 0; private HashSet _read_key_map = new HashSet(), _send_key_map = new HashSet(); - private readonly int _max_msg_size = PubSubConfig.Instance.MaxPacketSize.assert_value(); + private readonly int _max_msg_size = Int32.MaxValue;// PubSubConfig.Instance.MaxPacketSize.assert_value(); private readonly RemoteConnectionInfo _peer_conn_info, _self_conn_info; private readonly ITCPConnection.ConnectionSide _connection_side; private readonly Socket _sock; @@ -206,13 +179,33 @@ namespace bts { _hb_period = hb_period; _connection_side = connection_side; _sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true); - string ip; - int port; - getPeerAddress(out ip, out port); - _peer_conn_info = new RemoteConnectionInfo(ip, port); - getSelfAddress(out ip, out port); - _self_conn_info = new RemoteConnectionInfo(ip, port); + #region testing options + switch (connection_side) { + case ITCPConnection.ConnectionSide.Server: + TCPSocketUtils.set_server_socket_options(ref _sock); + break; + case ITCPConnection.ConnectionSide.Client: + TCPSocketUtils.set_client_socket_options(ref _sock); + break; + } + #endregion + _peer_conn_info = get_peer_address(); + _self_conn_info = get_self_address(); bts.Log.Gen.info($"AsyncTCPConnection_APM created {ToString()}"); + _detect_connection_status_change(); + } + + private void _detect_connection_status_change() { + Action testing_func = () => { + bool connected = _sock.Connected; + while (true) { + if (connected != _sock.Connected) + Log.Gen.error($"Connection status changed this: {ToString()} {connected} -> {_sock.Connected}"); + connected = _sock.Connected; + System.Threading.Thread.Sleep(50); + } + }; + ThreadPool.QueueUserWorkItem(_ => testing_func()); } public override string ToString() { @@ -228,7 +221,9 @@ namespace bts { } public void send(byte[] data) { + Log.Gen.info($"AsyncTCPConnection_APM.send data: {AsyncTCPConnection_AsyncWrapper.to_hex(data, 0, data.Length)}"); byte[] msg = _build_data_message(data); + Log.Gen.info($"AsyncTCPConnection_APM.send msg: {AsyncTCPConnection_AsyncWrapper.to_hex(msg, 0, msg.Length)}"); _begin_or_resume_sending(new AsyncSendMessage(msg, 0)); } @@ -241,24 +236,12 @@ namespace bts { _read_next_msg(); } - public void getPeerAddress(out string ip, out int port) { - ip = ""; - port = 0; - if (_is_alive) { - IPEndPoint ipEp = _sock.RemoteEndPoint as IPEndPoint; - port = ipEp.Port; - ip = ipEp.Address.ToString(); - } + public RemoteConnectionInfo get_peer_address() { + return TCPSocketUtils.get_peer_address(_sock); } - public void getSelfAddress(out string ip, out int port) { - ip = ""; - port = 0; - if (_is_alive) { - IPEndPoint ipEp = _sock.LocalEndPoint as IPEndPoint; - ip = ipEp.Address.ToString(); - port = ipEp.Port; - } + public RemoteConnectionInfo get_self_address() { + return TCPSocketUtils.get_peer_address(_sock); } public bool isDisposed { @@ -384,15 +367,17 @@ namespace bts { // writes size prefixed msg private byte[] _build_data_message(byte[] data) { // TODO, avoid unecessary memory creation (write directly into byte[] using BitConverter) - BinaryWriter bw = new BinaryWriter(new MemoryStream()); + MemoryStream str = new MemoryStream(); + BinaryWriter bw = new BinaryWriter(str); // the total msg =[msg_sz, msg_type, data] - var msg_sz = data.Length + 4; + Int32 msg_sz = data.Length + 4; bw.Write(msg_sz); // +4 for the type of msg - // to indicate data vs hb + // to indicate data vs hb bw.Write(DATA_MSG_SIGNAL); // for data - // if we want to id every message, this is the place to do it + // if we want to id every message, this is the place to do it bw.Write(data); - return (bw.BaseStream as MemoryStream).ToArray(); + str.Position = 0; //not needed really + return str.ToArray(); } #endregion @@ -437,8 +422,16 @@ namespace bts { } } + private void _log_incoming_msg(IAsyncResult res) { + AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage; + var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(rcv_message.buffer, 0, rcv_message.buffer.Length); + Log.Gen.info($"BeginReceive_callback_function _log_incoming_msg buff: {buff_txt}"); + + } + // this is the function passed to BeginReceive, we private void BeginReceive_callback_function(IAsyncResult res) { + _log_incoming_msg(res); if (res.CompletedSynchronously) { ThreadPool.QueueUserWorkItem(_ => _on_read_async(res)); } else @@ -465,6 +458,8 @@ namespace bts { Log.Gen.info($"AsyncTCPConnection_APM._on_read post-EndReceive rcv_message: {rcv_message} res: {IAsyncResult_txt(res)} bytes_read: {bytes_read}"); if (bytes_read == 0) { bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, throwing exception"); + //bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, sleeping and ignoring"); + //System.Threading.Thread.Sleep(50); _handle_error(new ErrorMessage(new ApplicationException("Read 0 bytes, peer socket is likely shutdown"), "_on_read")); return; } @@ -498,10 +493,10 @@ namespace bts { _begin_or_resume_reading(rcv_msg: rcv_message); } } else { - _handle_error(new ErrorMessage(read_err, "_on_read")); + _handle_error(new ErrorMessage(read_err, "_on_read_async")); } } catch (Exception ex) { - _handle_error(new ErrorMessage(ex, "_on_read")); + _handle_error(new ErrorMessage(ex, "_on_read_async")); } } @@ -512,10 +507,12 @@ namespace bts { bts.Log.assert(msg.fully_transfered()); bts.Log.assert(msg.original_size == 4); var msg_sz = msg.read_int32(); + var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(msg.buffer, 0, msg.buffer.Length); + bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely buff: {buff_txt}"); bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely msg: {msg} msg_sz: {msg_sz}"); - if (msg_sz > _max_msg_size) { + if (msg_sz < 0 || msg_sz > _max_msg_size) { string err_msg = $"Encountered malformed message, prefixed_size: {msg_sz}"; - _handle_error(new ErrorMessage(new ApplicationException(err_msg), "_on_read")); + _handle_error(new ErrorMessage(new ApplicationException(err_msg), "_on_read_size_completely")); return null; } else { // sz is determined, now we read that many bytes @@ -559,11 +556,13 @@ namespace bts { public void Dispose() { // socket is closed and disposed here lock (this) { + bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} Dispose called _dipose_impetus: {_dipose_impetus}"); if (_is_alive) { _master_worker.removeWorker(this); _is_alive = false; - bts.Log.Gen.info($"AsycnTCPConnection.Dispose this: {this} being disposed, last HB was received at {_last_hb_received_time}"); + bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} being disposed, last HB was received at {_last_hb_received_time}"); try { + bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} shutting down socket now"); _sock.Shutdown(SocketShutdown.Both); } catch (Exception ex) { String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket"); @@ -577,11 +576,13 @@ namespace bts { #endregion + private string _dipose_impetus = String.Empty; private void _handle_error(ErrorMessage err_msg) { lock (this) { - Log.Gen.info($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}"); + Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}"); if (err_msg.oom) throw err_msg.exception; + _dipose_impetus = $"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}"; Dispose(); } } diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs new file mode 100644 index 000000000..56a271da4 --- /dev/null +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs @@ -0,0 +1,269 @@ +namespace bts { + using BCG; + using BCG.Utilities.General; + + using bts.utils; + + + using System; + using System.IO; + using System.Net.Sockets; + using System.Threading.Tasks; + using System.Threading; + using System.Net; + using System.Text; + using System.ComponentModel; + + // this ITCPConnection uses the TCPClient class instead of a raw socket (in the hope, that it would configure the socket correctly) + + public class AsyncTCPConnection_AsyncWrapper : ITCPConnection, IWorker, IDisposable { + private const int DATA_MSG_SIGNAL = 0; + private const int HB_MSG_SIGNAL = 1; + private TcpClient _tcp_client; + private NetworkStream _nw_stream; + private ITCPMessageHandler _msg_handler; + private bool _is_alive = true; + RemoteConnectionInfo _peer_conn_info = null; + RemoteConnectionInfo _self_conn_info = null; + + public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) { + var len = end_pos - start_pos; + StringBuilder hex_builder = new StringBuilder(len * 2); + int i = 0; + + foreach (byte b in byte_arr) { + if (i >= start_pos && i < end_pos) + hex_builder.Append($"{b:X2} "); + i++; + } + return hex_builder.ToString().Trim(); + } + + public AsyncTCPConnection_AsyncWrapper(TcpClient tcp_client) { + _tcp_client = tcp_client; + _nw_stream = _tcp_client.GetStream(); + _peer_conn_info = get_peer_address(); + _self_conn_info = get_self_address(); + Log.Gen.info("AsyncTCPConnection_AsyncWrapper.cons _peer_conn_info: {_peer_conn_info} _self_conn_info: {_self_conn_info}"); + _detect_connection_status_change(); + } + + private void _detect_connection_status_change() { + Action testing_func = () => { + bool connected = _tcp_client.Connected; + while (true) { + if (connected != _tcp_client.Connected) + Log.Gen.error($"Connection status changed this: {ToString()} {connected} -> {_tcp_client.Connected}"); + connected = _tcp_client.Connected; + System.Threading.Thread.Sleep(50); + if (!connected) + break; + } + }; + ThreadPool.QueueUserWorkItem(_ => testing_func()); + } + + public void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del) { + + } + + public void _send_fixed_size_msg(byte[] buffer) { + string buff_txt = to_hex(buffer, 0, buffer.Length); + Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg buffer: {buff_txt}"); + + int total_bytes_sent = 0; + int n_bytes = buffer.Length; + + int n_transfers = 0; + while (total_bytes_sent < n_bytes) { + int remaining_bytes = n_bytes - total_bytes_sent; + _nw_stream.Write(buffer, total_bytes_sent, remaining_bytes); + total_bytes_sent += remaining_bytes; + n_transfers++; + } + Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg sent {buffer.Length} bytes in {n_transfers} transfers"); + } + + private void _send_one_msg(byte[] msg) { + try { + // the total msg = [msg_sz, msg_type, data] + int sz = msg.Length + 4; + MemoryStream str = new MemoryStream(); + BinaryWriter bw = new BinaryWriter(str); + bw.Write(sz); + bw.Write(DATA_MSG_SIGNAL); + bw.Write(msg); + str.Position = 0; + byte[] buff = str.ToArray(); + _send_fixed_size_msg(buff); + } catch (Exception ex) { + _handle_error(new ErrorMessage(ex, $"_send_one_msg call_stacK: {ex.StackTrace}")); + } + } + + public void send(byte[] data) { + var data_txt = to_hex(data, 0, data.Length); + Log.Gen.info($"AsyncTCPConnection_AsyncWrapper.send data: {data_txt}"); + //Task.Run(() => _send_one_msg(data)); + ThreadPool.QueueUserWorkItem(_ => _send_one_msg(data)); + } + + public void register(ITCPMessageHandler message_handler) { + _msg_handler = message_handler; + // start reading in a Task + ThreadPool.QueueUserWorkItem(_ => _read_msgs_forever()); + } + + public RemoteConnectionInfo get_peer_address() { + //if (_peer_conn_info == null) + return _peer_conn_info = TCPSocketUtils.get_peer_address(_tcp_client.Client); + //return _peer_conn_info; + } + + public RemoteConnectionInfo get_self_address() { + if (_self_conn_info == null) + _self_conn_info = TCPSocketUtils.get_self_address(_tcp_client.Client); + return _self_conn_info; + } + + public bool isDisposed { get { return !_is_alive; } } + + public void doWork() { + // send hb + } + + + // This makes the blocking IO call, the callstack is _read_msgs_forever -> _read_one_message -> _read_fixed_size_msg + private byte[] _read_fixed_size_msg(int n_bytes) { + if (n_bytes <= 0) { + throw new ArgumentException("Number of bytes to read must be greater than zero.", nameof(n_bytes)); + } + byte[] buffer = new byte[n_bytes]; + int total_bytes_read = 0; + int n_transfers = 0; + while (total_bytes_read < n_bytes) { + int bytes_read = _nw_stream.Read(buffer, total_bytes_read, n_bytes - total_bytes_read); + if (bytes_read <= 0) { + throw new IOException("Read 0 bytes, the connection was closed before all data could be read."); + } + n_transfers++; + total_bytes_read += bytes_read; + Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_fixed_size_msg n_bytes: {n_bytes} bytes_read: {bytes_read} total_bytes_read: {total_bytes_read} buff_len: {buffer.Length}"); + } + Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_fixed_size_msg read {n_bytes} bytes in {n_transfers} transfers"); + return buffer; + } + + private class OneRecvMessage { + public int msg_type; + public byte[] data; + public OneRecvMessage(int msg_type_, byte[] data_) { + msg_type = msg_type_; + data = data_; + } + public OneRecvMessage(byte[] msg) { + MemoryStream str = new MemoryStream(msg); + BinaryReader br = new BinaryReader(str); + int msg_type = br.ReadInt32(); + if (msg_type == DATA_MSG_SIGNAL) { + // TODO, polish it, too much memory bneing allocated + data = br.ReadBytes(msg.Length - 4); + } + } + + public override string ToString() { + string rv = $"OneMessage msg_type: {msg_type} data_len: {data.Length}"; + return rv; + } + } + + private OneRecvMessage _read_one_message() { + // read size first + byte[] sz_arr = _read_fixed_size_msg(4); + BinaryReader br = new BinaryReader(new MemoryStream(sz_arr)); + br.BaseStream.Position = 0; + int msg_sz = br.ReadInt32(); + Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg_sz: {msg_sz}"); + if (msg_sz < 0) { + throw new IOException("Read 0 bytes, the connection was closed before all data could be read."); + } + byte[] msg = _read_fixed_size_msg(msg_sz); + Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg sz: {msg.Length}"); + // TODO, read first 4 bytes and decide hb or data + OneRecvMessage rv = new OneRecvMessage(msg); + return rv; + } + + private void _read_msgs_forever() { + while (_is_alive) { + try { + OneRecvMessage one_msg = _read_one_message(); + if (one_msg.msg_type == DATA_MSG_SIGNAL) { + _msg_handler.onData(new RemoteMessage(one_msg.data, RemoteMessageType.Data, _peer_conn_info)); + } else if (one_msg.msg_type == HB_MSG_SIGNAL) { + _msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_conn_info)); + } else { + throw new ApplicationException($"Unknown message type: {one_msg.msg_type}"); + } + } catch (Exception ex) { + _handle_error(new ErrorMessage(ex, $"_read_msgs_forever, call_stack: {ex.StackTrace}")); + } + } + Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_msgs_forever exiting loop"); + } + + public bool is_disposed => !_is_alive; + + public class ErrorMessage { + public readonly SocketError? socket_error = null; + public readonly Exception exception = null; + public readonly bool is_exception = false; + public readonly string context = String.Empty; + public ErrorMessage(SocketError socket_error_, string context_) { + socket_error = socket_error_; + context = context_; + } + public ErrorMessage(Exception exception_, string context_) { + is_exception = true; + exception = exception_; + context = context_; + } + + public bool oom => exception != null && exception is OutOfMemoryException; + + public override string ToString() { + if (is_exception) + return $"ErrorMessage exception: {exception} context: {context}"; + return $"socket_error: {socket_error} context: {context}"; + } + } + + public void Dispose() { + // socket is closed and disposed here + lock (this) { + if (_is_alive) { + //_master_worker.removeWorker(this); + _is_alive = false; + try { + bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} closing tcp_client nin an orderly fashion now"); + _tcp_client.Close(); + } catch (Exception ex) { + String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket"); + if (ex is OutOfMemoryException) + throw ex; + } + _msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _peer_conn_info)); + } + } + } + + private void _handle_error(ErrorMessage err_msg) { + lock (this) { + Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}"); + if (err_msg.oom) + throw err_msg.exception; + Dispose(); + } + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_APM.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_APM.cs index fa3cae9c2..ebbda0262 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_APM.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_APM.cs @@ -31,13 +31,13 @@ namespace bts { } } - public static void connect_to(BlutoSubAddress server_address, ITCPConnectionHandler handler) { + public static void connect_to(RemoteConnectionInfo server_address, ITCPConnectionHandler handler) { bts.Log.Gen.info($"AsyncTCPConnector_APM.connectTo server: {server_address}"); // when connection is established, this sock passed to an ITCPCOnnection and sent back Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); - EndPoint ep = new IPEndPoint(IPAddress.Parse(server_address.ip), server_address.port); - sock.BeginConnect(ep, new AsyncCallback(_on_connect), - new SocketConnectionArgs(ep, sock, handler, server_address.ToString())); + EndPoint ip_ep = new IPEndPoint(IPAddress.Parse(server_address.ip), server_address.port); + sock.BeginConnect(ip_ep, new AsyncCallback(_on_connect), + new SocketConnectionArgs(ip_ep, sock, handler, server_address.ToString())); } private static void _on_connect(IAsyncResult res) { diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_AsyncWrapper.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_AsyncWrapper.cs new file mode 100644 index 000000000..1e79b5a73 --- /dev/null +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_AsyncWrapper.cs @@ -0,0 +1,83 @@ +using System.Net.Sockets; +using System.Net; + +using BCG; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using System.Threading; +using System.Reflection.Metadata.Ecma335; + + +namespace bts { + // Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests + // whereas AsyncTCPConnectionAcceptor_APM is always listening for incoming connections which lends itself to more + // statefullness and hence non static + public static class AsyncTCPConnector_AsyncWrapper { + public static void connect_to(RemoteConnectionInfo server_address, ITCPConnectionHandler handler) { + bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper.connect_to server: {server_address}"); + //Task.Run(() => _connect_to(server_address, handler)); + Action connect_once = () => { _connect_to(server_address, handler); }; + ThreadPool.QueueUserWorkItem(_ => _keep_trying_blocking_function(connect_once, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5))); + } + + // this is the asctual fuction thta calls TCPCLient sync methof and establishes a connection + private static void _connect_to(RemoteConnectionInfo server_rci, ITCPConnectionHandler handler) { + // TODO, random sleep + bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to server: {server_rci}"); + TcpClient tcp_client = new TcpClient(server_rci.ip, server_rci.port); + bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to server new TcpClient returned, connected: {tcp_client.Connected}"); + ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client); + var conn_rci = conn.get_peer_address(); + bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to new connection, server_rci: {server_rci} connection.peer_address: {conn_rci}"); + Log.assert(server_rci.Equals(conn_rci), $"{server_rci}!={conn_rci}"); + bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to new connection, self: {conn.get_self_address()} peer: {conn.get_peer_address()}, calling onConnection now"); + handler.onConnection(conn); + } + + public static void _keep_trying_blocking_function(Action action, TimeSpan timeout, TimeSpan wait_before_retrying) { + Func try_once = () => { + var cts = new CancellationTokenSource(); + var token = cts.Token; + + var task = Task.Run(action, token); + + if (Task.WaitAny(task, Task.Delay(timeout, token)) == 0) { + // Task completed within timeout + task.GetAwaiter().GetResult(); // Ensure any exceptions/cancellation are rethrown + return true; + } else { + // Timeout + cts.Cancel(); + return false; + } + }; + bool connected = false; + while (!connected) { + connected = try_once(); + System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1)); + } + } + + // public static void connect_to(RemoteConnectionInfo server_rci, TimeSpan socket_timeout, TimeSpan retry_interval, ITCPConnectionHandler handler) { + // while (true) { + // try { + // var client = new TcpClient(); + // var result = client.BeginConnect(server_rci.ip, server_rci.port, null, null); + // var success = result.AsyncWaitHandle.WaitOne(socket_timeout, true); + // if (success && client.Connected) { + // client.EndConnect(result); + // handler.onConnection(new AsyncTCPConnection_AsyncWrapper(client)); + // } else { + // throw new SocketException(); + // } + // } catch (Exception ex) { + // Log.Gen.info($"Connection attempt failed: {ex.Message}"); + // Log.Gen.info($"Retrying in {retry_interval.TotalSeconds} seconds..."); + // Thread.Sleep(retry_interval); + // } + // } + // } + + } +} diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs b/src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs new file mode 100644 index 000000000..b0115ec00 --- /dev/null +++ b/src/DataStreamSuite/PubSub/ManagedTCP/ClientToServerManagedTCPConnector.cs @@ -0,0 +1,82 @@ +// using System; +// using System.Collections.Generic; +// using System.Text; +// using System.Net; +// using System.Net.Sockets; +// using System.Threading; +// using System.Threading.Tasks; +// //using BCG.Utilities; +// //using BCG.Utilities.General; + +// namespace BCG { +// internal class SocketConnectionArgs { +// public EndPoint ep; +// public Socket sock; +// public ITCPConnectionHandler handler; +// public string serverURL; + +// internal SocketConnectionArgs(EndPoint theEp, Socket theSock, ITCPConnectionHandler theHandler, +// string theServerURL) { +// ep = theEp; +// sock = theSock; +// handler = theHandler; +// serverURL = theServerURL; +// } +// } + +// public class ClientToServerManagedTCPConnector { +// private static Random myRnd = new Random(DateTime.Now.Millisecond); +// private static TimeSpan myHBTimeSpan = new TimeSpan(0, 0, 0, 0, PubSub3Config.instance.PS3_HeartBeatInterval); +// private static int myWaitTimeBeforeRetryingUpperLimit = 5000, +// myWaitTimeBeforeRetryingLowerLimit = 1000; + +// public static void connectTo(BlutoSubAddress serverAddress, ITCPConnectionHandler handler) { +// Console.WriteLine($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}"); +// Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); +// EndPoint ep = new IPEndPoint(IPAddress.Parse(serverAddress.ip), serverAddress.port); +// sock.BeginConnect(ep, new AsyncCallback(myOnConnect), new SocketConnectionArgs(ep, sock, handler, serverAddress.ToString())); +// } + +// public static async Task connect_to_async(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) { +// bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}"); +// await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts); +// } + +// public static void connect_to(BlutoSubAddress serverAddress, ITCPConnectionHandler handler, TimeSpan wait_before_retrying, int max_n_attempts = 1024) { +// bts.Log.Gen.info($"ClientToServerManagedTCPConnector.connectTo server: {serverAddress}"); +// Task.Run(async () => { +// await TCPSocketUtils.connect_to_server_async(serverAddress.ip, serverAddress.port, handler, wait_before_retrying, max_n_attempts); +// }); +// } + +// private static void myOnConnect(IAsyncResult res) { +// Console.WriteLine("ClientToServerManagedTCPConnector.myOnConnect " + res.ToString() + " " + res.AsyncState.ToString()); +// SocketConnectionArgs args = res.AsyncState as SocketConnectionArgs; +// try { +// args.sock.EndConnect(res); +// } catch (SocketException sex) { +// string errMsg = String.Format("{0}: Connection to {1} failed.", DateTime.Now.ToString(), args.serverURL.ToString()); +// //BlueEventLog.logEvent(sex.Message + ";" + errMsg, PS3LogLevels.disconnection); +// //Console.WriteLine(errMsg); +// if (args.handler.isDisposed) { +// string doneMsg = String.Format("{0}: Connection to {1} was disposed. Not Retrying.", DateTime.Now.ToString(), args.serverURL.ToString()); +// //Console.WriteLine(doneMsg); +// // BlueEventLog.logEvent(doneMsg, PS3LogLevels.disconnection); +// return; +// } else { +// string retryMsg = String.Format("{0}: Retrying Connection to {1}.", DateTime.Now.ToString(), args.serverURL.ToString()); +// //Console.WriteLine(retryMsg); +// //BlueEventLog.logEvent(retryMsg, PS3LogLevels.disconnection); + +// int waitTime = myRnd.Next(myWaitTimeBeforeRetryingLowerLimit, myWaitTimeBeforeRetryingUpperLimit); +// Thread.Sleep(waitTime); +// args.sock.BeginConnect(args.ep, new AsyncCallback(myOnConnect), args); +// return; +// } +// } +// //BlueEventLog.logEvent(String.Format("Successful connection made to {0}.", args.serverURL.ToString()), PS3LogLevels.connection); +// args.handler.onConnection(new bts.AsyncTCPConnection_APM(args.sock, myHBTimeSpan, ITCPConnection.ConnectionSide.Client)); +// } + +// } +// } diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnection.cs b/src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnection.cs index bb6fe5f99..8a1511ecc 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnection.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/ITCPConnection.cs @@ -41,7 +41,7 @@ namespace BCG { /// /// /// - void getPeerAddress(out string ip, out int port); + RemoteConnectionInfo get_peer_address(); /// /// Gets its own ip and port, as seen by the peer. So if this @@ -51,7 +51,7 @@ namespace BCG { /// /// /// - void getSelfAddress(out string ip, out int port); + RemoteConnectionInfo get_self_address(); /// /// if the connection is disconnected, it disposes itself, /// hence this is an indicator of whether the connection is diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/RemoteConnectionInfo.cs b/src/DataStreamSuite/PubSub/ManagedTCP/RemoteConnectionInfo.cs index 91aff6fe8..4e3799100 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/RemoteConnectionInfo.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/RemoteConnectionInfo.cs @@ -3,37 +3,66 @@ using System.Collections.Generic; using System.Text; using System.Net; using bts.utils; +using System.Net.NetworkInformation; +using bts; namespace BCG { public class RemoteConnectionInfo { - private string myIp; - private int myPort; - private IPEndPoint myEndPoint; + private string _ip; + private int _port; + //private IPEndPoint _ip_endpoint; + public static string ipv4_compat_prefix = "::ffff:"; + public static RemoteConnectionInfo null_instance = new RemoteConnectionInfo(String.Empty, 0); + public static string prefix = "bts-cloud://"; + //IP addresses that start with ::ffff: are IPv6-mapped IPv4 addresses. This format is used to represent an IPv4 address within an IPv6 address space. + + public static string clean_ip(string ip) { + string rv = ip; + if (!String.IsNullOrEmpty(ip)) { + if (ip.StartsWith(ipv4_compat_prefix)) { + rv = ip.Substring(ipv4_compat_prefix.Length); + } + } + return rv; + } public RemoteConnectionInfo(string ip, int port) { - myIp = ip; - myPort = port; + _ip = clean_ip(ip); + _port = port; + Log.Gen.info($"RemoteConnectionInfo constructed ip_arg: {ip} _ip: {_ip} port: {_port}"); } - public RemoteConnectionInfo(string ip, short port, IPEndPoint endPoint) - : this(ip, port) { - myEndPoint = endPoint; + public static RemoteConnectionInfo parse_url(string url) { + string[] toks = url.Split("//", StringSplitOptions.RemoveEmptyEntries); + var prefix = toks[0]; + var ip_port = toks[1]; + var ip_port_toks = toks[1].Split(':'); + string ip = ip_port_toks[0]; + int port = Int32.Parse(ip_port_toks[1]); + return new RemoteConnectionInfo(ip, port); } + // public RemoteConnectionInfo(string ip, int port, IPEndPoint ip_endpoint) + // : this(ip, port) { + // _ip_endpoint = ip_endpoint; + // } + public string ip { - get { return myIp; } + get { return _ip; } } public int port { - get { return myPort; } + get { return _port; } } - public IPEndPoint endPoint { - get { return myEndPoint; } - } + // public IPEndPoint end_point { + // // new IPEndPoint(IPAddress.Parse(server_address.ip), server_address.port); + // get { return _ip_endpoint; } + // } public override string ToString() { - return ip + ":" + port; + //return ip + ":" + port; + return $"{prefix}{ip}:{port}"; } public override int GetHashCode() { diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/ServerToClientManagedTCPConnectionAcceptor.cs b/src/DataStreamSuite/PubSub/ManagedTCP/ServerToClientManagedTCPConnectionAcceptor.cs index 19c02ff35..879ad298a 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/ServerToClientManagedTCPConnectionAcceptor.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/ServerToClientManagedTCPConnectionAcceptor.cs @@ -7,6 +7,7 @@ using System.Net; using BCG; namespace bts { + // TODO: replace it with AsyncTCPConnectionAcceptor_APM public class ServerToClientManagedTCPConnectionAcceptor { private readonly int _port; @@ -53,12 +54,10 @@ namespace bts { ITCPConnection new_conn = new bts.AsyncTCPConnection_APM(new_sock, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Server); _handler.onConnection(new_conn); _accept_socket.BeginAccept(new AsyncCallback(_on_accept), new object[] { }); - string ip; int port; - new_conn.getPeerAddress(out ip, out port); - Log.Gen.info( - $"ServerToClientManagedTCPConnectionAcceptor._on_accept connection accepted successfully from ip: {ip} port: {port} (local_port: {_port})"); + var peer_rci = new_conn.get_peer_address(); + Log.Gen.info($"ServerToClientManagedTCPConnectionAcceptor._on_accept connection accepted successfully from rci: {peer_rci}"); } - - } + + } diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/SingleServerToClientManagedTCPConnectionAcceptor.cs b/src/DataStreamSuite/PubSub/ManagedTCP/SingleServerToClientManagedTCPConnectionAcceptor.cs index 7434688be..255937e3a 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/SingleServerToClientManagedTCPConnectionAcceptor.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/SingleServerToClientManagedTCPConnectionAcceptor.cs @@ -1,104 +1,104 @@ -using System; -using System.Collections.Generic; -using System.Text; -using System.Threading; -using System.Net; -using System.Net.Sockets; -using System.Threading.Tasks; -using bts; -using Bts.PubSub; - -//using BCG.Utilities; - -namespace BCG { - public class SingleServerToClientManagedTCPConnectionAcceptor { - private TimeSpan myHBTimeSpan, - myWaitTimeBeforeRetrying = new TimeSpan(0, 0, 0, 0, 250); - private ITCPConnectionHandler myHandler; - private Socket myAcceptSock; - private int myPort; - - - public int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan) { - myHBTimeSpan = clientHBTimeSpan; - myHandler = handler; - myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); - int port = 0; - bool madeItThrough = false; - // BlueEventLog.logEvent("Looking for open port to run the server on.", PS3LogLevels.connection); - Random randomizer = new Random(); - PubSubPortRange theRange = Log.notnull(PubSubConfig.Instance.PortRange); - - - //if the range of ports is 1000, then trying 5,000 times means - //your odds of missing an open port are less than 1% - - int maxAttempts = (theRange.maxPort - theRange.minPort) * 5; - int totalAttempts = 0; - while (!madeItThrough) { - port = randomizer.Next(); - port = theRange.minPort + (port % (theRange.maxPort - theRange.minPort)); - - try { - myAcceptSock.Bind(new IPEndPoint(IPAddress.Any, port)); - madeItThrough = true; - } catch (Exception) { - totalAttempts++; - if (totalAttempts < maxAttempts) { - //BlueEventLog.logEvent("Couldn't bind to port " + port.ToString() + "; retrying...", PS3LogLevels.connection); - } else { - // BlueEventLog2.instance().abort("Couldn't find an open port to advertise the server on. Consider modifying PS3_PortRange. Talk to Mark for details."); - } - } - } - myPort = port; - myAcceptSock.Listen(250); - myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); - - return myPort; - } - - public int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) { - myHBTimeSpan = clientHBTimeSpan; - myHandler = handler; - myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); - IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, port); - myAcceptSock.Bind(endpoint); - myAcceptSock.Listen(250); - myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); - myPort = endpoint.Port; - return myPort; // not sure why we need this class variable myPort, but I am just keeping it up with the existing code - } - - public int accept_on_port(ITCPConnectionHandler handler, string ip, int port, TimeSpan client_HB_timespan) { - Task.Run(async () => { - await TCPSocketUtils.accept_connections_async(ip, port, handler, TimeSpan.FromSeconds(1)); - }); - return port; - } - - private void myOnAccept(IAsyncResult res) { - Socket newSock; - try { - newSock = myAcceptSock.EndAccept(res); - } catch (Exception ex) { - // BlueEventLog.logEvent(ex); - Thread.Sleep(myWaitTimeBeforeRetrying); - myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); - if (ex is OutOfMemoryException) - throw ex; - return; - } - newSock.LingerState = new LingerOption(false, 0); - newSock.Blocking = false; - ITCPConnection newConn = new bts.AsyncTCPConnection_APM(newSock, myHBTimeSpan, ITCPConnection.ConnectionSide.Server); - myHandler.onConnection(newConn); - myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); - string ip; int port; - newConn.getPeerAddress(out ip, out port); - string connMsg = String.Format("Connection to port {2} made from {0}:{1}", ip, port, myPort); - // BlueEventLog.logEvent(connMsg, PS3LogLevels.connection); - //Console.WriteLine(connMsg); - } - } -} +// using System; +// using System.Collections.Generic; +// using System.Text; +// using System.Threading; +// using System.Net; +// using System.Net.Sockets; +// using System.Threading.Tasks; +// using bts; +// using Bts.PubSub; + +// //using BCG.Utilities; + +// namespace BCG { +// public class SingleServerToClientManagedTCPConnectionAcceptor { +// private TimeSpan myHBTimeSpan, +// myWaitTimeBeforeRetrying = new TimeSpan(0, 0, 0, 0, 250); +// private ITCPConnectionHandler myHandler; +// private Socket myAcceptSock; +// private int myPort; + + +// public int activateWithRandomPort(ITCPConnectionHandler handler, TimeSpan clientHBTimeSpan) { +// myHBTimeSpan = clientHBTimeSpan; +// myHandler = handler; +// myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); +// int port = 0; +// bool madeItThrough = false; +// // BlueEventLog.logEvent("Looking for open port to run the server on.", PS3LogLevels.connection); +// Random randomizer = new Random(); +// PubSubPortRange theRange = Log.notnull(PubSubConfig.Instance.PortRange); + + +// //if the range of ports is 1000, then trying 5,000 times means +// //your odds of missing an open port are less than 1% + +// int maxAttempts = (theRange.maxPort - theRange.minPort) * 5; +// int totalAttempts = 0; +// while (!madeItThrough) { +// port = randomizer.Next(); +// port = theRange.minPort + (port % (theRange.maxPort - theRange.minPort)); + +// try { +// myAcceptSock.Bind(new IPEndPoint(IPAddress.Any, port)); +// madeItThrough = true; +// } catch (Exception) { +// totalAttempts++; +// if (totalAttempts < maxAttempts) { +// //BlueEventLog.logEvent("Couldn't bind to port " + port.ToString() + "; retrying...", PS3LogLevels.connection); +// } else { +// // BlueEventLog2.instance().abort("Couldn't find an open port to advertise the server on. Consider modifying PS3_PortRange. Talk to Mark for details."); +// } +// } +// } +// myPort = port; +// myAcceptSock.Listen(250); +// myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); + +// return myPort; +// } + +// public int activate(ITCPConnectionHandler handler, int port, TimeSpan clientHBTimeSpan) { +// myHBTimeSpan = clientHBTimeSpan; +// myHandler = handler; +// myAcceptSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); +// IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, port); +// myAcceptSock.Bind(endpoint); +// myAcceptSock.Listen(250); +// myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); +// myPort = endpoint.Port; +// return myPort; // not sure why we need this class variable myPort, but I am just keeping it up with the existing code +// } + +// public int accept_on_port(ITCPConnectionHandler handler, string ip, int port, TimeSpan client_HB_timespan) { +// Task.Run(async () => { +// await TCPSocketUtils.accept_connections_async(ip, port, handler, TimeSpan.FromSeconds(1)); +// }); +// return port; +// } + +// private void myOnAccept(IAsyncResult res) { +// Socket newSock; +// try { +// newSock = myAcceptSock.EndAccept(res); +// } catch (Exception ex) { +// // BlueEventLog.logEvent(ex); +// Thread.Sleep(myWaitTimeBeforeRetrying); +// myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); +// if (ex is OutOfMemoryException) +// throw ex; +// return; +// } +// newSock.LingerState = new LingerOption(false, 0); +// newSock.Blocking = false; +// ITCPConnection newConn = new bts.AsyncTCPConnection_APM(newSock, myHBTimeSpan, ITCPConnection.ConnectionSide.Server); +// myHandler.onConnection(newConn); +// myAcceptSock.BeginAccept(new AsyncCallback(myOnAccept), new object[] { }); +// string ip; int port; +// newConn.get_peer_address(out ip, out port); +// string connMsg = String.Format("Connection to port {2} made from {0}:{1}", ip, port, myPort); +// // BlueEventLog.logEvent(connMsg, PS3LogLevels.connection); +// //Console.WriteLine(connMsg); +// } +// } +// } diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/TCPSocketUtils.cs b/src/DataStreamSuite/PubSub/ManagedTCP/TCPSocketUtils.cs index ab1673b59..15a58dfc4 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/TCPSocketUtils.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/TCPSocketUtils.cs @@ -6,14 +6,9 @@ using System.Net.Sockets; using System.Runtime.CompilerServices; using System.Text; using System.Threading.Tasks; -using BCG; -using System.Xml.Serialization; -using System.Reflection; -using BCG.Utilities.General; -using System.Net.WebSockets; using bts; using System.Threading; -using Google.Protobuf.WellKnownTypes; +using System.Runtime.InteropServices; namespace BCG { // utility functions for TCP socket async communications. Does the following @@ -22,6 +17,106 @@ namespace BCG { // send data // receive data public static class TCPSocketUtils { + + struct TcpKeepAlive { + public uint OnOff; + public uint KeepAliveTime; + public uint KeepAliveInterval; + } + + public static void set_server_socket_options(ref Socket server_socket) { + + // Enable KeepAlive with detailed settings + TcpKeepAlive keepAliveSettings = new TcpKeepAlive { + OnOff = 1, // Enable KeepAlive + KeepAliveTime = 60000, // 1 minute in milliseconds + KeepAliveInterval = 10000 // 10 seconds in milliseconds + }; + + server_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true); + + // Set keep-alive interval and time (Linux-specific method) + if (OperatingSystem.IsLinux()) { + ; + const int keepAliveTime = 60; // Seconds + const int keepAliveInterval = 10; // Seconds + server_socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, keepAliveTime); + server_socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveInterval, keepAliveInterval); + } else { + int size = Marshal.SizeOf(keepAliveSettings); + IntPtr ptr = Marshal.AllocHGlobal(size); + Marshal.StructureToPtr(keepAliveSettings, ptr, true); + byte[] inOptionValues = new byte[size]; + Marshal.Copy(ptr, inOptionValues, 0, size); + Marshal.FreeHGlobal(ptr); + + server_socket.IOControl(IOControlCode.KeepAliveValues, inOptionValues, null); + } + + // Set other socket options as needed + server_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); + server_socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.NoDelay, true); + server_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReceiveTimeout, 10000); + server_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.SendTimeout, 10000); + server_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReceiveBuffer, 8192); + server_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.SendBuffer, 8192); + + // Optionally configure linger option + LingerOption lingerOption = new LingerOption(true, 10); + server_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.Linger, lingerOption); + } + + public static RemoteConnectionInfo get_peer_address(Socket sock) { + IPEndPoint ip_ep = sock.RemoteEndPoint as IPEndPoint; + int port = ip_ep.Port; + string ip = ip_ep.Address.ToString(); + return new RemoteConnectionInfo(ip, port); + } + + public static RemoteConnectionInfo get_self_address(Socket socket) { + IPEndPoint ip_ep = socket.LocalEndPoint as IPEndPoint; + int port = ip_ep.Port; + string ip = ip_ep.Address.ToString(); + return new RemoteConnectionInfo(ip, port); + } + + + public static void set_client_socket_options(ref Socket client_socket) { + TcpKeepAlive keep_alive_settings = new TcpKeepAlive { + OnOff = 1, // Enable KeepAlive + KeepAliveTime = 60000, // 1 minute in milliseconds + KeepAliveInterval = 10000 // 10 seconds in milliseconds + }; + + if (OperatingSystem.IsLinux()) { + const int keepAliveTime = 60; // Seconds + const int keepAliveInterval = 10; // Seconds + client_socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, keepAliveTime); + client_socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveInterval, keepAliveInterval); + } else { + int size = Marshal.SizeOf(keep_alive_settings); + IntPtr ptr = Marshal.AllocHGlobal(size); + Marshal.StructureToPtr(keep_alive_settings, ptr, true); + byte[] inOptionValues = new byte[size]; + Marshal.Copy(ptr, inOptionValues, 0, size); + Marshal.FreeHGlobal(ptr); + + client_socket.IOControl(IOControlCode.KeepAliveValues, inOptionValues, null); + } + + // Set other socket options as needed + client_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); + client_socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.NoDelay, true); + client_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReceiveTimeout, 10000); + client_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.SendTimeout, 10000); + client_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReceiveBuffer, 8192); + client_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.SendBuffer, 8192); + + // Optionally configure linger option + LingerOption lingerOption = new LingerOption(true, 10); + client_socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.Linger, lingerOption); + } + #region socket opening and connecting // keeps accepting connections forever public static async Task accept_connections_async(string ip, int port, ITCPConnectionHandler handler, diff --git a/src/DataStreamSuite/PubSub/PublishedCache.cs b/src/DataStreamSuite/PubSub/PublishedCache.cs index 034a1fb9f..6cb392194 100644 --- a/src/DataStreamSuite/PubSub/PublishedCache.cs +++ b/src/DataStreamSuite/PubSub/PublishedCache.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; namespace bts { // This caches all the published data. This class is created and held by - // PublisherCore. Whend ata is publsihed , PublisherCore passes it to this + // PublisherCore. When data is publsihed , PublisherCore passes it to this // class via the publish call. On a reconnection to the PubSubServer, // PublisherCore calls get_content to get the data to re-publish, to // re-seed the PubSubServer @@ -14,6 +14,7 @@ namespace bts { public void publish(PublishMsg msg) { + Log.Gen.info($"PublishedCache.publish msg: {msg}"); SingleTagData tag_data = null; if (!_data_map.TryGetValue(msg.tag, out tag_data)) { tag_data = new SingleTagData(msg.tag, msg.tag_type); @@ -22,14 +23,15 @@ namespace bts { tag_data.handle_publish(msg); } - public byte[] get_content(string tag, out string tag_type) { + public Tuple get_content(string tag) { SingleTagData tag_data; + byte[] rv = new byte[] { }; if (_data_map.TryGetValue(tag, out tag_data)) { - tag_type = tag_data.tag_type; - return tag_data.get_current_content(); + string tag_type = tag_data.tag_type; + byte[] data = tag_data.get_current_content(); + return new Tuple(tag_type, data); } - tag_type = String.Empty; - return new byte[] { }; + return new Tuple(String.Empty, new byte[] { }); } public void clear(string tag) { diff --git a/src/DataStreamSuite/PubSub/PublisherCore.cs b/src/DataStreamSuite/PubSub/PublisherCore.cs index a10daeedf..a0fe85066 100644 --- a/src/DataStreamSuite/PubSub/PublisherCore.cs +++ b/src/DataStreamSuite/PubSub/PublisherCore.cs @@ -14,26 +14,24 @@ namespace BCG { // to be used when communicating to a pubsubserver protected PubSubServerConnectionFactory _connection_factory = new PubSubServerConnectionFactory(false); - // internal queue used to serialize pubsub method calls + // internal queue used to serialize server calls. All calls to PublisherCore are written to this quwue, + // dequeued by the message pump thread protected SyncQueue _main_queue = new SyncQueue(); // all the data published through this instance of PublisherCore - // is ketp there, in a form that can be used repopulate a pubsub server - // in the event of a crash + // is kept there, in a form that can be used repopulate a pubsub server + // in the event of a conection re-establishment to the PubSubServer protected PublishedCache _state = new PublishedCache(); - // a different IPubSubServerConnection object is used to interact - // with each of the pubsub servers used by this publishercore + // every PubSubServer is communicated with a adifferent IPubSubServerConnection object. // a dictionary for fast retrieval of publishers // tag -> IPubSubServerConnection protected IDictionary _tag_sender_map = new Dictionary(); - // url -> IPubSubServerConnection - private HashSet _connections = new HashSet(); - - // data waiting for transaction to end or abort - // tag->TagWaitingInTransaction + // pubsub server urls vs IPubSubServerConnection + private Dictionary _connection_map = + new Dictionary(); // router object, used to select which pubsub should be used for each tag protected IPubSubRouter _router = null; @@ -90,6 +88,7 @@ namespace BCG { public void publish(PubSubClientMessage msg) { Log.assert(msg.msg_type == PubSubMessageType.PUBLISH); + Log.assert(!String.IsNullOrEmpty(msg.pub_msg.tag_type)); bts.Log.Gen.info($"PublisherCore.publish msg: {msg}"); _main_queue.enqueue(msg); } @@ -122,6 +121,7 @@ namespace BCG { try { while (true) { PubSubClientMessage? msg = _main_queue.dequeue(100); + // only msg tyes expected right now (6/21/24) are PUBLISh type if (msg != null) { bts.Log.Gen.info($"PublisherCore._message_pump msg: {msg}"); switch (msg.msg_type) { @@ -138,8 +138,8 @@ namespace BCG { break; } } - - foreach (var conn in _connections) { + // all interactions with conn are called from the same thread + foreach (var conn in _connection_map.Values) { conn.doWork(); } } @@ -156,102 +156,14 @@ namespace BCG { } private void _commit_transaction() { - _end_transaction(true); + //_end_transaction(true); } private void _abort_transaction() { - _end_transaction(false); + //_end_transaction(false); } - - //based on success, sends a TA or TE, and sets myInTransaction to false - //TODO: if a transaction is sending to multiple pubsubs (in order to - //keep it consistent with the way it was done earlier), this should - //simply publish it non transactionally (easy way out), or - //transactionally with respect to each pubsub - protected virtual void _end_transaction(bool success) { - // lock (this) { - // // BlueEventLog.logAssert(inTransaction); - // //BlueEventLog.logEvent("PublisherCore Finishing Transaction.", BlueEventLevel.Information); - // _is_in_transaction = false; - // if (success) { - // // pubsub server -> tags with updates - // Dictionary> routesToTags = new Dictionary>(); - - // // group updates per pubsub server - // foreach (string tag in myWaitingInTransactionMessages.Keys) { - // string thisURL = _router.get_pub_url_for_tag(tag); - // if (!routesToTags.ContainsKey(thisURL)) - // routesToTags[thisURL] = new List(); - // routesToTags[thisURL].Add(tag); - // } - - // // more than one server means we won't be able to honor the transactionality request - // // log a warning if that is the case. - // if (routesToTags.Keys.Count > 1) { - // //this is going to be a long error message, but it's an importat one to be able to debug. - // StringBuilder errorMessageBuilder = new StringBuilder(); - // errorMessageBuilder.Append("Tags within this transaction are routed to different pubSub servers.\n"); - // errorMessageBuilder.Append("This will break the transactionality of clients receiving the data. Details:\n"); - - // foreach (string route in routesToTags.Keys) { - // foreach (string tag in routesToTags[route]) { - // errorMessageBuilder.Append("\tTag "); - // errorMessageBuilder.Append(tag); - // errorMessageBuilder.Append(" is routed to "); - // errorMessageBuilder.Append(route); - // errorMessageBuilder.Append(".\n"); - // } - - // } - // string errMsg = errorMessageBuilder.ToString(); - // // BlueEventLog.logEvent(errMsg, BlueEventLevel.Diagnosis); - // } - - // // iterate through each server with pending messages - // foreach (KeyValuePair> kvp in routesToTags) { - // IPubSubServerConnection transactionSender = null; - // string publishingApp = ClientInfo.thisClient.processString; - // string connection = kvp.Key; - // Stream transactionData = new MemoryStream(); - - // // iterate through each of the pending tag updates for one server - // foreach (string tag in kvp.Value) { - // ///it would be the same transaction sender that would - // ///return each time - // transactionSender = myGetOrBuildSender(tag); - - - // TagWaitingInTransaction waiting = myWaitingInTransactionMessages[tag]; - // // iterate through each of the updates for a specific tag, specific server - // foreach (byte[] data in waiting.data) - // // update internal state - // _state.publish(waiting.tag, waiting.tagType, data); - // // add a pubsub header and copy each message to the transactionData stream - // waiting.write_to(transactionData, connection, publishingApp); - // } - // // now transactionData holds all the updates for all the tags - // // that are to be sent to the PubSub represented by transactionSender - - // // add a transaction header, indicating how many tags in the transaction - // TransactionMetaData tMd = new TransactionMetaData(kvp.Value.Count); - - // // rewind transaction data to the beggining - // transactionData.Position = 0; - - // // send the entire thing to the specific pubsub server. - // // this is where the data is actually sent out ! - // var ps_msg = new PubSubMessage(new MetaData(tMd), transactionData); - // Console.WriteLine($"serverConn.send metadata: {ps_msg.meta_data}"); - // bts.Log.Gen.info($"PublisherCore.send ps_msg: {ps_msg}"); - // transactionSender.send(ps_msg); - // } - // } - // myWaitingInTransactionMessages.Clear(); - // } - } - protected virtual void _publish(PublishMsg msg) { bts.Log.Gen.info($"PublisherCore._publish msg: {msg}"); // update the state @@ -277,11 +189,13 @@ namespace BCG { protected IPubSubServerConnection _get_or_build_sender(string tag) { IPubSubServerConnection tag_conn = null; if (!_tag_sender_map.TryGetValue(tag, out tag_conn)) { - string route = _router.get_pub_url_for_tag(tag); - tag_conn = _connection_factory.getConnection(route); - if (!_connections.Contains(tag_conn)) { - tag_conn.register_connection_callback(new PubSubServerConnectionStatusChange(_handle_server_reboot)); - _connections.Add(tag_conn); + RemoteConnectionInfo server_rci = _router.get_pub_route_for_tag(tag); + if (!_connection_map.TryGetValue(server_rci, out tag_conn)) { + tag_conn = _connection_factory.get_connection(server_rci); + tag_conn.register_connection_callback((server_url, connected) => { + _handle_pubsubserver_reconnection(server_url, connected); + }); + _connection_map.Add(server_rci, tag_conn); } _tag_sender_map.Add(tag, tag_conn); } @@ -290,18 +204,19 @@ namespace BCG { ///in case of the server coming up, it publishes all the stored state ///for that server, back to it - protected void _handle_server_reboot(string server_url, bool connection) { - bts.Log.Gen.info($"PublisherCore._handle_server_reboot server_url: {server_url} connection: {connection}"); + private void _handle_pubsubserver_reconnection(string server_url, bool connection) { + bts.Log.Gen.info($"PublisherCore._handle_pubsubserver_reconnection server_url: {server_url} connection: {connection}"); if (connection) { // find all tags that are being published to this server foreach (var kv in _tag_sender_map) { var tag = kv.Key; var conn = kv.Value; if (conn.server_url == server_url) { - string tag_type; - var content_bytes = _state.get_content(tag, out tag_type); + var content = _state.get_content(tag); + byte[] content_bytes = content.Item2; + string tag_type = content.Item1; string content_bytes_log = content_bytes == null ? "NULL" : content_bytes.Length.ToString(); - Log.Gen.info($"PublisherCore._handle_server_reboot content_bytes: {content_bytes_log}"); + Log.Gen.info($"PublisherCore._handle_pubsubserver_reconnection server_url: {server_url} content_bytes: {content_bytes_log}"); bts.PublishMsg publish_msg = new bts.PublishMsg() { connection = kv.Key, publishing_app = ClientInfo.thisClient.processString, diff --git a/src/DataStreamSuite/PubSub/Server/Communication/IPubsubMessageHub.cs b/src/DataStreamSuite/PubSub/Server/Communication/IPubsubMessageHub.cs index ec152f9b9..b843e3460 100644 --- a/src/DataStreamSuite/PubSub/Server/Communication/IPubsubMessageHub.cs +++ b/src/DataStreamSuite/PubSub/Server/Communication/IPubsubMessageHub.cs @@ -15,11 +15,6 @@ namespace BCG { /// this event is prompted by the doWork. /// public interface IPubsubMessageHub : IWorker { - - /// - /// The port that the message hub is listening on. Return -1 if you're not listening on a port, i.e. for hte case of multicast - /// - int port { get; } /// /// register or unregister callback for data. The data can either be received on this callback /// on an arbitrary thread, or polled via getUpdates diff --git a/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs b/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs index 76f3f65a2..aab7c0549 100644 --- a/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs +++ b/src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs @@ -21,11 +21,7 @@ namespace BCG { ITCPMessageHandler { // interface to receive data from ITCPConnection private int _port; private bool _is_disposed = false; - public int port { - get { return _port; } - //property is read only - } private class ConnectionStatusInfo { public readonly string client_url; public readonly PubSubCallbacks.ConnectionStatus status; @@ -77,12 +73,12 @@ namespace BCG { private TimeSpan _collect_disposed_connections_period = new TimeSpan(0, 0, 0, 0, 1000); private DateTime _last_HB_collection_time = DateTime.Now; private DateTime _last_disposed_dead_collection_time = DateTime.Now; - private ServerToClientManagedTCPConnectionAcceptor _connection_acceptor; + private AsyncTCPConnectionAcceptor_AsyncWrapper _connection_acceptor; // TODO, get public ManagedTCPMessageHub(int port, TimeSpan hb_send_interval, int missing_HBs_for_gc, TimeSpan collect_disposed_connections_period) { _port = port; - _connection_acceptor = new ServerToClientManagedTCPConnectionAcceptor(port, this); + _connection_acceptor = new AsyncTCPConnectionAcceptor_AsyncWrapper(_port, this); _hb_send_interval = hb_send_interval; _subscriber_drop_threshold = new TimeSpan(0, 0, 0, 0, Convert.ToInt32(hb_send_interval.TotalMilliseconds * missing_HBs_for_gc)); _collect_disposed_connections_period = collect_disposed_connections_period; @@ -104,7 +100,7 @@ namespace BCG { } public void activate() { - _connection_acceptor.activate(); + _connection_acceptor.start(); bts.Log.Gen.info($"ManagedTCPMessageHub.activate TCP Component is ready, listening on port {_port} for incoming connections"); _activated = true; Thread do_it_all_thread = new Thread(new ThreadStart(_main_message_pump)); @@ -162,11 +158,8 @@ namespace BCG { /// /// public void onConnection(ITCPConnection connection) { - // this is where we get new connections, dead connections are collected in _collect_disposed_connections - string ip; - int port; - connection.getPeerAddress(out ip, out port); - bts.Log.Gen.info($"ManagedTCPMessageHub.onConnection ip: {ip} port: {port}"); + var client_rci = connection.get_peer_address(); + bts.Log.Gen.info($"ManagedTCPMessageHub.onConnection client_rci: {client_rci} connection_type: {connection.GetType()}"); _new_connections.enqueue(connection); } @@ -176,14 +169,14 @@ namespace BCG { // called by ITCPConnection on an arbitrary thread public void onData(RemoteMessage message) { bts.Log.Gen.info($"ManagedTCPMessgageHub.onData recv msg: {message}"); - string sender = new BlutoSubAddress(message.remoteConnectionInfo.ip, message.remoteConnectionInfo.port).ToString(); + var sender_rci = message.remoteConnectionInfo; switch (message.messageType) { case RemoteMessageType.Data: // if callback is registered fire it right away, else enqueue it if (_data_cb != null) - _data_cb(sender, message.message); + _data_cb(sender_rci.ToString(), message.message); else - _received_data_queue.enqueue(new Pair(sender, message.message)); + _received_data_queue.enqueue(new Pair(sender_rci.ToString(), message.message)); break; case RemoteMessageType.Connection: // wont happen @@ -264,17 +257,14 @@ namespace BCG { private void _handle_new_connections() { var new_conns = _new_connections.dequeueAll(); foreach (ITCPConnection conn in new_conns) { - string ip; - int port; - conn.getPeerAddress(out ip, out port); - string client_url = new BlutoSubAddress(ip, port).ToString(); - bts.Log.Gen.info($"ManagedTCPMessageHub._handle_new_connections client_url: {client_url}"); + var peer_rci = conn.get_peer_address(); + bts.Log.Gen.info($"ManagedTCPMessageHub._handle_new_connections peer_rci: {peer_rci}"); ///to stop it from being garbabge collected immediatly - _received_hbs.enqueue(client_url); + _received_hbs.enqueue(peer_rci.ToString()); conn.register(this); - _connection_map.Add(client_url, conn); + _connection_map.Add(peer_rci.ToString(), conn); // enqueued here to call the user sub in doWork - _conn_callback_queue.enqueue(new ConnectionStatusInfo(client_url, PubSubCallbacks.ConnectionStatus.CONNECTED)); + _conn_callback_queue.enqueue(new ConnectionStatusInfo(peer_rci.ToString(), PubSubCallbacks.ConnectionStatus.CONNECTED)); } } diff --git a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs index 40766243f..a22001349 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs @@ -6,6 +6,8 @@ using bts; using Bts.PubSub; using Spark.Core; +using System.Text; +using System.Diagnostics; namespace BCG { // This is the main PubSub Server class where it all starts (called by Program.cs). Its constructed and start is called and thats it. @@ -27,6 +29,7 @@ namespace BCG { private delegate void SingleTagTimeSliceDelegate(string tag, double processingTimeSeconds); private SingleTagTimeSliceDelegate onSingleTagPub; private static SyncQueue _internal_pub_queue = new SyncQueue(); + private Stopwatch _stop_watch = new Stopwatch(); //this thing keeps track of whether a given tag is compressed or not. It is used for logging messages. /* This delegate allows pubsub components to be notified when the identity ofe the pubsub is known*/ @@ -79,10 +82,13 @@ namespace BCG { /// hanlder /// private void _polling_thread_func() { + Log.Gen.info("PubSubMessageReactor._polling_thread_func starting thread function _polling_thread_func"); TimeSpan poll_timeout = new TimeSpan(0, 0, 0, 0, 2500); _msg_hub.activate(); Action handle_internal_pubs = () => { - var internal_pubs = _internal_pub_queue.dequeueMultipleElements(1, 64000); + Log.Gen.info($"PubSubMessageReactor._polling_thread_func handle_internal_pubs"); + var internal_pubs = _internal_pub_queue.dequeueMultipleElements(TimeSpan.FromSeconds(1), 64000); + //Log.Gen.info($"PubSubMessageReactor._polling_thread_func internal_pubs count: {internal_pubs.Count}"); foreach (var msg in internal_pubs) { try { _handle_pub("Internal", msg, true); @@ -94,11 +100,10 @@ namespace BCG { while (true) { try { + //bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func pre get_updates"); var received_data = _msg_hub.get_updates(poll_timeout, 1); var n_msgs = received_data.Count; - if (n_msgs > 0) { - bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func n_msgs: {n_msgs}"); - } + bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func post get_updates n_msgs: {n_msgs}"); foreach (var kv in received_data) { _handle_client_msg(kv.first, kv.second); } @@ -111,8 +116,25 @@ namespace BCG { } } } + public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) { + var len = end_pos - start_pos; + StringBuilder hex_builder = new StringBuilder(len * 2); + int i = 0; + + foreach (byte b in byte_arr) { + if (i >= start_pos && i < end_pos) + hex_builder.Append($"{b:X2} "); + i++; + } + return hex_builder.ToString().Trim(); + } + // all messages (polled from the message_hub) sent to the client are handled here private void _handle_client_msg(string sender, byte[] msg) { + //_stop_watch.Restart(); + var stop_watch = Stopwatch.StartNew(); + //string msg_txt = to_hex(msg, 0, msg.Length); + Log.Gen.info($"PubSubMessageReactor._handle_client_msg sender: {sender} msg: {msg.Length}"); // TODO: reduce object creation, C# has an ObjectPool PubSubClientMessage ps_client_msg = new PubSubClientMessage(); ps_client_msg.consruct_deserialize(msg); @@ -131,6 +153,8 @@ namespace BCG { _handle_pub(sender, ps_client_msg.pub_msg, false); break; } + stop_watch.Stop(); + Log.Gen.info($"PubSubMessageReactor._handle_client_msg processed ps_client_msg: {ps_client_msg} processing time: {stop_watch.ElapsedMilliseconds} msecs"); } private void _handle_sub(string sender, SubscribeMsg sub_msg) { @@ -170,7 +194,7 @@ namespace BCG { // this is where pubsub's data state is updated _state.update_pub_data(pub_msg); var sub_connections_for_tag = _state.get_subscriber_connections_for_tag(pub_msg.tag); - //bts.Log.Gen.info($"PubSubMessageReactor._handle_pub {sub_connections_for_tag.Count} subs for tag: {pub_msg.tag}"); + bts.Log.Gen.info($"PubSubMessageReactor._handle_pub {sub_connections_for_tag.Count} subs for tag: {pub_msg.tag}"); bts.PubSubServerMessage msg_to_send = new PubSubServerMessage() { tag = pub_msg.tag, tag_type = pub_msg.tag_type, diff --git a/src/DataStreamSuite/PubSub/Server/PubSubState.cs b/src/DataStreamSuite/PubSub/Server/PubSubState.cs index 8d4abd3c5..ed524157e 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubState.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubState.cs @@ -65,6 +65,7 @@ namespace BCG { // } public void handle_sub(bts.SubscribeMsg sub_msg) { + Log.Gen.info($"PubSubState.handle_sub sub_msg: {sub_msg}"); foreach (string tag in sub_msg.tags) { SingleTagConnectionData tag_conn_map = null; if (!_tag_to_connection_map.TryGetValue(tag, out tag_conn_map)) { diff --git a/src/DataStreamSuite/PubSub/Server/SingleTagConnectionData.cs b/src/DataStreamSuite/PubSub/Server/SingleTagConnectionData.cs index a64288fff..b4080953b 100644 --- a/src/DataStreamSuite/PubSub/Server/SingleTagConnectionData.cs +++ b/src/DataStreamSuite/PubSub/Server/SingleTagConnectionData.cs @@ -78,6 +78,7 @@ namespace BCG { } public bool handle_sub(SubscribeMsg sub_msg) { + Log.Gen.info($"SingleTagConnectionData.handle_sub sub_msg: {sub_msg}"); if (_sub_conn_to_app_map.ContainsKey(sub_msg.connection)) { bts.Log.Gen.info($"tag: {_tag} has already been subscribed by connection: {sub_msg.connection}"); return false; diff --git a/src/DataStreamSuite/PubSub/Server/SingleTagData.cs b/src/DataStreamSuite/PubSub/Server/SingleTagData.cs index 9bb3cd0ae..87cc5d9fc 100644 --- a/src/DataStreamSuite/PubSub/Server/SingleTagData.cs +++ b/src/DataStreamSuite/PubSub/Server/SingleTagData.cs @@ -10,7 +10,8 @@ using BCG; namespace bts { /// - /// this is the data (not pub/subs/connections) for one tag. This is owned by PubSubState and offers + /// this is the data (not pub/subs/connections) for one tag. This exists in both the PublisherCOre on the + /// publisher side, as well as the PubSubServer /// 2 main functions /// 1. handle_publish, called in response to a pubsub publish /// 2. get_current_content, to retreive the state built in response to 1. @@ -47,7 +48,7 @@ namespace bts { // PubsubState calls this public void handle_publish(PublishMsg pub_msg) { string handler_str = _tag_handler != null ? _tag_handler.GetType().ToString() : "NULL"; - bts.Log.Gen.info($"SingleTagData.handle_publish pub_msg: {pub_msg} handler: {handler_str} "); + bts.Log.Gen.info($"SingleTagData.handle_publish tag: {_tag} pub_msg: {pub_msg} handler: {handler_str} "); try { _tag_handler.update(_tag, pub_msg.data_to_publish); @@ -62,7 +63,9 @@ namespace bts { } public byte[] get_current_content() { - return _tag_handler.get_content(_tag); + byte[] rv = _tag_handler.get_content(_tag); + bts.Log.Gen.info($"SingleTagData.get_current_content tag: {_tag} rv: {rv}"); + return rv; } private void _on_handler_internal_publish(ICollection> cbItems) { diff --git a/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs b/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs index 882f28326..11bc9d23b 100644 --- a/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs +++ b/src/DataStreamSuite/PubSub/SingleServerSubscriberCore.cs @@ -3,7 +3,9 @@ using bts; using System; using System.Collections.Generic; +using System.ComponentModel; using System.Linq; +using System.Runtime.InteropServices; namespace BCG { /// @@ -27,9 +29,9 @@ namespace BCG { public event PubSubConnectionChange onPubSubConnectionChange; - public SingleServerSubscriberCore(string server_url, PubSubServerConnectionFactory connection_factory) { - bts.Log.Gen.info($"SingleServerSubscriberCore constructed for server_url: {server_url}"); - _connection = connection_factory.getConnection(server_url); + public SingleServerSubscriberCore(RemoteConnectionInfo server_rci, PubSubServerConnectionFactory connection_factory) { + bts.Log.Gen.info($"SingleServerSubscriberCore constructed for server_url: {server_rci}"); + _connection = connection_factory.get_connection(server_rci); _connection.register_connection_callback(new PubSubServerConnectionStatusChange(_handle_server_reboot)); _connection.register_data_callback(_on_data); } @@ -47,7 +49,11 @@ namespace BCG { foreach (var m in conn_data) { var msg = new PubSubServerMessage(); msg.consruct_deserialize(m); - rv.Add(msg); + if (_tags.Contains(msg.tag)) + rv.Add(msg); + else { + Log.Gen.warn("SingleSeverSubsciberCore.get_updates msg: {msg} does not have a subscription"); + } } return rv; } diff --git a/src/DataStreamSuite/PubSub/SubscriberCore.cs b/src/DataStreamSuite/PubSub/SubscriberCore.cs index acc21df6d..fb17bb243 100644 --- a/src/DataStreamSuite/PubSub/SubscriberCore.cs +++ b/src/DataStreamSuite/PubSub/SubscriberCore.cs @@ -1,125 +1,126 @@ -using bts; - -using Bts.PubSub; - -using System; -using System.Collections.Generic; - -namespace BCG { - - /// - /// This class serves as the subscrition interface to the pubsub. The only methods - /// it exposes are getUpdates and subscribe. Internally, it manages a SingleServerSubscriberCore - /// foreach server. So this class does the routing part, and each SingleServerSubscriberCore - /// handles dealing sending (sub/unsub) and receiving (data) messages from the transport. - /// There is no sharing with respect to this class. Ie, it creates new SingleServerSubscriberCore - /// to the servers even though there might be so created by other instances of the SubscriberCore - /// - /// - public class SubscriberCore : ISubscriberCore { - private Dictionary _server_url_to_subcore_map = new Dictionary(); - private IPubSubRouter _tag_router; - private PubSubServerConnectionFactory _connection_factory = new PubSubServerConnectionFactory(true); - - public event PubSubConnectionChange onPubSubConnectionChange; - - - #region constructors - - public SubscriberCore() { - //this defaults to a basic pubsubrouter if the config key is not filled out - _tag_router = new PubSubRouter(Log.notnull(PubSubConfig.Instance.SubscriberRoutes).ToArray()); - } - public SubscriberCore(string site_name) { - _tag_router = new PubSubRouter(PubSubConfig.Instance.SubscriberRoutesFor(site_name).ToArray()); - } - - private IPubSubRouter _get_router() => - new PubSubRouter(Log.notnull(PubSubConfig.Instance.SubscriberRoutes).ToArray()); - - public SubscriberCore(PubSubRoute[] routes) { - _tag_router = new PubSubRouter(routes); - } - - public SubscriberCore(IPubSubRouter router) { - _tag_router = router; - } - - #endregion - - public IEnumerable get_updates(TimeSpan timeout) { - List rv = new List(); - foreach (var kv in _server_url_to_subcore_map) { - var sub_core = kv.Value; - rv.AddRange(sub_core.get_updates(timeout)); - } - return rv; - } - - private IDictionary _n_update_map = new Dictionary(); - private void _log_n_updates(IEnumerable updates) { - foreach (PubSubUpdate u in updates) { - if (!_n_update_map.ContainsKey(u.tag)) { - _n_update_map.Add(u.tag, 0); - } - _n_update_map[u.tag] = _n_update_map[u.tag] + 1; - } - // log it now - foreach (var kv in _n_update_map) { - bts.Log.Gen.info($"StaticSubscrberCore tag: {kv.Key} n_updates: {kv.Value}"); - } - } - - public void subscribe(params string[] tags) { - lock (_server_url_to_subcore_map) { - foreach (string tag in tags) { - string server_url = _tag_router.get_sub_url_for_tag(tag); - var server_sub_core = _find_subscribercore(server_url); - server_sub_core.subscribe(tag); - } - } - } - - private ISingleServerSubscriberCore _find_subscribercore(string server_url) { - ISingleServerSubscriberCore rv = null; - if (!_server_url_to_subcore_map.TryGetValue(server_url, out rv)) { - rv = new SingleServerSubscriberCore(server_url, _connection_factory); - _server_url_to_subcore_map.Add(server_url, rv); - rv.onPubSubConnectionChange += _handle_pubsubserver_connection_change; - } - return rv; - } - - - public void _handle_pubsubserver_connection_change(bool is_connected, string server_url) { - if (onPubSubConnectionChange != null) - onPubSubConnectionChange(is_connected, server_url); - - } - - public void unsubscribe(params string[] tags) { - lock (_server_url_to_subcore_map) { - - foreach (string tag in tags) { - string server_url = _tag_router.get_sub_url_for_tag(tag); - var server_subcore = _find_subscribercore(server_url); - server_subcore.unsubscribe(tag); - } - } - } - - public IPubSubRouter get_router() { - return _tag_router; - } - - #region IDisposable Members - - public void Dispose() { - foreach (var sssc in _server_url_to_subcore_map.Values) - sssc.Dispose(); - _connection_factory.Dispose(); - } - - #endregion - } -} +using bts; + +using Bts.PubSub; + +using System; +using System.Collections.Generic; + +namespace BCG { + + /// + /// This class serves as the subscrition interface to the pubsub. The only methods + /// it exposes are getUpdates and subscribe. Internally, it manages a SingleServerSubscriberCore + /// foreach server. So this class does the routing part, and each SingleServerSubscriberCore + /// handles dealing sending (sub/unsub) and receiving (data) messages from the transport. + /// There is no sharing with respect to this class. Ie, it creates new SingleServerSubscriberCore + /// to the servers even though there might be so created by other instances of the SubscriberCore + /// + /// + public class SubscriberCore : ISubscriberCore { + private Dictionary _server_route_to_subcore_map = + new Dictionary(); + private IPubSubRouter _tag_router; + private PubSubServerConnectionFactory _connection_factory = new PubSubServerConnectionFactory(true); + + public event PubSubConnectionChange onPubSubConnectionChange; + + + #region constructors + + public SubscriberCore() { + //this defaults to a basic pubsubrouter if the config key is not filled out + _tag_router = new PubSubRouter(Log.notnull(PubSubConfig.Instance.SubscriberRoutes).ToArray()); + } + public SubscriberCore(string site_name) { + _tag_router = new PubSubRouter(PubSubConfig.Instance.SubscriberRoutesFor(site_name).ToArray()); + } + + private IPubSubRouter _get_router() => + new PubSubRouter(Log.notnull(PubSubConfig.Instance.SubscriberRoutes).ToArray()); + + public SubscriberCore(PubSubRoute[] routes) { + _tag_router = new PubSubRouter(routes); + } + + public SubscriberCore(IPubSubRouter router) { + _tag_router = router; + } + + #endregion + + public IEnumerable get_updates(TimeSpan timeout) { + List rv = new List(); + foreach (var kv in _server_route_to_subcore_map) { + var sub_core = kv.Value; + rv.AddRange(sub_core.get_updates(timeout)); + } + return rv; + } + + private IDictionary _n_update_map = new Dictionary(); + private void _log_n_updates(IEnumerable updates) { + foreach (PubSubUpdate u in updates) { + if (!_n_update_map.ContainsKey(u.tag)) { + _n_update_map.Add(u.tag, 0); + } + _n_update_map[u.tag] = _n_update_map[u.tag] + 1; + } + // log it now + foreach (var kv in _n_update_map) { + bts.Log.Gen.info($"StaticSubscrberCore tag: {kv.Key} n_updates: {kv.Value}"); + } + } + + public void subscribe(params string[] tags) { + lock (_server_route_to_subcore_map) { + foreach (string tag in tags) { + RemoteConnectionInfo server_rci = _tag_router.get_sub_route_for_tag(tag); + var server_sub_core = _find_subscribercore(server_rci); + server_sub_core.subscribe(tag); + } + } + } + + private ISingleServerSubscriberCore _find_subscribercore(RemoteConnectionInfo server_rci) { + ISingleServerSubscriberCore rv = null; + if (!_server_route_to_subcore_map.TryGetValue(server_rci, out rv)) { + rv = new SingleServerSubscriberCore(server_rci, _connection_factory); + _server_route_to_subcore_map.Add(server_rci, rv); + rv.onPubSubConnectionChange += _handle_pubsubserver_connection_change; + } + return rv; + } + + + public void _handle_pubsubserver_connection_change(bool is_connected, string server_url) { + if (onPubSubConnectionChange != null) + onPubSubConnectionChange(is_connected, server_url); + + } + + public void unsubscribe(params string[] tags) { + lock (_server_route_to_subcore_map) { + + foreach (string tag in tags) { + var server_rci = _tag_router.get_sub_route_for_tag(tag); + var server_subcore = _find_subscribercore(server_rci); + server_subcore.unsubscribe(tag); + } + } + } + + public IPubSubRouter get_router() { + return _tag_router; + } + + #region IDisposable Members + + public void Dispose() { + foreach (var sssc in _server_route_to_subcore_map.Values) + sssc.Dispose(); + _connection_factory.Dispose(); + } + + #endregion + } +} diff --git a/src/DataStreamSuite/PubSub/Testing/SimpleTCPClient.cs b/src/DataStreamSuite/PubSub/Testing/SimpleTCPClient.cs index c5fff1d8b..b923a52be 100644 --- a/src/DataStreamSuite/PubSub/Testing/SimpleTCPClient.cs +++ b/src/DataStreamSuite/PubSub/Testing/SimpleTCPClient.cs @@ -15,7 +15,7 @@ using System.Transactions; namespace BCG { public class SimpleTCPClient : ITCPConnectionHandler, ITCPMessageHandler { - private readonly BlutoSubAddress _server_addr; + private readonly RemoteConnectionInfo _server_addr; private int _m_msgs_received = 0; private readonly UInt32 _msg_sz = 1024;// * 1024; private readonly int _max_n_msgs = Int32.MaxValue; @@ -23,10 +23,11 @@ namespace BCG { int _send_wait_period_ms = 5000; Action _listener; - public SimpleTCPClient(string server_ip, int server_port, UInt32 msg_sz, int send_wait_period_ms) { + public SimpleTCPClient(string server_ip, int server_port, UInt32 msg_sz, int send_wait_period_ms, int max_n_msgs) { + _max_n_msgs = max_n_msgs; _msg_sz = msg_sz; _send_wait_period_ms = send_wait_period_ms; - _server_addr = new BlutoSubAddress(server_ip, server_port); + _server_addr = new RemoteConnectionInfo(server_ip, server_port); } public void start() { @@ -34,7 +35,10 @@ namespace BCG { } private void _connect() { - bts.AsyncTCPConnector_APM.connect_to(_server_addr, this); + Log.Gen.info($"SimpleTCPClient._connect to: {_server_addr}"); + + bts.AsyncTCPConnector_AsyncWrapper.connect_to(_server_addr, this); + //bts.AsyncTCPConnector_APM.connect_to(_server_addr, this); } public void register_listener(Action cb) { @@ -43,10 +47,10 @@ namespace BCG { public void onConnection(ITCPConnection connection) { string ip; int port; - connection.getPeerAddress(out ip, out port); - bts.Log.Gen.info($"SimpleTCPClientAsync.onConnection connected to server ip: {ip} port: {port}, starting task to send data in a loop"); + var peer_rci = connection.get_peer_address(); + bts.Log.Gen.info($"SimpleTCPClient.onConnection connected to server: {peer_rci}, starting task to send data in a loop"); if (_listener != null) { - _listener.Invoke($"Connected to server ip: {ip} port {port}"); + _listener.Invoke($"Connected to server {peer_rci}"); } connection.register(this); // start sending data @@ -56,14 +60,23 @@ namespace BCG { private void _keep_sending_data(ITCPConnection connection) { byte[] buff = new byte[_msg_sz]; - for (int i = 0; i < buff.Length; i++) { - buff[i] = (byte)i; - } + MemoryStream str = new MemoryStream(buff); + BinaryWriter bw = new BinaryWriter(str); + Action fill_msg = (rep_i) => { + str.Position = 0; + int n_ints = (int)_msg_sz / 4; + for (int i = 0; i < n_ints; i++) { + bw.Write(rep_i); + } + }; + + for (int i = 0; i < _max_n_msgs; i++) { if (!_send_flag) { return; } - bts.Log.Gen.info("Sending msg: {i} sz: {buff.Length}"); + fill_msg(1); + bts.Log.Gen.info($"Sending msg: {i} sz: {buff.Length}"); System.Threading.Thread.Sleep(_send_wait_period_ms); connection.send(buff); _listener.Invoke($"Data sent, sz: {buff.Length}"); @@ -72,9 +85,11 @@ namespace BCG { public void onData(RemoteMessage msg) { _m_msgs_received++; - Console.WriteLine($"SimpleTCPClientAsync.onData n_msgs: {_m_msgs_received} msg: {msg}"); + if (msg.messageType == RemoteMessageType.Connection) + Log.Gen.error($"SimpleTCPClient.onData n_msgs: {_m_msgs_received} msg: {msg}"); _listener.Invoke($"Message received msg: {msg}"); if (msg.messageType == RemoteMessageType.Disconnection) { + Log.Gen.error($"SimpleTCPClient.onData disconnected, reconnecting"); _send_flag = false; _connect(); } diff --git a/src/DataStreamSuite/PubSub/Testing/SimpleTCPServer.cs b/src/DataStreamSuite/PubSub/Testing/SimpleTCPServer.cs index 9be2682ab..a99086c35 100644 --- a/src/DataStreamSuite/PubSub/Testing/SimpleTCPServer.cs +++ b/src/DataStreamSuite/PubSub/Testing/SimpleTCPServer.cs @@ -8,25 +8,23 @@ namespace BCG { public class SimpleTCPServer : ITCPConnectionHandler, ITCPMessageHandler { int _n_msgs_received = 0; bool _isDisposed = false; - AsyncTCPConnectionAcceptor_APM _acceptor; + AsyncTCPConnectionAcceptor_AsyncWrapper _acceptor; IDictionary _conn_map = new Dictionary(); readonly bool _echo_msg_back; public SimpleTCPServer(int port, bool echo_msg_back) { - _acceptor = new AsyncTCPConnectionAcceptor_APM(port, this); + Log.Gen.info($"SimpleTCPServer.const port: {port} echo_msg_back: {echo_msg_back}"); + _acceptor = new AsyncTCPConnectionAcceptor_AsyncWrapper(port, this); _echo_msg_back = echo_msg_back; } public void init() { - //_acceptor = new AsyncTCPConnectionAcceptor_APM(12345, this); - _acceptor.init(); + _acceptor.start(); } public void onConnection(ITCPConnection connection) { - string ip; int port; - connection.getPeerAddress(out ip, out port); - RemoteConnectionInfo rci = new RemoteConnectionInfo(ip, port); + RemoteConnectionInfo rci = connection.get_peer_address(); _conn_map[rci] = connection; bts.Log.Gen.info($"SimpleTCPServer.onConnection new client connected conn: {rci}"); connection.register(this); @@ -34,7 +32,8 @@ namespace BCG { public void onData(RemoteMessage msg) { - var log_msg = $"SimpleTCPServer.onData msg: {msg}"; + _n_msgs_received++; + var log_msg = $"SimpleTCPServer.onData msgs_received: {_n_msgs_received} msg: {msg} _echo_msg_back: {_echo_msg_back}"; Log.Gen.info(log_msg); Console.WriteLine(log_msg); if (!_echo_msg_back) { @@ -44,11 +43,10 @@ namespace BCG { ITCPConnection this_conn; bool found = _conn_map.TryGetValue(rci, out this_conn); if (!found) { - Console.WriteLine("Could not find connection !!!!!!"); + Log.Gen.error($"Could not find connection: {rci}"); return; } - _n_msgs_received++; - Console.WriteLine($"\n\nSimpleTCPServerAsync.onData msgs_received: {_n_msgs_received} msg: {msg} ci: {rci} NOW echoing it back\n\n"); + Log.Gen.info($"\n\n\nSimpleTCPServerAsync.onData msgs_received: {_n_msgs_received} msg: {msg} ci: {rci} NOW echoing it back\n\n\n"); if (msg.messageType == RemoteMessageType.Data) { this_conn.send(msg.message); } else { diff --git a/src/DataStreamSuite/PubSub/documentation/readme.txt b/src/DataStreamSuite/PubSub/documentation/readme.txt index dc7a27efc..aaa5afda2 100644 --- a/src/DataStreamSuite/PubSub/documentation/readme.txt +++ b/src/DataStreamSuite/PubSub/documentation/readme.txt @@ -94,7 +94,7 @@ pubSubTesting(250,10000000,100000,"test_2")) Stream publishing: ------------------ -PublisherStream (publishesa any stream to the pubsubserver) +PublisherStream (publishes any stream to the pubsubserver) r=second(100) r=project(r,"id") r=math(r,"x = id*id") diff --git a/src/DataStreamSuite/PubSubServer/Program.cs b/src/DataStreamSuite/PubSubServer/Program.cs index 9df1c2c79..5b1be3303 100644 --- a/src/DataStreamSuite/PubSubServer/Program.cs +++ b/src/DataStreamSuite/PubSubServer/Program.cs @@ -12,6 +12,7 @@ using mbts.config; using Spark.Core; using System; +using System.Collections.Specialized; using System.Threading; using System.Threading.Tasks; @@ -24,8 +25,11 @@ int Run(PubSubServerConfig cfg) { cfg.MergeWith(PubSubServerConfig.Default()); PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); Configuration.initialize(cfg.Sparkd); + bool run_echo_server = false; + bool echo_msg_back = false; + cfg.SimpleEchoServer = run_echo_server; return cfg.SimpleEchoServer ?? false ? - RunEchoServer(cfg.PubSub.ServerPort.assert_value()) : + RunEchoServer(cfg.PubSub.ServerPort.assert_value(), echo_msg_back) : RunPubSubServer(); } catch (Exception ex) { // Write the error to console first, in case the logging system is compromised. @@ -58,11 +62,14 @@ int RunPubSubServer() { return 0; } -int RunEchoServer(short port) { - Console.WriteLine("Running as TCP Echo server, Hit enter to continue"); +int RunEchoServer(short port, bool echo) { + string log_msg = $"Running as TCP Echo server, port: {port} echo: {echo} Hit enter to continue:"; + Console.WriteLine(log_msg); Console.ReadLine(); - Console.WriteLine($"ready to listen to port: {port}"); - var sts = new SimpleTCPServer(port, true); + var ready_msg = $"ready to listen to port: {port}"; + Log.Gen.info(ready_msg); + Console.WriteLine(ready_msg); + var sts = new SimpleTCPServer(port, echo); sts.init(); Console.CancelKeyPress += OnCancelKeyPress; while (true) { diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/AllTagsStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/AllTagsStream.cs index 1a0f20281..786a38bca 100644 --- a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/AllTagsStream.cs +++ b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/AllTagsStream.cs @@ -1,50 +1,50 @@ -using System; -using System.Collections; -using System.IO; - -using BCG.Utilities.General; -using BCG.Utilities; -using Bts.PubSub; -using Spark.Core; - -namespace BCG { - - public class AllTagsStream : ViewStream, IWorker { - private IWorker myRealWorkerStream; - public AllTagsStream() - : this("") { } - - public AllTagsStream(string prefix) - : this(prefix, ViewStreamDatabase.instance) { } - - public AllTagsStream(ViewStreamDatabase vsd, string regExp) { - ViewStream rs; - mySetRealSource(rs = new TagSynopsisStream(vsd, regExp, true)); - myRealWorkerStream = rs as IWorker; - } - - public AllTagsStream(string prefix, ViewStreamDatabase db) - : this(prefix, db, db.subscriber.get_router()) { } - - public AllTagsStream(string prefix, ViewStreamDatabase db, IPubSubRouter router) { - ViewStream rs = null; - //if we're using substations, we don't care about the router - if (PubSubConfig.Instance.UseSubstations.assert_value()) { - mySetRealSource(rs = new TagSynopsisStream(db, "^" + prefix + ".*", true)); - } else { - mySetRealSource(rs = new TagSynopsisStream(db, router.get_all_pubsub_sub_urls(), "^" + prefix + ".*", true)); - } - myRealWorkerStream = rs as IWorker; - } - - #region IWorker Members - - public void doWork() { - if (myRealWorkerStream != null) - myRealWorkerStream.doWork(); - } - - #endregion - } - -} +// using System; +// using System.Collections; +// using System.IO; + +// using BCG.Utilities.General; +// using BCG.Utilities; +// using Bts.PubSub; +// using Spark.Core; + +// namespace BCG { + +// public class AllTagsStream : ViewStream, IWorker { +// private IWorker myRealWorkerStream; +// public AllTagsStream() +// : this("") { } + +// public AllTagsStream(string prefix) +// : this(prefix, ViewStreamDatabase.instance) { } + +// public AllTagsStream(ViewStreamDatabase vsd, string regExp) { +// ViewStream rs; +// mySetRealSource(rs = new TagSynopsisStream(vsd, regExp, true)); +// myRealWorkerStream = rs as IWorker; +// } + +// public AllTagsStream(string prefix, ViewStreamDatabase db) +// : this(prefix, db, db.subscriber.get_router()) { } + +// public AllTagsStream(string prefix, ViewStreamDatabase db, IPubSubRouter router) { +// ViewStream rs = null; +// //if we're using substations, we don't care about the router +// if (PubSubConfig.Instance.UseSubstations.assert_value()) { +// mySetRealSource(rs = new TagSynopsisStream(db, "^" + prefix + ".*", true)); +// } else { +// mySetRealSource(rs = new TagSynopsisStream(db, router.get_all_pubsub_sub_urls(), "^" + prefix + ".*", true)); +// } +// myRealWorkerStream = rs as IWorker; +// } + +// #region IWorker Members + +// public void doWork() { +// if (myRealWorkerStream != null) +// myRealWorkerStream.doWork(); +// } + +// #endregion +// } + +// } diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/TagSynopsisStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/TagSynopsisStream.cs index 62f9dd29b..b410762b2 100644 --- a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/TagSynopsisStream.cs +++ b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/TagSynopsisStream.cs @@ -1,117 +1,117 @@ -using System; -using System.Collections.Generic; -using System.Text; -using BCG.Utilities.General; -using System.Collections; -using System.IO; -using Bts.PubSub; -using bts; - -namespace BCG { - [DataStreamUserRespresentation("tagSynopsis")] - public class TagSynopsisStream : ViewStream, IStreamMapper, IWorker { - private List myWorkers = new List(); - IBlueTableSchema mySchema = new BlueTableSchema("AllTags", - new ColumnInfo("id", 0, typeof(int)), - new ColumnInfo("tag", 1, typeof(string)), - new ColumnInfo("subscribers", 2, typeof(int)), - new ColumnInfo("pubSub", 3, typeof(string)), - new ColumnInfo("dataPresent", 4, typeof(bool))); - - public TagSynopsisStream() { - mySetRealSource(new TagSynopsisStream(ViewStreamDatabase.instance, ViewStreamDatabase.instance.subscriber.get_router().get_all_pubsub_sub_urls())); - } - - public TagSynopsisStream(string tagRegExp) { - mySetRealSource(new TagSynopsisStream(ViewStreamDatabase.instance, ViewStreamDatabase.instance.subscriber.get_router().get_all_pubsub_sub_urls(), tagRegExp)); - } - - public TagSynopsisStream(ViewStreamDatabase vsd, string route) { - - ViewStream vs = new PubSubServerStateStream(/*route,*/ vsd); - vs = new MapStream(vs, this); - myWorkers.Add(vs as IWorker); - - mySetRealSource(vs); - } - - - public TagSynopsisStream(bool cons_overloader, string route) : - this(ViewStreamDatabase.instance, new string[] { route }) { - } - - public TagSynopsisStream(ViewStreamDatabase vsd, ICollection routes) { - List vss = new List(); - foreach (string r in routes) { - ViewStream vs = new PubSubServerStateStream(/*r,*/ vsd); - vs = new MapStream(vs, this); - vss.Add(vs); - myWorkers.Add(vs as IWorker); - } - mySetRealSource(new AppendStream(vss.ToArray())); - } - - public TagSynopsisStream(ViewStreamDatabase vsd, string tagRegExp, bool filterOutNonPublished) { - // if (PubSub3Config.instance.PS3_UseSubstations) - // { - // mySetRealSource(new ThreadSafeStream(new SubstationTagSynopsisStream(vsd, tagRegExp, filterOutNonPublished))); - - // } - // else - { - PubSubRouter theRouter = new(Log.notnull(PubSubConfig.Instance.SubscriberRoutes).ToArray()); - ICollection thisVSDRoutes = theRouter.get_all_pubsub_sub_urls(); - - ViewStream vs = new FilterStream(new TagSynopsisStream(vsd, thisVSDRoutes), "tag", tagRegExp); - - if (filterOutNonPublished) - vs = new FilterStream(vs, "dataPresent=true"); - mySetRealSource(vs); - } - - } - - public TagSynopsisStream(ViewStreamDatabase vsd, ICollection routes, string tagRegExp) { - mySetRealSource(new FilterStream(new TagSynopsisStream(vsd, routes), "tag", tagRegExp)); - } - - public TagSynopsisStream(ViewStreamDatabase vsd, ICollection routes, string tagRegExp, bool filterOutNonPublished) { - ViewStream vs = new TagSynopsisStream(vsd, routes, tagRegExp); - if (filterOutNonPublished) - vs = new FilterStream(vs, "dataPresent=true"); - mySetRealSource(vs); - } - - - #region IStreamMapper Members - - public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) { - return mySchema; - } - - #endregion - - #region IDynamicStreamMapper Members - - public IBlueRow map(IBlueRow row) { - string subValue = row["subscribers"].ToString().Trim(); - return new BlueRow(getOutputSchema(null), - row["id"], - row["tag"], - subValue.Length == 0 ? 0 : subValue.Split(',').Length, - row["server"], - row["publishers"].ToString().Length > 0 || row["deadPublishers"].ToString().Length > 0); - } - - #endregion - - #region IWorker Members - - public void doWork() { - foreach (IWorker w in myWorkers) - w.doWork(); - } - - #endregion - } -} +// using System; +// using System.Collections.Generic; +// using System.Text; +// using BCG.Utilities.General; +// using System.Collections; +// using System.IO; +// using Bts.PubSub; +// using bts; + +// namespace BCG { +// [DataStreamUserRespresentation("tagSynopsis")] +// public class TagSynopsisStream : ViewStream, IStreamMapper, IWorker { +// private List myWorkers = new List(); +// IBlueTableSchema mySchema = new BlueTableSchema("AllTags", +// new ColumnInfo("id", 0, typeof(int)), +// new ColumnInfo("tag", 1, typeof(string)), +// new ColumnInfo("subscribers", 2, typeof(int)), +// new ColumnInfo("pubSub", 3, typeof(string)), +// new ColumnInfo("dataPresent", 4, typeof(bool))); + +// public TagSynopsisStream() { +// mySetRealSource(new TagSynopsisStream(ViewStreamDatabase.instance, ViewStreamDatabase.instance.subscriber.get_router().get_all_pubsub_sub_urls())); +// } + +// public TagSynopsisStream(string tagRegExp) { +// mySetRealSource(new TagSynopsisStream(ViewStreamDatabase.instance, ViewStreamDatabase.instance.subscriber.get_router().get_all_pubsub_sub_urls(), tagRegExp)); +// } + +// public TagSynopsisStream(ViewStreamDatabase vsd, string route) { + +// ViewStream vs = new PubSubServerStateStream(/*route,*/ vsd); +// vs = new MapStream(vs, this); +// myWorkers.Add(vs as IWorker); + +// mySetRealSource(vs); +// } + + +// public TagSynopsisStream(bool cons_overloader, string route) : +// this(ViewStreamDatabase.instance, new string[] { route }) { +// } + +// public TagSynopsisStream(ViewStreamDatabase vsd, ICollection routes) { +// List vss = new List(); +// foreach (string r in routes) { +// ViewStream vs = new PubSubServerStateStream(/*r,*/ vsd); +// vs = new MapStream(vs, this); +// vss.Add(vs); +// myWorkers.Add(vs as IWorker); +// } +// mySetRealSource(new AppendStream(vss.ToArray())); +// } + +// public TagSynopsisStream(ViewStreamDatabase vsd, string tagRegExp, bool filterOutNonPublished) { +// // if (PubSub3Config.instance.PS3_UseSubstations) +// // { +// // mySetRealSource(new ThreadSafeStream(new SubstationTagSynopsisStream(vsd, tagRegExp, filterOutNonPublished))); + +// // } +// // else +// { +// PubSubRouter theRouter = new(Log.notnull(PubSubConfig.Instance.SubscriberRoutes).ToArray()); +// ICollection thisVSDRoutes = theRouter.get_all_pubsub_sub_urls(); + +// ViewStream vs = new FilterStream(new TagSynopsisStream(vsd, thisVSDRoutes), "tag", tagRegExp); + +// if (filterOutNonPublished) +// vs = new FilterStream(vs, "dataPresent=true"); +// mySetRealSource(vs); +// } + +// } + +// public TagSynopsisStream(ViewStreamDatabase vsd, ICollection routes, string tagRegExp) { +// mySetRealSource(new FilterStream(new TagSynopsisStream(vsd, routes), "tag", tagRegExp)); +// } + +// public TagSynopsisStream(ViewStreamDatabase vsd, ICollection routes, string tagRegExp, bool filterOutNonPublished) { +// ViewStream vs = new TagSynopsisStream(vsd, routes, tagRegExp); +// if (filterOutNonPublished) +// vs = new FilterStream(vs, "dataPresent=true"); +// mySetRealSource(vs); +// } + + +// #region IStreamMapper Members + +// public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) { +// return mySchema; +// } + +// #endregion + +// #region IDynamicStreamMapper Members + +// public IBlueRow map(IBlueRow row) { +// string subValue = row["subscribers"].ToString().Trim(); +// return new BlueRow(getOutputSchema(null), +// row["id"], +// row["tag"], +// subValue.Length == 0 ? 0 : subValue.Split(',').Length, +// row["server"], +// row["publishers"].ToString().Length > 0 || row["deadPublishers"].ToString().Length > 0); +// } + +// #endregion + +// #region IWorker Members + +// public void doWork() { +// foreach (IWorker w in myWorkers) +// w.doWork(); +// } + +// #endregion +// } +// }