Fixing the bug where messages were being sent by the pubsubserver out of order
This commit is contained in:
1 parent
6604eaaea7
commit
b55c4103df
8 files changed
+62
-135
No files matched your search
+1
-1
@@ -158,7 +158,7 @@ namespace BCG {
|
||||
/// </summary>
|
||||
/// <param name="message"></param>
|
||||
public void onData(RemoteMessage message) {
|
||||
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData message: {message}");
|
||||
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData connection: {_connection} message: {message}");
|
||||
// TODO check for disconenction
|
||||
switch (message.messageType) {
|
||||
case RemoteMessageType.Data:
|
||||
|
||||
@@ -41,12 +41,18 @@ namespace bts {
|
||||
// this is a blocking call
|
||||
Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port} starting blocking call AcceptTcpClient");
|
||||
TcpClient tcp_client = _server.AcceptTcpClient();
|
||||
ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client);
|
||||
ITCPConnection conn = _create_connection(tcp_client);
|
||||
Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port}, new connection, self: {conn.get_self_address()} peer: {conn.get_peer_address()} calling onConnection now");
|
||||
_connection_handler.onConnection(conn);
|
||||
}
|
||||
//_cbs(conn);
|
||||
}
|
||||
|
||||
private static ITCPConnection _create_connection(TcpClient tcp_client) {
|
||||
ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client);
|
||||
// ITCPConnection conn = new AsyncTCPConnection_APM(tcp_client.GetStream().Socket, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Client);
|
||||
return conn;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,10 +1,6 @@
|
||||
namespace bts {
|
||||
using BCG;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using bts.utils;
|
||||
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Net.Sockets;
|
||||
@@ -12,19 +8,30 @@ namespace bts {
|
||||
using System.Threading;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.ComponentModel;
|
||||
|
||||
// this ITCPConnection uses the TCPClient class instead of a raw socket (in the hope, that it would configure the socket correctly)
|
||||
|
||||
public class AsyncTCPConnection_AsyncWrapper : ITCPConnection, IWorker, IDisposable {
|
||||
public class ConnectionID {
|
||||
public readonly RemoteConnectionInfo self;
|
||||
public readonly RemoteConnectionInfo peer;
|
||||
public readonly Type conn_type;
|
||||
public override string ToString() {
|
||||
return $"{conn_type} self: {self} peer: {peer}";
|
||||
}
|
||||
public ConnectionID(RemoteConnectionInfo self_, RemoteConnectionInfo peer_, Type conn_type_) {
|
||||
self = self_;
|
||||
peer = peer_;
|
||||
conn_type = conn_type_;
|
||||
}
|
||||
}
|
||||
private const int DATA_MSG_SIGNAL = 0;
|
||||
private const int HB_MSG_SIGNAL = 1;
|
||||
private TcpClient _tcp_client;
|
||||
private NetworkStream _nw_stream;
|
||||
private ITCPMessageHandler _msg_handler;
|
||||
private bool _is_alive = true;
|
||||
RemoteConnectionInfo _peer_conn_info = null;
|
||||
RemoteConnectionInfo _self_conn_info = null;
|
||||
ConnectionID _connection_id;
|
||||
|
||||
public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) {
|
||||
var len = end_pos - start_pos;
|
||||
@@ -42,10 +49,8 @@ namespace bts {
|
||||
public AsyncTCPConnection_AsyncWrapper(TcpClient tcp_client) {
|
||||
_tcp_client = tcp_client;
|
||||
_nw_stream = _tcp_client.GetStream();
|
||||
_peer_conn_info = get_peer_address();
|
||||
_self_conn_info = get_self_address();
|
||||
Log.Gen.info("AsyncTCPConnection_AsyncWrapper.cons _peer_conn_info: {_peer_conn_info} _self_conn_info: {_self_conn_info}");
|
||||
//_detect_connection_status_change();
|
||||
_connection_id = new ConnectionID(TCPSocketUtils.get_self_address(_tcp_client.Client), TCPSocketUtils.get_peer_address(_tcp_client.Client), GetType());
|
||||
Log.Gen.info("AsyncTCPConnection_AsyncWrapper.cons connection: {_connection_id}");
|
||||
}
|
||||
|
||||
private void _detect_connection_status_change() {
|
||||
@@ -63,10 +68,7 @@ namespace bts {
|
||||
ThreadPool.QueueUserWorkItem(_ => testing_func());
|
||||
}
|
||||
|
||||
public void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del) {
|
||||
|
||||
}
|
||||
|
||||
#region Sending
|
||||
public void _send_fixed_size_msg(byte[] buffer) {
|
||||
string buff_txt = to_hex(buffer, 0, buffer.Length);
|
||||
Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg buffer: {buff_txt}");
|
||||
@@ -101,29 +103,38 @@ namespace bts {
|
||||
}
|
||||
}
|
||||
|
||||
private void _send_forever() {
|
||||
while (_is_alive) {
|
||||
byte[] data = _send_queue.dequeue();
|
||||
if (data != null) {
|
||||
_send_one_msg(data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private SyncQueue<byte[]> _send_queue = new SyncQueue<byte[]>();
|
||||
public void send(byte[] data) {
|
||||
var data_txt = to_hex(data, 0, data.Length);
|
||||
Log.Gen.info($"AsyncTCPConnection_AsyncWrapper.send data: {data_txt}");
|
||||
//Task.Run(() => _send_one_msg(data));
|
||||
ThreadPool.QueueUserWorkItem(_ => _send_one_msg(data));
|
||||
//ThreadPool.QueueUserWorkItem(_ => _send_one_msg(data));
|
||||
_send_queue.enqueue(data);
|
||||
}
|
||||
#endregion
|
||||
|
||||
public void register(ITCPMessageHandler message_handler) {
|
||||
_msg_handler = message_handler;
|
||||
// start reading in a Task
|
||||
// starting thread pool threads to keep reading and sending
|
||||
ThreadPool.QueueUserWorkItem(_ => _read_msgs_forever());
|
||||
ThreadPool.QueueUserWorkItem(_ => _send_forever());
|
||||
}
|
||||
|
||||
public RemoteConnectionInfo get_peer_address() {
|
||||
//if (_peer_conn_info == null)
|
||||
return _peer_conn_info = TCPSocketUtils.get_peer_address(_tcp_client.Client);
|
||||
//return _peer_conn_info;
|
||||
return _connection_id.peer;
|
||||
}
|
||||
|
||||
public RemoteConnectionInfo get_self_address() {
|
||||
if (_self_conn_info == null)
|
||||
_self_conn_info = TCPSocketUtils.get_self_address(_tcp_client.Client);
|
||||
return _self_conn_info;
|
||||
return _connection_id.self;
|
||||
}
|
||||
|
||||
public bool isDisposed { get { return !_is_alive; } }
|
||||
@@ -132,7 +143,7 @@ namespace bts {
|
||||
// send hb
|
||||
}
|
||||
|
||||
|
||||
#region Reading
|
||||
// This makes the blocking IO call, the callstack is _read_msgs_forever -> _read_one_message -> _read_fixed_size_msg
|
||||
private byte[] _read_fixed_size_msg(int n_bytes) {
|
||||
if (n_bytes <= 0) {
|
||||
@@ -199,10 +210,10 @@ namespace bts {
|
||||
try {
|
||||
OneRecvMessage one_msg = _read_one_message();
|
||||
if (one_msg.msg_type == DATA_MSG_SIGNAL) {
|
||||
_msg_handler.onData(new RemoteMessage(one_msg.data, RemoteMessageType.Data, _peer_conn_info));
|
||||
_msg_handler.onData(new RemoteMessage(one_msg.data, RemoteMessageType.Data, _connection_id.peer));
|
||||
} else if (one_msg.msg_type == HB_MSG_SIGNAL) {
|
||||
// keep a last hb time an dudpate it here atomically
|
||||
_msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_conn_info));
|
||||
_msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _connection_id.peer));
|
||||
} else {
|
||||
throw new ApplicationException($"Unknown message type: {one_msg.msg_type}");
|
||||
}
|
||||
@@ -212,6 +223,7 @@ namespace bts {
|
||||
}
|
||||
Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_msgs_forever exiting loop");
|
||||
}
|
||||
#endregion
|
||||
|
||||
public bool is_disposed => !_is_alive;
|
||||
|
||||
@@ -253,7 +265,7 @@ namespace bts {
|
||||
if (ex is OutOfMemoryException)
|
||||
throw ex;
|
||||
}
|
||||
_msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _peer_conn_info));
|
||||
_msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _connection_id.peer));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -268,9 +280,7 @@ namespace bts {
|
||||
}
|
||||
|
||||
public override string ToString() {
|
||||
RemoteConnectionInfo self = get_self_address(),
|
||||
peer = get_peer_address();
|
||||
return $"type: {GetType().ToString()} self: {self} peer: {peer} is_disposed: {isDisposed}";
|
||||
return _connection_id.ToString();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -27,7 +27,7 @@ namespace bts {
|
||||
bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to server: {server_rci}");
|
||||
TcpClient tcp_client = new TcpClient(server_rci.ip, server_rci.port);
|
||||
bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to server new TcpClient returned, connected: {tcp_client.Connected}");
|
||||
ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client);
|
||||
ITCPConnection conn = _create_connection(tcp_client);
|
||||
var conn_rci = conn.get_peer_address();
|
||||
bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper._connect_to new connection, server_rci: {server_rci} connection.peer_address: {conn_rci}");
|
||||
Log.assert(server_rci.Equals(conn_rci), $"{server_rci}!={conn_rci}");
|
||||
@@ -35,6 +35,12 @@ namespace bts {
|
||||
handler.onConnection(conn);
|
||||
}
|
||||
|
||||
private static ITCPConnection _create_connection(TcpClient tcp_client) {
|
||||
ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client);
|
||||
// ITCPConnection conn = new AsyncTCPConnection_APM(tcp_client.GetStream().Socket, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Client);
|
||||
return conn;
|
||||
}
|
||||
|
||||
public static void _keep_trying_blocking_function(Action action, TimeSpan timeout, TimeSpan wait_before_retrying) {
|
||||
Func<bool> try_once = () => {
|
||||
var cts = new CancellationTokenSource();
|
||||
|
||||
@@ -21,7 +21,7 @@ namespace BCG {
|
||||
/// <param name="data"></param>
|
||||
/// <param name="confId"></param>
|
||||
/// <param name="del"></param>
|
||||
void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del);
|
||||
//void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del);
|
||||
/// <summary>
|
||||
/// sends data to the other side. theres no guaranatee that
|
||||
/// data is delievered.
|
||||
|
||||
@@ -214,7 +214,7 @@ namespace BCG {
|
||||
_connection_map.TryGetValue(p.first, out conn);
|
||||
if (conn != null) {
|
||||
conn.send(p.second);
|
||||
Log.Gen.info($"ManagedTCPMessageHub._main_message_pump sent n_bytes : {p.second.Length} to: {p.first}");
|
||||
Log.Gen.info($"ManagedTCPMessageHub._main_message_pump sending data client: {p.first} size: {p.second.Length}");
|
||||
} else {
|
||||
string error_msg = $"sending data of sz: {p.second.Count()} to {p.first} failed, this client might have " +
|
||||
"disconnected, in which case it will be garbage collected within 60 secs";
|
||||
|
||||
@@ -20,27 +20,9 @@ namespace BCG {
|
||||
private Thread _msg_polling_thread = null;
|
||||
// all the data,list of subscribers etc are in the state
|
||||
private PubSubState _state = new PubSubState();
|
||||
private bool myStop = false;
|
||||
private MasterWorker _mainloop_master_worker = MasterWorker.getInstance(false);
|
||||
// private double myProcessingWarningThreshold =
|
||||
// Convert.ToDouble(PubSub3Config.instance.PS3_MessageProcessingWarningThreshold);
|
||||
// private HighPerformanceTimer myTimer = new HighPerformanceTimer();
|
||||
|
||||
private delegate void SingleTagTimeSliceDelegate(string tag, double processingTimeSeconds);
|
||||
private SingleTagTimeSliceDelegate onSingleTagPub;
|
||||
private static SyncQueue<PublishMsg> _internal_pub_queue = new SyncQueue<PublishMsg>();
|
||||
private Stopwatch _stop_watch = new Stopwatch();
|
||||
//this thing keeps track of whether a given tag is compressed or not. It is used for logging messages.
|
||||
|
||||
/* This delegate allows pubsub components to be notified when the identity ofe the pubsub is known*/
|
||||
// private delegate void IdentityFoundDelegate(string identity, bool isRoot);
|
||||
// private event IdentityFoundDelegate onIdentityFound;
|
||||
// private void fireIdentityFound(string identity, bool isRoot) {
|
||||
// if (onIdentityFound != null)
|
||||
// onIdentityFound(identity, isRoot);
|
||||
// }
|
||||
|
||||
//private Dictionary<string, bool> myTagCompressionState = new Dictionary<string, bool>();
|
||||
|
||||
public PubSubMessageReactor(int port_no) {
|
||||
bts.Log.Gen.info("PubSubMessageReactor.cons System Initialized");
|
||||
@@ -54,7 +36,7 @@ namespace BCG {
|
||||
TimeSpan.FromMilliseconds(PubSubConfig.Instance.HeartbeatInterval.assert_value()),
|
||||
PubSubConfig.Instance.MissingHeartbeatsBeforeGc.assert_value(),
|
||||
TimeSpan.FromMilliseconds(PubSubConfig.Instance.DisposedConnectionCheckPeriod.assert_value()));
|
||||
|
||||
// this is the only event this class gets from hub
|
||||
_msg_hub.register_connection_callback(new PubSubCallbacks.PubSubServerConnectionStatusChange(_handle_client_connection));
|
||||
|
||||
}
|
||||
@@ -69,10 +51,6 @@ namespace BCG {
|
||||
_state.remove_connection(client);
|
||||
}
|
||||
|
||||
private void stop() {
|
||||
myStop = true;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
_msg_polling_thread.Start();
|
||||
}
|
||||
@@ -104,6 +82,10 @@ namespace BCG {
|
||||
//bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func pre get_updates");
|
||||
var received_data = _msg_hub.get_all_updates();
|
||||
var n_msgs = received_data.Count;
|
||||
if (n_msgs == 0) {
|
||||
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(250));
|
||||
continue;
|
||||
}
|
||||
bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func post get_updates n_msgs: {n_msgs}");
|
||||
foreach (var kv in received_data) {
|
||||
// multiple pubs for the same tags should be batched here
|
||||
|
||||
@@ -42,28 +42,6 @@ namespace BCG {
|
||||
|
||||
#region subscription based state changes
|
||||
|
||||
/// <summary>
|
||||
/// everytime a subscription is made, this gets called
|
||||
/// by PubsubServer2. Adds a
|
||||
/// TODO : subscibers sending sub twice from the client
|
||||
/// </summary>
|
||||
/// <param name="sub"></param>
|
||||
/// <param name="md"></param>
|
||||
// public void sub(SubMetaData sub) {
|
||||
// ///tag based key ds update
|
||||
// foreach (string tag in sub.tags) {
|
||||
// if (!_tag_to_connection_map.ContainsKey(tag)) {
|
||||
// _tag_to_connection_map.Add(tag, new SingleTagConnectionData(tag));
|
||||
// }
|
||||
// _tag_to_connection_map[tag].onSub(sub);
|
||||
|
||||
// if (onSingleTagChange != null)
|
||||
// onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.subscription, sub.connection, sub.subscribing_app));
|
||||
// }
|
||||
// if (onTagMapChanged != null)
|
||||
// onTagMapChanged(sub.tags);
|
||||
// }
|
||||
|
||||
public void handle_sub(bts.SubscribeMsg sub_msg) {
|
||||
Log.Gen.info($"PubSubState.handle_sub sub_msg: {sub_msg}");
|
||||
foreach (string tag in sub_msg.tags) {
|
||||
@@ -82,24 +60,6 @@ namespace BCG {
|
||||
// TODO
|
||||
}
|
||||
|
||||
// public void unsub(UnSubMetaData unsub) {
|
||||
// foreach (string tag in unsub.tags) {
|
||||
// if (_tag_to_connection_map.ContainsKey(tag)) {
|
||||
// ///maybe check here, that if there are no subscribers
|
||||
// ///and publishers for the tag then remove it.
|
||||
// _tag_to_connection_map[tag].onUnsub(unsub);
|
||||
// if (onSingleTagChange != null)
|
||||
// onSingleTagChange(new PubSubSingleTagStateEvent(tag, PubSubSingleTagStateEvent.EventType.unsubscription, unsub.connection, unsub.unsubscribing_app));
|
||||
// } else {
|
||||
// string warningMsg = String.Format("{0} was never published or subscribed on this pubsub, hence" +
|
||||
// " the unsub call by app {1} is voided", tag, unsub.unsubscribing_app);
|
||||
// //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
|
||||
// }
|
||||
// }
|
||||
// if (onTagMapChanged != null)
|
||||
// onTagMapChanged(unsub.tags);
|
||||
// }
|
||||
|
||||
public void handle_unsub(bts.UnsubscribeMsg ubnsub_msg) {
|
||||
foreach (string tag in ubnsub_msg.tags) {
|
||||
SingleTagConnectionData conn_data = null;
|
||||
@@ -139,48 +99,11 @@ namespace BCG {
|
||||
}
|
||||
if (onTagMapChanged != null)
|
||||
onTagMapChanged(tags_effected.ToArray());
|
||||
|
||||
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region data based state changes
|
||||
/// <summary>
|
||||
/// this is the only place in the state class where the
|
||||
/// data is modified ; happens when data is published
|
||||
/// </summary>
|
||||
/// <param name="tag"></param>
|
||||
/// <param name="msg"></param>
|
||||
// public void updatePubData(PubMetaData pub, Stream msg) {
|
||||
// int sz = (int)(msg.Length - msg.Position);
|
||||
// bts.Log.Gen.info($"PubSubData.updatePubData pub: msg : {msg.Position} / {msg.Length} sz: {sz}");
|
||||
// ///this is the only place where SingleTagData should be created
|
||||
// if (!_tag_to_data_map.ContainsKey(pub.tag)) {
|
||||
// ///first time data is published for this tag
|
||||
// _tag_to_data_map.Add(pub.tag, new SingleTagData(pub.tag, pub.type));
|
||||
// }
|
||||
// _tag_to_data_map[pub.tag].pubUpdate(msg);
|
||||
|
||||
// if (!_tag_to_connection_map.ContainsKey(pub.tag)) {
|
||||
// _tag_to_connection_map.Add(pub.tag, new SingleTagConnectionData(pub.tag));
|
||||
// }
|
||||
// bool pubChanged = false;
|
||||
// _tag_to_connection_map[pub.tag].onPub(pub, newPublisher: out pubChanged);
|
||||
// if (pubChanged) {
|
||||
// if (onTagMapChanged != null)
|
||||
// onTagMapChanged(new string[] { pub.tag });
|
||||
|
||||
// if (onSingleTagChange != null)
|
||||
// onSingleTagChange(
|
||||
// new PubSubSingleTagStateEvent(pub.tag,
|
||||
// PubSubSingleTagStateEvent.EventType.publish,
|
||||
// pub.connection, pub.publishingApp));
|
||||
// }
|
||||
// if (onPublishStats != null)
|
||||
// onPublishStats(pub.tag, sz);
|
||||
// }
|
||||
|
||||
public void update_pub_data(bts.PublishMsg pub_msg) {
|
||||
try {
|
||||
bts.Log.Gen.info($"PubSubData.update_pub_data pub_msg: {pub_msg}");
|
||||
|
||||
Reference in new issue
Block a user