PubSuMessageReactor empties the message queue (instead oif one elem ata a time) in the main loop
This commit is contained in:
1 parent
b6eb727bc9
commit
cc43e9d3a4
7 files changed
+97
-80
No files matched your search
@@ -23,8 +23,10 @@ using System.Threading.Tasks;
|
||||
await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args)
|
||||
.MapResult(RunAsync, errs => Task.FromResult(1));
|
||||
|
||||
async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
|
||||
try {
|
||||
async Task<int> RunAsync(DataStreamPublisherConfig cfg)
|
||||
{
|
||||
try
|
||||
{
|
||||
cfg.MergeWithConfigFile();
|
||||
cfg.MergeWith(DataStreamPublisherConfig.Default());
|
||||
PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub));
|
||||
@@ -32,12 +34,17 @@ async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
|
||||
Log.Gen.info("Starting DataStreamPublisher");
|
||||
var service_ctx = ServiceContext.create(cfg.Sparkd);
|
||||
await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty);
|
||||
if (cfg.SimpleEchoClient ?? false) {
|
||||
if (cfg.SimpleEchoClient ?? false)
|
||||
{
|
||||
start_echo_client();
|
||||
} else {
|
||||
}
|
||||
else
|
||||
{
|
||||
_publish_streams(cfg.Publishers ?? new(0));
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Write the error to console first, in case the logging system is compromised.
|
||||
var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
|
||||
Console.Error.WriteLine(error_msg);
|
||||
@@ -48,41 +55,51 @@ async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
static ViewStream? build_stream(string? stream_expression) {
|
||||
if (stream_expression is null) {
|
||||
static ViewStream? build_stream(string? stream_expression)
|
||||
{
|
||||
if (stream_expression is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
try
|
||||
{
|
||||
bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
|
||||
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
||||
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||
return rv;
|
||||
|
||||
} catch (Exception ex) {
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
static void start_echo_client() {
|
||||
static void start_echo_client()
|
||||
{
|
||||
Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue");
|
||||
Console.ReadLine();
|
||||
SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345, 1000, 1000);
|
||||
SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345, 1000, 1000, Int32.MaxValue);
|
||||
client.start();
|
||||
while (true) {
|
||||
while (true)
|
||||
{
|
||||
System.Threading.Thread.Sleep(1000);
|
||||
}
|
||||
// await SimpleTCPClient.start("127.0.0.1", 12345);
|
||||
// Task.Delay(1000000000).Wait();
|
||||
}
|
||||
|
||||
void _publish_streams(IEnumerable<PublisherNode> publishers) {
|
||||
void _publish_streams(IEnumerable<PublisherNode> publishers)
|
||||
{
|
||||
PubTableBroker pb_broker = new PubTableBroker();
|
||||
RemoteQueryManager csm = new RemoteQueryManager();
|
||||
var csvPublishers = new List<CSVPublisher>();
|
||||
|
||||
foreach (var publisher in publishers) {
|
||||
if (Directory.Exists(publisher.CsvDir)) {
|
||||
foreach (var publisher in publishers)
|
||||
{
|
||||
if (Directory.Exists(publisher.CsvDir))
|
||||
{
|
||||
csvPublishers.Add(new(publisher.CsvDir, pb_broker));
|
||||
}
|
||||
}
|
||||
@@ -91,7 +108,8 @@ void _publish_streams(IEnumerable<PublisherNode> publishers) {
|
||||
.SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher))
|
||||
.ToList();
|
||||
|
||||
while (true) {
|
||||
while (true)
|
||||
{
|
||||
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
|
||||
ViewStreamDatabase.instance.doWork();
|
||||
csm.doWork();
|
||||
@@ -99,16 +117,19 @@ void _publish_streams(IEnumerable<PublisherNode> publishers) {
|
||||
ManagedSubscriber.instance.doWork();
|
||||
}
|
||||
|
||||
ViewPublisher? createViewPublisher(PublishedStream ps) {
|
||||
ViewPublisher? createViewPublisher(PublishedStream ps)
|
||||
{
|
||||
Log.Gen.info(
|
||||
$"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
|
||||
if (string.IsNullOrEmpty(ps.Name)) {
|
||||
if (string.IsNullOrEmpty(ps.Name))
|
||||
{
|
||||
Log.Gen.warn(
|
||||
$"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
|
||||
return null;
|
||||
}
|
||||
var vs = build_stream(ps.Expression);
|
||||
if (vs is null) {
|
||||
if (vs is null)
|
||||
{
|
||||
Log.Gen.warn(
|
||||
$"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
|
||||
return null;
|
||||
@@ -119,31 +140,39 @@ void _publish_streams(IEnumerable<PublisherNode> publishers) {
|
||||
}
|
||||
}
|
||||
|
||||
void on_change(object obj) {
|
||||
void on_change(object obj)
|
||||
{
|
||||
bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
|
||||
}
|
||||
|
||||
public class CSVPublisher : IWorker {
|
||||
public class CSVPublisher : IWorker
|
||||
{
|
||||
private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
|
||||
private readonly string _csv_directory;
|
||||
private PubTableBroker _pub_table_broker;
|
||||
|
||||
public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
|
||||
public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker)
|
||||
{
|
||||
_csv_directory = csv_directory;
|
||||
_pub_table_broker = pub_table_broker;
|
||||
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
public void doWork() {
|
||||
public void doWork()
|
||||
{
|
||||
var file_names = Directory.GetFiles(_csv_directory);
|
||||
foreach (var fn in file_names) {
|
||||
if (!_pubs.ContainsKey(fn)) {
|
||||
foreach (var fn in file_names)
|
||||
{
|
||||
if (!_pubs.ContainsKey(fn))
|
||||
{
|
||||
_handle_new_file(fn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void _handle_new_file(string file_name) {
|
||||
if (file_name.EndsWith(".csv")) {
|
||||
private void _handle_new_file(string file_name)
|
||||
{
|
||||
if (file_name.EndsWith(".csv"))
|
||||
{
|
||||
ViewStream vs = new CSVReaderStream(file_name);
|
||||
vs = new SchemaGuesserStream(vs);
|
||||
var toks = file_name.Split('\\');
|
||||
|
||||
+2
-4
@@ -78,7 +78,7 @@ namespace BCG {
|
||||
#region IPubSubServerConnection Members
|
||||
|
||||
private void _send(byte[] data) {
|
||||
bts.Log.Gen.info($"PubSubServerManagedTCPConnection._send server: {_server_rc_info} data: {data.Length} is_sub: {_is_sub} _connection : {_connection != null}");
|
||||
bts.Log.Gen.info($"PubSubServerManagedTCPConnection._send client: {client_url} server: {_server_rc_info} data: {data.Length} is_sub: {_is_sub} _connection : {_connection != null}");
|
||||
if (_connection != null) {
|
||||
_connection.send(data);
|
||||
bts.Log.Gen.info("PubSubServerManagedTCPConnection.send, data is sent, server: " + _server_rc_info + "data: " + data.Length);
|
||||
@@ -91,9 +91,7 @@ namespace BCG {
|
||||
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_rc_info} ps_msg: {ps_msg}");
|
||||
// whys it constructed again?
|
||||
var msg_bytes = ps_msg.serialize_into_bytes();
|
||||
// PubSubClientMessage msg_retrv = new PubSubClientMessage();
|
||||
// msg_retrv.consruct_deserialize(msg_bytes);
|
||||
// bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_rc_info} msg_retrv: {msg_retrv}");
|
||||
//bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_rc_info} msg_retrv: {msg_retrv}");
|
||||
_send(msg_bytes);
|
||||
}
|
||||
|
||||
|
||||
@@ -6,23 +6,20 @@ using System.Threading;
|
||||
using BCG;
|
||||
//using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
namespace BCG {
|
||||
|
||||
/// <summary>
|
||||
/// Use this class for inter thread communication. It assumed a thread safe
|
||||
/// queue (this is right now implemented using locks, but for better performance
|
||||
/// will be switched over to lock free code, using cas).
|
||||
/// </summary>
|
||||
public class SyncQueue <T> : ISyncQueue<T>
|
||||
{
|
||||
public class SyncQueue<T> : ISyncQueue<T> {
|
||||
private ThreadSafeQueue<T> myQueue = new ThreadSafeQueue<T>();
|
||||
private AutoResetEvent myEvent = new AutoResetEvent(false);
|
||||
|
||||
public SyncQueue() {}
|
||||
public SyncQueue() { }
|
||||
|
||||
public void enqueue(T t)
|
||||
{
|
||||
public void enqueue(T t) {
|
||||
myQueue.enqueue(t);
|
||||
myEvent.Set();
|
||||
//return true;
|
||||
@@ -35,42 +32,31 @@ namespace BCG
|
||||
// return true;
|
||||
//}
|
||||
|
||||
public T dequeue()
|
||||
{
|
||||
public T dequeue() {
|
||||
return dequeue(Int32.MaxValue);
|
||||
}
|
||||
|
||||
public T dequeue (int timeout)
|
||||
{
|
||||
public T dequeue(int timeout) {
|
||||
DateTime startTime = DateTime.Now;
|
||||
T rv = myQueue.dequeue();
|
||||
if (rv != null || timeout <= 0)
|
||||
{
|
||||
if (rv != null || timeout <= 0) {
|
||||
return rv;
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
int timeRemaining = timeout;
|
||||
while (true)
|
||||
{
|
||||
while (true) {
|
||||
bool res = myEvent.WaitOne(timeRemaining, false);
|
||||
if (res)
|
||||
{
|
||||
if (res) {
|
||||
///timeout has not expired, so thread has been signalled
|
||||
rv = myQueue.dequeue();
|
||||
if (rv == null)
|
||||
{
|
||||
if (rv == null) {
|
||||
timeRemaining = timeout - Convert.ToInt32(((TimeSpan)(DateTime.Now - startTime)).TotalMilliseconds);
|
||||
if (timeRemaining <= 0)
|
||||
return rv;
|
||||
else
|
||||
continue;
|
||||
}
|
||||
else
|
||||
} else
|
||||
return rv;
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
///timeout expired, return default;
|
||||
return default(T);
|
||||
}
|
||||
@@ -79,24 +65,18 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public List<T> dequeueMultipleElements(int timeout, int maxElements)
|
||||
{
|
||||
if (timeout == 0)
|
||||
{
|
||||
public List<T> dequeueMultipleElements(int timeout, int maxElements) {
|
||||
if (timeout == 0) {
|
||||
return myQueue.dequeueLimited(maxElements);
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
DateTime startTime = DateTime.Now;
|
||||
List<T> rv = new List<T>();
|
||||
rv = myQueue.dequeueLimited(maxElements);
|
||||
if (rv.Count == maxElements)
|
||||
return rv;
|
||||
else
|
||||
{
|
||||
else {
|
||||
int timeRemaining = timeout;
|
||||
while (true)
|
||||
{
|
||||
while (true) {
|
||||
T t = dequeue(timeRemaining);
|
||||
if (t != null)
|
||||
rv.Add(t);
|
||||
@@ -108,19 +88,20 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public List<T> dequeueMultipleElements(TimeSpan ts, int maxElements)
|
||||
{
|
||||
public List<T> dequeueMultipleElements(TimeSpan ts, int maxElements) {
|
||||
return dequeueMultipleElements(Convert.ToInt32(ts.TotalMilliseconds), maxElements);
|
||||
}
|
||||
|
||||
public int getCount()
|
||||
{
|
||||
public List<T> dequeue_all() {
|
||||
return myQueue.dequeueAll();
|
||||
}
|
||||
|
||||
public int getCount() {
|
||||
return myQueue.getCount();
|
||||
}
|
||||
|
||||
|
||||
public void terminate()
|
||||
{
|
||||
public void terminate() {
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,8 @@ namespace BCG {
|
||||
/// <param name="sender"></param>
|
||||
/// <param name="data"></param>
|
||||
List<Pair<string, byte[]>> get_updates(TimeSpan timeout, int maxElements);
|
||||
// same as above, but gets all updates that are queued up
|
||||
List<Pair<string, byte[]>> get_all_updates();
|
||||
/// <summary>
|
||||
/// sends data to the receiver.
|
||||
/// </summary>
|
||||
|
||||
@@ -112,6 +112,10 @@ namespace BCG {
|
||||
return _received_data_queue.dequeueMultipleElements(timeout, maxElements);
|
||||
}
|
||||
|
||||
public List<Pair<string, byte[]>> get_all_updates() {
|
||||
return _received_data_queue.dequeue_all();
|
||||
}
|
||||
|
||||
private bool _enqueue_send(string receiver, byte[] data) {
|
||||
bts.Log.Gen.info($"ManagedTCPMessageHub._enqueue_send receiver: {receiver} data len: {data.Length}");
|
||||
_send_data_queue.enqueue(new Pair<string, byte[]>(receiver, data));
|
||||
@@ -154,7 +158,8 @@ namespace BCG {
|
||||
/// Enqeues both the connection name (to be reported back
|
||||
/// via ConnectionStatusChange) and connection (to be
|
||||
/// stored in the hashtable and used for later communication).
|
||||
/// This is called on an arbitrary thread
|
||||
/// This is called on an arbitrary thread. This is the only place where
|
||||
/// clients connect to the pubsubserver
|
||||
/// </summary>
|
||||
/// <param name="connection"></param>
|
||||
public void onConnection(ITCPConnection connection) {
|
||||
|
||||
@@ -101,10 +101,11 @@ namespace BCG {
|
||||
while (true) {
|
||||
try {
|
||||
//bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func pre get_updates");
|
||||
var received_data = _msg_hub.get_updates(poll_timeout, 1);
|
||||
var received_data = _msg_hub.get_all_updates();
|
||||
var n_msgs = received_data.Count;
|
||||
bts.Log.Gen.info($"PubSubMessageReactor._polling_thread_func post get_updates n_msgs: {n_msgs}");
|
||||
foreach (var kv in received_data) {
|
||||
// multiple pubs for the same tags should be batched here
|
||||
_handle_client_msg(kv.first, kv.second);
|
||||
}
|
||||
handle_internal_pubs();
|
||||
@@ -158,7 +159,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
private void _handle_sub(string sender, SubscribeMsg sub_msg) {
|
||||
//TODO: we make this correction here, since non managed clients are ntosending this correctly
|
||||
//TODO: we make this correction here, since C++ clients are not sending this correctly
|
||||
bts.Log.Gen.info($"PubSubMessageReactor._handle_sub sender: {sender} sub_msg: {sub_msg}");
|
||||
if (String.IsNullOrEmpty(sub_msg.connection) || sub_msg.connection.Contains("unset")) {
|
||||
sub_msg.connection = sender;
|
||||
@@ -173,10 +174,10 @@ namespace BCG {
|
||||
//ie data has been published once
|
||||
if (data != null) {
|
||||
bts.Log.Gen.info($"ManagedTCPMessgeHub._handle_sub get_content tag: {tag} data len: {data.Length}");
|
||||
string tagType = _state.getTagType(tag);
|
||||
string tag_type_ = _state.getTagType(tag);
|
||||
var msg_to_send = new PubSubServerMessage() {
|
||||
tag = tag,
|
||||
tag_type = tagType,
|
||||
tag_type = tag_type_,
|
||||
data = data
|
||||
};
|
||||
bts.Log.Gen.info($"ManagedTCPMessgeHub._handle_sub sending msg_to_send: {msg_to_send} connection: {sub_msg.connection}");
|
||||
|
||||
@@ -127,6 +127,7 @@ namespace BCG {
|
||||
tags = tag_args
|
||||
}
|
||||
};
|
||||
Log.Gen.info($"SingleServerSubscriberCore._subscribe sending msg: {msg}");
|
||||
_connection.send(msg);
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user