Files
spark/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs
T
John Colvin a05c7410dc refactor: Use DependencyInjection paradigm in DataStreamSuite
This allows us to create loosely coupled services with automatic
dependency resolution
2024-07-02 11:30:14 -04:00

196 lines
8.0 KiB
C#

using BCG;
using BCG.Utilities.General;
using System.Collections.Generic;
//using MongoDB.Driver;
using System;
using System.IO;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
namespace bts {
public class PubSubTestingStream : ViewStream, IWorker {
public class PayloadMessage {
public int msg_id = 0;
public string guid;
public Int64 travel_epoch_ms = 0;
public byte[] payload = null;
public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte) {
msg_id = msg_id_args;
travel_epoch_ms = send_epoch_args;
payload = new byte[payload_size];
for (int i = 0; i < payload_size; i++) {
payload[i] = payload_byte;
}
}
private PayloadMessage() { }
public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte) { }
public byte[] serialize() {
var str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bw.Write(msg_id);
bw.Write(guid);
bw.Write(travel_epoch_ms);
bw.Write(payload.Length);
bw.Write(payload);
str.Position = 0;
var bytes = str.ToArray();
return bytes;
}
public static PayloadMessage deserialize(byte[] ser_bytes) {
var str = new MemoryStream(ser_bytes);
BinaryReader br = new BinaryReader(str);
int msg_id = br.ReadInt32();
string guid = br.ReadString();
Int64 epoch = br.ReadInt64();
int len = br.ReadInt32();
var payload = br.ReadBytes(len);
return new PayloadMessage() {
msg_id = msg_id,
guid = guid,
travel_epoch_ms = epoch,
payload = payload
};
}
}
//public void
private readonly int _msg_size;
private readonly TimeSpan _publish_period;
private DateTime _last_publish_time = DateTime.MinValue;
private IPublisherCore? _pub_core_backing;
private BlueTable _table;
private int _n_pubs = 0;
private readonly int _max_pubs;
private int _n_updates = 0;
private bool _continue_sub_thread = true;
private bool _fire_dirty = false;
private PayloadMessage _last_sent_msg, _last_recvd_msg;
private readonly int _payload_sz;
private const byte _payload_byte = (byte)'A';
private readonly string _tag_name;
public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name) {
_payload_sz = payload_size;
var schema = BlueDataUtility.form_schema("PubSubTest",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("n_pubs", TypeCode.Int32),
Tuple.Create("n_recvs", TypeCode.Int32),
Tuple.Create("last_sent_id", TypeCode.Int32),
Tuple.Create("last_recv_id", TypeCode.Int32),
Tuple.Create("last_sent_guid", TypeCode.String),
Tuple.Create("last_recv_guid", TypeCode.String),
Tuple.Create("match", TypeCode.Boolean),
Tuple.Create("echo_time", TypeCode.Int32));
_table = new BlueTable(schema);
_publish_period = TimeSpan.FromMilliseconds(publish_period_msecs);
_msg_size = payload_size;
_max_pubs = max_pubs;
_tag_name = tag_name;
}
IPublisherCore? _pub_core =>
_pub_core_backing ??= ActivatorUtilities.CreateInstance<IPublisherCore>(ServiceProvider);
protected override void myOnFirstSubscriber() {
base.myOnFirstSubscriber();
System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
th.IsBackground = true;
th.Start();
}
protected override void myOnLastSubscriber() {
base.myOnLastSubscriber();
_continue_sub_thread = false;
}
void _log_updates(IEnumerable<PubSubServerMessage> updates) {
var idx = 0;
foreach (var msg in updates) {
bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
idx++;
}
}
private void _subscribe_thread_func() {
// subscription is done in another thread
var sub_core = ActivatorUtilities.CreateInstance<SubscriberCore>(ServiceProvider);
sub_core.subscribe(_tag_name);
while (_continue_sub_thread) {
// TODO recv_id
var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
if (updates == null || updates.Count() == 0)
continue;
_log_updates(updates);
_n_updates += updates.Count();
if (updates.Count() > 0) {
var server_msg = updates.Last();
if (server_msg.data != null && server_msg.data.Length > 0) {
_last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
_last_recvd_msg.travel_epoch_ms = epoch_now();
} else {
// why are we getting empty or null data ? (server is sending data regardless?)
}
_fire_dirty = true;
}
}
}
private void _publish_next_packet() {
if (_last_sent_msg == null) {
_last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte) {
guid = Guid.NewGuid().ToString()
};
} else {
_last_sent_msg.msg_id++;
_last_sent_msg.guid = Guid.NewGuid().ToString();
_last_sent_msg.travel_epoch_ms = epoch_now();
}
var ser_bytes = _last_sent_msg.serialize();
_n_pubs++;
Log.notnull(_pub_core).publish(string.Empty, _tag_name, ser_bytes);
_fire_dirty = true;
}
private void _fire_dp() {
object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
object echo_time = DBNull.Value;
object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
if (_last_sent_msg != null && _last_recvd_msg != null) {
echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
}
var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
IBlueRow nr = new BlueRow(_table.schema, 0, _n_pubs, _n_updates,
last_sent_id, last_recv_id, last_sent_guid, last_recv_guid, match, echo_time);
BlueTable table = new BlueTable(_table.schema);
table.addRow(nr);
var dp = BlueDataUtility.build_dp_inc(table, myLastValue as DataPacket, true);
myFireEvent(dp);
}
public void doWork() {
if (DateTime.Now - _last_publish_time > _publish_period)
if (_n_pubs <= _max_pubs) {
_publish_next_packet();
_last_publish_time = DateTime.Now;
}
// publish
// TODO: send id
if (_fire_dirty) {
_fire_dirty = false;
_fire_dp();
}
}
public static long epoch_now() {
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
}
}
}