Files
spark/src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs
T
2024-07-17 02:02:01 -04:00

223 lines
7.4 KiB
C#

using BCG;
using BCG.Utilities.General;
using System.Collections.Generic;
using System;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using bts.api;
namespace bts
{
[DataStreamUserRespresentation("pss_echo")]
public class PubsubEchoTestStream : ViewStream, IWorker
{
public class PayloadMessage
{
public readonly string msg_id;
public readonly Int64 timestamp = 0;
public readonly byte[] payload = null;
public static byte[] create_data(int payload_size, byte payload_byte)
{
var rv = new byte[payload_size];
for (int i = 0; i < payload_size; i++)
{
rv[i] = payload_byte;
}
return rv;
}
// public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
// {
// guid = guid_;
// timestamp = timestamp_;
// payload = new byte[payload_size];
// for (int i = 0; i < payload_size; i++)
// {
// payload[i] = payload_byte;
// }
// }
public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
{
msg_id = guid_;
timestamp = timestamp_;
payload = payload_;
}
private PayloadMessage() { }
public byte[] serialize()
{
var str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bw.Write(msg_id);
bw.Write(timestamp);
bw.Write(payload.Length);
bw.Write(payload);
str.Position = 0;
var bytes = str.ToArray();
return bytes;
}
public static PayloadMessage deserialize(byte[] ser_bytes)
{
var str = new MemoryStream(ser_bytes);
BinaryReader br = new BinaryReader(str);
string guid = br.ReadString();
Int64 timestamp_ = br.ReadInt64();
int len = br.ReadInt32();
var payload = br.ReadBytes(len);
return new PayloadMessage(guid, timestamp_, payload);
}
}
//public void
private readonly int _msg_size;
private readonly TimeSpan _publish_period;
private DateTime _last_publish_time = DateTime.MinValue;
private PublisherCore _pub_core;
private ISubscriberCore _sub_core;
private BlueTable _table;
private int _n_pubs = 0;
private readonly int _max_pubs;
private int _n_updates = 0;
private ViewStreamDatabase _vsd;
private PayloadMessage _last_sent_msg, _last_recvd_msg;
private const byte _payload_byte = (byte)'A';
private readonly string _tag_name;
private readonly int _payload_sz;
private readonly IPubSubRouter _router;
private enum IOState
{
SEND = 0,
RECV = 1
};
IOState _state = IOState.SEND;
public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
{
var route = new PubSubRoute(server_url, ".*");
_router = new PubSubRouter(route);
_vsd = vsd;
_payload_sz = payload_size;
var schema = BlueDataUtility.form_schema("PubSubTest",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("sent_id", TypeCode.String),
Tuple.Create("sent_time", TypeCode.Int64),
Tuple.Create("recv_id", TypeCode.String),
Tuple.Create("recv_time", TypeCode.Int64),
Tuple.Create("match", TypeCode.Boolean),
Tuple.Create("echo_time", TypeCode.Int32));
_table = new BlueTable(schema);
_publish_period = publish_period;
_tag_name = tag_name;
}
public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
: this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
protected override void myOnFirstSubscriber()
{
_pub_core = new PublisherCore(_router);
_sub_core = new SubscriberCore(_router);
_sub_core.subscribe(_tag_name);
_vsd.addWorker(this);
}
protected override void myOnLastSubscriber()
{
_vsd.removeWorker(this);
_sub_core.unsubscribe(_tag_name);
}
private void _fire_dp_after_send()
{
// as soon as the data is published, its reflected in the stream state
Log.assert(_last_sent_msg != null);
BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
DBNull.Value, DBNull.Value);
_table.addRow(r);
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
myFireEvent(dp_out);
}
private void _fire_dp_after_recv()
{
Log.assert(_last_recvd_msg != null);
// update the last publish row
int id = _table.Count - 1;
IBlueRow last_row = _table[id];
_table.deleteRow(last_row);
var recv_ts = epoch_now();
BlueRow nr = new BlueRow(_table.schema, id, _last_sent_msg.msg_id, _last_sent_msg.timestamp, _last_recvd_msg.msg_id, recv_ts,
String.Compare(_last_recvd_msg.msg_id, _last_sent_msg.msg_id) == 0, recv_ts - _last_recvd_msg.timestamp);
_table.addRow(nr);
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
myFireEvent(dp_out);
}
private string _get_next_id()
{
return _table.Count.ToString();
}
private void _publish_next_packet()
{
var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
_last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
var ser_bytes = _last_sent_msg.serialize();
_n_pubs++;
_pub_core.publish("Blob", _tag_name, ser_bytes);
_last_publish_time = DateTime.Now;
}
public void doWork()
{
Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
{
foreach (var u in updates)
{
var recv_msg = PayloadMessage.deserialize(u.data);
if (recv_msg.msg_id == _last_sent_msg.msg_id)
{
return recv_msg;
}
}
return null;
};
switch (_state)
{
case IOState.SEND:
if (DateTime.Now - _last_publish_time >= _publish_period)
{
_publish_next_packet();
_last_publish_time = DateTime.Now;
_fire_dp_after_send();
_state = IOState.RECV;
return;
}
break;
case IOState.RECV:
var updates = _sub_core.get_updates(TimeSpan.Zero);
updates = updates.Where(u => u.tag == _tag_name);
var recv_msg = find_packet(updates);
if (recv_msg != null)
{
_last_recvd_msg = recv_msg;
_fire_dp_after_recv();
_state = IOState.SEND;
break;
}
break;
}
}
public static long epoch_now()
{
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
}
}
}