Files
spark/src/DataStreamSuite/DataStream/Primitives/Misc/DataPacketRecorderStream.cs
T
2024-06-06 09:20:05 -05:00

142 lines
5.3 KiB
C#

using BCG;
using System;
using System.IO;
using MongoDB.Bson;
using BCG.Utilities.General;
using System.Linq;
using System.Collections.Generic;
using System.Runtime.Intrinsics.X86;
namespace bts {
// listens to the underlying stream, and writes each seriialized packet in a
// seperate file
[DataStreamUserRespresentation("dp_recorder")]
public class DataPacketRecorderStream : ViewStream {
private ViewStream _base_stream;
private readonly string _folder;
StreamIOSerializationController _serializer = new StreamIOSerializationController();
private UInt32 _n_updates = 0;
public DataPacketRecorderStream(ViewStream base_stream, string folder) {
_base_stream = base_stream;
_folder = folder;
}
protected override void myOnFirstSubscriber() {
_base_stream.onChange += _handle_update;
}
protected override void myOnLastSubscriber() {
_base_stream.onChange -= _handle_update;
}
private void _handle_update(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
byte[] ser_data = null;
if (_n_updates == 0) {
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
ser_data = _serializer.serialize(new StreamIOMessage(cr_msg));
} else {
StreamIOMessage.MultipleRemoveAddMessage mra = new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows);
ser_data = _serializer.serialize(new StreamIOMessage(mra));
}
Log.assert(ser_data != null);
_n_updates++;
string file_name = $"{_folder}\\{_n_updates}.bin";
_write_to_file(ser_data, file_name);
myFireEvent(dp);
}
private static void _write_to_file(byte[] data, string file_name) {
using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
// Write the byte array to the file
fs.Write(data, 0, data.Length);
}
}
}
[DataStreamUserRespresentation("dp_replay")]
public class DataPacketReplayStream : ViewStream, IWorker {
private ViewStreamDatabase _vsd;
private readonly string _folder_path;
private readonly TimeSpan _fire_period;
IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
private int _file_ptr = 0;
private List<Tuple<string, byte[]>> _file_data = new List<Tuple<string, byte[]>>();
private BlueTable _state = null;
public DataPacketReplayStream(string folder_path, string fire_period)
:
this(ViewStreamDatabase.instance, folder_path, TimeBasedFilter.parse_duration(fire_period)) {
}
public DataPacketReplayStream(ViewStreamDatabase vsd, string folder_path, TimeSpan fire_period) {
_vsd = vsd;
_folder_path = folder_path;
_fire_period = fire_period;
}
protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, _fire_period);
_load_all_files();
}
protected override void myOnLastSubscriber() {
MasterWorker.getInstance(false).removeWorker(this);
}
public void doWork() {
if (_file_ptr >= _file_data.Count())
// we can set _file_ptr to 0 and keep going as well
return;
var curr_d = _file_data[_file_ptr++];
var hext_txt = StreamSerializationUtils.to_hex(curr_d.Item2);
StreamIOMessage msg = _deser.deserialize(curr_d.Item2);
Log.Gen.info($"DataPacketReplayStream.doWork packet no : {_file_ptr} file : {curr_d.Item1} data: {hext_txt}");
switch (msg.op) {
case StreamIOMessage.StreamOp.CREATE: {
_state = new BlueTable(msg.create_message.rows);
break;
}
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: {
var mra = msg.multiple_remove_add_message;
foreach (IBlueRow r in mra.removed_rows)
_state.deleteRow(r);
foreach (IBlueRow r in mra.added_rows)
_state.addRow(r);
break;
}
}
var dp = BlueDataUtility.build_dp_inc(_state, myLastValue == null ? null : myLastValue as DataPacket, false);
myFireEvent(dp);
}
private void _load_all_files() {
Func<string, string> extract_file_name = (string full_path) => {
string fn = full_path.Split("\\").Last();
fn = fn.Split(".").First();
return fn;
};
string[] bin_files = Directory.GetFiles(_folder_path, "*.bin");
// Iterate over the .bin files and perform an action
foreach (string file in bin_files) {
byte[] file_data = File.ReadAllBytes(file);
_file_data.Add(Tuple.Create(file, file_data));
}
_file_data.Sort((Tuple<string, byte[]> t1, Tuple<string, byte[]> t2) => {
string fn1 = extract_file_name(t1.Item1);
string fn2 = extract_file_name(t2.Item1);
int fn_i1 = Int32.Parse(fn1);
int fn_i2 = Int32.Parse(fn2);
if (fn_i1 < fn_i2)
return -1;
else if (fn_i1 == fn_i2)
return 0;
else if (fn_i1 > fn_i2)
return 1;
return 0;
});
}
}
}