Files
spark/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs
T

810 lines
39 KiB
C#

using bts;
using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
namespace BCG {
/// <summary>
/// This stream (not for prod) provides a way to run tests - that could otherwise be run in NUnits.
/// (truth is stream rpovide a RDE for quickly coding and running, so it was eaiser to run tests here)
/// </summary>
[DataStreamUserRespresentation("tests")]
public class TesStream : ViewStream {
static byte[] create_byte_array() {
Random random = new Random();
var sz = random.Next(1, 1000);// % 1000;
// Generate a random byte
byte randomByte = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
byte[] rv = new byte[sz];
for (int i = 0; i < sz; i++) {
rv[i] = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
}
return rv;
}
private IBlueTableSchema _schema;
class TestResult {
public string test_name = MethodBase.GetCurrentMethod()?.Name
?? throw new InvalidOperationException("Failed to get test name from method");
public bool test_result;
}
private List<Func<TestResult>> _tests = new
List<Func<TestResult>>();
public TesStream() {
_schema = BlueDataUtility.form_schema("Tests",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("test", TypeCode.String),
Tuple.Create("result", TypeCode.Boolean),
Tuple.Create("exception", TypeCode.String)
);
// register tests here
_tests.Add(new Func<TestResult>(_run_sub_msg_test));
_tests.Add(new Func<TestResult>(_run_pub_msg_test));
_tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
_tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
_tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
// _tests.Add(new Func<TestResult>(_run_schema_serialization_test));
// _tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls));
// _tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls));
//_tests.Add(new Func<TestResult>(_row_serialization_tests_all_types));
// _tests.Add(new Func<TestResult>(_run_full_stream_serialization_test));
// _tests.Add(new Func<TestResult>(_run_full_stream_serialization_with_controller_test));
// _tests.Add(new Func<TestResult>(_run_pubsub_server_stream_tag_handler_test));
// _tests.Add(new Func<TestResult>(_run_single_tag_data_test_clean));
//_tests.Add(new Func<TestResult>(_cpp_data_test));
//_tests.Add(new Func<TestResult>(_write_stream_updates_to_files));
_tests.Add(new Func<TestResult>(_deserialize_pub_msg_test));
}
protected override void myOnFirstSubscriber() {
BlueTable rows = new BlueTable(_schema);
// u the tests here
foreach (var test in _tests) {
string ex_msg = String.Empty;
TestResult tr = new TestResult() {
test_name = test.Method.Name,
test_result = false
};
try {
tr = test();
} catch (Exception ex) {
ex_msg = ex.Message + "\n" + ex.StackTrace;
}
IBlueRow r = new BlueRow(_schema, rows.Count, tr.test_name, tr.test_result, ex_msg);
rows.addRow(r);
}
var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
myFireEvent(dp);
}
protected override void myOnLastSubscriber() {
}
private TestResult _run_sub_msg_test() {
bts.SubscribeMsg sub_msg = new bts.SubscribeMsg() {
connection = Guid.NewGuid().ToString(),
subscribing_app = Guid.NewGuid().ToString(),
tags = new string[] { "one", "two", "three" }
};
MemoryStream str = new MemoryStream();
sub_msg.serialize_into_str(str);
str.Position = 0;
bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
sub_msg_.consruct_deserialize(str);
bool res = sub_msg.Equals(sub_msg_);
return new TestResult {
test_name = "SubscribeMsg",
test_result = res
};
}
private TestResult _run_pub_msg_test() {
var pub_msg = new bts.PublishMsg() {
connection = Guid.NewGuid().ToString(),
publishing_app = Guid.NewGuid().ToString(),
tag = Guid.NewGuid().ToString(),
tag_type = "DS_",
is_compressed = false,
publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
data_to_publish = create_byte_array(),
};
MemoryStream str = new MemoryStream();
pub_msg.serialize_into_str(str);
str.Position = 0;
var pub_msg_ = new bts.PublishMsg();
pub_msg_.consruct_deserialize(str);
bool res = pub_msg.Equals(pub_msg_);
return new TestResult {
test_name = "PublishMsg",
test_result = res
};
}
private TestResult _run_pubsub_server_msg_test() {
var msg = new bts.PubSubServerMessage() {
tag = Guid.NewGuid().ToString(),
tag_type = Guid.NewGuid().ToString(),
data = create_byte_array()
};
MemoryStream str = new MemoryStream();
msg.serialize_into_str(str);
str.Position = 0;
var msg_ = new bts.PubSubServerMessage();
msg_.consruct_deserialize(str);
bool res = msg.Equals(msg_);
return new TestResult {
test_name = "PubSubServerMessage",
test_result = res
};
}
private TestResult _run_pubsub_client_msg_test() {
PubSubClientMessage msg = new PubSubClientMessage() {
msg_type = PubSubMessageType.SUBSCRIBE,
sub_msg = new bts.SubscribeMsg() {
connection = Guid.NewGuid().ToString(),
subscribing_app = Guid.NewGuid().ToString(),
tags = new string[] { "one", "two", "three" }
}
};
var str = new MemoryStream();
msg.serialize_into_str(str);
str.Position = 0;
var msg_ = new PubSubClientMessage();
msg_.consruct_deserialize(str);
var res = msg.Equals(msg_);
return new TestResult {
test_name = "PubSubClientMessage",
test_result = res
};
}
private TestResult _run_columninfo_serialization_test() {
Func<ColumnInfo, bool> test_ci = (ci) => {
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
StreamSerializationUtils.serialize_column_info(ci, ref bw);
str.Position = 0;
//var ci_txt = BlueDataUtility.to_string(ci);
//var ci_ser =
//bts.Log.Gen.info("ci: {}")
BinaryReader br = new BinaryReader(str);
var ci_prime = StreamSerializationUtils.deserialize_column_info(ref br);
return BlueDataUtility.compute_equality(ci, ci_prime);
};
int i = 0;
foreach (Type t in BlueDataUtility.valid_column_types()) {
ColumnInfo ci = new ColumnInfo(Guid.NewGuid().ToString(),
i++, t);
var res = test_ci(ci);
if (!res) {
return new TestResult {
test_name = "ColumnInfoSerialization",
test_result = false
};
}
}
return new TestResult {
test_name = "ColumnInfoSerialization",
test_result = true
};
}
private TestResult _run_schema_serialization_test() {
var table_name = "test_table";
var excluded_types = new HashSet<Type>() { typeof(DateTime), typeof(Decimal) };
IBlueTableSchema schema = create_schema_with_all_types(excluded_types);
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
StreamSerializationUtils.serialize_schema(schema, ref bw);
_write_to_file(str, @"c:\Temp\tests\schema.bin");
var hex_txt = StreamSerializationUtils.to_hex(str);
bts.Log.Gen.info($"_run_schema_serialization_test ser: {hex_txt}");
str.Position = 0;
BinaryReader br = new BinaryReader(str);
var schema_prime = StreamSerializationUtils.deserialize_schema(ref br);
var res = BlueDataUtility.compute_equality(schema, schema_prime);
return new TestResult {
test_name = "SchemaSerialization",
test_result = res
};
}
private TestResult _run_datetime_seriliaztion_test() {
DateTime dt = DateTime.Now;
DateTimeOffset dt_off = new DateTimeOffset(dt);
long dt_epoch_ms = dt_off.ToUnixTimeMilliseconds();
DateTimeOffset dt_off_pr = DateTimeOffset.FromUnixTimeMilliseconds(dt_epoch_ms);
DateTime dt_p = dt_off_pr.DateTime;
var passes = dt.Equals(dt_p);
return new TestResult {
test_name = "_run_datetime_seriliaztion_test",
test_result = false
};
}
private TestResult _run_full_stream_serialization_test() {
// create message
var schema = create_schema_with_all_types();
BlueTable cr_rows = new BlueTable(schema);
// 100 rows
int n_rows = 100;
for (int i = 0; i < n_rows; i++) {
var nr = generate_random_row(schema, i);
cr_rows.addRow(nr);
}
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
UniversalStreamIOSerializer ser = new UniversalStreamIOSerializer();
ser.serialize(stream_io_msg, ref bw);
str.Position = 0;
BinaryReader br = new System.IO.BinaryReader(str);
UniversalStreamIODeserializer deser = new bts.UniversalStreamIODeserializer();
var stream_io_msg_prime = deser.deserialize(ref br);
bool passes = stream_io_msg.Equals(stream_io_msg_prime);
var n_dels = 50;
BlueTable del_rows = new BlueTable(cr_rows.schema);
foreach (IBlueRow r in cr_rows) {
del_rows.addRow(r);
if (del_rows.Count == n_dels)
break;
}
IBlueRowCollection added_rows = del_rows;
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
str = new MemoryStream();
bw = new BinaryWriter(str);
stream_io_msg = new bts.StreamIOMessage(mra_msg);
ser.serialize(stream_io_msg, ref bw);
bw.BaseStream.Position = 0;
br = new BinaryReader(str);
//deser = new UniversalStreamIODeserializer();
stream_io_msg_prime = deser.deserialize(ref br);
passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
return new TestResult {
test_name = "UniversalSerialization",
test_result = passes
};
}
private TestResult _run_full_stream_serialization_with_controller_test() {
// create message
var schema = create_schema_with_all_types();
BlueTable cr_rows = new BlueTable(schema);
// 100 rows
int n_rows = 100;
for (int i = 0; i < n_rows; i++) {
var nr = generate_random_row(schema, i);
cr_rows.addRow(nr);
}
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
IStreamIOMsgSerializer ser = new StreamIOSerializationController();
var ser_bytes = ser.serialize(stream_io_msg);
IStreamMsgDeserilizer deser = new bts.StreamIODeserializationController();
var stream_io_msg_prime = deser.deserialize(ser_bytes);
bool passes = stream_io_msg.Equals(stream_io_msg_prime);
//mult remove adds
// remove all rows and add 50 more
BlueTable del_rows = new BlueTable(cr_rows);
BlueTable added_rows = new BlueTable(cr_rows.schema);
var n_adds = 50;
for (int i = 0; i < n_adds; i++) {
var nr = generate_random_row(schema, i);
added_rows.addRow(nr);
}
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
stream_io_msg = new bts.StreamIOMessage(mra_msg);
ser_bytes = ser.serialize(stream_io_msg);
var ser_bytes_hex = StreamSerializationUtils.to_hex(ser_bytes);
Log.Gen.info($"ser_bytes: {ser_bytes_hex}");
stream_io_msg_prime = deser.deserialize(ser_bytes);
passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
return new TestResult {
test_name = "UniversalSerializationController",
test_result = passes
};
}
private TestResult _row_serialization_tests_no_dbnulls() {
// IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
// new Tuple<string, TypeCode>("id", TypeCode.Int32),
// new Tuple<string, TypeCode>("i1", TypeCode.Int32),
// new Tuple<string, TypeCode>("d1", TypeCode.Double),
// new Tuple<string, TypeCode>("string_1", TypeCode.String));
// bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
// IBlueRow r = new BlueRow(schema, 0, 100, 123.45, "test_string");
var schema = create_schema_with_all_types(new HashSet<Type>() { typeof(DateTime), typeof(Decimal) });
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
MemoryStream str = new MemoryStream();
IBlueRow r = generate_random_row(schema, 0, false);
BinaryWriter bw = new BinaryWriter(str);
row_ser.serialize_row(r, 0, ref bw);
_write_to_file(str, @"C:\Temp\tests\row.bin");
str.Position = 0;
byte[] raw_byte_array = str.ToArray();
var hex_txt = StreamSerializationUtils.to_hex(raw_byte_array);
Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls Row bytes: {hex_txt}");
BinaryReader br = new BinaryReader(str);
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
IBlueRow r_prime = des.deserialize_row(ref br);
string row_str = r_prime.to_string();
Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls r_prime: {row_str}");
var passes = BlueDataUtility.compute_equality(r, r_prime);
return new TestResult {
test_name = "UniversalRowSerialization_NonNull",
test_result = passes
};
}
private TestResult _deserialize_pub_msg_test() {
var pub_msg_ser = _read_from_file(@"C:\Temp\tests\pubsub_raw_msgs\publish_msg.bin");
PubSubClientMessage ps_client_msg = new PubSubClientMessage();
ps_client_msg.consruct_deserialize(pub_msg_ser);
bool passes = ps_client_msg.msg_type == PubSubMessageType.PUBLISH;
return new TestResult {
test_name = "_deserialize_pub_msg_test",
test_result = passes
};
}
private TestResult _cpp_data_test() {
//create and write schema
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
new Tuple<string, TypeCode>("id", TypeCode.Int32),
new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
new Tuple<string, TypeCode>("d_1", TypeCode.Double),
new Tuple<string, TypeCode>("string_1", TypeCode.String)
);
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
UniversalRowDeserializer row_deser = new UniversalRowDeserializer(schema);
StreamIOSerializationController ser_controller = new StreamIOSerializationController();
StreamIODeserializationController deser_controller = new StreamIODeserializationController();
IBlueTableSchema schema_p = null;
var schema_file = @"c:\Temp\tests\schema.bin";
var row_file = @"C:\Temp\tests\row.bin";
var create_op_file = @"C:\Temp\tests\create_op.bin";
var mra_op_file = @"C:\Temp\tests\mra_op.bin";
IBlueRow row = new BlueRow(schema, 0, 100, 200, 123.45, "test_string");
//serialize schema, write it to a file, read it back, deserialize it and match schemas
byte[] ser_schema_bytes = StreamSerializationUtils.serialize_schema(schema);
_write_to_file(ser_schema_bytes, schema_file);
byte[] schema_file_bytes = _read_from_file(schema_file);
schema_p = StreamSerializationUtils.deserialize_schema(schema_file_bytes);
bool passes = BlueDataUtility.compute_equality(schema, schema_p);
//serialize row, write it to a file, read it back, deserialize it and match schemas
byte[] ser_row_bytes = row_ser.serialize_row(row);
_write_to_file(ser_row_bytes, row_file);
byte[] row_file_bytes = _read_from_file(row_file);
var row_file_bytes_txt = StreamSerializationUtils.to_hex(row_file_bytes);
Log.Gen.info($"TestsStream._cpp_data_test row_file_bytes_txt: {row_file_bytes_txt}");
IBlueRow row_p = row_deser.deserialize_row(row_file_bytes);
passes = passes && BlueDataUtility.compute_equality(row, row_p);
{
// serialize a create operation
int n_rows = 100;
BlueTable table = new BlueTable(schema);
for (int i = 0; i < n_rows; i++) {
IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
table.addRow(nr);
}
StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
//StreamIOSerializationController ser_controller = new StreamIOSerializationController();
byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
_write_to_file(create_ser_bytes, create_op_file);
// read it and make sure its right
byte[] create_op_file_bytes = _read_from_file(create_op_file);
//StreamIODeserializationController deser_controller = new StreamIODeserializationController();
var cr_msg_p = deser_controller.deserialize(create_op_file_bytes);
_write_to_file(create_ser_bytes, mra_op_file);
passes = passes && cr_msg.Equals(cr_msg_p);
}
//multiple remove adds
{
// remove all rows and add them again
int n_rows = 100;
BlueTable table = new BlueTable(schema);
for (int i = 0; i < n_rows; i++) {
IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
table.addRow(nr);
}
StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(table, table));
//StreamIOSerializationController ser_controller = new StreamIOSerializationController();
byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
_write_to_file(mra_ser_bytes, mra_op_file);
// reasd it and make sure it matches
byte[] mra_op_file_bytes = _read_from_file(mra_op_file);
//StreamIODeserializationController deser_controller = new StreamIODeserializationController();
StreamIOMessage mra_msg_p = deser_controller.deserialize(mra_op_file_bytes);
passes = passes && mra_msg.Equals(mra_msg_p);
}
return new TestResult {
test_name = "_cpp_data_test",
test_result = passes
};
}
private TestResult _write_stream_updates_to_files() {
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
new Tuple<string, TypeCode>("id", TypeCode.Int32),
new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
new Tuple<string, TypeCode>("d_1", TypeCode.Double),
new Tuple<string, TypeCode>("string_1", TypeCode.String)
);
var packets_path = @"C:\Temp\tests\stream_packets\";
Func<long> now_epochs = () => {
DateTimeOffset now = DateTimeOffset.UtcNow;
// Convert to Unix time in milliseconds
long unix_time_milliseconds = now.ToUnixTimeMilliseconds();
return unix_time_milliseconds;
};
Func<int, bool, string> curr_file_path = (id, create) => {
//var epoch_ms = now_epochs();
var file_path = $"{packets_path}{id}.bin";
if (create)
file_path = $"{packets_path}create_{id}.bin";
return file_path;
};
// write 1000 stream updates
{
int init_sz = 100;
BlueTable table = new BlueTable(schema);
StreamIOSerializationController ser_controller = new StreamIOSerializationController();
// create table of 100 rows
for (int i = 0; i < init_sz; i++) {
IBlueRow nr = new BlueRow(schema, i, i * 100, i * 200, i * 123.45, "test_string");
table.addRow(nr);
}
// write create msg
StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
var create_file_path = curr_file_path(table.Count, true);
_write_to_file(create_ser_bytes, create_file_path);
// add 1000 rows and write an io msg for each update and write it to a file
int n_updates = 1000;
int period_ms = 50;
// write a create
for (int i = 0; i < n_updates; i++) {
// addition
BlueTable adds = new BlueTable(schema);
int id = table.Count;
IBlueRow nr = new BlueRow(schema, id, i * 100, i * 200, i * 123.45, "test_string");
adds.addRow(nr);
BlueTable dels = new BlueTable(schema);
table.addRow(nr);
StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(dels, adds));
byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
var mra_file_path = curr_file_path(id, false);
_write_to_file(mra_ser_bytes, mra_file_path);
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(period_ms));
}
// now start writing increments and add 1 row at a time
}
return new TestResult {
test_name = "_write_updates_as_files",
test_result = true
};
}
private TestResult _row_serialization_tests_with_dbnulls() {
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
new Tuple<string, TypeCode>("id", TypeCode.Int32),
new Tuple<string, TypeCode>("i1", TypeCode.Int32),
new Tuple<string, TypeCode>("d1", TypeCode.Double),
new Tuple<string, TypeCode>("string_1", TypeCode.String));
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
IBlueRow r = new BlueRow(schema, 0, DBNull.Value, 123.45, "test_string");
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
row_ser.serialize_row(r, 0, ref bw);
str.Position = 0;
BinaryReader br = new BinaryReader(str);
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
IBlueRow r_prime = des.deserialize_row(ref br);
var passes = BlueDataUtility.compute_equality(r, r_prime);
return new TestResult {
test_name = "UniversalRowSerialization_WithDBNulls",
test_result = passes
};
}
private TestResult _row_serialization_tests_all_types() {
var schema = create_schema_with_all_types();
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
IBlueRow r = generate_random_row(schema, 0, false);
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
row_ser.serialize_row(r, 0, ref bw);
_write_to_file(str, @"C:\Temp\tests\row.bin");
str.Position = 0;
BinaryReader br = new BinaryReader(str);
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
IBlueRow r_prime = des.deserialize_row(ref br);
var passes = BlueDataUtility.compute_equality(r, r_prime);
return new TestResult {
test_name = "UniversalRowSerialization_AllTypes",
test_result = passes
};
}
private static TestResult _run_pubsub_server_stream_tag_handler_test() {
// create message
var schema = create_schema_with_all_types();
var tag = "DS_" + schema.tableName;
BlueTable cr_rows = new BlueTable(schema);
// 100 rows
int n_rows = 100;
for (int i = 0; i < n_rows; i++) {
var nr = generate_random_row(schema, i);
cr_rows.addRow(nr);
}
PubsubServerSingleStreamTagHandler handler = new PubsubServerSingleStreamTagHandler();
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
bts.StreamIOMessage str_io_msg = new bts.StreamIOMessage(cr_msg);
IStreamIOMsgSerializer ser = new StreamIOSerializationController();
byte[] msg_bytes = ser.serialize(str_io_msg);
handler.update(tag, msg_bytes);
//now get it back for the handler
byte[] content = handler.get_content(tag);
IStreamMsgDeserilizer deser = new StreamIODeserializationController();
bts.StreamIOMessage str_io_msg_p = deser.deserialize(content);
// test here
var passes = str_io_msg.Equals(str_io_msg_p);
// delete all rows
BlueTable del_rows = new BlueTable(cr_rows);
cr_rows.clear();
// added 50 new rows
BlueTable added_rows = new BlueTable(del_rows.schema);
int new_state_sz = 50;
for (int i = 0; i < new_state_sz; i++) {
var nr = generate_random_row(schema, i);
added_rows.addRow(nr);
}
cr_rows.addRows(added_rows);
// the sequence of operations are ops, ser, update, get_content, deser, maake sure the get_content state is good
// 1. generate op
bts.StreamIOMessage.MultipleRemoveAddMessage mra = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
str_io_msg = new bts.StreamIOMessage(mra);
// 2. serialize it into bytes
msg_bytes = ser.serialize(str_io_msg);
// 3. send it to handler
handler.update(tag, msg_bytes);
// 4. get the state from the handler
byte[] content_p = handler.get_content(tag);
// 5. deserialize the state into a stream op
str_io_msg_p = deser.deserialize(content_p);
// 6. check to make sure the state's rows are good
passes = passes && str_io_msg_p.op == bts.StreamIOMessage.StreamOp.CREATE &&
BlueDataUtility.compute_equality(cr_rows, str_io_msg_p.create_message.rows);
return new TestResult {
test_result = passes,
test_name = "run_pubsub_server_stream_tag_handler_test"
};
}
private static TestResult _run_single_tag_data_test_clean() {
// create message
var test_name = MethodBase.GetCurrentMethod().Name;
var schema = create_schema_with_all_types();
var tag = "DS_" + schema.tableName;
BlueTable table = new BlueTable(schema);
IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
IStreamMsgDeserilizer deser = new StreamIODeserializationController();
bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
// create with 100 rows
int n_rows = 100;
bool is_succ = true;
for (int i = 0; i < n_rows; i++) {
var nr = generate_random_row(schema, i);
// perform the same action SingletagData, and verify the states match
PublishMsg pub_msg = build_pub_msg(nr, null, table.Count == 0, ref serializer);
table.addRow(nr);
st_data.handle_publish(pub_msg);
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
}
// change all rows
for (int i = 0; i < n_rows; i++) {
var nr = generate_random_row(schema, i);
var dr = table[i];
table.deleteRow(dr);
table.addRow(nr);
PublishMsg pub_msg = build_pub_msg(nr, dr, false, ref serializer);
st_data.handle_publish(pub_msg);
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
}
// delete all rows
var aux_table = new BlueTable(table);
foreach (IBlueRow r in aux_table) {
table.deleteRow(r);
PublishMsg pub_msg = build_pub_msg(null, r, false, ref serializer);
st_data.handle_publish(pub_msg);
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
}
return new TestResult() {
test_result = is_succ,
test_name = test_name
};
}
#region Helper functions
private static void _write_to_file(byte[] ser_bytes, string file_name) {
_write_to_file(new MemoryStream(ser_bytes), file_name);
}
private static void _write_to_file(MemoryStream str, string file_name) {
if (String.IsNullOrEmpty(file_name))
return;
byte[] str_bytes = str.ToArray();
using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
// Write the byte array to the file
fs.Write(str_bytes, 0, str_bytes.Length);
}
}
private static byte[] _read_from_file(string file_path) {
var rv = File.ReadAllBytes(file_path);
return rv;
}
public static IBlueRowCollection get_rows_from_single_tag_state(ref SingleTagData st_data, ref IStreamMsgDeserilizer deser) {
var content = st_data.get_current_content();
var msg_p = deser.deserialize(content);
Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE);
return msg_p.create_message.rows;
}
public static PublishMsg build_pub_msg(IBlueRow added_row, IBlueRow removed_row, bool is_create,
ref IStreamIOMsgSerializer serializer) {
Func<IBlueTableSchema> get_schema = () => {
return (added_row != null ? added_row : removed_row).schema;
};
bts.StreamIOMessage str_io_msg = null;
if (is_create) {
Log.assert(added_row != null);
var aux_new = new BlueTable(added_row.schema);
aux_new.addRow(added_row);
var cr_msg = new bts.StreamIOMessage.CreateMessage(aux_new);
str_io_msg = new bts.StreamIOMessage(cr_msg);
} else {
var schema = get_schema();
BlueTable added_table = new BlueTable(schema);
if (added_row != null)
added_table.addRow(added_row);
BlueTable rem_table = new BlueTable(schema);
if (removed_row != null)
rem_table.addRow(removed_row);
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg =
new bts.StreamIOMessage.MultipleRemoveAddMessage(rem_table, added_table);
str_io_msg = new bts.StreamIOMessage(mra_msg);
}
byte[] msg_bytes = serializer.serialize(str_io_msg);
PublishMsg pub_msg = new PublishMsg() {
connection = "test_connection",
tag = $"DS_{get_schema()}",
tag_type = "DS",
data_to_publish = msg_bytes
};
return pub_msg;
}
public static IBlueTableSchema create_schema_with_all_types(HashSet<Type> excluded_col_map = null) {
excluded_col_map = excluded_col_map == null ? new HashSet<Type>() : excluded_col_map;
var table_name = "test_table";
BlueTableSchema schema = new BlueTableSchema();
List<ColumnInfo> cis = new List<ColumnInfo>();
foreach (Type t in BlueDataUtility.valid_column_types()) {
if (t == null || t == typeof(DateTime) || excluded_col_map.Contains(t))
continue;
var col_name = t.ToString() + "_" + cis.Count.ToString(); // Guid.NewGuid().ToString();
ColumnInfo ci = new ColumnInfo(col_name, cis.Count, t);
cis.Add(ci);
}
cis.Add(new ColumnInfo("id", cis.Count, typeof(int)));
schema = new BlueTableSchema(table_name, cis.ToArray());
return schema;
}
public static IBlueRow generate_random_row(IBlueTableSchema schema, int id, bool include_dbnulls = true) {
Random rnd = new Random();
object[] row_vals = new object[schema.columnInfo.Length];
foreach (ColumnInfo ci in schema.columnInfo) {
if (ci.colName == "id") {
row_vals[ci.colNo] = id;
continue;
}
object v = get_random_object(ci.type_code());
if (include_dbnulls && rnd.NextSingle() % 4 == 0)
v = DBNull.Value;
row_vals[ci.colNo] = v;
}
return new BlueRow(schema, row_vals);
}
public static object get_random_object(TypeCode type_code) {
Random random = new Random();
switch (type_code) {
case TypeCode.Boolean:
return random.Next(2) == 0; // Random boolean value (true or false)
case TypeCode.Byte:
return (byte)random.Next(byte.MinValue, byte.MaxValue + 1); // Random byte value
case TypeCode.Char:
return (char)random.Next(65, 65 + 27); // Random char value
case TypeCode.Decimal:
Decimal dec = 100.123M;
return dec;
case TypeCode.DateTime:
return DateTime.Now;
case TypeCode.Double:
return random.NextDouble() * (double.MaxValue - double.MinValue) + double.MinValue; // Random double value
case TypeCode.Int16:
return (short)random.Next(short.MinValue, short.MaxValue + 1); // Random short value
case TypeCode.Int32:
return random.Next(int.MinValue, int.MaxValue); // Random int value
case TypeCode.Int64:
return random.NextInt64(long.MinValue, long.MaxValue); // Random long value
case TypeCode.SByte:
return (sbyte)random.Next(sbyte.MinValue, sbyte.MaxValue + 1); // Random sbyte value
case TypeCode.Single:
return (float)(random.NextDouble() * (float.MaxValue - float.MinValue) + float.MinValue); // Random float value
case TypeCode.String:
return Guid.NewGuid().ToString();
case TypeCode.UInt16:
return (ushort)random.Next(ushort.MinValue, ushort.MaxValue + 1); // Random ushort value
case TypeCode.UInt32:
return (uint)random.Next(int.MinValue, int.MaxValue) + (uint)int.MaxValue; // Random uint value
case TypeCode.UInt64:
byte[] ulongBuffer = new byte[8];
random.NextBytes(ulongBuffer);
return BitConverter.ToUInt64(ulongBuffer, 0);
default:
throw new ArgumentException($"Unsupported TypeCode: {type_code}");
return null;
}
}
#endregion
}
}