810 lines
39 KiB
C#
810 lines
39 KiB
C#
using bts;
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Reflection;
|
|
|
|
namespace BCG {
|
|
|
|
/// <summary>
|
|
/// This stream (not for prod) provides a way to run tests - that could otherwise be run in NUnits.
|
|
/// (truth is stream rpovide a RDE for quickly coding and running, so it was eaiser to run tests here)
|
|
/// </summary>
|
|
[DataStreamUserRespresentation("tests")]
|
|
public class TesStream : ViewStream {
|
|
static byte[] create_byte_array() {
|
|
Random random = new Random();
|
|
var sz = random.Next(1, 1000);// % 1000;
|
|
|
|
// Generate a random byte
|
|
byte randomByte = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
|
|
byte[] rv = new byte[sz];
|
|
for (int i = 0; i < sz; i++) {
|
|
rv[i] = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
|
|
}
|
|
return rv;
|
|
}
|
|
|
|
private IBlueTableSchema _schema;
|
|
class TestResult {
|
|
public string test_name = MethodBase.GetCurrentMethod()?.Name
|
|
?? throw new InvalidOperationException("Failed to get test name from method");
|
|
public bool test_result;
|
|
}
|
|
private List<Func<TestResult>> _tests = new
|
|
List<Func<TestResult>>();
|
|
|
|
public TesStream() {
|
|
_schema = BlueDataUtility.form_schema("Tests",
|
|
Tuple.Create("id", TypeCode.Int32),
|
|
Tuple.Create("test", TypeCode.String),
|
|
Tuple.Create("result", TypeCode.Boolean),
|
|
Tuple.Create("exception", TypeCode.String)
|
|
);
|
|
// register tests here
|
|
_tests.Add(new Func<TestResult>(_run_sub_msg_test));
|
|
_tests.Add(new Func<TestResult>(_run_pub_msg_test));
|
|
_tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
|
|
_tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
|
|
_tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
|
|
// _tests.Add(new Func<TestResult>(_run_schema_serialization_test));
|
|
// _tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls));
|
|
// _tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls));
|
|
//_tests.Add(new Func<TestResult>(_row_serialization_tests_all_types));
|
|
// _tests.Add(new Func<TestResult>(_run_full_stream_serialization_test));
|
|
// _tests.Add(new Func<TestResult>(_run_full_stream_serialization_with_controller_test));
|
|
// _tests.Add(new Func<TestResult>(_run_pubsub_server_stream_tag_handler_test));
|
|
// _tests.Add(new Func<TestResult>(_run_single_tag_data_test_clean));
|
|
//_tests.Add(new Func<TestResult>(_cpp_data_test));
|
|
//_tests.Add(new Func<TestResult>(_write_stream_updates_to_files));
|
|
_tests.Add(new Func<TestResult>(_deserialize_pub_msg_test));
|
|
|
|
}
|
|
|
|
protected override void myOnFirstSubscriber() {
|
|
BlueTable rows = new BlueTable(_schema);
|
|
// u the tests here
|
|
foreach (var test in _tests) {
|
|
string ex_msg = String.Empty;
|
|
TestResult tr = new TestResult() {
|
|
test_name = test.Method.Name,
|
|
test_result = false
|
|
};
|
|
try {
|
|
tr = test();
|
|
} catch (Exception ex) {
|
|
ex_msg = ex.Message + "\n" + ex.StackTrace;
|
|
}
|
|
IBlueRow r = new BlueRow(_schema, rows.Count, tr.test_name, tr.test_result, ex_msg);
|
|
rows.addRow(r);
|
|
}
|
|
var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
|
|
myFireEvent(dp);
|
|
}
|
|
protected override void myOnLastSubscriber() {
|
|
}
|
|
|
|
private TestResult _run_sub_msg_test() {
|
|
bts.SubscribeMsg sub_msg = new bts.SubscribeMsg() {
|
|
connection = Guid.NewGuid().ToString(),
|
|
subscribing_app = Guid.NewGuid().ToString(),
|
|
tags = new string[] { "one", "two", "three" }
|
|
};
|
|
MemoryStream str = new MemoryStream();
|
|
sub_msg.serialize_into_str(str);
|
|
str.Position = 0;
|
|
bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
|
|
sub_msg_.consruct_deserialize(str);
|
|
bool res = sub_msg.Equals(sub_msg_);
|
|
return new TestResult {
|
|
test_name = "SubscribeMsg",
|
|
test_result = res
|
|
};
|
|
}
|
|
|
|
private TestResult _run_pub_msg_test() {
|
|
var pub_msg = new bts.PublishMsg() {
|
|
connection = Guid.NewGuid().ToString(),
|
|
publishing_app = Guid.NewGuid().ToString(),
|
|
tag = Guid.NewGuid().ToString(),
|
|
tag_type = "DS_",
|
|
is_compressed = false,
|
|
publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
|
|
data_to_publish = create_byte_array(),
|
|
};
|
|
MemoryStream str = new MemoryStream();
|
|
pub_msg.serialize_into_str(str);
|
|
str.Position = 0;
|
|
var pub_msg_ = new bts.PublishMsg();
|
|
pub_msg_.consruct_deserialize(str);
|
|
bool res = pub_msg.Equals(pub_msg_);
|
|
return new TestResult {
|
|
test_name = "PublishMsg",
|
|
test_result = res
|
|
};
|
|
}
|
|
|
|
private TestResult _run_pubsub_server_msg_test() {
|
|
var msg = new bts.PubSubServerMessage() {
|
|
tag = Guid.NewGuid().ToString(),
|
|
tag_type = Guid.NewGuid().ToString(),
|
|
data = create_byte_array()
|
|
};
|
|
MemoryStream str = new MemoryStream();
|
|
msg.serialize_into_str(str);
|
|
str.Position = 0;
|
|
var msg_ = new bts.PubSubServerMessage();
|
|
msg_.consruct_deserialize(str);
|
|
bool res = msg.Equals(msg_);
|
|
return new TestResult {
|
|
test_name = "PubSubServerMessage",
|
|
test_result = res
|
|
};
|
|
}
|
|
|
|
private TestResult _run_pubsub_client_msg_test() {
|
|
PubSubClientMessage msg = new PubSubClientMessage() {
|
|
msg_type = PubSubMessageType.SUBSCRIBE,
|
|
sub_msg = new bts.SubscribeMsg() {
|
|
connection = Guid.NewGuid().ToString(),
|
|
subscribing_app = Guid.NewGuid().ToString(),
|
|
tags = new string[] { "one", "two", "three" }
|
|
}
|
|
};
|
|
var str = new MemoryStream();
|
|
msg.serialize_into_str(str);
|
|
str.Position = 0;
|
|
var msg_ = new PubSubClientMessage();
|
|
msg_.consruct_deserialize(str);
|
|
var res = msg.Equals(msg_);
|
|
return new TestResult {
|
|
test_name = "PubSubClientMessage",
|
|
test_result = res
|
|
};
|
|
}
|
|
|
|
private TestResult _run_columninfo_serialization_test() {
|
|
Func<ColumnInfo, bool> test_ci = (ci) => {
|
|
MemoryStream str = new MemoryStream();
|
|
BinaryWriter bw = new BinaryWriter(str);
|
|
StreamSerializationUtils.serialize_column_info(ci, ref bw);
|
|
str.Position = 0;
|
|
//var ci_txt = BlueDataUtility.to_string(ci);
|
|
//var ci_ser =
|
|
//bts.Log.Gen.info("ci: {}")
|
|
BinaryReader br = new BinaryReader(str);
|
|
var ci_prime = StreamSerializationUtils.deserialize_column_info(ref br);
|
|
return BlueDataUtility.compute_equality(ci, ci_prime);
|
|
};
|
|
int i = 0;
|
|
foreach (Type t in BlueDataUtility.valid_column_types()) {
|
|
ColumnInfo ci = new ColumnInfo(Guid.NewGuid().ToString(),
|
|
i++, t);
|
|
var res = test_ci(ci);
|
|
if (!res) {
|
|
return new TestResult {
|
|
test_name = "ColumnInfoSerialization",
|
|
test_result = false
|
|
};
|
|
}
|
|
}
|
|
return new TestResult {
|
|
test_name = "ColumnInfoSerialization",
|
|
test_result = true
|
|
};
|
|
}
|
|
|
|
|
|
private TestResult _run_schema_serialization_test() {
|
|
var table_name = "test_table";
|
|
var excluded_types = new HashSet<Type>() { typeof(DateTime), typeof(Decimal) };
|
|
IBlueTableSchema schema = create_schema_with_all_types(excluded_types);
|
|
MemoryStream str = new MemoryStream();
|
|
BinaryWriter bw = new BinaryWriter(str);
|
|
StreamSerializationUtils.serialize_schema(schema, ref bw);
|
|
_write_to_file(str, @"c:\Temp\tests\schema.bin");
|
|
var hex_txt = StreamSerializationUtils.to_hex(str);
|
|
bts.Log.Gen.info($"_run_schema_serialization_test ser: {hex_txt}");
|
|
|
|
str.Position = 0;
|
|
BinaryReader br = new BinaryReader(str);
|
|
var schema_prime = StreamSerializationUtils.deserialize_schema(ref br);
|
|
var res = BlueDataUtility.compute_equality(schema, schema_prime);
|
|
|
|
return new TestResult {
|
|
test_name = "SchemaSerialization",
|
|
test_result = res
|
|
};
|
|
}
|
|
|
|
|
|
|
|
|
|
private TestResult _run_datetime_seriliaztion_test() {
|
|
DateTime dt = DateTime.Now;
|
|
DateTimeOffset dt_off = new DateTimeOffset(dt);
|
|
long dt_epoch_ms = dt_off.ToUnixTimeMilliseconds();
|
|
DateTimeOffset dt_off_pr = DateTimeOffset.FromUnixTimeMilliseconds(dt_epoch_ms);
|
|
DateTime dt_p = dt_off_pr.DateTime;
|
|
var passes = dt.Equals(dt_p);
|
|
return new TestResult {
|
|
test_name = "_run_datetime_seriliaztion_test",
|
|
test_result = false
|
|
};
|
|
}
|
|
|
|
private TestResult _run_full_stream_serialization_test() {
|
|
// create message
|
|
var schema = create_schema_with_all_types();
|
|
BlueTable cr_rows = new BlueTable(schema);
|
|
// 100 rows
|
|
int n_rows = 100;
|
|
for (int i = 0; i < n_rows; i++) {
|
|
var nr = generate_random_row(schema, i);
|
|
cr_rows.addRow(nr);
|
|
}
|
|
MemoryStream str = new MemoryStream();
|
|
BinaryWriter bw = new BinaryWriter(str);
|
|
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
|
|
bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
|
|
UniversalStreamIOSerializer ser = new UniversalStreamIOSerializer();
|
|
ser.serialize(stream_io_msg, ref bw);
|
|
str.Position = 0;
|
|
BinaryReader br = new System.IO.BinaryReader(str);
|
|
UniversalStreamIODeserializer deser = new bts.UniversalStreamIODeserializer();
|
|
var stream_io_msg_prime = deser.deserialize(ref br);
|
|
bool passes = stream_io_msg.Equals(stream_io_msg_prime);
|
|
var n_dels = 50;
|
|
BlueTable del_rows = new BlueTable(cr_rows.schema);
|
|
foreach (IBlueRow r in cr_rows) {
|
|
del_rows.addRow(r);
|
|
if (del_rows.Count == n_dels)
|
|
break;
|
|
}
|
|
IBlueRowCollection added_rows = del_rows;
|
|
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
|
|
str = new MemoryStream();
|
|
bw = new BinaryWriter(str);
|
|
stream_io_msg = new bts.StreamIOMessage(mra_msg);
|
|
ser.serialize(stream_io_msg, ref bw);
|
|
bw.BaseStream.Position = 0;
|
|
br = new BinaryReader(str);
|
|
//deser = new UniversalStreamIODeserializer();
|
|
stream_io_msg_prime = deser.deserialize(ref br);
|
|
passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
|
|
return new TestResult {
|
|
test_name = "UniversalSerialization",
|
|
test_result = passes
|
|
};
|
|
}
|
|
|
|
private TestResult _run_full_stream_serialization_with_controller_test() {
|
|
// create message
|
|
var schema = create_schema_with_all_types();
|
|
BlueTable cr_rows = new BlueTable(schema);
|
|
// 100 rows
|
|
int n_rows = 100;
|
|
for (int i = 0; i < n_rows; i++) {
|
|
var nr = generate_random_row(schema, i);
|
|
cr_rows.addRow(nr);
|
|
}
|
|
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
|
|
bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
|
|
IStreamIOMsgSerializer ser = new StreamIOSerializationController();
|
|
var ser_bytes = ser.serialize(stream_io_msg);
|
|
IStreamMsgDeserilizer deser = new bts.StreamIODeserializationController();
|
|
var stream_io_msg_prime = deser.deserialize(ser_bytes);
|
|
bool passes = stream_io_msg.Equals(stream_io_msg_prime);
|
|
//mult remove adds
|
|
// remove all rows and add 50 more
|
|
|
|
BlueTable del_rows = new BlueTable(cr_rows);
|
|
BlueTable added_rows = new BlueTable(cr_rows.schema);
|
|
var n_adds = 50;
|
|
for (int i = 0; i < n_adds; i++) {
|
|
var nr = generate_random_row(schema, i);
|
|
added_rows.addRow(nr);
|
|
}
|
|
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
|
|
stream_io_msg = new bts.StreamIOMessage(mra_msg);
|
|
ser_bytes = ser.serialize(stream_io_msg);
|
|
var ser_bytes_hex = StreamSerializationUtils.to_hex(ser_bytes);
|
|
Log.Gen.info($"ser_bytes: {ser_bytes_hex}");
|
|
stream_io_msg_prime = deser.deserialize(ser_bytes);
|
|
passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
|
|
return new TestResult {
|
|
test_name = "UniversalSerializationController",
|
|
test_result = passes
|
|
};
|
|
}
|
|
|
|
private TestResult _row_serialization_tests_no_dbnulls() {
|
|
// IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
|
|
// new Tuple<string, TypeCode>("id", TypeCode.Int32),
|
|
// new Tuple<string, TypeCode>("i1", TypeCode.Int32),
|
|
// new Tuple<string, TypeCode>("d1", TypeCode.Double),
|
|
// new Tuple<string, TypeCode>("string_1", TypeCode.String));
|
|
// bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
|
// IBlueRow r = new BlueRow(schema, 0, 100, 123.45, "test_string");
|
|
var schema = create_schema_with_all_types(new HashSet<Type>() { typeof(DateTime), typeof(Decimal) });
|
|
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
|
MemoryStream str = new MemoryStream();
|
|
IBlueRow r = generate_random_row(schema, 0, false);
|
|
BinaryWriter bw = new BinaryWriter(str);
|
|
row_ser.serialize_row(r, 0, ref bw);
|
|
_write_to_file(str, @"C:\Temp\tests\row.bin");
|
|
str.Position = 0;
|
|
byte[] raw_byte_array = str.ToArray();
|
|
var hex_txt = StreamSerializationUtils.to_hex(raw_byte_array);
|
|
Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls Row bytes: {hex_txt}");
|
|
BinaryReader br = new BinaryReader(str);
|
|
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
|
|
IBlueRow r_prime = des.deserialize_row(ref br);
|
|
string row_str = r_prime.to_string();
|
|
Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls r_prime: {row_str}");
|
|
var passes = BlueDataUtility.compute_equality(r, r_prime);
|
|
return new TestResult {
|
|
test_name = "UniversalRowSerialization_NonNull",
|
|
test_result = passes
|
|
};
|
|
}
|
|
|
|
private TestResult _deserialize_pub_msg_test() {
|
|
var pub_msg_ser = _read_from_file(@"C:\Temp\tests\pubsub_raw_msgs\publish_msg.bin");
|
|
PubSubClientMessage ps_client_msg = new PubSubClientMessage();
|
|
ps_client_msg.consruct_deserialize(pub_msg_ser);
|
|
bool passes = ps_client_msg.msg_type == PubSubMessageType.PUBLISH;
|
|
return new TestResult {
|
|
test_name = "_deserialize_pub_msg_test",
|
|
test_result = passes
|
|
};
|
|
}
|
|
|
|
private TestResult _cpp_data_test() {
|
|
//create and write schema
|
|
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
|
|
new Tuple<string, TypeCode>("id", TypeCode.Int32),
|
|
new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
|
|
new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
|
|
new Tuple<string, TypeCode>("d_1", TypeCode.Double),
|
|
new Tuple<string, TypeCode>("string_1", TypeCode.String)
|
|
);
|
|
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
|
UniversalRowDeserializer row_deser = new UniversalRowDeserializer(schema);
|
|
StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
|
StreamIODeserializationController deser_controller = new StreamIODeserializationController();
|
|
|
|
IBlueTableSchema schema_p = null;
|
|
var schema_file = @"c:\Temp\tests\schema.bin";
|
|
var row_file = @"C:\Temp\tests\row.bin";
|
|
var create_op_file = @"C:\Temp\tests\create_op.bin";
|
|
var mra_op_file = @"C:\Temp\tests\mra_op.bin";
|
|
IBlueRow row = new BlueRow(schema, 0, 100, 200, 123.45, "test_string");
|
|
|
|
//serialize schema, write it to a file, read it back, deserialize it and match schemas
|
|
byte[] ser_schema_bytes = StreamSerializationUtils.serialize_schema(schema);
|
|
_write_to_file(ser_schema_bytes, schema_file);
|
|
|
|
byte[] schema_file_bytes = _read_from_file(schema_file);
|
|
schema_p = StreamSerializationUtils.deserialize_schema(schema_file_bytes);
|
|
bool passes = BlueDataUtility.compute_equality(schema, schema_p);
|
|
|
|
//serialize row, write it to a file, read it back, deserialize it and match schemas
|
|
byte[] ser_row_bytes = row_ser.serialize_row(row);
|
|
_write_to_file(ser_row_bytes, row_file);
|
|
|
|
byte[] row_file_bytes = _read_from_file(row_file);
|
|
var row_file_bytes_txt = StreamSerializationUtils.to_hex(row_file_bytes);
|
|
Log.Gen.info($"TestsStream._cpp_data_test row_file_bytes_txt: {row_file_bytes_txt}");
|
|
|
|
IBlueRow row_p = row_deser.deserialize_row(row_file_bytes);
|
|
passes = passes && BlueDataUtility.compute_equality(row, row_p);
|
|
|
|
{
|
|
// serialize a create operation
|
|
int n_rows = 100;
|
|
BlueTable table = new BlueTable(schema);
|
|
for (int i = 0; i < n_rows; i++) {
|
|
IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
|
|
table.addRow(nr);
|
|
}
|
|
StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
|
|
//StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
|
byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
|
|
_write_to_file(create_ser_bytes, create_op_file);
|
|
|
|
// read it and make sure its right
|
|
byte[] create_op_file_bytes = _read_from_file(create_op_file);
|
|
//StreamIODeserializationController deser_controller = new StreamIODeserializationController();
|
|
var cr_msg_p = deser_controller.deserialize(create_op_file_bytes);
|
|
_write_to_file(create_ser_bytes, mra_op_file);
|
|
|
|
passes = passes && cr_msg.Equals(cr_msg_p);
|
|
|
|
}
|
|
//multiple remove adds
|
|
{
|
|
// remove all rows and add them again
|
|
int n_rows = 100;
|
|
BlueTable table = new BlueTable(schema);
|
|
for (int i = 0; i < n_rows; i++) {
|
|
IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
|
|
table.addRow(nr);
|
|
}
|
|
StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(table, table));
|
|
|
|
//StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
|
byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
|
|
_write_to_file(mra_ser_bytes, mra_op_file);
|
|
|
|
// reasd it and make sure it matches
|
|
byte[] mra_op_file_bytes = _read_from_file(mra_op_file);
|
|
//StreamIODeserializationController deser_controller = new StreamIODeserializationController();
|
|
StreamIOMessage mra_msg_p = deser_controller.deserialize(mra_op_file_bytes);
|
|
passes = passes && mra_msg.Equals(mra_msg_p);
|
|
}
|
|
return new TestResult {
|
|
test_name = "_cpp_data_test",
|
|
test_result = passes
|
|
};
|
|
}
|
|
|
|
private TestResult _write_stream_updates_to_files() {
|
|
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
|
|
new Tuple<string, TypeCode>("id", TypeCode.Int32),
|
|
new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
|
|
new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
|
|
new Tuple<string, TypeCode>("d_1", TypeCode.Double),
|
|
new Tuple<string, TypeCode>("string_1", TypeCode.String)
|
|
);
|
|
var packets_path = @"C:\Temp\tests\stream_packets\";
|
|
Func<long> now_epochs = () => {
|
|
DateTimeOffset now = DateTimeOffset.UtcNow;
|
|
// Convert to Unix time in milliseconds
|
|
long unix_time_milliseconds = now.ToUnixTimeMilliseconds();
|
|
return unix_time_milliseconds;
|
|
};
|
|
Func<int, bool, string> curr_file_path = (id, create) => {
|
|
//var epoch_ms = now_epochs();
|
|
var file_path = $"{packets_path}{id}.bin";
|
|
if (create)
|
|
file_path = $"{packets_path}create_{id}.bin";
|
|
return file_path;
|
|
};
|
|
// write 1000 stream updates
|
|
{
|
|
int init_sz = 100;
|
|
BlueTable table = new BlueTable(schema);
|
|
StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
|
// create table of 100 rows
|
|
for (int i = 0; i < init_sz; i++) {
|
|
IBlueRow nr = new BlueRow(schema, i, i * 100, i * 200, i * 123.45, "test_string");
|
|
table.addRow(nr);
|
|
}
|
|
// write create msg
|
|
StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
|
|
byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
|
|
var create_file_path = curr_file_path(table.Count, true);
|
|
_write_to_file(create_ser_bytes, create_file_path);
|
|
|
|
// add 1000 rows and write an io msg for each update and write it to a file
|
|
int n_updates = 1000;
|
|
int period_ms = 50;
|
|
|
|
// write a create
|
|
for (int i = 0; i < n_updates; i++) {
|
|
// addition
|
|
BlueTable adds = new BlueTable(schema);
|
|
int id = table.Count;
|
|
IBlueRow nr = new BlueRow(schema, id, i * 100, i * 200, i * 123.45, "test_string");
|
|
adds.addRow(nr);
|
|
BlueTable dels = new BlueTable(schema);
|
|
table.addRow(nr);
|
|
StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(dels, adds));
|
|
byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
|
|
var mra_file_path = curr_file_path(id, false);
|
|
_write_to_file(mra_ser_bytes, mra_file_path);
|
|
|
|
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(period_ms));
|
|
}
|
|
// now start writing increments and add 1 row at a time
|
|
}
|
|
return new TestResult {
|
|
test_name = "_write_updates_as_files",
|
|
test_result = true
|
|
};
|
|
}
|
|
|
|
|
|
private TestResult _row_serialization_tests_with_dbnulls() {
|
|
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
|
|
new Tuple<string, TypeCode>("id", TypeCode.Int32),
|
|
new Tuple<string, TypeCode>("i1", TypeCode.Int32),
|
|
new Tuple<string, TypeCode>("d1", TypeCode.Double),
|
|
new Tuple<string, TypeCode>("string_1", TypeCode.String));
|
|
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
|
IBlueRow r = new BlueRow(schema, 0, DBNull.Value, 123.45, "test_string");
|
|
MemoryStream str = new MemoryStream();
|
|
BinaryWriter bw = new BinaryWriter(str);
|
|
row_ser.serialize_row(r, 0, ref bw);
|
|
str.Position = 0;
|
|
BinaryReader br = new BinaryReader(str);
|
|
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
|
|
IBlueRow r_prime = des.deserialize_row(ref br);
|
|
var passes = BlueDataUtility.compute_equality(r, r_prime);
|
|
return new TestResult {
|
|
test_name = "UniversalRowSerialization_WithDBNulls",
|
|
test_result = passes
|
|
};
|
|
}
|
|
|
|
private TestResult _row_serialization_tests_all_types() {
|
|
var schema = create_schema_with_all_types();
|
|
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
|
|
IBlueRow r = generate_random_row(schema, 0, false);
|
|
MemoryStream str = new MemoryStream();
|
|
BinaryWriter bw = new BinaryWriter(str);
|
|
row_ser.serialize_row(r, 0, ref bw);
|
|
_write_to_file(str, @"C:\Temp\tests\row.bin");
|
|
str.Position = 0;
|
|
BinaryReader br = new BinaryReader(str);
|
|
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
|
|
IBlueRow r_prime = des.deserialize_row(ref br);
|
|
var passes = BlueDataUtility.compute_equality(r, r_prime);
|
|
return new TestResult {
|
|
test_name = "UniversalRowSerialization_AllTypes",
|
|
test_result = passes
|
|
};
|
|
}
|
|
|
|
private static TestResult _run_pubsub_server_stream_tag_handler_test() {
|
|
// create message
|
|
var schema = create_schema_with_all_types();
|
|
var tag = "DS_" + schema.tableName;
|
|
BlueTable cr_rows = new BlueTable(schema);
|
|
// 100 rows
|
|
int n_rows = 100;
|
|
for (int i = 0; i < n_rows; i++) {
|
|
var nr = generate_random_row(schema, i);
|
|
cr_rows.addRow(nr);
|
|
}
|
|
|
|
PubsubServerSingleStreamTagHandler handler = new PubsubServerSingleStreamTagHandler();
|
|
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
|
|
bts.StreamIOMessage str_io_msg = new bts.StreamIOMessage(cr_msg);
|
|
IStreamIOMsgSerializer ser = new StreamIOSerializationController();
|
|
byte[] msg_bytes = ser.serialize(str_io_msg);
|
|
handler.update(tag, msg_bytes);
|
|
|
|
//now get it back for the handler
|
|
byte[] content = handler.get_content(tag);
|
|
IStreamMsgDeserilizer deser = new StreamIODeserializationController();
|
|
bts.StreamIOMessage str_io_msg_p = deser.deserialize(content);
|
|
// test here
|
|
var passes = str_io_msg.Equals(str_io_msg_p);
|
|
// delete all rows
|
|
BlueTable del_rows = new BlueTable(cr_rows);
|
|
cr_rows.clear();
|
|
// added 50 new rows
|
|
BlueTable added_rows = new BlueTable(del_rows.schema);
|
|
int new_state_sz = 50;
|
|
for (int i = 0; i < new_state_sz; i++) {
|
|
var nr = generate_random_row(schema, i);
|
|
added_rows.addRow(nr);
|
|
}
|
|
cr_rows.addRows(added_rows);
|
|
// the sequence of operations are ops, ser, update, get_content, deser, maake sure the get_content state is good
|
|
// 1. generate op
|
|
bts.StreamIOMessage.MultipleRemoveAddMessage mra = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
|
|
str_io_msg = new bts.StreamIOMessage(mra);
|
|
// 2. serialize it into bytes
|
|
msg_bytes = ser.serialize(str_io_msg);
|
|
// 3. send it to handler
|
|
handler.update(tag, msg_bytes);
|
|
// 4. get the state from the handler
|
|
byte[] content_p = handler.get_content(tag);
|
|
// 5. deserialize the state into a stream op
|
|
str_io_msg_p = deser.deserialize(content_p);
|
|
// 6. check to make sure the state's rows are good
|
|
passes = passes && str_io_msg_p.op == bts.StreamIOMessage.StreamOp.CREATE &&
|
|
BlueDataUtility.compute_equality(cr_rows, str_io_msg_p.create_message.rows);
|
|
return new TestResult {
|
|
test_result = passes,
|
|
test_name = "run_pubsub_server_stream_tag_handler_test"
|
|
};
|
|
}
|
|
|
|
|
|
private static TestResult _run_single_tag_data_test_clean() {
|
|
// create message
|
|
var test_name = MethodBase.GetCurrentMethod().Name;
|
|
var schema = create_schema_with_all_types();
|
|
var tag = "DS_" + schema.tableName;
|
|
BlueTable table = new BlueTable(schema);
|
|
IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
|
|
IStreamMsgDeserilizer deser = new StreamIODeserializationController();
|
|
bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
|
|
// create with 100 rows
|
|
int n_rows = 100;
|
|
bool is_succ = true;
|
|
for (int i = 0; i < n_rows; i++) {
|
|
var nr = generate_random_row(schema, i);
|
|
// perform the same action SingletagData, and verify the states match
|
|
PublishMsg pub_msg = build_pub_msg(nr, null, table.Count == 0, ref serializer);
|
|
table.addRow(nr);
|
|
st_data.handle_publish(pub_msg);
|
|
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
|
|
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
|
|
}
|
|
// change all rows
|
|
for (int i = 0; i < n_rows; i++) {
|
|
var nr = generate_random_row(schema, i);
|
|
var dr = table[i];
|
|
table.deleteRow(dr);
|
|
table.addRow(nr);
|
|
PublishMsg pub_msg = build_pub_msg(nr, dr, false, ref serializer);
|
|
st_data.handle_publish(pub_msg);
|
|
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
|
|
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
|
|
}
|
|
// delete all rows
|
|
var aux_table = new BlueTable(table);
|
|
foreach (IBlueRow r in aux_table) {
|
|
table.deleteRow(r);
|
|
PublishMsg pub_msg = build_pub_msg(null, r, false, ref serializer);
|
|
st_data.handle_publish(pub_msg);
|
|
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
|
|
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
|
|
}
|
|
return new TestResult() {
|
|
test_result = is_succ,
|
|
test_name = test_name
|
|
};
|
|
}
|
|
|
|
|
|
|
|
#region Helper functions
|
|
|
|
private static void _write_to_file(byte[] ser_bytes, string file_name) {
|
|
_write_to_file(new MemoryStream(ser_bytes), file_name);
|
|
}
|
|
|
|
private static void _write_to_file(MemoryStream str, string file_name) {
|
|
if (String.IsNullOrEmpty(file_name))
|
|
return;
|
|
byte[] str_bytes = str.ToArray();
|
|
using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
|
|
// Write the byte array to the file
|
|
fs.Write(str_bytes, 0, str_bytes.Length);
|
|
}
|
|
}
|
|
|
|
private static byte[] _read_from_file(string file_path) {
|
|
var rv = File.ReadAllBytes(file_path);
|
|
return rv;
|
|
}
|
|
|
|
public static IBlueRowCollection get_rows_from_single_tag_state(ref SingleTagData st_data, ref IStreamMsgDeserilizer deser) {
|
|
var content = st_data.get_current_content();
|
|
var msg_p = deser.deserialize(content);
|
|
Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE);
|
|
return msg_p.create_message.rows;
|
|
|
|
}
|
|
|
|
public static PublishMsg build_pub_msg(IBlueRow added_row, IBlueRow removed_row, bool is_create,
|
|
ref IStreamIOMsgSerializer serializer) {
|
|
Func<IBlueTableSchema> get_schema = () => {
|
|
return (added_row != null ? added_row : removed_row).schema;
|
|
};
|
|
bts.StreamIOMessage str_io_msg = null;
|
|
if (is_create) {
|
|
Log.assert(added_row != null);
|
|
var aux_new = new BlueTable(added_row.schema);
|
|
aux_new.addRow(added_row);
|
|
var cr_msg = new bts.StreamIOMessage.CreateMessage(aux_new);
|
|
str_io_msg = new bts.StreamIOMessage(cr_msg);
|
|
} else {
|
|
var schema = get_schema();
|
|
BlueTable added_table = new BlueTable(schema);
|
|
if (added_row != null)
|
|
added_table.addRow(added_row);
|
|
BlueTable rem_table = new BlueTable(schema);
|
|
if (removed_row != null)
|
|
rem_table.addRow(removed_row);
|
|
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg =
|
|
new bts.StreamIOMessage.MultipleRemoveAddMessage(rem_table, added_table);
|
|
str_io_msg = new bts.StreamIOMessage(mra_msg);
|
|
}
|
|
byte[] msg_bytes = serializer.serialize(str_io_msg);
|
|
PublishMsg pub_msg = new PublishMsg() {
|
|
connection = "test_connection",
|
|
tag = $"DS_{get_schema()}",
|
|
tag_type = "DS",
|
|
data_to_publish = msg_bytes
|
|
};
|
|
return pub_msg;
|
|
}
|
|
|
|
public static IBlueTableSchema create_schema_with_all_types(HashSet<Type> excluded_col_map = null) {
|
|
excluded_col_map = excluded_col_map == null ? new HashSet<Type>() : excluded_col_map;
|
|
var table_name = "test_table";
|
|
BlueTableSchema schema = new BlueTableSchema();
|
|
List<ColumnInfo> cis = new List<ColumnInfo>();
|
|
foreach (Type t in BlueDataUtility.valid_column_types()) {
|
|
if (t == null || t == typeof(DateTime) || excluded_col_map.Contains(t))
|
|
continue;
|
|
var col_name = t.ToString() + "_" + cis.Count.ToString(); // Guid.NewGuid().ToString();
|
|
ColumnInfo ci = new ColumnInfo(col_name, cis.Count, t);
|
|
cis.Add(ci);
|
|
}
|
|
cis.Add(new ColumnInfo("id", cis.Count, typeof(int)));
|
|
schema = new BlueTableSchema(table_name, cis.ToArray());
|
|
return schema;
|
|
}
|
|
|
|
public static IBlueRow generate_random_row(IBlueTableSchema schema, int id, bool include_dbnulls = true) {
|
|
Random rnd = new Random();
|
|
object[] row_vals = new object[schema.columnInfo.Length];
|
|
foreach (ColumnInfo ci in schema.columnInfo) {
|
|
if (ci.colName == "id") {
|
|
row_vals[ci.colNo] = id;
|
|
continue;
|
|
}
|
|
object v = get_random_object(ci.type_code());
|
|
if (include_dbnulls && rnd.NextSingle() % 4 == 0)
|
|
v = DBNull.Value;
|
|
row_vals[ci.colNo] = v;
|
|
}
|
|
return new BlueRow(schema, row_vals);
|
|
}
|
|
|
|
public static object get_random_object(TypeCode type_code) {
|
|
Random random = new Random();
|
|
switch (type_code) {
|
|
case TypeCode.Boolean:
|
|
return random.Next(2) == 0; // Random boolean value (true or false)
|
|
case TypeCode.Byte:
|
|
return (byte)random.Next(byte.MinValue, byte.MaxValue + 1); // Random byte value
|
|
case TypeCode.Char:
|
|
return (char)random.Next(65, 65 + 27); // Random char value
|
|
case TypeCode.Decimal:
|
|
Decimal dec = 100.123M;
|
|
return dec;
|
|
case TypeCode.DateTime:
|
|
return DateTime.Now;
|
|
case TypeCode.Double:
|
|
return random.NextDouble() * (double.MaxValue - double.MinValue) + double.MinValue; // Random double value
|
|
case TypeCode.Int16:
|
|
return (short)random.Next(short.MinValue, short.MaxValue + 1); // Random short value
|
|
case TypeCode.Int32:
|
|
return random.Next(int.MinValue, int.MaxValue); // Random int value
|
|
case TypeCode.Int64:
|
|
return random.NextInt64(long.MinValue, long.MaxValue); // Random long value
|
|
case TypeCode.SByte:
|
|
return (sbyte)random.Next(sbyte.MinValue, sbyte.MaxValue + 1); // Random sbyte value
|
|
case TypeCode.Single:
|
|
return (float)(random.NextDouble() * (float.MaxValue - float.MinValue) + float.MinValue); // Random float value
|
|
case TypeCode.String:
|
|
return Guid.NewGuid().ToString();
|
|
case TypeCode.UInt16:
|
|
return (ushort)random.Next(ushort.MinValue, ushort.MaxValue + 1); // Random ushort value
|
|
case TypeCode.UInt32:
|
|
return (uint)random.Next(int.MinValue, int.MaxValue) + (uint)int.MaxValue; // Random uint value
|
|
case TypeCode.UInt64:
|
|
byte[] ulongBuffer = new byte[8];
|
|
random.NextBytes(ulongBuffer);
|
|
return BitConverter.ToUInt64(ulongBuffer, 0);
|
|
default:
|
|
throw new ArgumentException($"Unsupported TypeCode: {type_code}");
|
|
return null;
|
|
}
|
|
}
|
|
#endregion
|
|
|
|
|
|
}
|
|
}
|