diff --git a/src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs b/src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs index bf4c10119..f6b220e41 100644 --- a/src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs +++ b/src/DataStreamSuite/DataStream/Offloading/StreamOffloadRequestor.cs @@ -1,13 +1,12 @@ using BCG; +using bts; + using System.Linq; -namespace BCG -{ - public class StreamOffloadRequestor - { - public class OffloadOp - { +namespace BCG { + public class StreamOffloadRequestor { + public class OffloadOp { string name; string stream_exp; } @@ -15,8 +14,7 @@ namespace BCG private IBlueRowCollection _rows; private PubTableBroker _broker = new PubTableBroker(); ViewStream _existing_offloads_vs = new TableStream("cloud_streams"); - public StreamOffloadRequestor() - { + public StreamOffloadRequestor() { _schema = new BlueTableSchema("cloud_streams", new ColumnInfo("id", 0, typeof(int)), new ColumnInfo("name", 1, typeof(string)), @@ -24,35 +22,31 @@ namespace BCG new ColumnInfo("stream_exp", 3, typeof(string))); _existing_offloads_vs.onChange += _on_str; } - private void _on_str(object obj) - { + private void _on_str(object obj) { DataPacket dp = obj as DataPacket; _rows = dp.newRows; // this is needed so the broker knows the state of the table - _broker.createTable(_rows); + _broker.create_table(_rows); } - public void add_to_cloud(string name, string stream_exp, bool persist) - { + public void add_to_cloud(string name, string stream_exp, bool persist) { // check if it exists - if (_rows == null) - { + if (_rows == null) { BlueTable table = new BlueTable(_schema); IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp); table.addRow(nr); - _broker.createTable(table); + _broker.create_table(table); // create table - } - else - { + } else { if (exists(name)) return; IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp); - _broker.addRow(nr); + // TODO: use make_batch_changes instead + Log.assert(false, "Fix addRow"); + //_broker.addRow(nr); } } - bool exists(string name) - { + bool exists(string name) { return _rows.to_ienum().Where(r => name == r.get("name")).Count() > 0; } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs index 9a5287ad5..e21afa9ec 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs @@ -12,6 +12,7 @@ using System.IO; using System; using Microsoft.VisualBasic; using Spark.Core.Collections; +using System.Reflection; namespace BCG { @@ -36,7 +37,7 @@ namespace BCG { private IBlueTableSchema _schema; class TestResult { - public string test_name; + public string test_name = MethodBase.GetCurrentMethod().Name; public bool test_result; } private List> _tests = new @@ -50,19 +51,19 @@ namespace BCG { Tuple.Create("exception", TypeCode.String) ); // resgiter tests here - // _tests.Add(new Func(_run_sub_msg_test)); - // _tests.Add(new Func(_run_pub_msg_test)); - // _tests.Add(new Func(_run_pubsub_server_msg_test)); - // _tests.Add(new Func(_run_pubsub_client_msg_test)); - // _tests.Add(new Func(_run_columninfo_serialization_test)); - // _tests.Add(new Func(_run_schema_serialization_test)); - // _tests.Add(new Func(_row_serialization_tests_no_dbnulls)); - // _tests.Add(new Func(_row_serialization_tests_with_dbnulls)); - // _tests.Add(new Func(_row_serialization_tests_all_types)); - // _tests.Add(new Func(_run_full_stream_serialization_test)); - // _tests.Add(new Func(_run_full_stream_serialization_with_controller_test)); - // _tests.Add(new Func(_run_pubsub_server_stream_tag_handler_test)); - _tests.Add(new Func(_run_single_tag_data_test)); + _tests.Add(new Func(_run_sub_msg_test)); + _tests.Add(new Func(_run_pub_msg_test)); + _tests.Add(new Func(_run_pubsub_server_msg_test)); + _tests.Add(new Func(_run_pubsub_client_msg_test)); + _tests.Add(new Func(_run_columninfo_serialization_test)); + _tests.Add(new Func(_run_schema_serialization_test)); + _tests.Add(new Func(_row_serialization_tests_no_dbnulls)); + _tests.Add(new Func(_row_serialization_tests_with_dbnulls)); + _tests.Add(new Func(_row_serialization_tests_all_types)); + _tests.Add(new Func(_run_full_stream_serialization_test)); + _tests.Add(new Func(_run_full_stream_serialization_with_controller_test)); + _tests.Add(new Func(_run_pubsub_server_stream_tag_handler_test)); + _tests.Add(new Func(_run_single_tag_data_test_clean)); } protected override void myOnFirstSubscriber() { @@ -128,7 +129,6 @@ namespace BCG { }; } - private TestResult _run_pubsub_server_msg_test() { var msg = new bts.PubSubServerMessage() { tag = Guid.NewGuid().ToString(), @@ -439,103 +439,194 @@ namespace BCG { }; } - private static TestResult _run_single_tag_data_test() { + // private static TestResult _run_single_tag_data_test() { + // // create message + // var schema = create_schema_with_all_types(); + // var tag = "DS_" + schema.tableName; + // BlueTable cr_rows = new BlueTable(schema); + // // 100 rows + // int n_rows = 100; + // for (int i = 0; i < n_rows; i++) { + // var nr = generate_random_row(schema, i); + // cr_rows.addRow(nr); + // } + // bts.SingleTagData st_data = new bts.SingleTagData(tag, "DS"); + + // bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows); + // bts.StreamIOMessage str_io_msg = new bts.StreamIOMessage(cr_msg); + // IStreamIOMsgSerializer serializer = new StreamIOSerializationController(); + // byte[] msg_bytes = serializer.serialize(str_io_msg); + // PublishMsg pub_msg = new PublishMsg() { + // connection = "test_connection", + // tag = tag, + // tag_type = "DS", + // data_to_publish = msg_bytes + // }; + // st_data.handle_publish(pub_msg); + // byte[] data_p = st_data.get_current_content(); + // IStreamMsgDeserilizer deser = new StreamIODeserializationController(); + // var msg_p = deser.deserialize(data_p); + // bool passes = msg_p.op == bts.StreamIOMessage.StreamOp.CREATE && + // BlueDataUtility.compute_equality(cr_rows, msg_p.create_message.rows); + + // Func build_pub_msg = (added_row, removed_row) => { + // BlueTable added_table = new BlueTable(cr_rows.schema); + // if (added_row != null) + // added_table.addRow(added_row); + // BlueTable rem_table = new BlueTable(cr_rows.schema); + // if (removed_row != null) + // rem_table.addRow(removed_row); + // bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = + // new bts.StreamIOMessage.MultipleRemoveAddMessage(rem_table, added_table); + // bts.StreamIOMessage str_io_msg = new bts.StreamIOMessage(mra_msg); + // byte[] msg_bytes = serializer.serialize(str_io_msg); + // PublishMsg pub_msg = new PublishMsg() { + // connection = "test_connection", + // tag = tag, + // tag_type = "DS", + // data_to_publish = msg_bytes + // }; + // return pub_msg; + // }; + + // Func get_state = (content) => { + // IStreamMsgDeserilizer deser = new StreamIODeserializationController(); + // var msg_p = deser.deserialize(content); + // Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE); + // return msg_p.create_message.rows; + // }; + // // 1. adding 100 rows + // int n_added_rows = 100; + // for (int i = 0; i < n_added_rows; i++) { + // var nr = generate_random_row(schema, cr_rows.Count); + // cr_rows.addRow(nr); + // // added + // PublishMsg new_pub_msg = build_pub_msg(nr, null); + // st_data.handle_publish(new_pub_msg); + // } + // var state_bytes = st_data.get_current_content(); + // IBlueRowCollection total_state = get_state(state_bytes); + // passes = passes && BlueDataUtility.compute_equality(cr_rows, total_state); + // // modifying 100 rows + // int n_mods = 100; + // int init_count = cr_rows.Count; + // for (int i = 0; i < n_mods; i++) { + // var dr = BlueDataUtility.extractRow(cr_rows, i); + // var nr = generate_random_row(schema, i + init_count); + // cr_rows.deleteRow(dr); + // cr_rows.addRow(nr); + // var new_pub_msg = build_pub_msg(nr, dr); + // st_data.handle_publish(new_pub_msg); + // } + // IBlueRowCollection new_total_state = get_state(st_data.get_current_content()); + // passes = passes && BlueDataUtility.compute_equality(cr_rows, new_total_state); + // // deleting all rows + // foreach (IBlueRow r in cr_rows) { + // pub_msg = build_pub_msg(null, r); + // st_data.handle_publish(pub_msg); + // } + // cr_rows.clear(); + // new_total_state = get_state(st_data.get_current_content()); + // passes = passes && BlueDataUtility.compute_equality(new_total_state, cr_rows); + // return new TestResult() { + // test_name = "SingleTagData", + // test_result = passes + // }; + // } + + private static TestResult _run_single_tag_data_test_clean() { // create message + var test_name = MethodBase.GetCurrentMethod().Name; var schema = create_schema_with_all_types(); var tag = "DS_" + schema.tableName; - BlueTable cr_rows = new BlueTable(schema); - // 100 rows + BlueTable table = new BlueTable(schema); + IStreamIOMsgSerializer serializer = new StreamIOSerializationController(); + IStreamMsgDeserilizer deser = new StreamIODeserializationController(); + bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType); + // create with 100 rows int n_rows = 100; + bool is_succ = true; for (int i = 0; i < n_rows; i++) { var nr = generate_random_row(schema, i); - cr_rows.addRow(nr); + // perform the same action SingletagData, and verify the states match + PublishMsg pub_msg = build_pub_msg(nr, null, table.Count == 0, ref serializer); + table.addRow(nr); + st_data.handle_publish(pub_msg); + var table_p = get_rows_from_single_tag_state(ref st_data, ref deser); + is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p); } - bts.SingleTagData st_data = new bts.SingleTagData(tag, "DS"); - - bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows); - bts.StreamIOMessage str_io_msg = new bts.StreamIOMessage(cr_msg); - IStreamIOMsgSerializer serializer = new StreamIOSerializationController(); - byte[] msg_bytes = serializer.serialize(str_io_msg); - PublishMsg pub_msg = new PublishMsg() { - connection = "test_connection", - tag = tag, - tag_type = "DS", - data_to_publish = msg_bytes + // change all rows + for (int i = 0; i < n_rows; i++) { + var nr = generate_random_row(schema, i); + var dr = table[i]; + table.deleteRow(dr); + table.addRow(nr); + PublishMsg pub_msg = build_pub_msg(nr, dr, false, ref serializer); + st_data.handle_publish(pub_msg); + var table_p = get_rows_from_single_tag_state(ref st_data, ref deser); + is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p); + } + // delete all rows + var aux_table = new BlueTable(table); + foreach (IBlueRow r in aux_table) { + table.deleteRow(r); + PublishMsg pub_msg = build_pub_msg(null, r, false, ref serializer); + st_data.handle_publish(pub_msg); + var table_p = get_rows_from_single_tag_state(ref st_data, ref deser); + is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p); + } + return new TestResult() { + test_result = is_succ, + test_name = test_name }; - st_data.handle_publish(pub_msg); - byte[] data_p = st_data.get_current_content(); - IStreamMsgDeserilizer deser = new StreamIODeserializationController(); - var msg_p = deser.deserialize(data_p); - bool passes = msg_p.op == bts.StreamIOMessage.StreamOp.CREATE && - BlueDataUtility.compute_equality(cr_rows, msg_p.create_message.rows); + } - Func build_pub_msg = (added_row, removed_row) => { - BlueTable added_table = new BlueTable(cr_rows.schema); + + + #region Helper functions + + public static IBlueRowCollection get_rows_from_single_tag_state(ref SingleTagData st_data, ref IStreamMsgDeserilizer deser) { + var content = st_data.get_current_content(); + var msg_p = deser.deserialize(content); + Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE); + return msg_p.create_message.rows; + + } + + public static PublishMsg build_pub_msg(IBlueRow added_row, IBlueRow removed_row, bool is_create, + ref IStreamIOMsgSerializer serializer) { + Func get_schema = () => { + return (added_row != null ? added_row : removed_row).schema; + }; + bts.StreamIOMessage str_io_msg = null; + if (is_create) { + Log.assert(added_row != null); + var aux_new = new BlueTable(added_row.schema); + aux_new.addRow(added_row); + var cr_msg = new bts.StreamIOMessage.CreateMessage(aux_new); + str_io_msg = new bts.StreamIOMessage(cr_msg); + } else { + var schema = get_schema(); + BlueTable added_table = new BlueTable(schema); if (added_row != null) added_table.addRow(added_row); - BlueTable rem_table = new BlueTable(cr_rows.schema); + BlueTable rem_table = new BlueTable(schema); if (removed_row != null) rem_table.addRow(removed_row); bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(rem_table, added_table); - bts.StreamIOMessage str_io_msg = new bts.StreamIOMessage(mra_msg); - byte[] msg_bytes = serializer.serialize(str_io_msg); - PublishMsg pub_msg = new PublishMsg() { - connection = "test_connection", - tag = tag, - tag_type = "DS", - data_to_publish = msg_bytes - }; - return pub_msg; - }; - - Func get_state = (content) => { - IStreamMsgDeserilizer deser = new StreamIODeserializationController(); - var msg_p = deser.deserialize(content); - Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE); - return msg_p.create_message.rows; - }; - // 1. adding 100 rows - int n_added_rows = 100; - for (int i = 0; i < n_added_rows; i++) { - var nr = generate_random_row(schema, cr_rows.Count); - cr_rows.addRow(nr); - // added - PublishMsg new_pub_msg = build_pub_msg(nr, null); - st_data.handle_publish(new_pub_msg); + str_io_msg = new bts.StreamIOMessage(mra_msg); } - var state_bytes = st_data.get_current_content(); - IBlueRowCollection total_state = get_state(state_bytes); - passes = passes && BlueDataUtility.compute_equality(cr_rows, total_state); - // modifying 100 rows - int n_mods = 100; - int init_count = cr_rows.Count; - for (int i = 0; i < n_mods; i++) { - var dr = BlueDataUtility.extractRow(cr_rows, i); - var nr = generate_random_row(schema, i + init_count); - cr_rows.deleteRow(dr); - cr_rows.addRow(nr); - var new_pub_msg = build_pub_msg(nr, dr); - st_data.handle_publish(new_pub_msg); - } - IBlueRowCollection new_total_state = get_state(st_data.get_current_content()); - passes = passes && BlueDataUtility.compute_equality(cr_rows, new_total_state); - // deleting all rows - foreach (IBlueRow r in cr_rows) { - pub_msg = build_pub_msg(null, r); - st_data.handle_publish(pub_msg); - } - cr_rows.clear(); - new_total_state = get_state(st_data.get_current_content()); - passes = passes && BlueDataUtility.compute_equality(new_total_state, cr_rows); - return new TestResult() { - test_name = "SingleTagData", - test_result = passes + byte[] msg_bytes = serializer.serialize(str_io_msg); + PublishMsg pub_msg = new PublishMsg() { + connection = "test_connection", + tag = $"DS_{get_schema()}", + tag_type = "DS", + data_to_publish = msg_bytes }; + return pub_msg; } - #region Helper functions - public static IBlueTableSchema create_schema_with_all_types() { var table_name = "test_table"; BlueTableSchema schema = new BlueTableSchema(); diff --git a/src/DataStreamSuite/DataStream/Publish/CircularPublisher.cs b/src/DataStreamSuite/DataStream/Publish/CircularPublisher.cs index a776c6177..464e30dcc 100644 --- a/src/DataStreamSuite/DataStream/Publish/CircularPublisher.cs +++ b/src/DataStreamSuite/DataStream/Publish/CircularPublisher.cs @@ -1,71 +1,61 @@ using System; -namespace BCG -{ +namespace BCG { /// /// Publishes the rows in the form of a circular buffer, ie if the buffer size is n /// then the n'th row being puclished results in the deletion of the 0th row /// - public class CircularPublisher - { + public class CircularPublisher { private CircularRowCollection myBuffer; private PubTableBroker myBroker; private int myBufferSize; - public CircularPublisher (int bufferSize) - { + public CircularPublisher(int bufferSize) { myBufferSize = bufferSize; - } - - public CircularPublisher (int bufferSize, IBlueRowCollection rows) - : this (bufferSize) - { - publishRows (rows); } - public void publishRow (IBlueRow nr) - { - if (myBuffer == null) - { - myBuffer = new CircularRowCollection ( - new BlueTableSchema (nr.schema.tableName, nr.schema.columnInfo), - myBufferSize); - myBroker = new PubTableBroker (); - myBroker.createTable (myBuffer); - } - IBlueRow br = myBuffer.addRowWithReturn (nr); - myBroker.makeBatchChanges (myBuffer, - myReturnCollection (br), - myReturnCollection (nr)); + public CircularPublisher(int bufferSize, IBlueRowCollection rows) + : this(bufferSize) { + publishRows(rows); } - public void publishRows (IBlueRowCollection nrs) - { - if (myBuffer == null) - { - myBuffer = new CircularRowCollection ( - new BlueTableSchema (BlueDataUtility.extractRow (nrs, 0).schema.tableName, - BlueDataUtility.extractRow (nrs, 0).schema.columnInfo), + public void publishRow(IBlueRow nr) { + if (myBuffer == null) { + myBuffer = new CircularRowCollection( + new BlueTableSchema(nr.schema.tableName, nr.schema.columnInfo), myBufferSize); - myBroker = new PubTableBroker (); - myBroker.createTable (myBuffer); + myBroker = new PubTableBroker(); + myBroker.create_table(myBuffer); } - BlueTable deletes = new BlueTable (myBuffer.schema); - foreach (IBlueRow nr in nrs) - { - IBlueRow dr = myBuffer.addRowWithReturn (nr); + IBlueRow br = myBuffer.addRowWithReturn(nr); + myBroker.make_batch_changes(myBuffer, + myReturnCollection(br), + myReturnCollection(nr)); + } + + public void publishRows(IBlueRowCollection nrs) { + if (myBuffer == null) { + myBuffer = new CircularRowCollection( + new BlueTableSchema(BlueDataUtility.extractRow(nrs, 0).schema.tableName, + BlueDataUtility.extractRow(nrs, 0).schema.columnInfo), + myBufferSize); + myBroker = new PubTableBroker(); + myBroker.create_table(myBuffer); + } + BlueTable deletes = new BlueTable(myBuffer.schema); + foreach (IBlueRow nr in nrs) { + IBlueRow dr = myBuffer.addRowWithReturn(nr); if (dr != null) - deletes.addRow (dr); + deletes.addRow(dr); } - myBroker.makeBatchChanges (myBuffer, deletes, nrs); + myBroker.make_batch_changes(myBuffer, deletes, nrs); } - private IBlueRowCollection myReturnCollection (IBlueRow r) - { + private IBlueRowCollection myReturnCollection(IBlueRow r) { if (r == null) - return new BlueTable (myBuffer.schema); - BlueTable retVal = new BlueTable (r.schema); - retVal.addRow (r); + return new BlueTable(myBuffer.schema); + BlueTable retVal = new BlueTable(r.schema); + retVal.addRow(r); return retVal; } - } + } } diff --git a/src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs b/src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs index 123233c86..e7d20c6ca 100644 --- a/src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs +++ b/src/DataStreamSuite/DataStream/Publish/PubTableBroker.cs @@ -6,70 +6,41 @@ using System.Xml.Serialization; using BCG.Utilities; +using bts; + namespace BCG { /// /// This class will be used by the clients to publish tabular data to the pubsub /// public class PubTableBroker { protected PublisherCore _publisher_core; - protected IDictionary _publisher_map = new Dictionary(); - protected Dictionary _schema_map = new Dictionary(); + protected IDictionary _publisher_map = + new Dictionary(); protected bool myWaitingForFirstOp = false; public PubTableBroker() : this(new PublisherCore()) { } - public PubTableBroker(PublisherCore publisher) { _publisher_core = publisher; } - - public PubTableBroker(IBlueRowCollection[] tablesToPublish) - : this(new PublisherCore(), tablesToPublish) { } - - public PubTableBroker(PublisherCore publisher, IBlueRowCollection[] tablesToPublish) { + public PubTableBroker(PublisherCore publisher) { _publisher_core = publisher; - - _publisher_core.begin_transaction(); - foreach (IBlueRowCollection table in tablesToPublish) - createTable(table); - _publisher_core.commit_transaction(); + Log.Gen.info("PubTableBroker constructed"); } - public void beginTransaction() { - myWaitingForFirstOp = true; - } - - public void commitTransaction() { - bts.Log.Gen.info("PubTableBroker.commitTransaction"); - if (myWaitingForFirstOp) - myWaitingForFirstOp = false; - else - _publisher_core.commit_transaction(); - } - - public void rollbackTransaction() { - bts.Log.Gen.info("PubTableBroker.rollbackTransaction"); - if (myWaitingForFirstOp) - myWaitingForFirstOp = false; - else - _publisher_core.abort_transaction(); - } - - public virtual void createTable(IBlueRowCollection newTable) { - string log_msg = $"PubTableBroker.createTable table: {newTable.schema.tableName}\n"; - foreach (IBlueRow r in newTable) { + public virtual void create_table(IBlueRowCollection table) { + string log_msg = $"PubTableBroker.createTable table: {table.schema.tableName}\n"; + foreach (IBlueRow r in table) { log_msg += $"id: {r["id"]}"; } bts.Log.Gen.info(log_msg); - _check_schema_consistency(newTable.schema); myCheckTransaction(); - IPubTablePublisher table_publisher = null; - var name_key = newTable.schema.tableName.ToLower(); + SinglePubTableBroker table_publisher = null; + var name_key = table.schema.tableName.ToLower(); if (!_publisher_map.TryGetValue(name_key, out table_publisher)) { - table_publisher = _create_pubtable_publisher(name_key); + table_publisher = new SinglePubTableBroker(name_key, _publisher_core); _publisher_map.Add(name_key, table_publisher); - _schema_map.Add(name_key, newTable.schema); } - table_publisher.createTable(newTable); + table_publisher.create_table(table); } /// @@ -77,108 +48,28 @@ namespace BCG { /// will be thrown /// /// - public virtual void addRow(IBlueRow r) { - string log_msg = $"PubTableBroker.addRow table: {r.schema.tableName} id: {r["id"]}"; - bts.Log.Gen.info(log_msg); - _check_schema_consistency(r.schema); - myCheckTransaction(); - IPubTablePublisher table_publisher = null; - var name_key = r.schema.tableName.ToLower(); - if (!_publisher_map.TryGetValue(name_key, out table_publisher)) - throw new ApplicationException("Row can not be added, as the table that has not been created yet"); - else - table_publisher.addRow(r); - } - - /// - /// If a table has not been created with the same instance of a PubTableBroker an ApplicationException - /// will be thrown - /// - /// - public virtual void deleteRow(IBlueRow r) { - string log_msg = $"PubTableBroker.deleteRow table: {r.schema.tableName} id: {r["id"]}"; - bts.Log.Gen.info(log_msg); - _check_schema_consistency(r.schema); - myCheckTransaction(); - IPubTablePublisher table_publisher = null; - var name_key = r.schema.tableName.ToLower(); - if (!_publisher_map.TryGetValue(name_key, out table_publisher)) - throw new ApplicationException("Row can not be removed, as the table has not been created yet"); - else - table_publisher.deleteRow(r); - } - - /// - /// If a table has not been created with the same instance of a PubTableBroker an ApplicationException - /// will be thrown - /// - /// - public virtual void changeRow(IBlueRow r) { - string log_msg = $"PubTableBroker.changeRow table: {r.schema.tableName} id: {r["id"]}"; - bts.Log.Gen.info(log_msg); - _check_schema_consistency(r.schema); - myCheckTransaction(); - IPubTablePublisher table_publisher = null; - var name_key = r.schema.tableName.ToLower(); - if (!_publisher_map.TryGetValue(name_key, out table_publisher)) - throw new ApplicationException("Row can not be changed, as the table has not been created yet"); - else - table_publisher.changeRow(r); - } - - /// - /// If a table has not been created with the same instance of a PubTableBroker an ApplicationException - /// will be thrown - /// - /// - public virtual void makeBatchChanges(IBlueRowCollection newRows, - IBlueRowCollection deletedRows, - IBlueRowCollection addedRows) { - string log_msg = $"PubTableBroker.makeBatchChanges {deletedRows.Count} {addedRows.Count} table: {newRows.schema.tableName} deleted_rows: "; - foreach (IBlueRow r in deletedRows) + public virtual void make_batch_changes(IBlueRowCollection new_rows, + IBlueRowCollection del_rows, + IBlueRowCollection added_rows) { + string log_msg = $"PubTableBroker.make_batch_changes table: {new_rows.schema.tableName} del: {del_rows.Count} added: {added_rows.Count}"; + foreach (IBlueRow r in del_rows) log_msg += $"id: {r["id"]} "; log_msg += "addedRows: "; - foreach (IBlueRow r in addedRows) + foreach (IBlueRow r in added_rows) log_msg += $"id: {r["id"]}"; bts.Log.Gen.info(log_msg); - _check_schema_consistency(newRows.schema); - _check_schema_consistency(deletedRows.schema); - _check_schema_consistency(addedRows.schema); - IPubTablePublisher table_publisher = null; - var name_key = newRows.schema.tableName.ToLower(); + SinglePubTableBroker table_publisher = null; + var name_key = new_rows.schema.tableName.ToLower(); if (!_publisher_map.TryGetValue(name_key, out table_publisher)) throw new ApplicationException("Batch changes can not be made, as the table has not been created yet"); else - table_publisher.batchRemoveAdd(deletedRows, addedRows); + table_publisher.make_batch_changes(del_rows, added_rows); } - public virtual void deleteTableRows(string tableName) { - myCheckTransaction(); - (_publisher_map[tableName.ToLower()] as IPubTablePublisher).createTable( - new BlueTable(_schema_map[tableName.ToLower()] as IBlueTableSchema)); - } - - public int getNumberOfWaitingMessages() { return _publisher_core.get_number_of_waiting_messages(); } - - public void clearTable(string tableName) { _publisher_core.clear_state(PubStreamPrefixer.prefix(tableName)); } - protected void myCheckTransaction() { if (myWaitingForFirstOp) _publisher_core.begin_transaction(); myWaitingForFirstOp = false; } - - protected virtual IPubTablePublisher _create_pubtable_publisher(string tag) { - bts.Log.Gen.info($"PubTableBroker._create_pubtable_publisher tag: {tag}"); - return new PubTablePublisher(tag, _publisher_core); - } - - private void _check_schema_consistency(IBlueTableSchema schema_arg) { - IBlueTableSchema schema; - if (_schema_map.TryGetValue(schema_arg.tableName, out schema)) { - bts.Log.assert(BlueDataUtility.compareSchemas(schema, schema_arg)); - } - } - } } diff --git a/src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs b/src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs index 906647b52..d9a923497 100644 --- a/src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs +++ b/src/DataStreamSuite/DataStream/Publish/ViewPublisher.cs @@ -7,48 +7,32 @@ using BCG.Utilities; namespace BCG { public class ViewPublisher : IDisposable { protected ViewPublisher() { } + private readonly PubTableBroker _broker; + protected readonly ViewStream _stream; + private int _n_updates = 0; /// If an name IsNullOrEmpty then it uses the name /// in the schema (be careful with that because this /// could lead to double publishes if the schema name /// is not uniquely chosen) - public ViewPublisher(string name, PubTableBroker br, ViewStream vs) { - myName = name; - myView = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name); - myBroker = br; - myFirstTime = true; - myView.onChange += new DataStream.ChangeDelegate(myChangeHandler); + public ViewPublisher(string name, PubTableBroker broker, ViewStream vs) { + _stream = String.IsNullOrEmpty(name) ? vs : new ChangeTableNameStream(vs, name); + _broker = broker; + _stream.onChange += new DataStream.ChangeDelegate(_handle_update); } public void Dispose() { - myView.onChange -= new DataStream.ChangeDelegate(myChangeHandler); - if (!String.IsNullOrEmpty(myName)) - myBroker.clearTable(myName); + _stream.onChange -= new DataStream.ChangeDelegate(_handle_update); } - private void myChangeHandler(object o) { - DataPacket dp = (DataPacket)o; - if (myFirstTime) { - myName = String.IsNullOrEmpty(myName) ? dp.schema.tableName : myName; - myLastRowCount = dp.newRows.Count; - myBroker.createTable(dp.newRows); - myFirstTime = false; + private void _handle_update(object o) { + DataPacket dp = o as DataPacket; + if (_n_updates == 0) { + _broker.create_table(dp.newRows); + _n_updates++; } else { - if (((dp.addedRows.Count + dp.deletedRows.Count) != 0)) { - myLastRowCount = dp.newRows.Count; - myBroker.makeBatchChanges(dp.newRows, dp.deletedRows, dp.addedRows); - } + _broker.make_batch_changes(dp.newRows, dp.deletedRows, dp.addedRows); } } - - public int getLastRowPublishedCount() { return myLastRowCount; } - - public bool hasPublished() { return !myFirstTime; } - - private int myLastRowCount = -1; - private string myName; - private PubTableBroker myBroker; - protected ViewStream myView; - private bool myFirstTime; } } diff --git a/src/DataStreamSuite/DataStream/Publish/ViewStreamPublisher.cs b/src/DataStreamSuite/DataStream/Publish/ViewStreamPublisher.cs index d38c15491..8d75b5036 100644 --- a/src/DataStreamSuite/DataStream/Publish/ViewStreamPublisher.cs +++ b/src/DataStreamSuite/DataStream/Publish/ViewStreamPublisher.cs @@ -1,92 +1,93 @@ -using System; -using BCG.Utilities; +// using System; +// using BCG.Utilities; -namespace BCG -{ - public class ViewStreamPublisher - { - private IDynamicStreamMapper myMapper; - private PubTableBroker myBroker; - private IBlueRowCollection myOutputTable; - private IBlueRowCollection myInputTable; +// TODO: remove +// namespace BCG +// { +// public class ViewStreamPublisher +// { +// private IDynamicStreamMapper myMapper; +// private PubTableBroker myBroker; +// private IBlueRowCollection myOutputTable; +// private IBlueRowCollection myInputTable; - public ViewStreamPublisher (PubTableBroker broker) - { - myBroker = broker; - } +// public ViewStreamPublisher (PubTableBroker broker) +// { +// myBroker = broker; +// } - public ViewStreamPublisher (PubTableBroker broker, IDynamicStreamMapper mapper) - : this (broker) - { - myMapper = mapper; - } +// public ViewStreamPublisher (PubTableBroker broker, IDynamicStreamMapper mapper) +// : this (broker) +// { +// myMapper = mapper; +// } - public ViewStreamPublisher (PubTableBroker broker, IDynamicStreamMapper mapper, ViewStream vs) - : this (broker, mapper) - { - vs.onChange += new BCG.DataStream.ChangeDelegate(myHandleChanges); - } +// public ViewStreamPublisher (PubTableBroker broker, IDynamicStreamMapper mapper, ViewStream vs) +// : this (broker, mapper) +// { +// vs.onChange += new BCG.DataStream.ChangeDelegate(myHandleChanges); +// } - private void myHandleChanges (object newValue) - { - DataPacket dp = newValue as DataPacket; - if (myMapper == null) - { - if (myOutputTable == null) - { - myBroker.createTable (dp.newRows); - } - else - { - myBroker.makeBatchChanges (dp.newRows, dp.deletedRows, dp.addedRows); - } - myOutputTable = dp.newRows; - } - else - { - if (myOutputTable != null) - { - BlueTable dRows = new BlueTable (myOutputTable.schema), - aRows = new BlueTable (myOutputTable.schema); - foreach (IBlueRow dRow in dp.deletedRows) - { - (myOutputTable as BlueTable).deleteRow ((int)dRow["id"]); - dRows.addRow (myMapper.map (dRow)); - } - foreach (IBlueRow aRow in dp.addedRows) - { - IBlueRow r = myMapper.map (aRow); - (myOutputTable as BlueTable).addRow (r); - aRows.addRow (r); - } - myBroker.makeBatchChanges (myOutputTable, dRows, aRows); - } - else - { - if (dp.newRows.Count > 0) - { - myOutputTable = new BlueTable ( - myMapper.map (BlueDataUtility.extractRow (dp.newRows, 0)).schema); - myHandleChanges (newValue); - } - } - } - } +// private void myHandleChanges (object newValue) +// { +// DataPacket dp = newValue as DataPacket; +// if (myMapper == null) +// { +// if (myOutputTable == null) +// { +// myBroker.createTable (dp.newRows); +// } +// else +// { +// myBroker.makeBatchChanges (dp.newRows, dp.deletedRows, dp.addedRows); +// } +// myOutputTable = dp.newRows; +// } +// else +// { +// if (myOutputTable != null) +// { +// BlueTable dRows = new BlueTable (myOutputTable.schema), +// aRows = new BlueTable (myOutputTable.schema); +// foreach (IBlueRow dRow in dp.deletedRows) +// { +// (myOutputTable as BlueTable).deleteRow ((int)dRow["id"]); +// dRows.addRow (myMapper.map (dRow)); +// } +// foreach (IBlueRow aRow in dp.addedRows) +// { +// IBlueRow r = myMapper.map (aRow); +// (myOutputTable as BlueTable).addRow (r); +// aRows.addRow (r); +// } +// myBroker.makeBatchChanges (myOutputTable, dRows, aRows); +// } +// else +// { +// if (dp.newRows.Count > 0) +// { +// myOutputTable = new BlueTable ( +// myMapper.map (BlueDataUtility.extractRow (dp.newRows, 0)).schema); +// myHandleChanges (newValue); +// } +// } +// } +// } - public void addRows (IBlueRowCollection aRows) - { - if (myInputTable != null) - (myInputTable as BlueTable).addRows (aRows); - else - myInputTable = new BlueTable (aRows); - myHandleChanges (new DataPacket(myInputTable, aRows, new BlueTable (aRows.schema))); - } +// public void addRows (IBlueRowCollection aRows) +// { +// if (myInputTable != null) +// (myInputTable as BlueTable).addRows (aRows); +// else +// myInputTable = new BlueTable (aRows); +// myHandleChanges (new DataPacket(myInputTable, aRows, new BlueTable (aRows.schema))); +// } - public void deleteRows (IBlueRowCollection dRows) - { - // BlueEventLog.logAssert (myInputTable != null); - (myInputTable as BlueTable).deleteRows (dRows); - myHandleChanges (new DataPacket (myInputTable, new BlueTable (myInputTable.schema), dRows)); - } - } -} +// public void deleteRows (IBlueRowCollection dRows) +// { +// // BlueEventLog.logAssert (myInputTable != null); +// (myInputTable as BlueTable).deleteRows (dRows); +// myHandleChanges (new DataPacket (myInputTable, new BlueTable (myInputTable.schema), dRows)); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Misc/PubStreamPrefixer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Misc/PubStreamPrefixer.cs deleted file mode 100644 index 44733ce15..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Misc/PubStreamPrefixer.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; - -namespace BCG -{ - public class PubStreamPrefixer - { - private static readonly string myPrefixString = "DS"; - - public static string prefix (string tableName) - { - return myPrefixString + "_" + tableName.ToLower(); - } - - public static string extractTableNameFromTag (string tag) - { - return tag.Substring ((myPrefixString + "_").Length); - } - - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/IPubTablePublisher.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/IPubTablePublisher.cs deleted file mode 100644 index 6e9d07725..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/IPubTablePublisher.cs +++ /dev/null @@ -1,44 +0,0 @@ -using System; - -namespace BCG -{ - /// - /// The table publishing happens here - /// - public interface IPubTablePublisher - { - /// - /// Adds a row to the existing table - /// - /// - void addRow (IBlueRow row); - /// - /// Deletes a row from an existing table - /// - /// - void deleteRow (IBlueRow row); - /// - /// Changes an existing row - /// - /// - void changeRow (IBlueRow row); - /// - /// Creates a table. if already exists overwrites it. - /// - /// - void createTable (IBlueRowCollection rows); - /// - /// Removes and adds more than one row; however the assumption here is that delete is - /// applied first and then add. If that violates the current state, exception is thrown - /// - /// - /// - void batchRemoveAdd (IBlueRowCollection delRows, IBlueRowCollection addRows); - /// - /// Changes more than one row at a time. This is specially helpful, since changes make a huge - /// percentage of the pubsub traffic. - /// - /// - void batchChange (IBlueRowCollection rows); - } -} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTablePublisher.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTablePublisher.cs deleted file mode 100644 index c9a87286f..000000000 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/PubTablePublisher.cs +++ /dev/null @@ -1,162 +0,0 @@ -using System; -using System.Collections; -using System.IO; - -namespace BCG { - /// - /// The only instance of IPubTableBroker. It handles the publishing of one tag, and holds a specialized serializer - /// that it initializes according to the table, to optimize serialiazation, hence bw and performance - /// This class is held (as an interface) by the PubTableBroker, which guarantees the sanity of calls, - /// ie the first call will always be create. - /// - public class PubTablePublisher : IPubTablePublisher { - private readonly string _tag; - protected IBlueDataSerializer mySerializer; - protected PublisherCore myPubCore; - - public PubTablePublisher(string tag, PublisherCore publisherCore) { - _tag = tag; - bts.Log.Gen.info($"PubTablePublisher.cons created for tag: {tag}"); - myPubCore = publisherCore; - mySerializer = new BlueDataSerializationController(); - } - - #region IPubTablePublisher Members - - public virtual void addRow(IBlueRow row) { - byte[] data; - mySerializer.serialize(StreamOpCode.add, - myGetRowCollectionArray(row), - out data); - myPublish(data, row.schema.tableName); - } - - public virtual void deleteRow(IBlueRow row) { - byte[] data; - mySerializer.serialize(StreamOpCode.remove, - myGetRowCollectionArray(row), - out data); - myPublish(data, row.schema.tableName); - } - - public virtual void changeRow(IBlueRow row) { - byte[] data; - mySerializer.serialize(StreamOpCode.change, - myGetRowCollectionArray(row), - out data); - myPublish(data, row.schema.tableName); - } - - public virtual void createTable(IBlueRowCollection rows) { - byte[] data; - mySerializer.serialize(StreamOpCode.create, - new IBlueRowCollection[] { rows }, - out data); - bts.Log.Gen.info($"PubTablePublisher.createTable tag: {_tag} rows: {rows.Count} data: {data.Length}"); - myPublish(data, rows.schema.tableName); - // byte[] ops; - // Array rows_ds; - // _test_deserializer.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows_ds); - } - - //private ClientDeserializationController _test_deserializer = new ClientDeserializationController(); - public virtual void batchRemoveAdd(IBlueRowCollection delRows, IBlueRowCollection addRows) { - byte[] data = null; - if (myIsRealChange(delRows, addRows)) - batchChange(addRows); - else { - mySerializer.serialize(StreamOpCode.multipleRemoveAdd, - new IBlueRowCollection[] { delRows, addRows }, - out data); - myPublish(data, delRows.schema.tableName); - } - // byte[] ops; - // Array rows_ds; - // _test_deserializer.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows_ds); - } - - public virtual void batchChange(IBlueRowCollection rows) { - byte[] data; - - mySerializer.serialize(StreamOpCode.multipleChange, - new IBlueRowCollection[] { rows }, - out data); - myPublish(data, rows.schema.tableName); - } - - #endregion - - #region private methods - private void myPublish(byte[] data, string tableName) { - bts.Log.Gen.info($"PubTablePublisher.myPublish table: {tableName} data: {data.Length}"); - if (data.Length > 0) - myPubCore.publish("DataStreamTag", - PubStreamPrefixer.prefix(tableName), - data); - } - - private IBlueRowCollection[] myGetRowCollectionArray(IBlueRow r) { - return new IBlueRowCollection[]{ - BlueDataUtility.toBlueRowCollection (new IBlueRow[] {r}) - }; - } - - private bool myIsRealChange(IBlueRowCollection delRows, IBlueRowCollection addRows) { - if (delRows.Count == addRows.Count) { - foreach (IBlueRow dRow in delRows) - if (!addRows.containsID((int)dRow["id"])) - return false; - return true; - } else - return false; - } - #endregion - } - - public class IndependentTablePublisher : PubTablePublisher { - bool myIsCreated = false; - - public IndependentTablePublisher(string tag, PublisherCore pubCore) : base(tag, pubCore) { } - - public override void addRow(IBlueRow row) { - myCheckCreate(row.schema); - base.addRow(row); - } - - public override void batchChange(IBlueRowCollection rows) { - myCheckCreate(rows.schema); - base.batchChange(rows); - } - - public override void batchRemoveAdd(IBlueRowCollection delRows, IBlueRowCollection addRows) { - myCheckCreate(delRows.schema); - base.batchRemoveAdd(delRows, addRows); - } - - public override void changeRow(IBlueRow row) { - myCheckCreate(row.schema); - base.changeRow(row); - } - - public override void deleteRow(IBlueRow row) { - myCheckCreate(row.schema); - base.deleteRow(row); - } - - public override void createTable(IBlueRowCollection rows) { - myIsCreated = true; - base.createTable(rows); - } - - - private void myCheckCreate(IBlueTableSchema schema) { - if (!myIsCreated) { - //so that the initiazliation work is done on the serializer - byte[] placeHolder; - mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { new BlueTable(schema) }, out placeHolder); - myIsCreated = true; - } - } - - } -} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/SinglePubTableBroker.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/SinglePubTableBroker.cs new file mode 100644 index 000000000..f9fa7fa64 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/SinglePubTableBroker.cs @@ -0,0 +1,58 @@ +using bts; + +using System; +using System.Collections; +using System.IO; + +namespace BCG { + /// + /// The only instance of IPubTableBroker. It handles the publishing of one tag, and holds a specialized serializer + /// that it initializes according to the table, to optimize serialiazation, hence bw and performance + /// This class is held (as an interface) by the PubTableBroker, which guarantees the sanity of calls, + /// ie the first call will always be create. + /// + public class SinglePubTableBroker { + private readonly string _tag; + protected IStreamIOMsgSerializer _serializer; + protected PublisherCore _publisher_core; + + public SinglePubTableBroker(string tag, PublisherCore publisher_core) { + bts.Log.Gen.info($"SinglePubTableBroker.cons created for tag: {tag}"); + _tag = tag; + _publisher_core = publisher_core; + _serializer = new UniversalStreamIOSerializer(); + } + + public void create_table(IBlueRowCollection rows) { + bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(rows); + bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg); + byte[] msg_bytes = _serializer.serialize(stream_io_msg); + PublishMsg pub_msg = new PublishMsg() { + tag = DSTagManager.get_stream_tag(rows.schema.tableName), + tag_type = DSTagManager.DataStreamTagType, + data_to_publish = msg_bytes + }; + PubSubClientMessage msg = new PubSubClientMessage() { + pub_msg = pub_msg + }; + _publisher_core.publish(msg); + } + + public void make_batch_changes(IBlueRowCollection del_rows, IBlueRowCollection added_rows) { + bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = + new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows); + bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(mra_msg); + byte[] msg_bytes = _serializer.serialize(stream_io_msg); + PublishMsg pub_msg = new PublishMsg() { + tag = DSTagManager.get_stream_tag(added_rows.schema.tableName), + tag_type = DSTagManager.DataStreamTagType, + data_to_publish = msg_bytes + }; + PubSubClientMessage msg = new PubSubClientMessage() { + pub_msg = pub_msg + }; + _publisher_core.publish(msg); + } + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/TableStream.cs b/src/DataStreamSuite/DataStreamCore/TableStream.cs index fef2b2524..211894d58 100644 --- a/src/DataStreamSuite/DataStreamCore/TableStream.cs +++ b/src/DataStreamSuite/DataStreamCore/TableStream.cs @@ -4,89 +4,74 @@ using System.Data; using BCG.Utilities; -namespace BCG -{ - public class TableStream : ViewStream - { +namespace BCG { + public class TableStream : ViewStream { public string tableName; private ITableAccumulator myTA; private bool myGotData = false; public TableStream(string tableName) : - this(tableName, null, null) - { + this(tableName, null, null) { } public TableStream(string tableName, ViewStreamDatabase db) : - this(tableName, db, null) - {} + this(tableName, db, null) { } public TableStream(string tableName, string colExp) : - this(tableName, - null, - colExp) - {} + this(tableName, + null, + colExp) { } - - public TableStream(string ttableName, ViewStreamDatabase db, string colExp) - { - tableName = PubStreamPrefixer.prefix (ttableName); + + public TableStream(string ttableName, ViewStreamDatabase db, string colExp) { + tableName = DSTagManager.get_stream_tag(ttableName); // if (colExp != null) // mySetRealSource (new ProjectStream (new TableStream (tableName, db), colExp)); // else { if (db == null) - myTA = ViewStreamDatabase.instance.getITableAccumulator (tableName); + myTA = ViewStreamDatabase.instance.getITableAccumulator(tableName); else - myTA = db.getITableAccumulator (tableName); + myTA = db.getITableAccumulator(tableName); } } - - protected override void myOnFirstSubscriber() - { - myTA.onUpdate += new TableUpdateDelegate (myHandleUpdates); + + protected override void myOnFirstSubscriber() { + myTA.onUpdate += new TableUpdateDelegate(myHandleUpdates); } - protected override void myOnLastSubscriber() - { - myTA.onUpdate -= new TableUpdateDelegate (myHandleUpdates); + protected override void myOnLastSubscriber() { + myTA.onUpdate -= new TableUpdateDelegate(myHandleUpdates); } - protected virtual void myHandleUpdates (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows) - { - if (!myGotData) - { + protected virtual void myHandleUpdates(IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows) { + if (!myGotData) { myGotData = true; // BlueEventLog.logEvent("Received data for " + tableName + " for the first time", BlueEventLevel.Information); } - myFireEvent (new DataPacket (newRows, addedRows, deletedRows)); + myFireEvent(new DataPacket(newRows, addedRows, deletedRows)); } - public override string ToString() - { + public override string ToString() { return GetType() + " table = " + tableName; } } - public class ThreadedTableStream : TableStream, IExportableDataStream - { + public class ThreadedTableStream : TableStream, IExportableDataStream { static private Lazy _vsd = new Lazy(() => new ViewStreamDatabase(), true); - public ThreadedTableStream(string table) : base(table, _vsd.Value) - { + public ThreadedTableStream(string table) : base(table, _vsd.Value) { } - public void begin_subscription() - { + public void begin_subscription() { this.onChange += _stream_onChange; } public event ExportableChangeDelegate on_change; - void _stream_onChange(object newValue) - { - if(on_change != null) + void _stream_onChange(object newValue) { + if (on_change != null) on_change(newValue); } } diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs index 4fee092a0..a82fe1f9c 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs @@ -623,12 +623,12 @@ namespace BCG { SortedList sl = new SortedList(); if (string.IsNullOrEmpty(myTableFilterBox.Text)) foreach (IBlueRow r in rows) { - sl[r["tag"].ToString().Substring(PubStreamPrefixer.prefix("").Length)] = true; + sl[r["tag"].ToString().Substring(DSTagManager.get_stream_tag("").Length)] = true; } else { Regex tableRegex = new Regex(myTableFilterBox.Text); foreach (IBlueRow r in rows) { - string thisTable = r["tag"].ToString().Substring(PubStreamPrefixer.prefix("").Length); + string thisTable = r["tag"].ToString().Substring(DSTagManager.get_stream_tag("").Length); if (tableRegex.IsMatch(thisTable)) sl[thisTable] = true; } diff --git a/src/DataStreamSuite/PubSub/AllPublishedTagState.cs b/src/DataStreamSuite/PubSub/AllPublishedTagState.cs deleted file mode 100644 index 8f6137c1c..000000000 --- a/src/DataStreamSuite/PubSub/AllPublishedTagState.cs +++ /dev/null @@ -1,83 +0,0 @@ -using System; -using System.IO; -using System.Collections; -using System.Collections.Generic; -using BCG.Utilities; -using System.Linq; -using bts; - -namespace BCG { - /// - /// This isa client side class (for the publisher) that holds the state for a single tag (on the publisher side). - /// If the tag_type has an IPubSubServerTagHandler, it is updated and the state stored - /// there. Otherwise, this class saves the last published data. - /// - internal class SingleTagHandler { - private IPubSubServerTagHandler _handler = null; - private string _tag, _tag_type; - - public SingleTagHandler(string tag, string tag_type) { - _tag = tag; - _tag_type = tag_type; - _handler = bts.PubsubTagHandlerFactory.get_handler(tag_type, false); - Log.assert(_handler != null); - } - - public string tag_type { get { return _tag_type; } } - - public void publish(byte[] msg) { - _handler.update(_tag, msg); - } - - public byte[] get_content() { - return _handler.get_content(_tag); - } - } - - /// - /// Keeps the state of all the tags published. - /// - public class AllPublishedTagState { - private Dictionary _tag_map = new Dictionary(); - - public AllPublishedTagState() { } - - - public void publish(string tag, string tag_type, byte[] data) { - SingleTagHandler tag_handler = null; - if (!_tag_map.TryGetValue(tag, out tag_handler)) { - tag_handler = new SingleTagHandler(tag, tag_type); - _tag_map.Add(tag, tag_handler); - } - tag_handler.publish(data); - } - - public string[] publishedTags { - get { - return _tag_map.Keys.ToArray(); - } - } - - public byte[] getContent(string tag, out string tag_type) { - SingleTagHandler tag_handler = null; - if (!_tag_map.TryGetValue(tag, out tag_handler)) { - tag_type = tag_handler.tag_type; - return tag_handler.get_content(); - } - tag_type = String.Empty; - return new byte[] { }; - } - - public bool contains(string tag) { - return _tag_map.ContainsKey(tag); - } - public void clear(string tag) { - if (_tag_map.ContainsKey(tag)) { - _tag_map.Remove(tag); - } else { - bts.Log.Gen.warn($"AllPublishTagState told to clear nonexisting tag: {tag} "); - } - - } - } -} diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs b/src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs index d7bc20f22..0703093cc 100644 --- a/src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs +++ b/src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs @@ -121,7 +121,7 @@ namespace bts { public string tag_type; public bool is_compressed = false; // this is the size that of the msg, regarless of it being serialized or not - public long publish_time_epoch_ms; + public long publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); public byte[] data_to_publish; public void serialize_into_str(Stream str) { diff --git a/src/DataStreamSuite/PubSub/Client/TagType2HandlerMap.cs b/src/DataStreamSuite/PubSub/Client/PubsubTagHandlerFactory.cs similarity index 62% rename from src/DataStreamSuite/PubSub/Client/TagType2HandlerMap.cs rename to src/DataStreamSuite/PubSub/Client/PubsubTagHandlerFactory.cs index c94bcd1d5..e2e888082 100644 --- a/src/DataStreamSuite/PubSub/Client/TagType2HandlerMap.cs +++ b/src/DataStreamSuite/PubSub/Client/PubsubTagHandlerFactory.cs @@ -8,51 +8,6 @@ using BCG; namespace bts { - - // Simple wrapper that hold multiple IPubsubServerTagHandlers, one for each tag - // public class PubsubServerMultiTagHandler : BCG.IPubSubServerTagHandler where T : IPubSubServerTagHandler { - // Dictionary _handler_map = new Dictionary(); - // private readonly string _tag; - // Func _tag_creator; - - // public PubsubServerMultiTagHandler(Func tag_creator) { - // _tag_creator = tag_creator; - // } - // public void update(string tag, byte[] msg) { - // BCG.IPubSubServerTagHandler rv = _get_handler(tag); - // rv.update(tag, msg); - // } - // public byte[] get_content(string tag) { - // BCG.IPubSubServerTagHandler rv = _get_handler(tag); - // return rv.get_content(tag); - // } - // public string[] supportedTagTypes { - // get { - // List rv = new List(); - // foreach (var h in _handler_map.Values) - // rv.AddRange(h.supportedTagTypes); - // return rv.ToArray(); - // } - // } - - // private BCG.IPubSubServerTagHandler _get_handler(string tag) { - // BCG.IPubSubServerTagHandler rv = null; - // if (!_handler_map.TryGetValue(tag, out rv)) { - // rv = _tag_creator(tag); - // _handler_map.Add(tag, rv); - // } - // return rv; - // } - // public void register(InternalPublishDelegate callback) { - // foreach (var h in _handler_map.Values) - // h.register(callback); - // } - // public void doWork() { - // foreach (var h in _handler_map.Values) - // h.doWork(); - // } - // } - public static class PubsubTagHandlerFactory { // each tag type is represented by this public class TagHandlerEntry { @@ -67,7 +22,7 @@ namespace bts { } private static Dictionary _config_map = new Dictionary() { - {"DS", new TagHandlerEntry("DS" , "DataStreamCore", "bts.PubsubServerStreamTagHandler")}, + {DSTagManager.DataStreamTagType, new TagHandlerEntry(DSTagManager.DataStreamTagType , "DataStreamCore", "bts.PubsubServerStreamTagHandler")}, {"System_Sub", new TagHandlerEntry("System_Sub", "PubSub", "BCG.SystemSubscriptionMapPublisher")}, {"System_Stats", new TagHandlerEntry("System_Stats", "PubSub", "BCG.SystemStatsPublisher")} }; diff --git a/src/DataStreamSuite/PubSub/Config/DSTagManager.cs b/src/DataStreamSuite/PubSub/Config/DSTagManager.cs new file mode 100644 index 000000000..12f4ca8e4 --- /dev/null +++ b/src/DataStreamSuite/PubSub/Config/DSTagManager.cs @@ -0,0 +1,18 @@ +using System; + +namespace BCG { + public static class DSTagManager { + private static readonly string _ds_tag_pefix = "DS"; + + public static string get_stream_tag(string table_name) { + return _ds_tag_pefix + "_" + table_name.ToLower(); + } + + public static string build_table_name_from_tag(string tag) { + return tag.Substring((_ds_tag_pefix + "_").Length); + } + + public static string DataStreamTagType => "DataStream"; + + } +} diff --git a/src/DataStreamSuite/PubSub/PublishedCache.cs b/src/DataStreamSuite/PubSub/PublishedCache.cs new file mode 100644 index 000000000..6c02d5848 --- /dev/null +++ b/src/DataStreamSuite/PubSub/PublishedCache.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; + +namespace bts { + // This caches all the published data. This class is created and held by + // PublisherCore. Whend ata is publsihed , PublisherCore passes it to this + // class via the publish call. On a reconnection to the PubSubServer, + // PublisherCore calls get_content to get the data to re-publish, to + // re-seed the PubSubServer + public class PublishedCache { + private Dictionary _data_map = new Dictionary(); + + public PublishedCache() { } + + + public void publish(PublishMsg msg) { + SingleTagData tag_data = null; + if (!_data_map.TryGetValue(msg.tag, out tag_data)) { + tag_data = new SingleTagData(msg.tag, msg.tag_type); + _data_map.Add(msg.tag, tag_data); + } + tag_data.handle_publish(msg); + } + + public byte[] get_content(string tag, out string tag_type) { + SingleTagData tag_data; + if (!_data_map.TryGetValue(tag, out tag_data)) { + tag_type = tag_data.tag_type; + return tag_data.get_current_content(); + } + tag_type = String.Empty; + return new byte[] { }; + } + + public void clear(string tag) { + if (_data_map.ContainsKey(tag)) { + _data_map.Remove(tag); + } else { + bts.Log.Gen.warn($"AllPublishTagState told to clear nonexisting tag: {tag} "); + } + } + } +} diff --git a/src/DataStreamSuite/PubSub/PublisherCore.cs b/src/DataStreamSuite/PubSub/PublisherCore.cs index 687df550b..074c75b3a 100644 --- a/src/DataStreamSuite/PubSub/PublisherCore.cs +++ b/src/DataStreamSuite/PubSub/PublisherCore.cs @@ -20,7 +20,7 @@ namespace BCG { // all the data published through this object in snapshot form // i.e. in a form that can be used repopulate a pubsub server // in the event of a crash - protected AllPublishedTagState _state = new AllPublishedTagState(); + protected PublishedCache _state = new PublishedCache(); // a different IPubSubServerConnection object is used to interact // with each of the pubsub servers used by this publishercore @@ -255,7 +255,7 @@ namespace BCG { protected virtual void _publish(PublishMsg msg) { bts.Log.Gen.info($"PublisherCore._publish msg: {msg}"); // update the state - _state.publish(msg.tag, msg.tag_type, msg.data_to_publish); + _state.publish(msg); // send the data out var sender = _get_or_build_sender(msg.tag); // actually send the data out @@ -299,7 +299,7 @@ namespace BCG { var conn = kv.Value; if (conn.server_url == server_url) { string tag_type; - var content_bytes = _state.getContent(tag, out tag_type); + var content_bytes = _state.get_content(tag, out tag_type); //_send_data(tag, tag_type, tag_data); bts.PublishMsg publish_msg = new bts.PublishMsg() { connection = kv.Key,