diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs index 73c3e72f2..0244b053d 100644 --- a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs +++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsRow.cs @@ -1,17 +1,14 @@ using System; using System.Collections; -namespace BCG -{ - internal class AddColumnsRow : IBlueRow - { +namespace BCG { + internal class AddColumnsRow : IBlueRow { private IBlueTableSchema mySchema; private IBlueRow myRow; //private object[] myCachedValues; #region Constructors - public AddColumnsRow (IBlueTableSchema schema, IBlueRow row) - { + public AddColumnsRow(IBlueTableSchema schema, IBlueRow row) { mySchema = schema; myRow = row; //myCachedValues = new object[schema.columnNames.Length]; @@ -19,85 +16,78 @@ namespace BCG #endregion #region IBlueRow members - public IBlueTableSchema schema{get{return mySchema;}} + public IBlueTableSchema schema { get { return mySchema; } } #endregion #region ICollection members - public int Count {get{return mySchema.columnNames.Length;}} + public int Count { get { return mySchema.columnNames.Length; } } - public bool IsSynchronized {get{return false;}} + public bool IsSynchronized { get { return false; } } - public object SyncRoot {get{return this;}} + public object SyncRoot { get { return this; } } - public void CopyTo (Array array, int index) - { + public void CopyTo(Array array, int index) { int i = index; foreach (object o in this as IEnumerable) - array.SetValue (o, i++); + array.SetValue(o, i++); } #endregion #region IDictionary members - public bool IsFixedSize {get{return true;}} + public bool IsFixedSize { get { return true; } } - public bool IsReadOnly {get{return true;}} + public bool IsReadOnly { get { return true; } } /// /// The key can either be the string corrsponding to the column name or /// the order of the column /// - public object this [object key] - { - get - { + public object this[object key] { + get { if (key is string) return this[(string)key]; return this[(int)key]; } - set {throw new NotSupportedException ("The IBlueRow is readonly");} + set { throw new NotSupportedException("The IBlueRow is readonly"); } } - public object this[int index] - { - get - { + public object this[int index] { + get { //if (myCachedValues[index] == null) // myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index]; //return myCachedValues[index]; - return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index]; + return mySchema[index].isConstant ? DBNull.Value : myRow[index]; } } - public object this[string colName] {get{return this[mySchema[colName].colNo];}} + public object this[string colName] { get { return this[mySchema[colName].colNo]; } } - public ICollection Keys{get{return mySchema.columnNames;}} + public ICollection Keys { get { return mySchema.columnNames; } } - public ICollection Values - { - get - { + public ICollection Values { + get { object[] retVal = new object[mySchema.columnNames.Length]; - CopyTo (retVal, 0); + CopyTo(retVal, 0); return retVal; } } - public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");} + public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); } - public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");} + public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); } - public bool Contains (object key){return mySchema[(string)key] != null;} + public bool Contains(object key) { return mySchema[(string)key] != null; } /// /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to /// iterate through the values, typecast this into an IEnumerable /// /// - public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);} - - IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);} + public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } - public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");} + IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); } + + public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); } #endregion } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs index 0d520578c..496059a3c 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs @@ -4,18 +4,14 @@ using BCG.Utilities; using System.Collections.Generic; using System.Linq; -namespace BCG -{ - public class GroupByStream : MethodDrivenStream - { - private class MyFuncGroupByEvaluator : IGroupByEvaluator - { +namespace BCG { + public class GroupByStream : MethodDrivenStream { + private class MyFuncGroupByEvaluator : IGroupByEvaluator { private string[] _groupby_cols; private Func _groupby_func; - public MyFuncGroupByEvaluator(IEnumerable groupby_cols, Func groupby_func) - { + public MyFuncGroupByEvaluator(IEnumerable groupby_cols, Func groupby_func) { _groupby_cols = groupby_cols.ToArray(); - _groupby_func = groupby_func; + _groupby_func = groupby_func; } public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); } @@ -29,7 +25,7 @@ namespace BCG /// Serves as a quick way to access all rows belonging to the same group /// private bool myIdGuard = true; - private RowDictionaryTable myBookKeepingTable ; + private RowDictionaryTable myBookKeepingTable; private IGroupByEvaluator myGroupByEvaluator; private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's //can map a schema to another @@ -37,112 +33,98 @@ namespace BCG private int myIdGenerator = 0; public GroupByStream(ViewStream stream, IEnumerable groupby_cols, Func groupby_func) - : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) - { } + : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { } - public GroupByStream (ViewStream stream, IGroupByEvaluator groupByEvaluator) - : this (stream, groupByEvaluator, true) - {} + public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator) + : this(stream, groupByEvaluator, true) { } - public GroupByStream (ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard) - : base (stream) - { + public GroupByStream(ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard) + : base(stream) { myGroupByEvaluator = groupbyEvaluator; myIdGuard = idGuard; } - protected override void myHandleFirstCall () - { + protected override void myHandleFirstCall() { mySchema = myLastArrivedPacket.schema; - myOutputSchema = myGroupByEvaluator.getOutputSchema (mySchema); - myBookKeepingTable = new RowDictionaryTable (myLastArrivedPacket.newRows, + myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema); + myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows, myGroupByEvaluator.groupByColumns); - GroupByOutputTable temp = new GroupByOutputTable (myOutputSchema); - foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) - { - IBlueRow newRow = myEvaluate (new DataPacket (rowGroup, - rowGroup, new BlueTable (mySchema))); + GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema); + foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) { + IBlueRow newRow = myEvaluate(new DataPacket(rowGroup, + rowGroup, new BlueTable(mySchema))); if (newRow != null) - temp.addRow (newRow, myGetGroupString (rowGroup)); + temp.addRow(newRow, myGetGroupString(rowGroup)); } - myLastFiredPacket = new DataPacket (temp, new BlueTable(temp), new BlueTable (temp.schema)); + myLastFiredPacket = new DataPacket(temp, new BlueTable(temp), new BlueTable(temp.schema)); } /// /// Adding need not to anything as deletion already takes care /// of addition /// - protected override void myHandleIncrementalAdd() {} - + protected override void myHandleIncrementalAdd() { } + /// /// Both deletes and adds are handled here to make the intermediate results not /// appear in the spitted fired DataPacket (note this is done only in GroupByStream /// because of the nature of it) /// - protected override void myHandleIncrementalDelete() - { + protected override void myHandleIncrementalDelete() { myFireSwitch = false; RowDictionaryTable delGroups, addGroups; //form groups for deleted and added rows - myInitIncrementalChangeData (out delGroups, out addGroups); - ArrayList handledAddGroups = new ArrayList (); + myInitIncrementalChangeData(out delGroups, out addGroups); + ArrayList handledAddGroups = new ArrayList(); string groupRepString; - foreach (IBlueRowCollection delGroup in delGroups.getGroups ()) - { + foreach (IBlueRowCollection delGroup in delGroups.getGroups()) { // BlueEventLog.logAssert (delGroup.Count > 0); - groupRepString = myGetGroupString (delGroup); - IBlueRowCollection addGroup = addGroups.getGroup (groupRepString) == null ? - new BlueTable (mySchema) : addGroups.getGroup (groupRepString); - handledAddGroups.Add (addGroup); - IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString) == null ? - new BlueTable (mySchema) : myBookKeepingTable.getGroup (groupRepString); - IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, delGroup)); - myFireSwitch |= myMergeRow (newRow, groupRepString); + groupRepString = myGetGroupString(delGroup); + IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ? + new BlueTable(mySchema) : addGroups.getGroup(groupRepString); + handledAddGroups.Add(addGroup); + IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ? + new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString); + IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup)); + myFireSwitch |= myMergeRow(newRow, groupRepString); } - foreach (IBlueRowCollection addGroup in addGroups.getGroups ()) - { + foreach (IBlueRowCollection addGroup in addGroups.getGroups()) { // BlueEventLog.logAssert (addGroup.Count > 0); - if (!handledAddGroups.Contains (addGroup)) - { + if (!handledAddGroups.Contains(addGroup)) { //this is the case when there are rows added to a group and none //have been removed, hence it 'escapes' from being dealt with in //the loop above - groupRepString = myGetGroupString (addGroup); - IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString); + groupRepString = myGetGroupString(addGroup); + IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString); // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0); - IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, - new BlueTable (aggGroup.schema))); - myFireSwitch |= myMergeRow (newRow, groupRepString); + IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, + new BlueTable(aggGroup.schema))); + myFireSwitch |= myMergeRow(newRow, groupRepString); } } } - private bool myMergeRow (IBlueRow newRow, string groupRepString) - { - IBlueRow existingRow = ((GroupByOutputTable) myLastFiredPacket.newRows).getRow (groupRepString); + private bool myMergeRow(IBlueRow newRow, string groupRepString) { + IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString); //BlueEventLog.logAssert (newRow != null || existingRow != null); if (existingRow == null && newRow == null) return false; - else if (existingRow != null && newRow != null && BlueDataUtility.computeRowEquality (existingRow, newRow)) + else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow)) return false; - else if (existingRow == null) - { + else if (existingRow == null) { //row added for the first time - ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString); - ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow); + ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString); + ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow); return true; - } - else - { - ((GroupByOutputTable) myLastFiredPacket.newRows).removeRow (groupRepString); - ((BlueTable) myLastFiredPacket.deletedRows).addRow (existingRow); - if (newRow != null) - { - ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString); - ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow); + } else { + ((GroupByOutputTable)myLastFiredPacket.newRows).removeRow(groupRepString); + ((BlueTable)myLastFiredPacket.deletedRows).addRow(existingRow); + if (newRow != null) { + ((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString); + ((BlueTable)myLastFiredPacket.addedRows).addRow(newRow); } return true; } @@ -154,77 +136,68 @@ namespace BCG /// /// /// - private void myInitIncrementalChangeData (out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) - { + private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) { delGroups = - new RowDictionaryTable (myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns); + new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns); addGroups = - new RowDictionaryTable (myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns); + new RowDictionaryTable(myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns); foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows) - myBookKeepingTable.removeRow (deletedRow); + myBookKeepingTable.removeRow(deletedRow); foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows) - myBookKeepingTable.addRow (addedRow); - myLastFiredPacket.addedRows = new BlueTable (myOutputSchema); - myLastFiredPacket.deletedRows = new BlueTable (myOutputSchema); + myBookKeepingTable.addRow(addedRow); + myLastFiredPacket.addedRows = new BlueTable(myOutputSchema); + myLastFiredPacket.deletedRows = new BlueTable(myOutputSchema); } - protected override void myOnTableClear () - { + protected override void myOnTableClear() { if (myBookKeepingTable.Count == 0) return; //ie there arent any rows, so no need to optimize the 'cleansing' process - myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows); - myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema); - (myLastFiredPacket.newRows as GroupByOutputTable).clear (); - myBookKeepingTable.clear (); - myGroupByEvaluator.clear (); + myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows); + myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema); + (myLastFiredPacket.newRows as GroupByOutputTable).clear(); + myBookKeepingTable.clear(); + myGroupByEvaluator.clear(); } - private IBlueRow myEvaluate (DataPacket dp) - { - IBlueRow evalRow = myGroupByEvaluator.evaluate (dp); - return evalRow == null ? null : + private IBlueRow myEvaluate(DataPacket dp) { + IBlueRow evalRow = myGroupByEvaluator.evaluate(dp); + return evalRow == null ? null : myIdGuard ? - BlueDataUtility.getNewReplacedRow (evalRow, myIdGenerator++, "id") : evalRow; + BlueDataUtility.getNewReplacedRow(evalRow, myIdGenerator++, "id") : evalRow; } - private string myGetGroupString (IBlueRowCollection group) - { - return BlueDataUtility.rowToString(BlueDataUtility.extractRow (group, 0), myGroupByEvaluator.groupByColumns); + private string myGetGroupString(IBlueRowCollection group) { + return BlueDataUtility.rowToString(BlueDataUtility.extractRow(group, 0), myGroupByEvaluator.groupByColumns); } - protected override void myOnLastSubscriber () - { + protected override void myOnLastSubscriber() { //the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream //gets resubscribed, we start the process of building data again, and treat the callback as the first //callback. - myInit (); - myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler); + myInit(); + myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler); } - protected override void myInit () - { - base.myInit (); - if (myBookKeepingTable != null) - { + protected override void myInit() { + base.myInit(); + if (myBookKeepingTable != null) { //to avoid calling init before the first callback - myBookKeepingTable.clear (); - myGroupByEvaluator.clear (); + myBookKeepingTable.clear(); + myGroupByEvaluator.clear(); } } - public static string getRepString (DataPacket dp, string[] colNames) - { - return BlueDataUtility.rowToString (getRepRow (dp), colNames); + public static string getRepString(DataPacket dp, string[] colNames) { + return BlueDataUtility.rowToString(getRepRow(dp), colNames); } - public static IBlueRow getRepRow (DataPacket dp) - { + public static IBlueRow getRepRow(DataPacket dp) { if (dp.newRows.Count > 0) - return BlueDataUtility.extractRow (dp.newRows, 0); + return BlueDataUtility.extractRow(dp.newRows, 0); else if (dp.addedRows.Count > 0) - return BlueDataUtility.extractRow (dp.addedRows, 0); + return BlueDataUtility.extractRow(dp.addedRows, 0); else if (dp.deletedRows.Count > 0) - return BlueDataUtility.extractRow (dp.deletedRows, 0); + return BlueDataUtility.extractRow(dp.deletedRows, 0); // BlueEventLog.logAssert (false, "Illegal Data Packet"); return null; } @@ -237,22 +210,21 @@ namespace BCG /// /// /// - public static object incrementalAdd (object lastValue, object newValue, bool isAdd) - { - if (lastValue == null || lastValue.Equals (DBNull.Value)) + public static object incrementalAdd(object lastValue, object newValue, bool isAdd) { + if (lastValue == null || lastValue.Equals(DBNull.Value)) return newValue; // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) || - // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) || - // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal))); + // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) || + // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal))); Object retVal = null; - if (newValue.GetType().Equals (typeof(int))) + if (newValue.GetType().Equals(typeof(int))) retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue; - else if (newValue.GetType().Equals (typeof(Double))) + else if (newValue.GetType().Equals(typeof(Double))) retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue; - else if (lastValue.GetType().Equals (typeof(Decimal))) + else if (lastValue.GetType().Equals(typeof(Decimal))) retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue; - else{} - // BlueEventLog.logAssert (false, "Illegal type"); + else { } + // BlueEventLog.logAssert (false, "Illegal type"); return retVal; } diff --git a/src/DataStreamSuite/DataStream/Primitives/Map/DynamicUntypedToTypedMapper.cs b/src/DataStreamSuite/DataStream/Primitives/Map/DynamicUntypedToTypedMapper.cs index c7eeb2855..a08410df2 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Map/DynamicUntypedToTypedMapper.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Map/DynamicUntypedToTypedMapper.cs @@ -1,24 +1,24 @@ -using System; +// using System; -namespace BCG -{ - public class DynamicUntypedToTypedMapper : IDynamicStreamMapper - { - IBlueTableSchema mySchema; +// namespace BCG +// { +// public class DynamicUntypedToTypedMapper : IDynamicStreamMapper +// { +// IBlueTableSchema mySchema; - public IBlueRow map (IBlueRow row) - { - if (mySchema == null) - { - ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length]; - int i = 0; - foreach (ColumnInfo ci in row.schema.columnInfo) - cis[i++] = ci.colType == null ? - new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) : - new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue); - mySchema = new BlueTableSchema (row.schema.tableName, cis); - } - return new BlueRow (mySchema, row); - } - } -} +// public IBlueRow map (IBlueRow row) +// { +// if (mySchema == null) +// { +// ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length]; +// int i = 0; +// foreach (ColumnInfo ci in row.schema.columnInfo) +// cis[i++] = ci.colType == null ? +// new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) : +// new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue); +// mySchema = new BlueTableSchema (row.schema.tableName, cis); +// } +// return new BlueRow (mySchema, row); +// } +// } +// } diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs index 29e87e7a2..958ef6f40 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs @@ -73,7 +73,7 @@ namespace bts { private readonly int _payload_sz; private const byte _payload_byte = (byte)'A'; private readonly string _tag_name; - public PubSubTestingStream(int publish_period_seconds, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) { + public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) { _vsd = vsd; _payload_sz = payload_size; var schema = BlueDataUtility.form_schema("PubSubTest", @@ -87,7 +87,7 @@ namespace bts { Tuple.Create("match", TypeCode.Boolean), Tuple.Create("echo_time", TypeCode.Int32)); _table = new BlueTable(schema); - _publish_period = TimeSpan.FromSeconds(publish_period_seconds); + _publish_period = TimeSpan.FromMilliseconds(publish_period_msecs); _msg_size = payload_size; _max_pubs = max_pubs; _tag_name = tag_name; diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs index 2f9fa9ca6..a95926fb2 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs @@ -1,15 +1,21 @@ +using Amazon.Runtime.Internal; + using bts; +using DnsClient.Protocol; + using System; using System.Collections.Generic; using System.IO; +using System; +using Microsoft.VisualBasic; namespace BCG { /// - /// This stream (not for prod) provies a way to run tests - that could otherwise be run in NUnits. + /// This stream (not for prod) provides a way to run tests - that could otherwise be run in NUnits. /// (truth is stream rpovide a RDE for quickly coding and running, so it was eaiser to run tests here) /// [DataStreamUserRespresentation("tests")] @@ -39,21 +45,38 @@ namespace BCG { _schema = BlueDataUtility.form_schema("Tests", Tuple.Create("id", TypeCode.Int32), Tuple.Create("test", TypeCode.String), - Tuple.Create("result", TypeCode.Boolean) + Tuple.Create("result", TypeCode.Boolean), + Tuple.Create("exception", TypeCode.String) ); // resgiter tests here _tests.Add(new Func(_run_sub_msg_test)); _tests.Add(new Func(_run_pub_msg_test)); _tests.Add(new Func(_run_pubsub_server_msg_test)); _tests.Add(new Func(_run_pubsub_client_msg_test)); + _tests.Add(new Func(_run_columninfo_serialization_test)); + _tests.Add(new Func(_run_schema_serialization_test)); + _tests.Add(new Func(_row_serialization_tests_no_dbnulls)); + _tests.Add(new Func(_row_serialization_tests_with_dbnulls)); + _tests.Add(new Func(_row_serialization_tests_all_types)); + _tests.Add(new Func(_run_full_stream_serialization_test)); + _tests.Add(new Func(_run_full_stream_serialization_with_controller_test)); } protected override void myOnFirstSubscriber() { BlueTable rows = new BlueTable(_schema); // u the tests here foreach (var test in _tests) { - var tr = test(); - IBlueRow r = new BlueRow(_schema, rows.Count, tr.test_name, tr.test_result); + string ex_msg = String.Empty; + TestResult tr = new TestResult() { + test_name = test.Method.Name, + test_result = false + }; + try { + tr = test(); + } catch (Exception ex) { + ex_msg = ex.Message + "\n" + ex.StackTrace; + } + IBlueRow r = new BlueRow(_schema, rows.Count, tr.test_name, tr.test_result, ex_msg); rows.addRow(r); } var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true); @@ -141,5 +164,303 @@ namespace BCG { test_result = res }; } + + private TestResult _run_columninfo_serialization_test() { + Func test_ci = (ci) => { + MemoryStream str = new MemoryStream(); + BinaryWriter bw = new BinaryWriter(str); + StreamSerializationUtils.serialize_column_info(ci, ref bw); + str.Position = 0; + //var ci_txt = BlueDataUtility.to_string(ci); + //var ci_ser = + //bts.Log.Gen.info("ci: {}") + BinaryReader br = new BinaryReader(str); + var ci_prime = StreamSerializationUtils.deserialize_column_info(ref br); + return BlueDataUtility.compute_equality(ci, ci_prime); + }; + int i = 0; + foreach (Type t in BlueDataUtility.valid_column_types()) { + ColumnInfo ci = new ColumnInfo(Guid.NewGuid().ToString(), + i++, t); + var res = test_ci(ci); + if (!res) { + return new TestResult { + test_name = "ColumnInfoSerialization", + test_result = false + }; + } + } + return new TestResult { + test_name = "ColumnInfoSerialization", + test_result = true + }; + } + + private TestResult _run_schema_serialization_test() { + var table_name = "test_table";//Guid.NewGuid().ToString(); + BlueTableSchema schema = new BlueTableSchema(); + List cis = new List(); + foreach (Type t in BlueDataUtility.valid_column_types()) { + if (t == null) + continue; + var col_name = t.ToString() + "_" + cis.Count.ToString(); // Guid.NewGuid().ToString(); + ColumnInfo ci = new ColumnInfo(col_name, cis.Count, t); + cis.Add(ci); + } + schema = new BlueTableSchema(table_name, cis.ToArray()); + MemoryStream str = new MemoryStream(); + BinaryWriter bw = new BinaryWriter(str); + StreamSerializationUtils.serialize_schema(schema, ref bw); + var hex_txt = StreamSerializationUtils.to_hex(str); + bts.Log.Gen.info($"_run_schema_serialization_test ser: {hex_txt}"); + + str.Position = 0; + BinaryReader br = new BinaryReader(str); + var schema_prime = StreamSerializationUtils.deserialize_schema(ref br); + var res = BlueDataUtility.compute_equality(schema, schema_prime); + + return new TestResult { + test_name = "SchemaSerialization", + test_result = res + }; + } + + private TestResult _run_universal_row_serialization_test() { + return new TestResult { + test_name = "UniversalRowwSerialization", + test_result = false + }; + } + + private TestResult _run_full_stream_serialization_test() { + // create message + var schema = create_schema_with_all_types(); + BlueTable cr_rows = new BlueTable(schema); + // 100 rows + int n_rows = 100; + for (int i = 0; i < n_rows; i++) { + var nr = generate_random_row(schema, i); + cr_rows.addRow(nr); + } + MemoryStream str = new MemoryStream(); + BinaryWriter bw = new BinaryWriter(str); + bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows); + bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg); + UniversalStreamIOSerializer ser = new UniversalStreamIOSerializer(); + ser.serialize(stream_io_msg, ref bw); + str.Position = 0; + BinaryReader br = new System.IO.BinaryReader(str); + UniversalStreamIODeserializer deser = new bts.UniversalStreamIODeserializer(); + var stream_io_msg_prime = deser.deserialize(ref br); + bool passes = stream_io_msg.Equals(stream_io_msg_prime); + var n_dels = 50; + BlueTable del_rows = new BlueTable(cr_rows.schema); + foreach (IBlueRow r in cr_rows) { + del_rows.addRow(r); + if (del_rows.Count == n_dels) + break; + } + IBlueRowCollection added_rows = del_rows; + bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows); + str = new MemoryStream(); + bw = new BinaryWriter(str); + stream_io_msg = new bts.StreamIOMessage(mra_msg); + ser.serialize(stream_io_msg, ref bw); + bw.BaseStream.Position = 0; + br = new BinaryReader(str); + //deser = new UniversalStreamIODeserializer(); + stream_io_msg_prime = deser.deserialize(ref br); + passes = passes && stream_io_msg.Equals(stream_io_msg_prime); + return new TestResult { + test_name = "UniversalSerialization", + test_result = passes + }; + } + + private TestResult _run_full_stream_serialization_with_controller_test() { + // create message + var schema = create_schema_with_all_types(); + BlueTable cr_rows = new BlueTable(schema); + // 100 rows + int n_rows = 100; + for (int i = 0; i < n_rows; i++) { + var nr = generate_random_row(schema, i); + cr_rows.addRow(nr); + } + MemoryStream str = new MemoryStream(); + BinaryWriter bw = new BinaryWriter(str); + bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows); + bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg); + IStreamIOMsgSerializer ser = new StreamIOSerializationController(); + ser.serialize(stream_io_msg, ref bw); + str.Position = 0; + BinaryReader br = new System.IO.BinaryReader(str); + IStreamMsgDeserilizer deser = new bts.StreamIODeserializationController(); + var stream_io_msg_prime = deser.deserialize(ref br); + bool passes = stream_io_msg.Equals(stream_io_msg_prime); + //mult remove adds + var n_dels = 50; + BlueTable del_rows = new BlueTable(cr_rows.schema); + foreach (IBlueRow r in cr_rows) { + del_rows.addRow(r); + if (del_rows.Count == n_dels) + break; + } + IBlueRowCollection added_rows = del_rows; + bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows); + str = new MemoryStream(); + bw = new BinaryWriter(str); + stream_io_msg = new bts.StreamIOMessage(mra_msg); + ser.serialize(stream_io_msg, ref bw); + bw.BaseStream.Position = 0; + br = new BinaryReader(str); + stream_io_msg_prime = deser.deserialize(ref br); + passes = passes && stream_io_msg.Equals(stream_io_msg_prime); + return new TestResult { + test_name = "UniversalSerializationController", + test_result = passes + }; + } + + private TestResult _row_serialization_tests_no_dbnulls() { + IBlueTableSchema schema = BlueDataUtility.form_schema("test_table", + new Tuple("id", TypeCode.Int32), + new Tuple("i1", TypeCode.Int32), + new Tuple("d1", TypeCode.Double), + new Tuple("string_1", TypeCode.String)); + bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema); + IBlueRow r = new BlueRow(schema, 0, 100, 123.45, "test_string"); + MemoryStream str = new MemoryStream(); + BinaryWriter bw = new BinaryWriter(str); + row_ser.serialize_row(r, 0, ref bw); + str.Position = 0; + BinaryReader br = new BinaryReader(str); + bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema); + IBlueRow r_prime = des.deserialize_row(ref br); + var passes = BlueDataUtility.compute_equality(r, r_prime); + return new TestResult { + test_name = "UniversalRowSerialization_NonNull", + test_result = passes + }; + } + + private TestResult _row_serialization_tests_with_dbnulls() { + IBlueTableSchema schema = BlueDataUtility.form_schema("test_table", + new Tuple("id", TypeCode.Int32), + new Tuple("i1", TypeCode.Int32), + new Tuple("d1", TypeCode.Double), + new Tuple("string_1", TypeCode.String)); + bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema); + IBlueRow r = new BlueRow(schema, 0, DBNull.Value, 123.45, "test_string"); + MemoryStream str = new MemoryStream(); + BinaryWriter bw = new BinaryWriter(str); + row_ser.serialize_row(r, 0, ref bw); + str.Position = 0; + BinaryReader br = new BinaryReader(str); + bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema); + IBlueRow r_prime = des.deserialize_row(ref br); + var passes = BlueDataUtility.compute_equality(r, r_prime); + return new TestResult { + test_name = "UniversalRowSerialization_WithDBNulls", + test_result = passes + }; + } + + private TestResult _row_serialization_tests_all_types() { + var schema = create_schema_with_all_types(); + bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema); + IBlueRow r = generate_random_row(schema, 0); + MemoryStream str = new MemoryStream(); + BinaryWriter bw = new BinaryWriter(str); + row_ser.serialize_row(r, 0, ref bw); + str.Position = 0; + BinaryReader br = new BinaryReader(str); + bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema); + IBlueRow r_prime = des.deserialize_row(ref br); + var passes = BlueDataUtility.compute_equality(r, r_prime); + return new TestResult { + test_name = "UniversalRowSerialization_AllTypes", + test_result = passes + }; + } + + #region Helper functions + + public static IBlueTableSchema create_schema_with_all_types() { + var table_name = "test_table"; + BlueTableSchema schema = new BlueTableSchema(); + List cis = new List(); + foreach (Type t in BlueDataUtility.valid_column_types()) { + if (t == null || t == typeof(DateTime)) + continue; + var col_name = t.ToString() + "_" + cis.Count.ToString(); // Guid.NewGuid().ToString(); + ColumnInfo ci = new ColumnInfo(col_name, cis.Count, t); + cis.Add(ci); + } + cis.Add(new ColumnInfo("id", cis.Count, typeof(int))); + schema = new BlueTableSchema(table_name, cis.ToArray()); + return schema; + } + + public static IBlueRow generate_random_row(IBlueTableSchema schema, int id) { + Random rnd = new Random(); + object[] row_vals = new object[schema.columnInfo.Length]; + foreach (ColumnInfo ci in schema.columnInfo) { + if (ci.colName == "id") { + row_vals[ci.colNo] = id; + continue; + } + object v = get_random_object(ci.type_code()); + if (rnd.NextSingle() % 4 == 0) + v = DBNull.Value; + row_vals[ci.colNo] = v; + } + return new BlueRow(schema, row_vals); + } + + public static object get_random_object(TypeCode type_code) { + Random random = new Random(); + switch (type_code) { + case TypeCode.Boolean: + return random.Next(2) == 0; // Random boolean value (true or false) + case TypeCode.Byte: + return (byte)random.Next(byte.MinValue, byte.MaxValue + 1); // Random byte value + case TypeCode.Char: + return (char)random.Next(65, 65 + 27); // Random char value + case TypeCode.Decimal: + Decimal dec = 100.123M; + return dec; + // object obj_d = get_random_object(TypeCode.Double); + // return Convert.ToDecimal(obj_d); + case TypeCode.Double: + return random.NextDouble() * (double.MaxValue - double.MinValue) + double.MinValue; // Random double value + case TypeCode.Int16: + return (short)random.Next(short.MinValue, short.MaxValue + 1); // Random short value + case TypeCode.Int32: + return random.Next(int.MinValue, int.MaxValue); // Random int value + case TypeCode.Int64: + return random.NextInt64(long.MinValue, long.MaxValue); // Random long value + case TypeCode.SByte: + return (sbyte)random.Next(sbyte.MinValue, sbyte.MaxValue + 1); // Random sbyte value + case TypeCode.Single: + return (float)(random.NextDouble() * (float.MaxValue - float.MinValue) + float.MinValue); // Random float value + case TypeCode.String: + return Guid.NewGuid().ToString(); + case TypeCode.UInt16: + return (ushort)random.Next(ushort.MinValue, ushort.MaxValue + 1); // Random ushort value + case TypeCode.UInt32: + return (uint)random.Next(int.MinValue, int.MaxValue) + (uint)int.MaxValue; // Random uint value + case TypeCode.UInt64: + byte[] ulongBuffer = new byte[8]; + random.NextBytes(ulongBuffer); + return BitConverter.ToUInt64(ulongBuffer, 0); + default: + throw new ArgumentException($"Unsupported TypeCode: {type_code}"); + return null; + } + } + #endregion + + } } \ No newline at end of file diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/AlphabetizeColumnsStream.cs b/src/DataStreamSuite/DataStream/Primitives/Union/AlphabetizeColumnsStream.cs index 032d89ebf..498160658 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Union/AlphabetizeColumnsStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Union/AlphabetizeColumnsStream.cs @@ -8,38 +8,36 @@ namespace BCG { private ViewStream mySrc; private IBlueTableSchema mySchema = null; - public AlphabetizeColumnsStream (ViewStream src) { + public AlphabetizeColumnsStream(ViewStream src) { mySrc = src; } - protected override void myOnFirstSubscriber() - { + protected override void myOnFirstSubscriber() { mySrc.onChange += new ChangeDelegate(mySrc_onChange); } - protected override void myOnLastSubscriber () - { - mySrc.onChange -= new ChangeDelegate (mySrc_onChange); + protected override void myOnLastSubscriber() { + mySrc.onChange -= new ChangeDelegate(mySrc_onChange); } - private void mySrc_onChange (object newValue) { + private void mySrc_onChange(object newValue) { DataPacket dp = (DataPacket)newValue; if (mySchema == null) { - SortedList alphabetical = new SortedList (); + SortedList alphabetical = new SortedList(); foreach (ColumnInfo cinfo in dp.newRows.schema.columnInfo) - alphabetical.Add (cinfo.colName,cinfo); + alphabetical.Add(cinfo.colName, cinfo); ColumnInfo[] cinfos = new ColumnInfo[alphabetical.Count]; - alphabetical.Values.CopyTo (cinfos,0); + alphabetical.Values.CopyTo(cinfos, 0); for (int i = 0; i < cinfos.Length; ++i) - cinfos[i] = new ColumnInfo (cinfos[i].colName,i,cinfos[i].colType,cinfos[i].colValue); - mySchema = new BlueTableSchema (cinfos,dp.schema.tableName); + cinfos[i] = new ColumnInfo(cinfos[i].colName, i, cinfos[i].colType); + mySchema = new BlueTableSchema(cinfos, dp.schema.tableName); } BlueTable - deleted = new BlueTable (mySchema), - added = new BlueTable (mySchema), - newrows = new BlueTable (mySchema); - deleted.addRows (dp.deletedRows); - added.addRows (dp.addedRows); - newrows.addRows (dp.newRows); - - myFireEvent (new DataPacket (newrows,added,deleted)); + deleted = new BlueTable(mySchema), + added = new BlueTable(mySchema), + newrows = new BlueTable(mySchema); + deleted.addRows(dp.deletedRows); + added.addRows(dp.addedRows); + newrows.addRows(dp.newRows); + + myFireEvent(new DataPacket(newrows, added, deleted)); } } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs index a29191e2a..2e2bc758c 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs @@ -201,20 +201,21 @@ namespace BCG { } return rv; } + #region Schema types + // these are the only types allowed in schemas private static IReadOnlyDictionary _typecode_to_type = new Dictionary { { TypeCode.Boolean, typeof(bool) }, { TypeCode.Byte, typeof(byte) }, { TypeCode.Char, typeof(char) }, - { TypeCode.DateTime, typeof(DateTime) }, - { TypeCode.DBNull, typeof(DBNull) }, + // TODO + // { TypeCode.DateTime, typeof(DateTime) }, { TypeCode.Decimal, typeof(decimal) }, { TypeCode.Double, typeof(double) }, - { TypeCode.Empty, null }, + //{ TypeCode.Empty, null }, { TypeCode.Int16, typeof(short) }, { TypeCode.Int32, typeof(int) }, { TypeCode.Int64, typeof(long) }, - { TypeCode.Object, typeof(object) }, { TypeCode.SByte, typeof(sbyte) }, { TypeCode.Single, typeof(Single) }, { TypeCode.String, typeof(string) }, @@ -223,15 +224,42 @@ namespace BCG { { TypeCode.UInt64, typeof(UInt64) } }; + public static IEnumerable valid_column_typecodes() { + return _typecode_to_type.Keys; + } + + public static IEnumerable valid_column_types() { + return _typecode_to_type.Values; + } + + public static bool is_valid_col_type(Type t) { + TypeCode type_code = get_typecode(t); + return type_code != TypeCode.Empty; + } + public static TypeCode get_typecode(Type t) { foreach (var kv in _typecode_to_type) { if (kv.Value == t) return kv.Key; } - bts.Log.assert_d(false, $"type: {t} not found"); + //bts.Log.assert_d(false, $"type: {t} not found"); return TypeCode.Empty; } + public static Type get_type(TypeCode type_code) { + Type rv = null; + if (_typecode_to_type.TryGetValue(type_code, out rv)) { + return rv; + } + return null; + } + + public static TypeCode type_code(this ColumnInfo ci) { + return get_typecode(ci.colType); + } + + #endregion + public static IBlueTableSchema form_schema(string table_name, params Tuple[] cols) { List cis = new List(); foreach (Tuple c in cols) @@ -375,12 +403,12 @@ namespace BCG { foreach (PropertyInfo pi in t.GetProperties()) if (myIsActiveProperty(pi)) colInfos.Add(new ColumnInfo(pi.Name, i++, - ColumnInfo.isAllowedType(pi.PropertyType) ? + is_valid_col_type(pi.PropertyType) ? pi.PropertyType : typeof(String))); foreach (FieldInfo fi in t.GetFields()) if (myIsActiveField(fi)) colInfos.Add(new ColumnInfo(fi.Name, i++, - ColumnInfo.isAllowedType(fi.FieldType) ? fi.FieldType : typeof(String))); + is_valid_col_type(fi.FieldType) ? fi.FieldType : typeof(String))); if (createIdField) colInfos.Add(new ColumnInfo("id", i++, typeof(int))); @@ -405,12 +433,12 @@ namespace BCG { foreach (PropertyInfo pi in t.GetProperties()) if (myIsActiveProperty(pi)) rowArray[schema[pi.Name].colNo] = - ColumnInfo.isAllowedType(pi.PropertyType) ? + is_valid_col_type(pi.PropertyType) ? pi.GetValue(o, null) : pi.GetValue(o, null).ToString(); foreach (FieldInfo fi in t.GetFields()) if (myIsActiveField(fi)) rowArray[schema[fi.Name].colNo] = - ColumnInfo.isAllowedType(fi.FieldType) ? + is_valid_col_type(fi.FieldType) ? fi.GetValue(o) : fi.GetValue(o).ToString(); if (idVal != null) rowArray[schema["id"].colNo] = idVal; @@ -424,12 +452,12 @@ namespace BCG { foreach (PropertyInfo pi in t.GetProperties()) if (myIsActiveProperty(pi)) rowArray[schema[pi.Name].colNo] = - ColumnInfo.isAllowedType(pi.PropertyType) ? + is_valid_col_type(pi.PropertyType) ? pi.GetValue(o, null) : type_converter(pi.PropertyType, pi.GetValue(o, null)); foreach (FieldInfo fi in t.GetFields()) if (myIsActiveField(fi)) rowArray[schema[fi.Name].colNo] = - ColumnInfo.isAllowedType(fi.FieldType) ? + is_valid_col_type(fi.FieldType) ? fi.GetValue(o) : type_converter(fi.FieldType, fi.GetValue(o)); if (idVal != null) rowArray[schema["id"].colNo] = idVal; @@ -618,15 +646,7 @@ namespace BCG { /// /// /// - public static bool computeRowEquality(IBlueRow r1, IBlueRow r2) { - return computeRowEquality(r1, r2, r1.schema.columnNames); - } - public static bool computeRowEquality(IBlueRow r1, - IBlueRow r2, - params string[] colNames) { - return compareRows(r1, r2, colNames) == 0 ? true : false; - } /// /// Computes a string equivelant for the row, using the columns provided @@ -664,7 +684,7 @@ namespace BCG { /// /// /// - public static int compareRows(IBlueRow r1, + public static int compare_rows(IBlueRow r1, IBlueRow r2, params string[] colNames) { int retVal = 0; @@ -815,7 +835,7 @@ namespace BCG { public static IBlueRow computeMaxRow(IBlueRowCollection rows, params string[] cols) { IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null; foreach (IBlueRow r in rows) - retVal = compareRows(r, retVal, cols) > 0 ? r : retVal; + retVal = compare_rows(r, retVal, cols) > 0 ? r : retVal; return retVal; } @@ -828,7 +848,7 @@ namespace BCG { public static IBlueRow computeMinRow(IBlueRowCollection rows, params string[] cols) { IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null; foreach (IBlueRow r in rows) - retVal = compareRows(r, retVal, cols) < 0 ? r : retVal; + retVal = compare_rows(r, retVal, cols) < 0 ? r : retVal; return retVal; } @@ -907,6 +927,43 @@ namespace BCG { out aRows); } + public static bool compute_equality(IBlueRowCollection rows1, IBlueRowCollection rows2) { + IBlueRowCollection d_rows, a_rows; + compute_differences_fast(rows2, rows1, out d_rows, out a_rows); + return d_rows.Count == 0 && a_rows.Count == 0; + } + + public static bool compute_equality(ColumnInfo ci1, ColumnInfo ci2) { + return (ci1.colNo == ci2.colNo) && (ci1.colName == ci2.colName) && (ci1.colType == ci2.colType); + } + + public static bool compute_equality(IBlueTableSchema s1, IBlueTableSchema s2) { + if (s1.tableName != s1.tableName) + return false; + if (s1.columnInfo.Length != s2.columnInfo.Length) + return false; + for (int i = 0; i < s1.columnInfo.Length; i++) { + ColumnInfo ci1 = s2.columnInfo[i], ci2 = s2.columnInfo[i]; + if (!compute_equality(ci1, ci2)) + return false; + } + return true; + } + + public static bool compute_equality(IBlueRow r1, IBlueRow r2) { + return compute_row_equality(r1, r2, r1.schema.columnNames); + } + + public static bool compute_row_equality(IBlueRow r1, + IBlueRow r2, + params string[] col_names) { + return compare_rows(r1, r2, col_names) == 0 ? true : false; + } + + public static string to_string(ColumnInfo ci) { + return ci.colNo + ";" + ci.colName + ";" + ci.colType; + } + public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) { int[] col_indices = new int[new_rows.schema.columnNames.Length]; @@ -960,7 +1017,7 @@ namespace BCG { foreach (IBlueRow nr in c1) if (!BlueDataUtility.rowPresent(c2, nr)) (aRows as BlueTable).addRow(nr); - else if (BlueDataUtility.compareRows(nr, c2[(int) + else if (BlueDataUtility.compare_rows(nr, c2[(int) nr["id"]], columns) != 0) { (aRows as BlueTable).addRow(nr); (dRows as BlueTable).addRow(c2[(int)nr["id"]]); @@ -1040,7 +1097,7 @@ namespace BCG { truelyAdds = new BlueTable(adds); foreach (IBlueRow r in dels) { if (adds.containsID((int)r["id"]) && - BlueDataUtility.computeRowEquality(adds[(int)r["id"]], r, cols)) { + BlueDataUtility.compute_row_equality(adds[(int)r["id"]], r, cols)) { (truelyDels as BlueTable).deleteRow((int)r["id"]); (truelyAdds as BlueTable).deleteRow((int)r["id"]); } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataValidTypes.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataValidTypes.cs index 9425ba613..696ffa6c3 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataValidTypes.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataValidTypes.cs @@ -1,45 +1,42 @@ using System; -namespace BCG -{ - /// - /// Allowed types - /// - public class BlueDataValidTypes - { - public static readonly Type BOOL = typeof(bool); - public static readonly Type BYTE = typeof(byte); - public static readonly Type CHAR = typeof(char); - public static readonly Type DECIMAL = typeof(decimal); - public static readonly Type DOUBLE = typeof(double); - public static readonly Type SHORT = typeof(short); - public static readonly Type INT = typeof(int); - public static readonly Type LONG = typeof(long); - public static readonly Type SBYTE = typeof(sbyte); - //public static readonly Type FLOAT = typeof(float); - public static readonly Type STRING = typeof(string); - //public static readonly Type USHORT = typeof(ushort); - public static readonly Type UINT = typeof(uint); - public static readonly Type ULONG = typeof(ulong); - public static readonly Type DATETIME = typeof(DateTime); - public static readonly Type ISERIALIZEDTYPE = typeof(ISerializedType); - public static readonly Type SINGLE = typeof(Single); - public static readonly Type BYTEARRAY = typeof(byte[]); +namespace BCG { + /// + /// Allowed types + /// + public class BlueDataValidTypes { + public static readonly Type BOOL = typeof(bool); + public static readonly Type BYTE = typeof(byte); + public static readonly Type CHAR = typeof(char); + public static readonly Type DECIMAL = typeof(decimal); + public static readonly Type DOUBLE = typeof(double); + public static readonly Type SHORT = typeof(short); + public static readonly Type INT = typeof(int); + public static readonly Type LONG = typeof(long); + public static readonly Type SBYTE = typeof(sbyte); + //public static readonly Type FLOAT = typeof(float); + public static readonly Type STRING = typeof(string); + //public static readonly Type USHORT = typeof(ushort); + public static readonly Type UINT = typeof(uint); + public static readonly Type ULONG = typeof(ulong); + public static readonly Type DATETIME = typeof(DateTime); + public static readonly Type ISERIALIZEDTYPE = typeof(ISerializedType); + public static readonly Type SINGLE = typeof(Single); + public static readonly Type BYTEARRAY = typeof(byte[]); - public static int getValueTypeSize (Type t) - { - if (t == BOOL || t == BYTE || t == SBYTE) - return 1; - else if (t == SHORT || t == CHAR) - return 2; - else if (t == INT || t == UINT || t == SINGLE) - return 4; - else if (t == DOUBLE || t == LONG || t == ULONG || t == DATETIME) - return 8; - else if (t == DECIMAL) - return 16; - else - throw new ApplicationException("Unsupported Data Type"); - } - } + public static int getValueTypeSize(Type t) { + if (t == BOOL || t == BYTE || t == SBYTE) + return 1; + else if (t == SHORT || t == CHAR) + return 2; + else if (t == INT || t == UINT || t == SINGLE) + return 4; + else if (t == DOUBLE || t == LONG || t == ULONG || t == DATETIME) + return 8; + else if (t == DECIMAL) + return 16; + else + throw new ApplicationException("Unsupported Data Type"); + } + } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowLookupGroup.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowLookupGroup.cs index d368587b6..cd8e6a087 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowLookupGroup.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowLookupGroup.cs @@ -1,116 +1,117 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; +// using System; +// using System.Collections.Generic; +// using System.Linq; +// using System.Text; -/* - * This class reprsents one lookup group that is stored together in a lookup table because it contains repeated data patterns. - * if groupNo is zero, then the group is not stord in a lookup table at all. - * - * - * - */ -namespace BCG -{ - public class BlueRowLookupGroup - { - BlueTableSchema myEntrySchema; - public IBlueTableSchema entrySchema - { - get { return myEntrySchema; } - //cant' set this value, have to build it from the row's schema +//TODORemove +// /* +// * This class reprsents one lookup group that is stored together in a lookup table because it contains repeated data patterns. +// * if groupNo is zero, then the group is not stord in a lookup table at all. +// * +// * +// * +// */ +// namespace BCG +// { +// public class BlueRowLookupGroup +// { +// BlueTableSchema myEntrySchema; +// public IBlueTableSchema entrySchema +// { +// get { return myEntrySchema; } +// //cant' set this value, have to build it from the row's schema - } +// } - //for faster access and such - private int[] myGroupColNos; - public int[] groupColNos - { - get { return myGroupColNos; } +// //for faster access and such +// private int[] myGroupColNos; +// public int[] groupColNos +// { +// get { return myGroupColNos; } - } - private string[] myGroupColNames; - public string[] groupColNames - { - get { return myGroupColNames; } - } +// } +// private string[] myGroupColNames; +// public string[] groupColNames +// { +// get { return myGroupColNames; } +// } - private short myGroupNumber; - public short groupNumber - { - get { return myGroupNumber; } +// private short myGroupNumber; +// public short groupNumber +// { +// get { return myGroupNumber; } - } - - /* how many bits are the ids into the table? - * In c# everything is stored as integers, but when it goes across the wire - * the data can be represented as a byte, a short, or an int - */ - private short myTableSize; - public short tableSize - { - get { return myTableSize; } +// } - } +// /* how many bits are the ids into the table? +// * In c# everything is stored as integers, but when it goes across the wire +// * the data can be represented as a byte, a short, or an int +// */ +// private short myTableSize; +// public short tableSize +// { +// get { return myTableSize; } - //if the group number is 0, the group is not stored in tables for serialization - public bool isStoredInTable - { - get { return myGroupNumber != 0; } +// } - } - public BlueRowLookupGroup(short num, short sizeInBits, params string[] cols) - { +// //if the group number is 0, the group is not stored in tables for serialization +// public bool isStoredInTable +// { +// get { return myGroupNumber != 0; } - myGroupNumber = num; - myTableSize = sizeInBits; - myGroupColNames = cols ; - } +// } +// public BlueRowLookupGroup(short num, short sizeInBits, params string[] cols) +// { - /*Takes a LookupTableSchema for the rows this group caches for, and builds the schema representing - each of these entries in the lookup table. This function should be called by the LookupTableSchema - during its constructor. - - This function also caches the column numbers for this group, for quicker access*/ - public void buildEntrySchema(LookupTableSchema theSchema) - { - ColumnInfo[] columns = new ColumnInfo[myGroupColNames.Length + 1]; - myGroupColNos = new int[myGroupColNames.Length]; - //the first column in our table is always the ID - int idColumn = 0; - columns[idColumn] = new ColumnInfo("id", idColumn, typeof(int)); - //go through each of the columns - for (int col = 1; col < columns.Length; ++col) - { - ColumnInfo columnInInputTable = theSchema[groupColNames[col - 1]]; - if (columnInInputTable.isConstant) - columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colValue); - else - columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colType); +// myGroupNumber = num; +// myTableSize = sizeInBits; +// myGroupColNames = cols ; +// } - myGroupColNos[col - 1] = columnInInputTable.colNo; - } +// /*Takes a LookupTableSchema for the rows this group caches for, and builds the schema representing +// each of these entries in the lookup table. This function should be called by the LookupTableSchema +// during its constructor. - +// This function also caches the column numbers for this group, for quicker access*/ +// public void buildEntrySchema(LookupTableSchema theSchema) +// { +// ColumnInfo[] columns = new ColumnInfo[myGroupColNames.Length + 1]; +// myGroupColNos = new int[myGroupColNames.Length]; +// //the first column in our table is always the ID +// int idColumn = 0; +// columns[idColumn] = new ColumnInfo("id", idColumn, typeof(int)); +// //go through each of the columns +// for (int col = 1; col < columns.Length; ++col) +// { +// ColumnInfo columnInInputTable = theSchema[groupColNames[col - 1]]; +// if (columnInInputTable.isConstant) +// columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colValue); +// else +// columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colType); - myEntrySchema = new BlueTableSchema(columns, "Lookup Group " + theSchema.tableName + ":" + myGroupNumber); - - } - - /*takes a row as input, and extracts the members of the group from the row */ - public object[] extractGroupFromRow(IBlueRow input) - { - int numCols = myGroupColNames.Length; - object[] results = new object[numCols]; - - for (int col = 0; col < numCols; ++col) - results[col] = input[myGroupColNos[col]]; - - return results; - - } +// myGroupColNos[col - 1] = columnInInputTable.colNo; +// } - } -} +// myEntrySchema = new BlueTableSchema(columns, "Lookup Group " + theSchema.tableName + ":" + myGroupNumber); + +// } + +// /*takes a row as input, and extracts the members of the group from the row */ +// public object[] extractGroupFromRow(IBlueRow input) +// { +// int numCols = myGroupColNames.Length; +// object[] results = new object[numCols]; + +// for (int col = 0; col < numCols; ++col) +// results[col] = input[myGroupColNos[col]]; + +// return results; + +// } + + + +// } +// } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchema.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchema.cs index 1a69c2103..4b3f3fa6c 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchema.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchema.cs @@ -8,8 +8,7 @@ using System.Linq; using BCG.Utilities.General; -namespace BCG -{ +namespace BCG { /// /// The BlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it @@ -17,112 +16,105 @@ namespace BCG /// the respective data, sorted according to the ColumnOrder. /// [Serializable] - public class BlueTableSchema : IBlueTableSchema - { - public class BlueTableSchemaSerializationHelper - { - public static object parse(Type t, string data) - { - if (t == BlueDataValidTypes.BOOL) - return Boolean.Parse(data); - else if (t == BlueDataValidTypes.BYTE) - return Byte.Parse(data); - else if (t == BlueDataValidTypes.CHAR) - return char.Parse(data); - else if (t == BlueDataValidTypes.DATETIME) - return DateTime.Parse(data); - else if (t == BlueDataValidTypes.DOUBLE) - return Double.Parse(data); - else if (t == BlueDataValidTypes.DECIMAL) - return Decimal.Parse(data); - else if (t == BlueDataValidTypes.INT) - return int.Parse(data); - else if (t == BlueDataValidTypes.LONG) - return long.Parse(data); - else if (t == BlueDataValidTypes.SBYTE) - return SByte.Parse(data); - else if (t == BlueDataValidTypes.STRING) - return data; - else if (t == BlueDataValidTypes.UINT) - return uint.Parse(data); - else if (t == BlueDataValidTypes.ULONG) - return ulong.Parse(data); - else if (t == BlueDataValidTypes.SINGLE) - return Single.Parse(data); - else - throw new ApplicationException("Can not be parsed"); - } - public static void serialize(BinaryWriter bw, Type t, object o) - { - if (t == BlueDataValidTypes.BOOL) - bw.Write((bool)o); - else if (t == BlueDataValidTypes.BYTE) - bw.Write((byte)o); - else if (t == BlueDataValidTypes.CHAR) - bw.Write((char)o); - else if (t == BlueDataValidTypes.DATETIME) - bw.Write(((DateTime)o).Ticks); - else if (t == BlueDataValidTypes.DOUBLE) - bw.Write((double)o); - else if (t == BlueDataValidTypes.DECIMAL) - bw.Write((decimal)o); - else if (t == BlueDataValidTypes.INT) - bw.Write((int)o); - else if (t == BlueDataValidTypes.LONG) - bw.Write((long)o); - else if (t == BlueDataValidTypes.SBYTE) - bw.Write((sbyte)o); - else if (t == BlueDataValidTypes.STRING) - bw.Write(o as string); - else if (t == BlueDataValidTypes.SINGLE) - bw.Write((Single)o); - else if (t == BlueDataValidTypes.BYTEARRAY) - { - byte[] bs = o as byte[]; - bw.Write(bs.Length); - bw.Write(bs); - } - else if (t == BlueDataValidTypes.UINT) - bw.Write((uint)o); - else if (t == BlueDataValidTypes.ULONG) - bw.Write((ulong)o); - else - throw new ApplicationException("Unsupported type"); - } + public class BlueTableSchema : IBlueTableSchema { + public class BlueTableSchemaSerializationHelper { + // public static object parse(Type t, string data) { + // if (t == BlueDataValidTypes.BOOL) + // return Boolean.Parse(data); + // else if (t == BlueDataValidTypes.BYTE) + // return Byte.Parse(data); + // else if (t == BlueDataValidTypes.CHAR) + // return char.Parse(data); + // else if (t == BlueDataValidTypes.DATETIME) + // return DateTime.Parse(data); + // else if (t == BlueDataValidTypes.DOUBLE) + // return Double.Parse(data); + // else if (t == BlueDataValidTypes.DECIMAL) + // return Decimal.Parse(data); + // else if (t == BlueDataValidTypes.INT) + // return int.Parse(data); + // else if (t == BlueDataValidTypes.LONG) + // return long.Parse(data); + // else if (t == BlueDataValidTypes.SBYTE) + // return SByte.Parse(data); + // else if (t == BlueDataValidTypes.STRING) + // return data; + // else if (t == BlueDataValidTypes.UINT) + // return uint.Parse(data); + // else if (t == BlueDataValidTypes.ULONG) + // return ulong.Parse(data); + // else if (t == BlueDataValidTypes.SINGLE) + // return Single.Parse(data); + // else + // throw new ApplicationException("Can not be parsed"); + // } + // public static void serialize(BinaryWriter bw, Type t, object o) { + // if (t == BlueDataValidTypes.BOOL) + // bw.Write((bool)o); + // else if (t == BlueDataValidTypes.BYTE) + // bw.Write((byte)o); + // else if (t == BlueDataValidTypes.CHAR) + // bw.Write((char)o); + // else if (t == BlueDataValidTypes.DATETIME) + // bw.Write(((DateTime)o).Ticks); + // else if (t == BlueDataValidTypes.DOUBLE) + // bw.Write((double)o); + // else if (t == BlueDataValidTypes.DECIMAL) + // bw.Write((decimal)o); + // else if (t == BlueDataValidTypes.INT) + // bw.Write((int)o); + // else if (t == BlueDataValidTypes.LONG) + // bw.Write((long)o); + // else if (t == BlueDataValidTypes.SBYTE) + // bw.Write((sbyte)o); + // else if (t == BlueDataValidTypes.STRING) + // bw.Write(o as string); + // else if (t == BlueDataValidTypes.SINGLE) + // bw.Write((Single)o); + // else if (t == BlueDataValidTypes.BYTEARRAY) { + // byte[] bs = o as byte[]; + // bw.Write(bs.Length); + // bw.Write(bs); + // } else if (t == BlueDataValidTypes.UINT) + // bw.Write((uint)o); + // else if (t == BlueDataValidTypes.ULONG) + // bw.Write((ulong)o); + // else + // throw new ApplicationException("Unsupported type"); + // } - public static object deserialize(BinaryReader br, Type t) - { - if (t == BlueDataValidTypes.BOOL) - return br.ReadBoolean(); - else if (t == BlueDataValidTypes.BYTE) - return br.ReadByte(); - else if (t == BlueDataValidTypes.CHAR) - return br.ReadChar(); - else if (t == BlueDataValidTypes.DATETIME) - return new DateTime(br.ReadInt64()); - else if (t == BlueDataValidTypes.DOUBLE) - return br.ReadDouble(); - else if (t == BlueDataValidTypes.DECIMAL) - return br.ReadDecimal(); - else if (t == BlueDataValidTypes.INT) - return br.ReadInt32(); - else if (t == BlueDataValidTypes.LONG) - return br.ReadInt64(); - else if (t == BlueDataValidTypes.SBYTE) - return br.ReadSByte(); - else if (t == BlueDataValidTypes.STRING) - return br.ReadString(); - else if (t == BlueDataValidTypes.BYTEARRAY) - return br.ReadBytes(br.ReadInt32()); - else if (t == BlueDataValidTypes.UINT) - return br.ReadUInt32(); - else if (t == BlueDataValidTypes.ULONG) - return br.ReadUInt64(); - else if (t == BlueDataValidTypes.SINGLE) - return br.ReadSingle(); - else - throw new ApplicationException("Unsupported type"); - } + // public static object deserialize(BinaryReader br, Type t) { + // if (t == BlueDataValidTypes.BOOL) + // return br.ReadBoolean(); + // else if (t == BlueDataValidTypes.BYTE) + // return br.ReadByte(); + // else if (t == BlueDataValidTypes.CHAR) + // return br.ReadChar(); + // else if (t == BlueDataValidTypes.DATETIME) + // return new DateTime(br.ReadInt64()); + // else if (t == BlueDataValidTypes.DOUBLE) + // return br.ReadDouble(); + // else if (t == BlueDataValidTypes.DECIMAL) + // return br.ReadDecimal(); + // else if (t == BlueDataValidTypes.INT) + // return br.ReadInt32(); + // else if (t == BlueDataValidTypes.LONG) + // return br.ReadInt64(); + // else if (t == BlueDataValidTypes.SBYTE) + // return br.ReadSByte(); + // else if (t == BlueDataValidTypes.STRING) + // return br.ReadString(); + // else if (t == BlueDataValidTypes.BYTEARRAY) + // return br.ReadBytes(br.ReadInt32()); + // else if (t == BlueDataValidTypes.UINT) + // return br.ReadUInt32(); + // else if (t == BlueDataValidTypes.ULONG) + // return br.ReadUInt64(); + // else if (t == BlueDataValidTypes.SINGLE) + // return br.ReadSingle(); + // else + // throw new ApplicationException("Unsupported type"); + // } } protected Hashtable myIndexByColName = new Hashtable(); @@ -132,29 +124,24 @@ namespace BCG public BlueTableSchema() { } - public BlueTableSchema(DataTable table) - { + public BlueTableSchema(DataTable table) { myIndexByColOrder = new ColumnInfo[table.Columns.Count]; myTableName = table.TableName; - foreach (DataColumn col in table.Columns) - { + foreach (DataColumn col in table.Columns) { ColumnInfo ci = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType); myIndexByColName[ci.colName] = ci; myIndexByColOrder[ci.colNo] = ci; } } - public BlueTableSchema(ColumnInfo[] cis, string tableName) - { + public BlueTableSchema(ColumnInfo[] cis, string tableName) { myConstruct(cis, tableName); } - private void myConstruct(ColumnInfo[] cis, string tableName) - { + private void myConstruct(ColumnInfo[] cis, string tableName) { myIndexByColOrder = new ColumnInfo[cis.Length]; myTableName = tableName; - foreach (ColumnInfo ci in cis) - { + foreach (ColumnInfo ci in cis) { myIndexByColOrder[ci.colNo] = ci; myIndexByColName[ci.colName] = ci; } @@ -169,15 +156,13 @@ namespace BCG /// /// the table name /// the plain old data type whose fields will be mirrored in the schema - public BlueTableSchema(string tableName, Type fromType) - { + public BlueTableSchema(string tableName, Type fromType) { List cis = new List(); int i = 0; // we sort the field name so that we have a deterministic order given // the same type. bool foundId = false; - foreach (FieldInfo fi in fromType.GetFields().OrderBy((f) => { return f.Name; })) - { + foreach (FieldInfo fi in fromType.GetFields().OrderBy((f) => { return f.Name; })) { if (fi.Name.ToLower().Equals("id")) foundId = true; cis.Add(new ColumnInfo(fi.Name, i++, fi.FieldType)); @@ -188,32 +173,25 @@ namespace BCG myConstruct(cis.ToArray(), tableName); } - public ColumnInfo this[string colName] - { - get - { + public ColumnInfo this[string colName] { + get { object retVal = myIndexByColName[colName]; return retVal != null ? retVal as ColumnInfo : null; } } - public ColumnInfo this[int colNo] - { + public ColumnInfo this[int colNo] { get { return myIndexByColOrder[colNo]; } } /// /// The returned Array is sorted according to the Column No /// - public string[] columnNames - { - get - { - if (myTableColNames == null) - { + public string[] columnNames { + get { + if (myTableColNames == null) { myTableColNames = new string[myIndexByColName.Count]; - foreach (DictionaryEntry de in myIndexByColName) - { + foreach (DictionaryEntry de in myIndexByColName) { ColumnInfo temp = (ColumnInfo)de.Value; myTableColNames[temp.colNo] = temp.colName; } @@ -225,8 +203,7 @@ namespace BCG /// /// The returned Array is sorted according to the Column No /// - public ColumnInfo[] columnInfo - { + public ColumnInfo[] columnInfo { get { return myIndexByColOrder; } } @@ -234,49 +211,30 @@ namespace BCG #region ISerializedType methods - public void serialize(BinaryWriter bw) - { + public void serialize(BinaryWriter bw) { bw.Write(columnNames.Length); bw.Write(tableName); - foreach (ColumnInfo ci in columnInfo) - { + foreach (ColumnInfo ci in columnInfo) { bw.Write(ci.colName); bw.Write(ci.colType.ToString()); - if (ci.isConstant) - { - bw.Write(true); - BlueTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue); - } - else { bw.Write(false); } } } - public void deserialze(BinaryReader br) - { + public void deserialze(BinaryReader br) { ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()]; string name = br.ReadString(); - for (int i = 0; i < cis.Length; i++) - { + for (int i = 0; i < cis.Length; i++) { string colName = br.ReadString(); Type t = TypeInstanceLoader.loadType(br.ReadString(), true); - bool isConstant = br.ReadBoolean(); - if (isConstant) - { - cis[i] = new ColumnInfo(colName, i, t, BlueTableSchemaSerializationHelper.deserialize(br, t)); - } - else - { - cis[i] = new ColumnInfo(colName, i, t); - } + cis[i] = new ColumnInfo(colName, i, t); } myConstruct(cis, name); } - public int getSerializedSize() - { + public int getSerializedSize() { throw new NotImplementedException("Feature for provided constant values to" + "non-primitive types in schema, not provided as of now"); } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs index 9337d9420..71a645076 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs @@ -1,56 +1,25 @@ using System; -namespace BCG -{ +namespace BCG { /// /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable /// [Serializable] - public class ColumnInfo : IComparable - { - public static bool isAllowedType (Type t) - { - return - t.Equals (typeof (System.Int32)) || - t.Equals (typeof (System.Int64)) || - t.Equals (typeof (System.Double)) || - t.Equals (typeof (System.String)) || - t.Equals (typeof (System.DateTime)) ? true : - false; - } + public class ColumnInfo : IComparable { //the default column info does not cach values in a lookup table, therefore it is in group 0 - public ColumnInfo(string colName, int colNo, Type colType) - : this(colName, colNo, colType, 0) - { - - - } - - public ColumnInfo(string colName, int colNo, Type colType, short colGroup) - { + public ColumnInfo(string colName, int colNo, Type colType) { myColName = colName; myColNo = colNo; myColType = colType; - myLookupTableGroup = colGroup; - myColValue = null; + } + + public ColumnInfo(string col_name, int col_no, TypeCode type_code) + : this(col_name, col_no, BlueDataUtility.get_type(type_code)) { } - public ColumnInfo (string colName, int colNo, object colValue) - : this (colName, colNo, colValue.GetType()) {myColValue = colValue == null ? DBNull.Value : colValue;} - - public ColumnInfo (string colName, int colNo, Type colType, object colValue) - : this (colName, colNo, colType) - { - - myColValue = colValue == null ? DBNull.Value : colValue; - - } - - - public int CompareTo (object obj) - { + public int CompareTo(object obj) { int otherColNo = ((ColumnInfo)(obj)).colNo; if (myColNo < otherColNo) return -1; @@ -59,37 +28,29 @@ namespace BCG return 0; } - //public bool isCalculated{get{return myExpression != null;}} + public override bool Equals(object obj) { + if (obj is ColumnInfo) { + ColumnInfo other = obj as ColumnInfo; + return (colNo == other.colNo) && (colName == other.colName) && (colType == other.colType); + } + return false; + } - public bool isConstant {get{return myColValue != null;}} + public override string ToString() { + return colName + " " + colNo + " " + colType; + } + + + // for backwards compatibility to allow build + public bool isConstant { get { return false; } } private int myColNo; private string myColName; private Type myColType; - //private string myExpression; - private object myColValue; - //no table will ever have more than 65,536 columns - //if it does, we are in serious trouble - private short myLookupTableGroup; + public int colNo { get { return myColNo; } } + public string colName { get { return myColName; } } + public Type colType { get { return myColType; } } - public int colNo{get{return myColNo;}} - public string colName {get{return myColName;}} - public Type colType {get{return myColType;}} - - - //this value corresponds to how the data in the column is serialized. - //if the group number is 0, the values are not stored in a table - //if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together - //are stored together in a lookup table - - - public short lookupTableGroup - { - get { return myLookupTableGroup; } - set { myLookupTableGroup = value; } - } - //public string expression {get{return myExpression;}} - public object colValue {get{return myColValue;}} } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/LookupTableSchema.cs b/src/DataStreamSuite/DataStreamCore/BlueData/LookupTableSchema.cs index 426050353..61c249d8b 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/LookupTableSchema.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/LookupTableSchema.cs @@ -1,324 +1,324 @@ -using System; -using System.Data; -using System.Collections; -using System.IO; -using System.Reflection; -using System.Collections.Generic; -using System.Linq; +// using System; +// using System.Data; +// using System.Collections; +// using System.IO; +// using System.Reflection; +// using System.Collections.Generic; +// using System.Linq; -using BCG.Utilities.General; +// using BCG.Utilities.General; -namespace BCG -{ - /// - /// The LookupTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created - /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it - /// are the indexers (both order and name based), and the property columnInfo and columnNames, that return - /// the respective data, sorted according to the ColumnOrder. - /// - [Serializable] - public class LookupTableSchema : IBlueTableSchema - { - public class LookupTableSchemaSerializationHelper - { - public static object parse(Type t, string data) - { - if (t == BlueDataValidTypes.BOOL) - return Boolean.Parse(data); - else if (t == BlueDataValidTypes.BYTE) - return Byte.Parse(data); - else if (t == BlueDataValidTypes.CHAR) - return char.Parse(data); - else if (t == BlueDataValidTypes.DATETIME) - return DateTime.Parse(data); - else if (t == BlueDataValidTypes.DOUBLE) - return Double.Parse(data); - else if (t == BlueDataValidTypes.DECIMAL) - return Decimal.Parse(data); - else if (t == BlueDataValidTypes.INT) - return int.Parse(data); - else if (t == BlueDataValidTypes.LONG) - return long.Parse(data); - else if (t == BlueDataValidTypes.SBYTE) - return SByte.Parse(data); - else if (t == BlueDataValidTypes.STRING) - return data; - else if (t == BlueDataValidTypes.UINT) - return uint.Parse(data); - else if (t == BlueDataValidTypes.ULONG) - return ulong.Parse(data); - else if (t == BlueDataValidTypes.SINGLE) - return Single.Parse(data); - else - throw new ApplicationException("Can not be parsed"); - } - public static void serialize(BinaryWriter bw, Type t, object o) - { - if (t == BlueDataValidTypes.BOOL) - bw.Write((bool)o); - else if (t == BlueDataValidTypes.BYTE) - bw.Write((byte)o); - else if (t == BlueDataValidTypes.CHAR) - bw.Write((char)o); - else if (t == BlueDataValidTypes.DATETIME) - bw.Write(((DateTime)o).Ticks); - else if (t == BlueDataValidTypes.DOUBLE) - bw.Write((double)o); - else if (t == BlueDataValidTypes.DECIMAL) - bw.Write((decimal)o); - else if (t == BlueDataValidTypes.INT) - bw.Write((int)o); - else if (t == BlueDataValidTypes.LONG) - bw.Write((long)o); - else if (t == BlueDataValidTypes.SBYTE) - bw.Write((sbyte)o); - else if (t == BlueDataValidTypes.STRING) - bw.Write(o as string); - else if (t == BlueDataValidTypes.SINGLE) - bw.Write((Single)o); - else if (t == BlueDataValidTypes.BYTEARRAY) - { - byte[] bs = o as byte[]; - bw.Write(bs.Length); - bw.Write(bs); - } - else if (t == BlueDataValidTypes.UINT) - bw.Write((uint)o); - else if (t == BlueDataValidTypes.ULONG) - bw.Write((ulong)o); - else - throw new ApplicationException("Unsupported type"); - } +// namespace BCG +// { +// /// +// /// The LookupTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created +// /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it +// /// are the indexers (both order and name based), and the property columnInfo and columnNames, that return +// /// the respective data, sorted according to the ColumnOrder. +// /// +// [Serializable] +// public class LookupTableSchema : IBlueTableSchema +// { +// public class LookupTableSchemaSerializationHelper +// { +// public static object parse(Type t, string data) +// { +// if (t == BlueDataValidTypes.BOOL) +// return Boolean.Parse(data); +// else if (t == BlueDataValidTypes.BYTE) +// return Byte.Parse(data); +// else if (t == BlueDataValidTypes.CHAR) +// return char.Parse(data); +// else if (t == BlueDataValidTypes.DATETIME) +// return DateTime.Parse(data); +// else if (t == BlueDataValidTypes.DOUBLE) +// return Double.Parse(data); +// else if (t == BlueDataValidTypes.DECIMAL) +// return Decimal.Parse(data); +// else if (t == BlueDataValidTypes.INT) +// return int.Parse(data); +// else if (t == BlueDataValidTypes.LONG) +// return long.Parse(data); +// else if (t == BlueDataValidTypes.SBYTE) +// return SByte.Parse(data); +// else if (t == BlueDataValidTypes.STRING) +// return data; +// else if (t == BlueDataValidTypes.UINT) +// return uint.Parse(data); +// else if (t == BlueDataValidTypes.ULONG) +// return ulong.Parse(data); +// else if (t == BlueDataValidTypes.SINGLE) +// return Single.Parse(data); +// else +// throw new ApplicationException("Can not be parsed"); +// } +// public static void serialize(BinaryWriter bw, Type t, object o) +// { +// if (t == BlueDataValidTypes.BOOL) +// bw.Write((bool)o); +// else if (t == BlueDataValidTypes.BYTE) +// bw.Write((byte)o); +// else if (t == BlueDataValidTypes.CHAR) +// bw.Write((char)o); +// else if (t == BlueDataValidTypes.DATETIME) +// bw.Write(((DateTime)o).Ticks); +// else if (t == BlueDataValidTypes.DOUBLE) +// bw.Write((double)o); +// else if (t == BlueDataValidTypes.DECIMAL) +// bw.Write((decimal)o); +// else if (t == BlueDataValidTypes.INT) +// bw.Write((int)o); +// else if (t == BlueDataValidTypes.LONG) +// bw.Write((long)o); +// else if (t == BlueDataValidTypes.SBYTE) +// bw.Write((sbyte)o); +// else if (t == BlueDataValidTypes.STRING) +// bw.Write(o as string); +// else if (t == BlueDataValidTypes.SINGLE) +// bw.Write((Single)o); +// else if (t == BlueDataValidTypes.BYTEARRAY) +// { +// byte[] bs = o as byte[]; +// bw.Write(bs.Length); +// bw.Write(bs); +// } +// else if (t == BlueDataValidTypes.UINT) +// bw.Write((uint)o); +// else if (t == BlueDataValidTypes.ULONG) +// bw.Write((ulong)o); +// else +// throw new ApplicationException("Unsupported type"); +// } - public static object deserialize(BinaryReader br, Type t) - { - if (t == BlueDataValidTypes.BOOL) - return br.ReadBoolean(); - else if (t == BlueDataValidTypes.BYTE) - return br.ReadByte(); - else if (t == BlueDataValidTypes.CHAR) - return br.ReadChar(); - else if (t == BlueDataValidTypes.DATETIME) - return new DateTime(br.ReadInt64()); - else if (t == BlueDataValidTypes.DOUBLE) - return br.ReadDouble(); - else if (t == BlueDataValidTypes.DECIMAL) - return br.ReadDecimal(); - else if (t == BlueDataValidTypes.INT) - return br.ReadInt32(); - else if (t == BlueDataValidTypes.LONG) - return br.ReadInt64(); - else if (t == BlueDataValidTypes.SBYTE) - return br.ReadSByte(); - else if (t == BlueDataValidTypes.STRING) - return br.ReadString(); - else if (t == BlueDataValidTypes.BYTEARRAY) - return br.ReadBytes(br.ReadInt32()); - else if (t == BlueDataValidTypes.UINT) - return br.ReadUInt32(); - else if (t == BlueDataValidTypes.ULONG) - return br.ReadUInt64(); - else if (t == BlueDataValidTypes.SINGLE) - return br.ReadSingle(); - else - throw new ApplicationException("Unsupported type"); - } - } +// public static object deserialize(BinaryReader br, Type t) +// { +// if (t == BlueDataValidTypes.BOOL) +// return br.ReadBoolean(); +// else if (t == BlueDataValidTypes.BYTE) +// return br.ReadByte(); +// else if (t == BlueDataValidTypes.CHAR) +// return br.ReadChar(); +// else if (t == BlueDataValidTypes.DATETIME) +// return new DateTime(br.ReadInt64()); +// else if (t == BlueDataValidTypes.DOUBLE) +// return br.ReadDouble(); +// else if (t == BlueDataValidTypes.DECIMAL) +// return br.ReadDecimal(); +// else if (t == BlueDataValidTypes.INT) +// return br.ReadInt32(); +// else if (t == BlueDataValidTypes.LONG) +// return br.ReadInt64(); +// else if (t == BlueDataValidTypes.SBYTE) +// return br.ReadSByte(); +// else if (t == BlueDataValidTypes.STRING) +// return br.ReadString(); +// else if (t == BlueDataValidTypes.BYTEARRAY) +// return br.ReadBytes(br.ReadInt32()); +// else if (t == BlueDataValidTypes.UINT) +// return br.ReadUInt32(); +// else if (t == BlueDataValidTypes.ULONG) +// return br.ReadUInt64(); +// else if (t == BlueDataValidTypes.SINGLE) +// return br.ReadSingle(); +// else +// throw new ApplicationException("Unsupported type"); +// } +// } - protected Hashtable myIndexByColName = new Hashtable(); - protected ColumnInfo[] myIndexByColOrder; - private string[] myTableColNames; - private BlueRowLookupGroup[] myLookupGroups; - protected string myTableName; +// protected Hashtable myIndexByColName = new Hashtable(); +// protected ColumnInfo[] myIndexByColOrder; +// private string[] myTableColNames; +// private BlueRowLookupGroup[] myLookupGroups; +// protected string myTableName; - public LookupTableSchema() { } +// public LookupTableSchema() { } - public LookupTableSchema(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups) - { - myConstruct(tableName, cis, groups); - } +// public LookupTableSchema(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups) +// { +// myConstruct(tableName, cis, groups); +// } - private void myConstruct(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups) - { - myIndexByColOrder = new ColumnInfo[cis.Length]; - myTableName = tableName; - foreach (ColumnInfo ci in cis) - { - myIndexByColOrder[ci.colNo] = ci; - myIndexByColName[ci.colName] = ci; - } - myLookupGroups = groups; +// private void myConstruct(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups) +// { +// myIndexByColOrder = new ColumnInfo[cis.Length]; +// myTableName = tableName; +// foreach (ColumnInfo ci in cis) +// { +// myIndexByColOrder[ci.colNo] = ci; +// myIndexByColName[ci.colName] = ci; +// } +// myLookupGroups = groups; - foreach (BlueRowLookupGroup group in groups) - { - group.buildEntrySchema(this); +// foreach (BlueRowLookupGroup group in groups) +// { +// group.buildEntrySchema(this); - } - } +// } +// } - public ColumnInfo this[string colName] - { - get - { - object retVal = myIndexByColName[colName]; - return retVal != null ? retVal as ColumnInfo : null; - } - } +// public ColumnInfo this[string colName] +// { +// get +// { +// object retVal = myIndexByColName[colName]; +// return retVal != null ? retVal as ColumnInfo : null; +// } +// } - public BlueRowLookupGroup[] lookupGroups - { - get { return myLookupGroups; } +// public BlueRowLookupGroup[] lookupGroups +// { +// get { return myLookupGroups; } - } - public ColumnInfo this[int colNo] - { - get { return myIndexByColOrder[colNo]; } - } +// } +// public ColumnInfo this[int colNo] +// { +// get { return myIndexByColOrder[colNo]; } +// } - /// - /// The returned Array is sorted according to the Column No - /// - public string[] columnNames - { - get - { - if (myTableColNames == null) - { - myTableColNames = new string[myIndexByColName.Count]; - foreach (DictionaryEntry de in myIndexByColName) - { - ColumnInfo temp = (ColumnInfo)de.Value; - myTableColNames[temp.colNo] = temp.colName; - } - } - return myTableColNames; - } - } +// /// +// /// The returned Array is sorted according to the Column No +// /// +// public string[] columnNames +// { +// get +// { +// if (myTableColNames == null) +// { +// myTableColNames = new string[myIndexByColName.Count]; +// foreach (DictionaryEntry de in myIndexByColName) +// { +// ColumnInfo temp = (ColumnInfo)de.Value; +// myTableColNames[temp.colNo] = temp.colName; +// } +// } +// return myTableColNames; +// } +// } - /// - /// The returned Array is sorted according to the Column No - /// - public ColumnInfo[] columnInfo - { - get { return myIndexByColOrder; } - } +// /// +// /// The returned Array is sorted according to the Column No +// /// +// public ColumnInfo[] columnInfo +// { +// get { return myIndexByColOrder; } +// } - public string tableName { get { return myTableName; } } +// public string tableName { get { return myTableName; } } - #region ISerializedType methods - public void serialize(BinaryWriter bw) - { - bw.Write(columnNames.Length); - bw.Write(tableName); +// #region ISerializedType methods +// public void serialize(BinaryWriter bw) +// { +// bw.Write(columnNames.Length); +// bw.Write(tableName); - //first, write the column names - foreach (ColumnInfo ci in columnInfo) - { - bw.Write(ci.colName); - bw.Write(ci.colType.ToString()); - if (ci.isConstant) - { - bw.Write(true); - LookupTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue); - } - else - { - bw.Write(false); - bw.Write(ci.lookupTableGroup); - } +// //first, write the column names +// foreach (ColumnInfo ci in columnInfo) +// { +// bw.Write(ci.colName); +// bw.Write(ci.colType.ToString()); +// if (ci.isConstant) +// { +// bw.Write(true); +// LookupTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue); +// } +// else +// { +// bw.Write(false); +// bw.Write(ci.lookupTableGroup); +// } - } +// } - //this is here for error checking - bw.Write(myLookupGroups.Length); - //now, write the size of each lookup table group - foreach (BlueRowLookupGroup group in myLookupGroups) - { - bw.Write(group.groupNumber); - bw.Write(group.tableSize); - bw.Write(group.groupColNames.Length); - foreach (string colName in group.groupColNames) - { - //write the index of this column in the table - bw.Write(this[colName].colNo); +// //this is here for error checking +// bw.Write(myLookupGroups.Length); +// //now, write the size of each lookup table group +// foreach (BlueRowLookupGroup group in myLookupGroups) +// { +// bw.Write(group.groupNumber); +// bw.Write(group.tableSize); +// bw.Write(group.groupColNames.Length); +// foreach (string colName in group.groupColNames) +// { +// //write the index of this column in the table +// bw.Write(this[colName].colNo); - } +// } - } +// } - } +// } - public void deserialze(BinaryReader br) - { - int numCols = br.ReadInt32(); - //these are the final output columns that will be created - ColumnInfo[] columnInfos = new ColumnInfo[numCols]; +// public void deserialze(BinaryReader br) +// { +// int numCols = br.ReadInt32(); +// //these are the final output columns that will be created +// ColumnInfo[] columnInfos = new ColumnInfo[numCols]; - Dictionary> colLookupGroupMap = new Dictionary>(); +// Dictionary> colLookupGroupMap = new Dictionary>(); - string tableName = br.ReadString(); - for (int colNo = 0; colNo < numCols; ++colNo) - { - string colName = br.ReadString(); - Type colType = TypeInstanceLoader.loadType(br.ReadString(), true); - bool colIsConst = br.ReadBoolean(); - if (colIsConst) - { - columnInfos[colNo] = new ColumnInfo(colName, colNo, LookupTableSchemaSerializationHelper.deserialize(br, colType)); +// string tableName = br.ReadString(); +// for (int colNo = 0; colNo < numCols; ++colNo) +// { +// string colName = br.ReadString(); +// Type colType = TypeInstanceLoader.loadType(br.ReadString(), true); +// bool colIsConst = br.ReadBoolean(); +// if (colIsConst) +// { +// columnInfos[colNo] = new ColumnInfo(colName, colNo, LookupTableSchemaSerializationHelper.deserialize(br, colType)); - } - else - { - short thisColGroup = br.ReadInt16(); - columnInfos[colNo] = new ColumnInfo(colName, colNo, colType, thisColGroup); - } - } +// } +// else +// { +// short thisColGroup = br.ReadInt16(); +// columnInfos[colNo] = new ColumnInfo(colName, colNo, colType, thisColGroup); +// } +// } - int numGroups = br.ReadInt32(); - BlueRowLookupGroup[] theGroups = new BlueRowLookupGroup[numGroups]; +// int numGroups = br.ReadInt32(); +// BlueRowLookupGroup[] theGroups = new BlueRowLookupGroup[numGroups]; - for (int group = 0; group < numGroups; ++group) - { - short groupNo = br.ReadInt16(); - short groupSizeInBits = br.ReadInt16(); - int numColsInGroup = br.ReadInt32(); - int[] groupCols = new int[numColsInGroup]; - string[] groupColNames = new string[numColsInGroup]; - for (int col = 0; col < numColsInGroup; ++col) - { - groupCols[col] = br.ReadInt32(); - groupColNames[col] = columnInfos[groupCols[col]].colName; - } +// for (int group = 0; group < numGroups; ++group) +// { +// short groupNo = br.ReadInt16(); +// short groupSizeInBits = br.ReadInt16(); +// int numColsInGroup = br.ReadInt32(); +// int[] groupCols = new int[numColsInGroup]; +// string[] groupColNames = new string[numColsInGroup]; +// for (int col = 0; col < numColsInGroup; ++col) +// { +// groupCols[col] = br.ReadInt32(); +// groupColNames[col] = columnInfos[groupCols[col]].colName; +// } - //create the group - theGroups[groupNo] = new BlueRowLookupGroup(groupNo, groupSizeInBits, groupColNames); +// //create the group +// theGroups[groupNo] = new BlueRowLookupGroup(groupNo, groupSizeInBits, groupColNames); - } +// } - myConstruct(tableName, columnInfos, theGroups); +// myConstruct(tableName, columnInfos, theGroups); - } +// } - public int getSerializedSize() - { - throw new NotImplementedException("Feature for provided constant values to" + - "non-primitive types in schema, not provided as of now"); - } +// public int getSerializedSize() +// { +// throw new NotImplementedException("Feature for provided constant values to" + +// "non-primitive types in schema, not provided as of now"); +// } - #endregion - } -} +// #endregion +// } +// } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/SerializedBlueTableSchema.cs b/src/DataStreamSuite/DataStreamCore/BlueData/SerializedBlueTableSchema.cs index 958a6f431..6c5256149 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/SerializedBlueTableSchema.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/SerializedBlueTableSchema.cs @@ -5,8 +5,7 @@ using System.IO; using BCG.Utilities.General; //using BCG.Utilities; -namespace BCG -{ +namespace BCG { /// /// The SerializedBlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it @@ -17,12 +16,9 @@ namespace BCG /// memory streams, and uses this schema to unpack the values when they are needed. /// [Serializable] - public class SerializedBlueTableSchema : IBlueTableSchema - { - public class SerializedBlueTableSchemaSerializationHelper - { - public static object parse(Type t, string data) - { + public class SerializedBlueTableSchema : IBlueTableSchema { + public class SerializedBlueTableSchemaSerializationHelper { + public static object parse(Type t, string data) { if (t == BlueDataValidTypes.BOOL) return Boolean.Parse(data); else if (t == BlueDataValidTypes.BYTE) @@ -52,8 +48,7 @@ namespace BCG else throw new ApplicationException("Can not be parsed"); } - public static void serialize(BinaryWriter bw, Type t, object o) - { + public static void serialize(BinaryWriter bw, Type t, object o) { if (t == BlueDataValidTypes.BOOL) bw.Write((bool)o); else if (t == BlueDataValidTypes.BYTE) @@ -76,13 +71,11 @@ namespace BCG bw.Write(o as string); else if (t == BlueDataValidTypes.SINGLE) bw.Write((Single)o); - else if (t == BlueDataValidTypes.BYTEARRAY) - { + else if (t == BlueDataValidTypes.BYTEARRAY) { byte[] bs = o as byte[]; bw.Write(bs.Length); bw.Write(bs); - } - else if (t == BlueDataValidTypes.UINT) + } else if (t == BlueDataValidTypes.UINT) bw.Write((uint)o); else if (t == BlueDataValidTypes.ULONG) bw.Write((ulong)o); @@ -90,8 +83,7 @@ namespace BCG throw new ApplicationException("Unsupported type"); } - public static object deserialize(BinaryReader br, Type t) - { + public static object deserialize(BinaryReader br, Type t) { if (t == BlueDataValidTypes.BOOL) return br.ReadBoolean(); else if (t == BlueDataValidTypes.BYTE) @@ -132,45 +124,37 @@ namespace BCG public SerializedBlueTableSchema() { } //copies the schema from an existing schema, building the dispatch table - public SerializedBlueTableSchema(IBlueTableSchema toCopy) - { + public SerializedBlueTableSchema(IBlueTableSchema toCopy) { int numCols = toCopy.columnInfo.Length; ColumnInfo[] theColumns = new ColumnInfo[toCopy.columnInfo.Length]; - foreach (ColumnInfo toCopyCI in toCopy.columnInfo) - { + foreach (ColumnInfo toCopyCI in toCopy.columnInfo) { theColumns[toCopyCI.colNo] = toCopyCI; } myConstruct(theColumns, toCopy.tableName); } - public SerializedBlueTableSchema(DataTable table) - { + public SerializedBlueTableSchema(DataTable table) { ColumnInfo[] theseCols = new ColumnInfo[table.Columns.Count]; myTableName = table.TableName; - foreach (DataColumn col in table.Columns) - { + foreach (DataColumn col in table.Columns) { theseCols[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType); } myConstruct(theseCols, myTableName); } - public SerializedBlueTableSchema(ColumnInfo[] cis, string tableName) - { + public SerializedBlueTableSchema(ColumnInfo[] cis, string tableName) { myConstruct(cis, tableName); } - public void serializeObject(object data, BinaryWriter bw, int colNo) - { + public void serializeObject(object data, BinaryWriter bw, int colNo) { mySerializeDispatchTable[colNo](data, bw); } - private void myConstruct(ColumnInfo[] cis, string tableName) - { + private void myConstruct(ColumnInfo[] cis, string tableName) { myIndexByColOrder = new ColumnInfo[cis.Length]; myTableName = tableName; - foreach (ColumnInfo ci in cis) - { + foreach (ColumnInfo ci in cis) { myIndexByColOrder[ci.colNo] = ci; myIndexByColName[ci.colName] = ci; } @@ -179,32 +163,25 @@ namespace BCG public SerializedBlueTableSchema(string tableName, params ColumnInfo[] cis) : this(cis, tableName) { } - public ColumnInfo this[string colName] - { - get - { + public ColumnInfo this[string colName] { + get { object retVal = myIndexByColName[colName]; return retVal != null ? retVal as ColumnInfo : null; } } - public ColumnInfo this[int colNo] - { + public ColumnInfo this[int colNo] { get { return myIndexByColOrder[colNo]; } } /// /// The returned Array is sorted according to the Column No /// - public string[] columnNames - { - get - { - if (myTableColNames == null) - { + public string[] columnNames { + get { + if (myTableColNames == null) { myTableColNames = new string[myIndexByColName.Count]; - foreach (DictionaryEntry de in myIndexByColName) - { + foreach (DictionaryEntry de in myIndexByColName) { ColumnInfo temp = (ColumnInfo)de.Value; myTableColNames[temp.colNo] = temp.colName; } @@ -216,64 +193,46 @@ namespace BCG /// /// The returned Array is sorted according to the Column No /// - public ColumnInfo[] columnInfo - { + public ColumnInfo[] columnInfo { get { return myIndexByColOrder; } } public string tableName { get { return myTableName; } } - public int IdColumn - { + public int IdColumn { get { return myIdCol; } } #region ISerializedType methods - public void serialize(BinaryWriter bw) - { + public void serialize(BinaryWriter bw) { bw.Write(columnNames.Length); bw.Write(tableName); - foreach (ColumnInfo ci in columnInfo) - { + foreach (ColumnInfo ci in columnInfo) { bw.Write(ci.colName); bw.Write(ci.colType.ToString()); - if (ci.isConstant) - { - bw.Write(true); - SerializedBlueTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue); - } - else - { - bw.Write(false); - } + bw.Write(false); } } - public void deserialze(BinaryReader br) - { + public void deserialze(BinaryReader br) { ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()]; string name = br.ReadString(); - for (int i = 0; i < cis.Length; i++) - { + for (int i = 0; i < cis.Length; i++) { string colName = br.ReadString(); Type t = TypeInstanceLoader.loadType(br.ReadString(), true); bool isConstant = br.ReadBoolean(); - if (isConstant) - { - cis[i] = new ColumnInfo(colName, i, t, SerializedBlueTableSchemaSerializationHelper.deserialize(br, t)); - } - else - { + if (isConstant) { + cis[i] = new ColumnInfo(colName, i, t); + } else { cis[i] = new ColumnInfo(colName, i, t); } } myConstruct(cis, name); } - public int getSerializedSize() - { + public int getSerializedSize() { throw new NotImplementedException("Feature for provided constant values to" + "non-primitive types in schema, not provided as of now"); } @@ -282,8 +241,7 @@ namespace BCG //builds a dispatch table used to pull values out of the memory streams in the rows. - private void myBuildDispatchTable() - { + private void myBuildDispatchTable() { int numCols = myIndexByColOrder.Length; mySerializeDispatchTable = new SerializeDelegate[numCols]; myPullFromByteBufferDispatchTable = new PullFromByteBufferDelegate[numCols]; @@ -292,18 +250,14 @@ namespace BCG myIdCol = idColumn.colNo; - for (int col = 0; col < numCols; ++col) - { + for (int col = 0; col < numCols; ++col) { Type thisColType = myIndexByColOrder[col].colType; //we don't store serializers for the id column, or string columns. - if (col == myIdCol || thisColType == typeof(string)) - { + if (col == myIdCol || thisColType == typeof(string)) { mySerializeDispatchTable[col] = null; myColumnSizesTable[col] = 0; myPullFromByteBufferDispatchTable[col] = null; - } - else - { + } else { mySerializeDispatchTable[col] = myGetDispatchMethodSerialize(thisColType); myPullFromByteBufferDispatchTable[col] = myGetDispatchMethodPullFromBuffer(thisColType); myColumnSizesTable[col] = BlueDataValidTypes.getValueTypeSize(thisColType); @@ -315,16 +269,14 @@ namespace BCG } - public int getNullMapSize() - { + public int getNullMapSize() { int numCols = myIndexByColOrder.Length; return Convert.ToInt32(Math.Ceiling((Convert.ToDouble(numCols) / 8.0))); } //this function reads the null array from inside the byte array. //it has to copy the bytes over. - public BitArray getNullMap(byte[] theBytes) - { + public BitArray getNullMap(byte[] theBytes) { int nullMapSize = getNullMapSize(); //the null map starts after the id, so that we have instant access to the id. int byteCursor = BlueDataValidTypes.getValueTypeSize(typeof(Int32)); @@ -336,8 +288,7 @@ namespace BCG } //gets the value either from the bytes stream but only if the value is not a string. //doesn't work if you're getting the id. - public unsafe object getValueNonString(byte[] theBytes, int colNo) - { + public unsafe object getValueNonString(byte[] theBytes, int colNo) { if (colNo == myIdCol) throw new BlueDataException("Error: do not use getValueNonString to try to fetch the id of a row"); BitArray nullMap = getNullMap(theBytes); @@ -345,8 +296,7 @@ namespace BCG return DBNull.Value; //the rest of the row starts after the id and the null map int thisOffset = BlueDataValidTypes.getValueTypeSize(typeof(Int32)) + getNullMapSize(); - for (int currentColumn = 0; currentColumn < colNo; ++currentColumn) - { + for (int currentColumn = 0; currentColumn < colNo; ++currentColumn) { //pull the data from the binary reader, but only if this type is not null or a string //if the type wer'e looking for is a string, it would have been returned already by the accesor //in the bluerow. @@ -357,16 +307,14 @@ namespace BCG thisOffset += myColumnSizesTable[currentColumn]; } - fixed (byte* thisBytePtr = &theBytes[thisOffset]) - { + fixed (byte* thisBytePtr = &theBytes[thisOffset]) { return myPullFromByteBufferDispatchTable[colNo](thisBytePtr); } } - public unsafe object[] getAllValues(byte[] theBytes, string[] stringArray) - { + public unsafe object[] getAllValues(byte[] theBytes, string[] stringArray) { int numCols = myColumnSizesTable.Length; object[] theObjs = new Object[numCols]; @@ -374,8 +322,7 @@ namespace BCG int thisOffset = BlueDataValidTypes.getValueTypeSize(typeof(Int32)) + getNullMapSize(); int thisStringNo = 0; - for (int currentColumn = 0; currentColumn < numCols; ++currentColumn) - { + for (int currentColumn = 0; currentColumn < numCols; ++currentColumn) { //pull the data from the binary reader, but only if this type is not null or a string //if the type wer'e looking for is a string, it would have been returned already by the accesor //in the bluerow. @@ -384,10 +331,8 @@ namespace BCG theObjs[currentColumn] = DBNull.Value; else if (myIndexByColOrder[currentColumn].colType == typeof(string)) theObjs[currentColumn] = stringArray[thisStringNo++]; - else if (currentColumn != myIdCol) - { - fixed (byte* thisGuy = &theBytes[thisOffset]) - { + else if (currentColumn != myIdCol) { + fixed (byte* thisGuy = &theBytes[thisOffset]) { theObjs[currentColumn] = myPullFromByteBufferDispatchTable[currentColumn](thisGuy); } thisOffset += myColumnSizesTable[currentColumn]; @@ -416,8 +361,7 @@ namespace BCG private void mySerializeDateTime(object data, BinaryWriter bw) { bw.Write(((DateTime)data).ToOADate()); } private void mySerializeSingle(object data, BinaryWriter bw) { bw.Write((Single)data); } - private SerializeDelegate myGetDispatchMethodSerialize(Type t) - { + private SerializeDelegate myGetDispatchMethodSerialize(Type t) { if (t.Equals(BlueDataValidTypes.INT)) return new SerializeDelegate(mySerializeInt32); else if (t.Equals(BlueDataValidTypes.DOUBLE)) return new SerializeDelegate(mySerializeDouble); else if (t.Equals(BlueDataValidTypes.CHAR)) return new SerializeDelegate(mySerializeChar); @@ -448,16 +392,14 @@ namespace BCG private unsafe object myPullDecimal(byte* bytePtr) { return *((decimal*)bytePtr); } private unsafe object myPullUInt(byte* bytePtr) { return *((UInt32*)bytePtr); } private unsafe object myPullULong(byte* bytePtr) { return *((UInt64*)bytePtr); } - private unsafe object myPullDateTime(byte* bytePtr) - { + private unsafe object myPullDateTime(byte* bytePtr) { double thisDateTime = *((double*)bytePtr); return DateTime.FromOADate(thisDateTime); } private unsafe object myPullSingle(byte* bytePtr) { return *((Single*)bytePtr); ; } - private unsafe PullFromByteBufferDelegate myGetDispatchMethodPullFromBuffer(Type t) - { + private unsafe PullFromByteBufferDelegate myGetDispatchMethodPullFromBuffer(Type t) { if (t.Equals(BlueDataValidTypes.INT)) return new PullFromByteBufferDelegate(myPullInt32); else if (t.Equals(BlueDataValidTypes.DOUBLE)) return new PullFromByteBufferDelegate(myPullDouble); else if (t.Equals(BlueDataValidTypes.CHAR)) return new PullFromByteBufferDelegate(myPullChar); diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueDataSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueDataSerializer.cs index 3e9bc628a..125911537 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueDataSerializer.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Publish/Serialization/IBlueDataSerializer.cs @@ -1,21 +1,21 @@ using System; +using BCG; using System.Collections; +using System.IO; -namespace BCG -{ - /// - /// All the data serialized to be send over to PubSub happens here. All it does is f(op, rows) = byte[]. - /// - public interface IBlueDataSerializer - { +namespace BCG { /// - /// f(op, rows) = byte[] + /// All the data serialized to be send over to PubSub happens here. All it does is f(op, rows) = byte[]. /// - /// Operation that can be add, delete, change etc - /// The data to serialize. The only case when it will be an array of more than 1 element - /// is when there are multiple deletes and adds, ie StreamOpCode.multipleRemoveAdd. - /// - /// The data serialized as bytes to be send over the wire - void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData); - } -} + public interface IBlueDataSerializer { + /// + /// f(op, rows) = byte[] + /// + /// Operation that can be add, delete, change etc + /// The data to serialize. The only case when it will be an array of more than 1 element + /// is when there are multiple deletes and adds, ie StreamOpCode.multipleRemoveAdd. + /// + /// The data serialized as bytes to be send over the wire + void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData); + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientDeserializer.cs index 61f6c2014..555005587 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientDeserializer.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientDeserializer.cs @@ -1,171 +1,171 @@ -using System; -using System.Collections; -using System.IO; -using System.IO.Compression; -using BCG.Utilities; -//using BCG.Utilities.BlueConfiguration; +// using System; +// using System.Collections; +// using System.IO; +// using System.IO.Compression; +// using BCG.Utilities; +// //using BCG.Utilities.BlueConfiguration; -namespace BCG -{ - public class LookupTableClientDeserializer : IClientDeserializer - { - public LookupTableClientDeserializer() { +// namespace BCG +// { +// public class LookupTableClientDeserializer : IClientDeserializer +// { +// public LookupTableClientDeserializer() { - mySchema = new LookupTableSchema(); - myLookupTables = null; - } +// mySchema = new LookupTableSchema(); +// myLookupTables = null; +// } - private IClientRowDeserializer myRowDeserializer; - private LookupTableSchema mySchema; - private LookupTable[] myLookupTables; +// private IClientRowDeserializer myRowDeserializer; +// private LookupTableSchema mySchema; +// private LookupTable[] myLookupTables; - public BinaryReader myGetReaderForRestOfStream(Stream theStream) - { - //teh rest of the stream is only compressed data - long compressedBytesLength = theStream.Length - theStream.Position; - byte[] compressedBytes = new byte[compressedBytesLength]; - theStream.Read(compressedBytes, 0, (int)compressedBytesLength); +// public BinaryReader myGetReaderForRestOfStream(Stream theStream) +// { +// //teh rest of the stream is only compressed data +// long compressedBytesLength = theStream.Length - theStream.Position; +// byte[] compressedBytes = new byte[compressedBytesLength]; +// theStream.Read(compressedBytes, 0, (int)compressedBytesLength); - //get a stream to read the uncompressed data - SharpZipCompressor deKompressor = new SharpZipCompressor(); +// //get a stream to read the uncompressed data +// SharpZipCompressor deKompressor = new SharpZipCompressor(); - MemoryStream decompressed = new MemoryStream(); - deKompressor.decompress(new MemoryStream(compressedBytes), decompressed); - decompressed.Position = 0; +// MemoryStream decompressed = new MemoryStream(); +// deKompressor.decompress(new MemoryStream(compressedBytes), decompressed); +// decompressed.Position = 0; - return new BinaryReader(decompressed); - } - #region IClientDeserializer methods - - public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) - { - ArrayList ops = new ArrayList(); - ArrayList rows = new ArrayList(); - IBlueRowCollection[] result; - foreach (Stream theStream in streams) - { - byte op; +// return new BinaryReader(decompressed); +// } +// #region IClientDeserializer methods - BinaryReader br = new BinaryReader(theStream); +// public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) +// { +// ArrayList ops = new ArrayList(); +// ArrayList rows = new ArrayList(); +// IBlueRowCollection[] result; +// foreach (Stream theStream in streams) +// { +// byte op; - op = br.ReadByte(); +// BinaryReader br = new BinaryReader(theStream); - if (op == StreamOpCode.create) - { - br.ReadString(); //pubsub deserializer - br.ReadString(); //client deserializer +// op = br.ReadByte(); + +// if (op == StreamOpCode.create) +// { +// br.ReadString(); //pubsub deserializer +// br.ReadString(); //client deserializer - br = myGetReaderForRestOfStream(theStream); +// br = myGetReaderForRestOfStream(theStream); - result = new IBlueRowCollection[] { myHandleCreate(br) }; - ops.Add(op); - rows.Add(result); +// result = new IBlueRowCollection[] { myHandleCreate(br) }; +// ops.Add(op); +// rows.Add(result); - } - else - { - //the schema and lookup tables should already be created. We just need to update everything. - //figure out how long the rest of the compressed data is - br = myGetReaderForRestOfStream(theStream); +// } +// else +// { +// //the schema and lookup tables should already be created. We just need to update everything. +// //figure out how long the rest of the compressed data is +// br = myGetReaderForRestOfStream(theStream); - if (mySchema != null) - { - if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove) - { - myDeserializeLookupTableUpdates(br); - result = new IBlueRowCollection[] { myGetRows(br, op, 1) }; - ops.Add(op); - rows.Add(result); - } - else if (op == StreamOpCode.multipleChange) - { - myDeserializeLookupTableUpdates(br); - int numRows = br.ReadInt32(); - result = new IBlueRowCollection[] { myGetRows(br, op, numRows) }; - ops.Add(op); - rows.Add(result); +// if (mySchema != null) +// { +// if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove) +// { +// myDeserializeLookupTableUpdates(br); +// result = new IBlueRowCollection[] { myGetRows(br, op, 1) }; +// ops.Add(op); +// rows.Add(result); +// } +// else if (op == StreamOpCode.multipleChange) +// { +// myDeserializeLookupTableUpdates(br); +// int numRows = br.ReadInt32(); +// result = new IBlueRowCollection[] { myGetRows(br, op, numRows) }; +// ops.Add(op); +// rows.Add(result); - } - else if (op == StreamOpCode.multipleRemoveAdd) - { - myDeserializeLookupTableUpdates(br); - result = new IBlueRowCollection[2]; - int numDeletedRows = br.ReadInt32(); - result[0] = myGetRows(br, StreamOpCode.remove, numDeletedRows); - int numAddedRows = br.ReadInt32(); - result[1] = myGetRows(br, StreamOpCode.add, numAddedRows); - ops.Add(op); - rows.Add(result); - } - else{} - // BlueEventLog.logEvent("Illegal op-code recieved", BlueEventLevel.Error); +// } +// else if (op == StreamOpCode.multipleRemoveAdd) +// { +// myDeserializeLookupTableUpdates(br); +// result = new IBlueRowCollection[2]; +// int numDeletedRows = br.ReadInt32(); +// result[0] = myGetRows(br, StreamOpCode.remove, numDeletedRows); +// int numAddedRows = br.ReadInt32(); +// result[1] = myGetRows(br, StreamOpCode.add, numAddedRows); +// ops.Add(op); +// rows.Add(result); +// } +// else{} +// // BlueEventLog.logEvent("Illegal op-code recieved", BlueEventLevel.Error); - } - else - { - // BlueEventLog.logEvent("Received a non create operation even though the schema" + - // "has never been created", BlueEventLevel.Error); - } +// } +// else +// { +// // BlueEventLog.logEvent("Received a non create operation even though the schema" + +// // "has never been created", BlueEventLevel.Error); +// } - } +// } - } +// } - theOps = ops.ToArray(typeof(byte)) as byte[]; - theRows = rows.ToArray(); +// theOps = ops.ToArray(typeof(byte)) as byte[]; +// theRows = rows.ToArray(); - } +// } - #endregion +// #endregion - IBlueRowCollection myHandleCreate(BinaryReader br) - { +// IBlueRowCollection myHandleCreate(BinaryReader br) +// { - mySchema.deserialze(br); +// mySchema.deserialze(br); - //build the lookup tables now - // There is one less than the length of the lookupTableGroups because the first one - //is always the default group which isn't stored in a lookup table. - myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1]; +// //build the lookup tables now +// // There is one less than the length of the lookupTableGroups because the first one +// //is always the default group which isn't stored in a lookup table. +// myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1]; - //start with 1 because we don't store the 0th lookup table - for (int i = 0; i < myLookupTables.Length; ++i) - myLookupTables[i] = new LookupTable(mySchema, i+1); +// //start with 1 because we don't store the 0th lookup table +// for (int i = 0; i < myLookupTables.Length; ++i) +// myLookupTables[i] = new LookupTable(mySchema, i+1); - myDeserializeLookupTableUpdates(br); - //now read the rows - int numRows = br.ReadInt32(); - return myGetRows(br, StreamOpCode.create, numRows); +// myDeserializeLookupTableUpdates(br); +// //now read the rows +// int numRows = br.ReadInt32(); +// return myGetRows(br, StreamOpCode.create, numRows); - } +// } - private IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows) - { - BlueTable retVal = new BlueTable(mySchema); - myRowDeserializer = myRowDeserializer == null ? new LookupTableClientRowDeserializer(mySchema,myLookupTables) : myRowDeserializer; - for (int i = 0; i < nRows; i++) - { - IBlueRow row; - myRowDeserializer.deserializeRow(null, br, op, out row); - retVal.addRow(row); - } - return retVal; - } +// private IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows) +// { +// BlueTable retVal = new BlueTable(mySchema); +// myRowDeserializer = myRowDeserializer == null ? new LookupTableClientRowDeserializer(mySchema,myLookupTables) : myRowDeserializer; +// for (int i = 0; i < nRows; i++) +// { +// IBlueRow row; +// myRowDeserializer.deserializeRow(null, br, op, out row); +// retVal.addRow(row); +// } +// return retVal; +// } - private void myDeserializeLookupTableUpdates(BinaryReader br) - { - for (int i = 0; i < myLookupTables.Length; ++i) - myLookupTables[i].deserializeUpdates(br); - } +// private void myDeserializeLookupTableUpdates(BinaryReader br) +// { +// for (int i = 0; i < myLookupTables.Length; ++i) +// myLookupTables[i].deserializeUpdates(br); +// } - - } -} + +// } +// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientlRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientlRowDeserializer.cs index d8f794a13..d9426f951 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientlRowDeserializer.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/ClientDeserialization/LookupTableClientlRowDeserializer.cs @@ -1,107 +1,107 @@ -using System; -using System.Collections; -using System.IO; +// using System; +// using System.Collections; +// using System.IO; -using BCG.Utilities; -//using BCG.Utilities.BlueConfiguration; +// using BCG.Utilities; +// //using BCG.Utilities.BlueConfiguration; -namespace BCG -{ - /// - /// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process - /// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations) - /// - internal class LookupTableClientRowDeserializer : IClientRowDeserializer - { - private LookupTableSchema mySchema; - private LookupTable[] myTables; - private int myNullMapLength = 0; - private PrimitiveDeserializationHelper myDeserializationHelper; - private BitArray myNullMap; - private int myIdColumn; - public LookupTableClientRowDeserializer (LookupTableSchema schema, LookupTable[] theTables) - { - mySchema = schema; - myTables = theTables; +// namespace BCG +// { +// /// +// /// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process +// /// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations) +// /// +// internal class LookupTableClientRowDeserializer : IClientRowDeserializer +// { +// private LookupTableSchema mySchema; +// private LookupTable[] myTables; +// private int myNullMapLength = 0; +// private PrimitiveDeserializationHelper myDeserializationHelper; +// private BitArray myNullMap; +// private int myIdColumn; +// public LookupTableClientRowDeserializer (LookupTableSchema schema, LookupTable[] theTables) +// { +// mySchema = schema; +// myTables = theTables; - myDeserializationHelper = new PrimitiveDeserializationHelper (mySchema); - myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8))); - myIdColumn = schema["id"].colNo; +// myDeserializationHelper = new PrimitiveDeserializationHelper (mySchema); +// myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8))); +// myIdColumn = schema["id"].colNo; - } +// } - #region IClientRowDeserializer Members +// #region IClientRowDeserializer Members - public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) - { - int rowLength = br.ReadInt32 (); //length information for the row not needed - //rad the null map - myNullMap = new BitArray(br.ReadBytes (myNullMapLength)); - object[] values = new object [mySchema.columnNames.Length]; - //now read the id - values[myIdColumn] = br.ReadInt32(); - //now read the lookup groups - for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo) - { - BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo]; - LookupTable thisTable = myTables[groupNo - 1]; - //read the id in, depending upon the size - int idRead = -1; - switch (thisGroup.tableSize) - { - case 8: - idRead = br.ReadByte(); - break; - case 16: - idRead = br.ReadInt16(); - break; - case 32: - idRead = br.ReadInt32(); - break; - default: - throw new BlueDataException("Error: invalid size " + thisGroup.tableSize.ToString() + " specific for lookup group."); - } +// public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) +// { +// int rowLength = br.ReadInt32 (); //length information for the row not needed +// //rad the null map +// myNullMap = new BitArray(br.ReadBytes (myNullMapLength)); +// object[] values = new object [mySchema.columnNames.Length]; +// //now read the id +// values[myIdColumn] = br.ReadInt32(); +// //now read the lookup groups +// for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo) +// { +// BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo]; +// LookupTable thisTable = myTables[groupNo - 1]; +// //read the id in, depending upon the size +// int idRead = -1; +// switch (thisGroup.tableSize) +// { +// case 8: +// idRead = br.ReadByte(); +// break; +// case 16: +// idRead = br.ReadInt16(); +// break; +// case 32: +// idRead = br.ReadInt32(); +// break; +// default: +// throw new BlueDataException("Error: invalid size " + thisGroup.tableSize.ToString() + " specific for lookup group."); +// } - object[] thisGroupValues = thisTable.getValuesForIndex(idRead); +// object[] thisGroupValues = thisTable.getValuesForIndex(idRead); - if (thisGroupValues == null) - throw new BlueDataException("Error: Couldnt' find values for index " + idRead.ToString()); - if (thisGroupValues.Length != thisGroup.groupColNos.Length) - throw new BlueDataException("Error: Values extracted for index " + idRead.ToString() + " had length " + thisGroupValues.Length.ToString() + " but length " + thisGroup.groupColNos.Length.ToString() + " was expected."); +// if (thisGroupValues == null) +// throw new BlueDataException("Error: Couldnt' find values for index " + idRead.ToString()); +// if (thisGroupValues.Length != thisGroup.groupColNos.Length) +// throw new BlueDataException("Error: Values extracted for index " + idRead.ToString() + " had length " + thisGroupValues.Length.ToString() + " but length " + thisGroup.groupColNos.Length.ToString() + " was expected."); - //insert those values into our object array +// //insert those values into our object array - for (int groupCol = 0; groupCol < thisGroup.groupColNos.Length; ++groupCol) - { - int thisColInOutputRow = thisGroup.groupColNos[groupCol]; - values[thisColInOutputRow] = thisGroupValues[groupCol]; - } - } - for (int colNo = 0; colNo < values.Length; colNo++) - { +// for (int groupCol = 0; groupCol < thisGroup.groupColNos.Length; ++groupCol) +// { +// int thisColInOutputRow = thisGroup.groupColNos[groupCol]; +// values[thisColInOutputRow] = thisGroupValues[groupCol]; +// } +// } +// for (int colNo = 0; colNo < values.Length; colNo++) +// { - if (myNullMap[colNo]) - { - values[colNo] = DBNull.Value; - } - //if it's non null, not the id column, and not part of a lookup group, get it here. - else if (mySchema[colNo].lookupTableGroup == 0 && colNo != myIdColumn) - { +// if (myNullMap[colNo]) +// { +// values[colNo] = DBNull.Value; +// } +// //if it's non null, not the id column, and not part of a lookup group, get it here. +// else if (mySchema[colNo].lookupTableGroup == 0 && colNo != myIdColumn) +// { - values[colNo] = myDeserializationHelper.deserializeColumn(colNo, br); +// values[colNo] = myDeserializationHelper.deserializeColumn(colNo, br); - } - } +// } +// } - r = new BlueRow(mySchema, values); +// r = new BlueRow(mySchema, values); - } +// } - #endregion - } +// #endregion +// } -} +// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/LookupTable.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/LookupTable.cs index 08a755f52..0761ae9cc 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/LookupTable.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/LookupTable.cs @@ -1,194 +1,194 @@ -using System; -using System.Collections.Generic; -using System.Collections; -using System.IO; -using System.Linq; -using System.Text; +// using System; +// using System.Collections.Generic; +// using System.Collections; +// using System.IO; +// using System.Linq; +// using System.Text; -namespace BCG -{ +// namespace BCG +// { - /* - * - * LookupTable - * - * This class represents a lookup table used to store values inside data streams. This allows indexes into the table - * to be passed over the wire, instead of actual values of the rows. - * - * Each table can have multiple lookup table groups. This class represents just one of those groups. This means that all the entries - * contained in the table will have the same schema. - * - * - * - * */ - public class LookupTable - { - //each of the entries has the same schema - that is, each of the entries contains the same number of values, with the same - IBlueTableSchema myEntrySchema; - UniversalRowSerializer mySerializer; - ClientUniversalRowDeserializer myDeserializer; - private const int myIdColumn = 0; +// /* +// * +// * LookupTable +// * +// * This class represents a lookup table used to store values inside data streams. This allows indexes into the table +// * to be passed over the wire, instead of actual values of the rows. +// * +// * Each table can have multiple lookup table groups. This class represents just one of those groups. This means that all the entries +// * contained in the table will have the same schema. +// * +// * +// * +// * */ +// public class LookupTable +// { +// //each of the entries has the same schema - that is, each of the entries contains the same number of values, with the same +// IBlueTableSchema myEntrySchema; +// UniversalRowSerializer mySerializer; +// ClientUniversalRowDeserializer myDeserializer; +// private const int myIdColumn = 0; - //the table is only allowed to reach a maximum size based upon the number of bits you give to it. - //it throws an error if you try to put too much data into it. - private long myMaxSize; - private int mySizeInBits; +// //the table is only allowed to reach a maximum size based upon the number of bits you give to it. +// //it throws an error if you try to put too much data into it. +// private long myMaxSize; +// private int mySizeInBits; - public int sizeInBits - { - get { return mySizeInBits; } - } +// public int sizeInBits +// { +// get { return mySizeInBits; } +// } - KeysBasedTable2 myValuesToEntryMap; +// KeysBasedTable2 myValuesToEntryMap; - //keep track of the last time somebody published you, so you know what values to publish. - int myIdOfLastPublish; +// //keep track of the last time somebody published you, so you know what values to publish. +// int myIdOfLastPublish; - int myIdToUse; +// int myIdToUse; - /* builds the local schema, serializer, and desrializer - * initializes to an empty table - */ - public LookupTable(LookupTableSchema theSchema, int lookupGroupNumber) - { - BlueRowLookupGroup group = theSchema.lookupGroups[lookupGroupNumber]; +// /* builds the local schema, serializer, and desrializer +// * initializes to an empty table +// */ +// public LookupTable(LookupTableSchema theSchema, int lookupGroupNumber) +// { +// BlueRowLookupGroup group = theSchema.lookupGroups[lookupGroupNumber]; - myEntrySchema = group.entrySchema; +// myEntrySchema = group.entrySchema; - //compute how much room you have in the table - mySizeInBits = group.tableSize; - myMaxSize = (long)(Math.Pow(2, group.tableSize) - 1); +// //compute how much room you have in the table +// mySizeInBits = group.tableSize; +// myMaxSize = (long)(Math.Pow(2, group.tableSize) - 1); - myDeserializer = new ClientUniversalRowDeserializer(myEntrySchema); - mySerializer = new UniversalRowSerializer(myEntrySchema); +// myDeserializer = new ClientUniversalRowDeserializer(myEntrySchema); +// mySerializer = new UniversalRowSerializer(myEntrySchema); - //create the map - myValuesToEntryMap = new KeysBasedTable2(myEntrySchema, group.groupColNames); - myIdOfLastPublish = 0; +// //create the map +// myValuesToEntryMap = new KeysBasedTable2(myEntrySchema, group.groupColNames); +// myIdOfLastPublish = 0; - } +// } - /*Checks to see if the given values are in the table. IF they are in there, it returns the index to those values. - If they are not present, it adds them to teh table and returns the new index. */ - public int getIndexForValues(params object[] values) - { +// /*Checks to see if the given values are in the table. IF they are in there, it returns the index to those values. +// If they are not present, it adds them to teh table and returns the new index. */ +// public int getIndexForValues(params object[] values) +// { - int index = 0; - //see if the key is present - IBlueRow thisRowEntry = myValuesToEntryMap.getRow(values); +// int index = 0; +// //see if the key is present +// IBlueRow thisRowEntry = myValuesToEntryMap.getRow(values); - if (thisRowEntry == null) - { - //check to see if we even have room in the table - if (myIdToUse > myMaxSize) - throw new BlueDataException("Error: LookupTable has reached maximum size. Make a bigger table."); - //we need to entrer this new value into the table. Create a blue row using the next ID - //and the values they supplied to us - object[] thisRowValues = new object[myEntrySchema.columnInfo.Length]; - index = myIdToUse; - thisRowValues[myIdColumn] = index; - myIdToUse ++; - //now add in the values they supplied - for (int colNo = myIdColumn+1; colNo < myEntrySchema.columnInfo.Length; ++colNo) - { - thisRowValues[colNo] = values[colNo - 1]; +// if (thisRowEntry == null) +// { +// //check to see if we even have room in the table +// if (myIdToUse > myMaxSize) +// throw new BlueDataException("Error: LookupTable has reached maximum size. Make a bigger table."); +// //we need to entrer this new value into the table. Create a blue row using the next ID +// //and the values they supplied to us +// object[] thisRowValues = new object[myEntrySchema.columnInfo.Length]; +// index = myIdToUse; +// thisRowValues[myIdColumn] = index; +// myIdToUse ++; +// //now add in the values they supplied +// for (int colNo = myIdColumn+1; colNo < myEntrySchema.columnInfo.Length; ++colNo) +// { +// thisRowValues[colNo] = values[colNo - 1]; - } - IBlueRow newEntryRow = new BlueRow(myEntrySchema, thisRowValues); +// } +// IBlueRow newEntryRow = new BlueRow(myEntrySchema, thisRowValues); - myValuesToEntryMap.update(newEntryRow); - } - else - index = (int) thisRowEntry["id"]; +// myValuesToEntryMap.update(newEntryRow); +// } +// else +// index = (int) thisRowEntry["id"]; - return index; +// return index; - } +// } - //queries the table to get the values that were stored on a given index. - //returns null if the values are not found - public object[] getValuesForIndex(int index) - { - object[] results = null; +// //queries the table to get the values that were stored on a given index. +// //returns null if the values are not found +// public object[] getValuesForIndex(int index) +// { +// object[] results = null; - //get the row from the table - if (myValuesToEntryMap.containsID(index)) - { - IBlueRow thisRow = myValuesToEntryMap[index]; - results = new object[myEntrySchema.columnInfo.Length - 1]; +// //get the row from the table +// if (myValuesToEntryMap.containsID(index)) +// { +// IBlueRow thisRow = myValuesToEntryMap[index]; +// results = new object[myEntrySchema.columnInfo.Length - 1]; - //copy the values from the row into the array - for (int colNo = myIdColumn+1; colNo < myEntrySchema.columnInfo.Length; ++ colNo) - { - results[colNo - 1] = thisRow[colNo]; +// //copy the values from the row into the array +// for (int colNo = myIdColumn+1; colNo < myEntrySchema.columnInfo.Length; ++ colNo) +// { +// results[colNo - 1] = thisRow[colNo]; - } +// } - } +// } - //if the index isn't in the tablem, can just return the null value +// //if the index isn't in the tablem, can just return the null value - return results; +// return results; - } +// } - public void deserializeUpdates(BinaryReader br) - { +// public void deserializeUpdates(BinaryReader br) +// { - //get the number of updates that are being published. - int numUpdates = br.ReadInt32(); +// //get the number of updates that are being published. +// int numUpdates = br.ReadInt32(); - for (int i = 0; i < numUpdates; ++i) - { +// for (int i = 0; i < numUpdates; ++i) +// { - IBlueRow theNewUpdate = null; - myDeserializer.deserializeRow(null, br, StreamOpCode.create, out theNewUpdate); - myValuesToEntryMap.update(theNewUpdate); +// IBlueRow theNewUpdate = null; +// myDeserializer.deserializeRow(null, br, StreamOpCode.create, out theNewUpdate); +// myValuesToEntryMap.update(theNewUpdate); - int thisID = (int) theNewUpdate[myIdColumn]; - if (thisID != myIdToUse) - throw new BlueDataException("Error: bad serialized update given to lookup table. Expected id " + myIdToUse.ToString() + ", found " + thisID.ToString()); - ++myIdToUse; +// int thisID = (int) theNewUpdate[myIdColumn]; +// if (thisID != myIdToUse) +// throw new BlueDataException("Error: bad serialized update given to lookup table. Expected id " + myIdToUse.ToString() + ", found " + thisID.ToString()); +// ++myIdToUse; - } +// } - } +// } - public void serializeUpdates(BinaryWriter bw) - { - int numUpdates = myIdToUse - myIdOfLastPublish; +// public void serializeUpdates(BinaryWriter bw) +// { +// int numUpdates = myIdToUse - myIdOfLastPublish; - //serialize the number of updates - bw.Write(numUpdates); +// //serialize the number of updates +// bw.Write(numUpdates); - //serialize each update - for (int i = myIdOfLastPublish; i < myIdToUse; ++i) - mySerializer.serializeRow(myValuesToEntryMap[i], StreamOpCode.add, bw); +// //serialize each update +// for (int i = myIdOfLastPublish; i < myIdToUse; ++i) +// mySerializer.serializeRow(myValuesToEntryMap[i], StreamOpCode.add, bw); - //store the last id you published so that next time we don't serialize the same updates - myIdOfLastPublish = myIdToUse; - } +// //store the last id you published so that next time we don't serialize the same updates +// myIdOfLastPublish = myIdToUse; +// } - /*this function serializes EVERYTHING in the table into the writer. This is - used by the pubsub deserializer, which needs to be able to supply the new state - to any clients connection. */ - public void serializeWholeTable(BinaryWriter bw) - { - //serialize the number of updates - bw.Write(myIdToUse); +// /*this function serializes EVERYTHING in the table into the writer. This is +// used by the pubsub deserializer, which needs to be able to supply the new state +// to any clients connection. */ +// public void serializeWholeTable(BinaryWriter bw) +// { +// //serialize the number of updates +// bw.Write(myIdToUse); - //serialize each update - for (int i = 0; i < myIdToUse; ++i) - mySerializer.serializeRow(myValuesToEntryMap[i], StreamOpCode.add, bw); +// //serialize each update +// for (int i = 0; i < myIdToUse; ++i) +// mySerializer.serializeRow(myValuesToEntryMap[i], StreamOpCode.add, bw); - } - } -} +// } +// } +// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/PubSubDeserialization/LookupTablePubSubDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/PubSubDeserialization/LookupTablePubSubDeserializer.cs index ab8adc94c..0eb191733 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/PubSubDeserialization/LookupTablePubSubDeserializer.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/PubSubDeserialization/LookupTablePubSubDeserializer.cs @@ -1,226 +1,226 @@ -using System; -using System.Collections; -using System.IO; -using BCG.Utilities; -using System.IO.Compression; +// using System; +// using System.Collections; +// using System.IO; +// using BCG.Utilities; +// using System.IO.Compression; -namespace BCG -{ +// namespace BCG +// { - public class LookupTablePubSubDeserializer : IPubsubDeserializer - { - protected IDictionary myState = new Hashtable(); - protected LookupTableSchema mySchema; - protected string myClientDeserializerName; //for the getState method - protected IPubsubRowDeserializer myDeserializer; - LookupTable[] myLookupTables; +// public class LookupTablePubSubDeserializer : IPubsubDeserializer +// { +// protected IDictionary myState = new Hashtable(); +// protected LookupTableSchema mySchema; +// protected string myClientDeserializerName; //for the getState method +// protected IPubsubRowDeserializer myDeserializer; +// LookupTable[] myLookupTables; - public LookupTablePubSubDeserializer() { +// public LookupTablePubSubDeserializer() { - mySchema = new LookupTableSchema(); - - } +// mySchema = new LookupTableSchema(); + +// } - public BinaryReader myGetReaderForCompressedPortion(Stream theStream) - { - //teh rest of the stream is only compressed data - long compressedBytesLength = theStream.Length - theStream.Position; - byte[] compressedBytes = new byte[compressedBytesLength]; - theStream.Read(compressedBytes, 0, (int)compressedBytesLength); +// public BinaryReader myGetReaderForCompressedPortion(Stream theStream) +// { +// //teh rest of the stream is only compressed data +// long compressedBytesLength = theStream.Length - theStream.Position; +// byte[] compressedBytes = new byte[compressedBytesLength]; +// theStream.Read(compressedBytes, 0, (int)compressedBytesLength); - //get a stream to read the uncompressed data - SharpZipCompressor deKompressor = new SharpZipCompressor(); +// //get a stream to read the uncompressed data +// SharpZipCompressor deKompressor = new SharpZipCompressor(); - MemoryStream decompressed = new MemoryStream(); - deKompressor.decompress(new MemoryStream(compressedBytes), decompressed); - decompressed.Position = 0; +// MemoryStream decompressed = new MemoryStream(); +// deKompressor.decompress(new MemoryStream(compressedBytes), decompressed); +// decompressed.Position = 0; - return new BinaryReader(decompressed); - } +// return new BinaryReader(decompressed); +// } - #region IPubsubDeserializer Members - public virtual void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) - { - ArrayList ops = new ArrayList(), rows = new ArrayList(); - foreach (Stream theStream in streams) - { - BinaryReader br = new BinaryReader(theStream); - IDictionary[] result; - byte op; - op = br.ReadByte(); - - - - if (op == StreamOpCode.create) - { - myState.Clear(); - br.ReadString(); //the pubsub-deserializer name, should be the same as this's name - myClientDeserializerName = br.ReadString(); //the client-deserializer name - - //at this point, the rest of the bytes in the stream are from a GZIP stream - //so we need to read into a GZipStream and decompress - //figure out how long the rest of the compressed data is - br = myGetReaderForCompressedPortion(theStream); +// #region IPubsubDeserializer Members +// public virtual void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) +// { +// ArrayList ops = new ArrayList(), rows = new ArrayList(); +// foreach (Stream theStream in streams) +// { +// BinaryReader br = new BinaryReader(theStream); +// IDictionary[] result; +// byte op; +// op = br.ReadByte(); - mySchema.deserialze(br); - //now build the lookup tables from the knowledge gleaned at the schema - myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1]; +// if (op == StreamOpCode.create) +// { +// myState.Clear(); +// br.ReadString(); //the pubsub-deserializer name, should be the same as this's name +// myClientDeserializerName = br.ReadString(); //the client-deserializer name - //start with 1 because we don't store the 0th lookup table - for (int i = 0; i < myLookupTables.Length; ++i) - myLookupTables[i] = new LookupTable(mySchema, i + 1); - - myDeserializeLookupTableUpdates(br); - - int numRows = br.ReadInt32(); - result = new IDictionary[] { myGetRows(br, op, numRows) }; - myEncorporatRows(result[0], op); - ops.Add(op); - rows.Add(result); - } - else - { - //figure out how long the rest of the compressed data is - br = myGetReaderForCompressedPortion(theStream); - if (mySchema != null) - { - if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove) - { - myDeserializeLookupTableUpdates(br); - result = new IDictionary[] { myGetRows(br, op, 1) }; - myEncorporatRows(result[0], op); - ops.Add(op); - rows.Add(result); - } - else if (op == StreamOpCode.multipleChange) - { - myDeserializeLookupTableUpdates(br); - int numRows = br.ReadInt32(); - result = new IDictionary[] { myGetRows(br, op, numRows) }; - myEncorporatRows(result[0], op); - ops.Add(op); - rows.Add(result); - } - else if (op == StreamOpCode.multipleRemoveAdd) - { - myDeserializeLookupTableUpdates(br); - - int numRemoved = br.ReadInt32(); - result = new IDictionary[2]; - - result[0] = myGetRows(br, StreamOpCode.remove, numRemoved); - int numAdded = br.ReadInt32(); - result[1] = myGetRows (br, StreamOpCode.add, numAdded); - - myEncorporatRows(result[0], StreamOpCode.remove); - myEncorporatRows(result[1], StreamOpCode.add); - ops.Add(op); - rows.Add(result); - } - else - { - // BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error); - } - } - else - { - // BlueEventLog.logEvent("Received a non create operation even though the schema" + - // "has never been created", BlueEventLevel.Warning); - } - } - } - theOps = ops.ToArray(typeof(byte)) as byte[]; - theRows = rows.ToArray(); - } - - /// - /// Sends in a create signal, so that new clients can get a create everytime and start - /// afresh from there - /// - /// - public virtual byte[] getState() - { - if (mySchema == null) - return new byte[] { }; - else - { - BinaryWriter bw = new BinaryWriter(new MemoryStream()); - bw.Write(StreamOpCode.create); - bw.Write(GetType().FullName); - bw.Write(myClientDeserializerName); - - //the rest of this data has to be serialized. - BinaryWriter compressedBw = new BinaryWriter(new MemoryStream()); - mySchema.serialize(compressedBw); - //serialize all the tables - for (int i = 0; i < myLookupTables.Length; ++i) - myLookupTables[i].serializeWholeTable(compressedBw); - compressedBw.Write(myState.Count); - foreach (byte[] row in myState.Values) - compressedBw.Write(row); - - //now compress the bytes - MemoryStream compressedStream = new MemoryStream(); - SharpZipCompressor Kompressor = new SharpZipCompressor(); - compressedBw.BaseStream.Position = 0; - - Kompressor.compress(compressedBw.BaseStream, compressedStream); - - //append the two streams together - byte[] output = new byte[bw.BaseStream.Length + compressedStream.Length]; - bw.BaseStream.Position = 0; - compressedStream.Position = 0; +// //at this point, the rest of the bytes in the stream are from a GZIP stream +// //so we need to read into a GZipStream and decompress +// //figure out how long the rest of the compressed data is +// br = myGetReaderForCompressedPortion(theStream); - bw.BaseStream.Read(output, 0, (int) bw.BaseStream.Length); - compressedStream.Read(output, (int) bw.BaseStream.Length, (int) compressedStream.Length); +// mySchema.deserialze(br); - +// //now build the lookup tables from the knowledge gleaned at the schema +// myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1]; - return output; - } - } - #endregion +// //start with 1 because we don't store the 0th lookup table +// for (int i = 0; i < myLookupTables.Length; ++i) +// myLookupTables[i] = new LookupTable(mySchema, i + 1); - private void myDeserializeLookupTableUpdates(BinaryReader br) - { - for (int i = 0; i < myLookupTables.Length; ++i) - myLookupTables[i].deserializeUpdates(br); - } - /// - /// Every row that has been deserialized passes through this method, so that it can update the state. - /// - /// - /// - protected virtual void myEncorporatRows(IDictionary rows, byte op) - { - if (op == StreamOpCode.add || op == StreamOpCode.create) - foreach (DictionaryEntry de in rows) - myState.Add(de.Key, de.Value); - else if (op == StreamOpCode.remove) - foreach (DictionaryEntry de in rows) - myState.Remove(de.Key); - else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange) - foreach (DictionaryEntry de in rows) - myState[de.Key] = de.Value; - } +// myDeserializeLookupTableUpdates(br); - private IDictionary myGetRows(BinaryReader br, byte op, int noOfRows) - { - //the row deserializer doesn't need to know anything about the lookup tables - myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer(mySchema) : myDeserializer; - Hashtable retVal = new Hashtable(); - for (int i = 0; i < noOfRows; i++) - { - int id; object row; - myDeserializer.deserializeRow(myState, br, op, out id, out row); - retVal.Add(id, row); - } - return retVal; - } +// int numRows = br.ReadInt32(); +// result = new IDictionary[] { myGetRows(br, op, numRows) }; +// myEncorporatRows(result[0], op); +// ops.Add(op); +// rows.Add(result); +// } +// else +// { +// //figure out how long the rest of the compressed data is +// br = myGetReaderForCompressedPortion(theStream); +// if (mySchema != null) +// { +// if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove) +// { +// myDeserializeLookupTableUpdates(br); +// result = new IDictionary[] { myGetRows(br, op, 1) }; +// myEncorporatRows(result[0], op); +// ops.Add(op); +// rows.Add(result); +// } +// else if (op == StreamOpCode.multipleChange) +// { +// myDeserializeLookupTableUpdates(br); +// int numRows = br.ReadInt32(); +// result = new IDictionary[] { myGetRows(br, op, numRows) }; +// myEncorporatRows(result[0], op); +// ops.Add(op); +// rows.Add(result); +// } +// else if (op == StreamOpCode.multipleRemoveAdd) +// { +// myDeserializeLookupTableUpdates(br); - } -} +// int numRemoved = br.ReadInt32(); +// result = new IDictionary[2]; + +// result[0] = myGetRows(br, StreamOpCode.remove, numRemoved); +// int numAdded = br.ReadInt32(); +// result[1] = myGetRows (br, StreamOpCode.add, numAdded); + +// myEncorporatRows(result[0], StreamOpCode.remove); +// myEncorporatRows(result[1], StreamOpCode.add); +// ops.Add(op); +// rows.Add(result); +// } +// else +// { +// // BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error); +// } +// } +// else +// { +// // BlueEventLog.logEvent("Received a non create operation even though the schema" + +// // "has never been created", BlueEventLevel.Warning); +// } +// } +// } +// theOps = ops.ToArray(typeof(byte)) as byte[]; +// theRows = rows.ToArray(); +// } + +// /// +// /// Sends in a create signal, so that new clients can get a create everytime and start +// /// afresh from there +// /// +// /// +// public virtual byte[] getState() +// { +// if (mySchema == null) +// return new byte[] { }; +// else +// { +// BinaryWriter bw = new BinaryWriter(new MemoryStream()); +// bw.Write(StreamOpCode.create); +// bw.Write(GetType().FullName); +// bw.Write(myClientDeserializerName); + +// //the rest of this data has to be serialized. +// BinaryWriter compressedBw = new BinaryWriter(new MemoryStream()); +// mySchema.serialize(compressedBw); +// //serialize all the tables +// for (int i = 0; i < myLookupTables.Length; ++i) +// myLookupTables[i].serializeWholeTable(compressedBw); +// compressedBw.Write(myState.Count); +// foreach (byte[] row in myState.Values) +// compressedBw.Write(row); + +// //now compress the bytes +// MemoryStream compressedStream = new MemoryStream(); +// SharpZipCompressor Kompressor = new SharpZipCompressor(); +// compressedBw.BaseStream.Position = 0; + +// Kompressor.compress(compressedBw.BaseStream, compressedStream); + +// //append the two streams together +// byte[] output = new byte[bw.BaseStream.Length + compressedStream.Length]; +// bw.BaseStream.Position = 0; +// compressedStream.Position = 0; + + +// bw.BaseStream.Read(output, 0, (int) bw.BaseStream.Length); +// compressedStream.Read(output, (int) bw.BaseStream.Length, (int) compressedStream.Length); + + + +// return output; +// } +// } +// #endregion + +// private void myDeserializeLookupTableUpdates(BinaryReader br) +// { +// for (int i = 0; i < myLookupTables.Length; ++i) +// myLookupTables[i].deserializeUpdates(br); +// } +// /// +// /// Every row that has been deserialized passes through this method, so that it can update the state. +// /// +// /// +// /// +// protected virtual void myEncorporatRows(IDictionary rows, byte op) +// { +// if (op == StreamOpCode.add || op == StreamOpCode.create) +// foreach (DictionaryEntry de in rows) +// myState.Add(de.Key, de.Value); +// else if (op == StreamOpCode.remove) +// foreach (DictionaryEntry de in rows) +// myState.Remove(de.Key); +// else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange) +// foreach (DictionaryEntry de in rows) +// myState[de.Key] = de.Value; +// } + +// private IDictionary myGetRows(BinaryReader br, byte op, int noOfRows) +// { +// //the row deserializer doesn't need to know anything about the lookup tables +// myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer(mySchema) : myDeserializer; +// Hashtable retVal = new Hashtable(); +// for (int i = 0; i < noOfRows; i++) +// { +// int id; object row; +// myDeserializer.deserializeRow(myState, br, op, out id, out row); +// retVal.Add(id, row); +// } +// return retVal; +// } + +// } +// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableRowSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableRowSerializer.cs index 34ce1f207..e866ac3be 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableRowSerializer.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableRowSerializer.cs @@ -1,211 +1,211 @@ -using System; -using System.Collections; -using System.IO; -using BCG.Utilities; -using System.Collections.Generic; -namespace BCG -{ - /// - /// Helps the LookupTableRowSerializer by serializing one row. - /// - /// It does NOT sort the schema, even though the sorted schema doesn't do anything. - /// That is because the schema sorting utility is agnostic of schema type, and so it would destory the grouping information - /// which is VITAL - /// - /// It performs a mapping : (row)->byte[] - /// +// /// Helps the LookupTableRowSerializer by serializing one row. +// /// +// /// It does NOT sort the schema, even though the sorted schema doesn't do anything. +// /// That is because the schema sorting utility is agnostic of schema type, and so it would destory the grouping information +// /// which is VITAL +// /// +// /// It performs a mapping : (row)->byte[] +// /// - /// Row serialized in the following order - /// row size (nullBitMap size + actual row size) - /// null bit map - /// row id - /// row lookup group ids - /// rest of row values - /// - /// - /// - public void serializeRow(IBlueRow r, byte op, BinaryWriter bw) - { - myNullMap.SetAll(false); - int sz = myFixedSize; - //first, build the null map - for (int i = 0; i < mySchema.columnNames.Length; i++) - { - if (r[i] == DBNull.Value || r[i] == null) - { - myNullMap[i] = true; +// } +// #region IBlueRowSerializer methods +// /// +// /// Row serialized in the following order +// /// row size (nullBitMap size + actual row size) +// /// null bit map +// /// row id +// /// row lookup group ids +// /// rest of row values +// /// +// /// +// /// +// public void serializeRow(IBlueRow r, byte op, BinaryWriter bw) +// { +// myNullMap.SetAll(false); +// int sz = myFixedSize; +// //first, build the null map +// for (int i = 0; i < mySchema.columnNames.Length; i++) +// { +// if (r[i] == DBNull.Value || r[i] == null) +// { +// myNullMap[i] = true; - //if this thing isn't in a lookup group, it's no longer going to be accounted for in the fixed size of the table - //therefore we need to decrement the size of this row by the size of the type - if (mySchema[i].lookupTableGroup == 0) - { - if (SerializationUtilityMethods.isFixedLengthType(mySchema[i].colType)) - sz -= SerializationUtilityMethods.getFixedLengthTypeSize(mySchema[i].colType); - } - } +// //if this thing isn't in a lookup group, it's no longer going to be accounted for in the fixed size of the table +// //therefore we need to decrement the size of this row by the size of the type +// if (mySchema[i].lookupTableGroup == 0) +// { +// if (SerializationUtilityMethods.isFixedLengthType(mySchema[i].colType)) +// sz -= SerializationUtilityMethods.getFixedLengthTypeSize(mySchema[i].colType); +// } +// } - } - //now serialize the null map - myNullMap.CopyTo(myNullMapArray, 0); - int thisRowSize = myGetRowSizeInBytes(sz, r); - bw.Write(thisRowSize); - //tehcnically the size calculatinos are all off by 4 becuase tehy don't account for integer telling how big the row is - long startPos = bw.BaseStream.Position; - bw.Write(myNullMapArray); - //write the ID first, for easy access - bw.Write((int)r[myIdColumn]); - //now write all the value stored in lookup table groups - //we don't do this with the first lookup group becuase it's the default one that isn't stored in lookup tables - for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo) - { - BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo]; - LookupTable thisTable = myTables[groupNo - 1]; - int thisLookupID = thisTable.getIndexForValues(thisGroup.extractGroupFromRow(r)); +// } +// //now serialize the null map +// myNullMap.CopyTo(myNullMapArray, 0); +// int thisRowSize = myGetRowSizeInBytes(sz, r); +// bw.Write(thisRowSize); +// //tehcnically the size calculatinos are all off by 4 becuase tehy don't account for integer telling how big the row is +// long startPos = bw.BaseStream.Position; +// bw.Write(myNullMapArray); +// //write the ID first, for easy access +// bw.Write((int)r[myIdColumn]); +// //now write all the value stored in lookup table groups +// //we don't do this with the first lookup group becuase it's the default one that isn't stored in lookup tables +// for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo) +// { +// BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo]; +// LookupTable thisTable = myTables[groupNo - 1]; +// int thisLookupID = thisTable.getIndexForValues(thisGroup.extractGroupFromRow(r)); - //now, depending on the group size, write this value as either a byte, a short, or an int. +// //now, depending on the group size, write this value as either a byte, a short, or an int. - switch (thisGroup.tableSize) - { - case 8: - bw.Write((byte)thisLookupID); - break; - case 16: - bw.Write((short)thisLookupID); - break; - case 32: - bw.Write(thisLookupID); - break; - default: - throw new BlueDataException("Error: invalid size " + thisGroup.tableSize.ToString() + " specified for lookup group."); +// switch (thisGroup.tableSize) +// { +// case 8: +// bw.Write((byte)thisLookupID); +// break; +// case 16: +// bw.Write((short)thisLookupID); +// break; +// case 32: +// bw.Write(thisLookupID); +// break; +// default: +// throw new BlueDataException("Error: invalid size " + thisGroup.tableSize.ToString() + " specified for lookup group."); - } +// } - } - //now write the rest of the individual columns - foreach (ColumnInfo ci in mySchema.columnInfo) - //we have already serialized the id column, as well as all the column groups, and we don't serialize any columns that are null - if ( (!myNullMap[ci.colNo]) - && ci.colNo != myIdColumn - && ci.lookupTableGroup == 0) - { +// } +// //now write the rest of the individual columns +// foreach (ColumnInfo ci in mySchema.columnInfo) +// //we have already serialized the id column, as well as all the column groups, and we don't serialize any columns that are null +// if ( (!myNullMap[ci.colNo]) +// && ci.colNo != myIdColumn +// && ci.lookupTableGroup == 0) +// { - myPrimitiveSerializationHelper.serializeColumn(r, ci.colNo, bw); +// myPrimitiveSerializationHelper.serializeColumn(r, ci.colNo, bw); - } +// } - long stopPos = bw.BaseStream.Position; +// long stopPos = bw.BaseStream.Position; - if (stopPos- startPos != thisRowSize) - throw new BlueDataException("Error: Bad Size Calculation in LookupTableRowSerializer. Expected size was "+thisRowSize.ToString()+", actual size was "+(stopPos-startPos).ToString()); - } +// if (stopPos- startPos != thisRowSize) +// throw new BlueDataException("Error: Bad Size Calculation in LookupTableRowSerializer. Expected size was "+thisRowSize.ToString()+", actual size was "+(stopPos-startPos).ToString()); +// } - public void serializeSchema(BinaryWriter bw) - { - mySchema.serialize(bw); +// public void serializeSchema(BinaryWriter bw) +// { +// mySchema.serialize(bw); - } - #endregion +// } +// #endregion - #region size calculations - /// - /// Sets up the fixed length (size in bytes occupied by each of the primitive type) - /// and an index for the non primitive type - /// - private void myInitSizeCalculations() - { - ArrayList temp = new ArrayList(); - myFixedSize = myNullMapArray.Length; - //go through all the columns that aren't in lookup groups, and add up their sizes - foreach (ColumnInfo ci in mySchema.columnInfo) - { - //if you're in a lookup table group, your size is always fixed. - if (ci.lookupTableGroup == 0) - { - if (SerializationUtilityMethods.isFixedLengthType(ci.colType)) - myFixedSize += SerializationUtilityMethods.getFixedLengthTypeSize(ci.colType); - else - temp.Add(ci.colNo); - } - } - //now add on the size of the lookup groups - foreach (BlueRowLookupGroup group in mySchema.lookupGroups) - myFixedSize += (group.tableSize / 8); +// #region size calculations +// /// +// /// Sets up the fixed length (size in bytes occupied by each of the primitive type) +// /// and an index for the non primitive type +// /// +// private void myInitSizeCalculations() +// { +// ArrayList temp = new ArrayList(); +// myFixedSize = myNullMapArray.Length; +// //go through all the columns that aren't in lookup groups, and add up their sizes +// foreach (ColumnInfo ci in mySchema.columnInfo) +// { +// //if you're in a lookup table group, your size is always fixed. +// if (ci.lookupTableGroup == 0) +// { +// if (SerializationUtilityMethods.isFixedLengthType(ci.colType)) +// myFixedSize += SerializationUtilityMethods.getFixedLengthTypeSize(ci.colType); +// else +// temp.Add(ci.colNo); +// } +// } +// //now add on the size of the lookup groups +// foreach (BlueRowLookupGroup group in mySchema.lookupGroups) +// myFixedSize += (group.tableSize / 8); - myNonFixedColIndexes = temp.ToArray(typeof(int)) as int[]; - } +// myNonFixedColIndexes = temp.ToArray(typeof(int)) as int[]; +// } - /// - /// Find the size of the serialized row, assumes that nullMap has already - /// been built - /// - /// - /// - /// - private int myGetRowSizeInBytes(int fixedSize, IBlueRow r) - { - foreach (int colNo in myNonFixedColIndexes) - { - if (!myNullMap[colNo]) - fixedSize += //myGetStringSizeInBytes (r[colNo] as string); - SerializationUtilityMethods.getObjectSerializedSize(r[colNo]); - } - return fixedSize; - } - #endregion - } -} +// /// +// /// Find the size of the serialized row, assumes that nullMap has already +// /// been built +// /// +// /// +// /// +// /// +// private int myGetRowSizeInBytes(int fixedSize, IBlueRow r) +// { +// foreach (int colNo in myNonFixedColIndexes) +// { +// if (!myNullMap[colNo]) +// fixedSize += //myGetStringSizeInBytes (r[colNo] as string); +// SerializationUtilityMethods.getObjectSerializedSize(r[colNo]); +// } +// return fixedSize; +// } +// #endregion +// } +// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableSerializer.cs index bc5e130d2..adb17f6e9 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableSerializer.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/SpecializedSerialization/LookupTable/Serialization/LookupTableSerializer.cs @@ -1,279 +1,279 @@ -using System; -using System.IO; -using BCG.Utilities; -using System.Collections.Generic; -using System.IO.Compression; +// using System; +// using System.IO; +// using BCG.Utilities; +// using System.Collections.Generic; +// using System.IO.Compression; -namespace BCG -{ - /// - /// Serializes data stored in all IBlueRowCollections that use a LookupTableSchema. - /// - public class LookupTableSerializer : IBlueDataSerializer - { - private const string myPubSubDeser = "BCG.LookupTablePubSubDeserializer", - myClientDeser = "BCG.LookupTableClientDeserializer"; +// namespace BCG +// { +// /// +// /// Serializes data stored in all IBlueRowCollections that use a LookupTableSchema. +// /// +// public class LookupTableSerializer : IBlueDataSerializer +// { +// private const string myPubSubDeser = "BCG.LookupTablePubSubDeserializer", +// myClientDeser = "BCG.LookupTableClientDeserializer"; - private LookupTable[] myLookupTables; - private LookupTableSchema mySchema; - private LookupTableRowSerializer myRowSerializer; +// private LookupTable[] myLookupTables; +// private LookupTableSchema mySchema; +// private LookupTableRowSerializer myRowSerializer; - //the header writer writes into a regular stream - private BinaryWriter myHeaderWriter; - //the body writer writes into a compressed binary stream - //that way, we shrink the size of the data that we send over the wire - private BinaryWriter myBodyWriter; - public LookupTableSerializer (IBlueTableSchema inSchema, object[] groups) - { - //get the column infos from the input schema - ColumnInfo[] inputColumns = inSchema.columnInfo; +// //the header writer writes into a regular stream +// private BinaryWriter myHeaderWriter; +// //the body writer writes into a compressed binary stream +// //that way, we shrink the size of the data that we send over the wire +// private BinaryWriter myBodyWriter; +// public LookupTableSerializer (IBlueTableSchema inSchema, object[] groups) +// { +// //get the column infos from the input schema +// ColumnInfo[] inputColumns = inSchema.columnInfo; - //build the lookup table groups - //there is one more group than input string because we have the 'default' group, which isn't stored in a table - int numGroups = groups.Length + 1; - BlueRowLookupGroup[] lookupGroups = new BlueRowLookupGroup[numGroups]; - //we start the groups with group no 1, becuase groups with 0 are not stored in lookup tables - short groupNo = 1; - Dictionary colNameToGroupNo = new Dictionary(); +// //build the lookup table groups +// //there is one more group than input string because we have the 'default' group, which isn't stored in a table +// int numGroups = groups.Length + 1; +// BlueRowLookupGroup[] lookupGroups = new BlueRowLookupGroup[numGroups]; +// //we start the groups with group no 1, becuase groups with 0 are not stored in lookup tables +// short groupNo = 1; +// Dictionary colNameToGroupNo = new Dictionary(); - foreach (string group in groups) - { - string[] sizeAndColumns = group.Split(';'); +// foreach (string group in groups) +// { +// string[] sizeAndColumns = group.Split(';'); - if (sizeAndColumns.Length != 2) - throw new ArgumentException("Error, bad Lookup Table Group found: " + group); +// if (sizeAndColumns.Length != 2) +// throw new ArgumentException("Error, bad Lookup Table Group found: " + group); - short tableSize = short.Parse(sizeAndColumns[0]); +// short tableSize = short.Parse(sizeAndColumns[0]); - string[] columns = sizeAndColumns[1].Split(','); +// string[] columns = sizeAndColumns[1].Split(','); - BlueRowLookupGroup thisGroup = new BlueRowLookupGroup(groupNo, tableSize, columns); - lookupGroups[groupNo] = thisGroup; - foreach (string column in columns) - colNameToGroupNo[column] = groupNo; +// BlueRowLookupGroup thisGroup = new BlueRowLookupGroup(groupNo, tableSize, columns); +// lookupGroups[groupNo] = thisGroup; +// foreach (string column in columns) +// colNameToGroupNo[column] = groupNo; - ++groupNo; +// ++groupNo; - } +// } - //make the lookup group correpsonding to columns that aren't stored in the lookup table - List nonLookupTableColumns = new List(); - foreach (ColumnInfo ci in inputColumns) - { - if (!colNameToGroupNo.ContainsKey(ci.colName)) - { - nonLookupTableColumns.Add(ci.colName); - } - } +// //make the lookup group correpsonding to columns that aren't stored in the lookup table +// List nonLookupTableColumns = new List(); +// foreach (ColumnInfo ci in inputColumns) +// { +// if (!colNameToGroupNo.ContainsKey(ci.colName)) +// { +// nonLookupTableColumns.Add(ci.colName); +// } +// } - lookupGroups[0] = new BlueRowLookupGroup(0, 0, nonLookupTableColumns.ToArray()); +// lookupGroups[0] = new BlueRowLookupGroup(0, 0, nonLookupTableColumns.ToArray()); - //now build new column infos to replace the old ones - int numCols = inputColumns.Length; - ColumnInfo[] newColumns = new ColumnInfo[numCols]; +// //now build new column infos to replace the old ones +// int numCols = inputColumns.Length; +// ColumnInfo[] newColumns = new ColumnInfo[numCols]; - for (int col = 0; col < numCols; ++col) - { - ColumnInfo inCol = inputColumns[col]; - if (inCol.isConstant) - newColumns[col] = new ColumnInfo(inCol.colName, col, inCol.colType, inCol.colValue); - else - { - string colName = inCol.colName; - if (!colNameToGroupNo.ContainsKey(inCol.colName)) - newColumns[col] = new ColumnInfo(colName, col, inCol.colType); - else - newColumns[col] = new ColumnInfo(colName, col, inCol.colType, colNameToGroupNo[colName]); - } +// for (int col = 0; col < numCols; ++col) +// { +// ColumnInfo inCol = inputColumns[col]; +// if (inCol.isConstant) +// newColumns[col] = new ColumnInfo(inCol.colName, col, inCol.colType, inCol.colValue); +// else +// { +// string colName = inCol.colName; +// if (!colNameToGroupNo.ContainsKey(inCol.colName)) +// newColumns[col] = new ColumnInfo(colName, col, inCol.colType); +// else +// newColumns[col] = new ColumnInfo(colName, col, inCol.colType, colNameToGroupNo[colName]); +// } - } +// } - //now build the schema - mySchema = new LookupTableSchema(inSchema.tableName, newColumns, lookupGroups); - - - //build the lookup tables. There is one less than the length of the lookupTableGroups because the first one - //is always the default group which isn't stored in a lookup table. - myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1]; - - //start with 1 because we don't store the 0th lookup table - for (int i = 0; i < myLookupTables.Length; ++i) - { - myLookupTables[i] = new LookupTable(mySchema, i+1); - } - - //now create the row serializer - myRowSerializer = new LookupTableRowSerializer(mySchema, myLookupTables); - - } - - #region IBlueDataSerializer Members - /// - /// Serializes the data related to a single op. - /// Supports All Ops - /// - /// First, a LookupTableUpdate, which is contained in each op, looks like this: - /// - /// - /// -LookupTableUpdate: - /// group 1 values size \ - /// group 1 value |- all serialized by a single LookupTableObject - /// group 1 value / - /// .... - /// group 2 values size - /// group 2 value key - /// .... - /// - /// - /// Next, the stream formats of the different op codes - /// - /// - /// -create : - /// op - /// clientDeser name - /// pubsubsDeser name - /// schema (which is a LookupTableSchema) - /// LookupTableUpdate - /// num rows - /// row 1 - /// row 2 - /// .... - /// - /// - /// - add, remove, change: - /// op - /// LookupTableUpdate - /// the row - /// - /// -multipleRemoveAdd : - /// op - /// LookupTableUpdate - /// num deleted rows - /// deleted row 1 - /// deleted row 2 - /// .... - /// num added rows - /// added row 1 - /// added row 2 - /// ... - /// - /// -multipleChange - /// op - /// LookupTableUpdate - /// num changed rows - /// changd row 1 - /// changed row 2 - /// ... - /// changd row n - /// - /// - /// - /// - /// - public void serialize(byte op, BCG.IBlueRowCollection[] rows, out byte[] serializedData) - { - myHeaderWriter = new BinaryWriter(new MemoryStream()); - myBodyWriter = new BinaryWriter(new MemoryStream()); - //everything start with an op - myHeaderWriter.Write(op); - - /*go through each of the rows we're about the serialize, and populate the lookup table - * that way, when we serialize the lookup tables, well have something to report*/ - foreach (IBlueRowCollection collection in rows) - foreach (IBlueRow r in collection) - myRowSerializer.updateLookupTables(r); - - if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change) - mySerializeSingleRow(op, BlueDataUtility.extractRow(rows[0], 0)); - else if (op == StreamOpCode.create) - mySerializeCreate(rows[0]); - else if (op == StreamOpCode.multipleChange) - mySerializeMultipleChange(rows[0]); - else if (op == StreamOpCode.multipleRemoveAdd) - mySerializeMultipleRemoveAdd(rows[0], rows[1]); - else - { - serializedData = null; - // BlueEventLog.logAssert(false, String.Format("{0} is not a valid op", op)); - } +// //now build the schema +// mySchema = new LookupTableSchema(inSchema.tableName, newColumns, lookupGroups); - - //copy the data from the header and the body of the message - myHeaderWriter.BaseStream.Position = 0; - myBodyWriter.BaseStream.Position = 0; - int headerLength = (int) myHeaderWriter.BaseStream.Length; +// //build the lookup tables. There is one less than the length of the lookupTableGroups because the first one +// //is always the default group which isn't stored in a lookup table. +// myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1]; - - - MemoryStream zippedMemStream = new MemoryStream(); - SharpZipCompressor Kompressor = new SharpZipCompressor(); +// //start with 1 because we don't store the 0th lookup table +// for (int i = 0; i < myLookupTables.Length; ++i) +// { +// myLookupTables[i] = new LookupTable(mySchema, i+1); +// } - Kompressor.compress(myBodyWriter.BaseStream,zippedMemStream); - byte[] compressedBytes = zippedMemStream.ToArray(); - - - serializedData = new byte[headerLength + zippedMemStream.Length]; +// //now create the row serializer +// myRowSerializer = new LookupTableRowSerializer(mySchema, myLookupTables); - Console.WriteLine(String.Format("Lookuptable Serializer: Header is {0}. Body compresesd from {1} to {2} bytes.", headerLength , myBodyWriter.BaseStream.Length, zippedMemStream.Length)); +// } - - myHeaderWriter.BaseStream.Read(serializedData, 0, headerLength); - BinaryWriter sizeWriter = new BinaryWriter(new MemoryStream(serializedData)); - zippedMemStream.Position = 0; - zippedMemStream.Read(serializedData, headerLength, (int) zippedMemStream.Length); +// #region IBlueDataSerializer Members +// /// +// /// Serializes the data related to a single op. +// /// Supports All Ops +// /// +// /// First, a LookupTableUpdate, which is contained in each op, looks like this: +// /// +// /// +// /// -LookupTableUpdate: +// /// group 1 values size \ +// /// group 1 value |- all serialized by a single LookupTableObject +// /// group 1 value / +// /// .... +// /// group 2 values size +// /// group 2 value key +// /// .... +// /// +// /// +// /// Next, the stream formats of the different op codes +// /// +// /// +// /// -create : +// /// op +// /// clientDeser name +// /// pubsubsDeser name +// /// schema (which is a LookupTableSchema) +// /// LookupTableUpdate +// /// num rows +// /// row 1 +// /// row 2 +// /// .... +// /// +// /// +// /// - add, remove, change: +// /// op +// /// LookupTableUpdate +// /// the row +// /// +// /// -multipleRemoveAdd : +// /// op +// /// LookupTableUpdate +// /// num deleted rows +// /// deleted row 1 +// /// deleted row 2 +// /// .... +// /// num added rows +// /// added row 1 +// /// added row 2 +// /// ... +// /// +// /// -multipleChange +// /// op +// /// LookupTableUpdate +// /// num changed rows +// /// changd row 1 +// /// changed row 2 +// /// ... +// /// changd row n +// /// +// /// +// /// +// /// +// /// +// public void serialize(byte op, BCG.IBlueRowCollection[] rows, out byte[] serializedData) +// { +// myHeaderWriter = new BinaryWriter(new MemoryStream()); +// myBodyWriter = new BinaryWriter(new MemoryStream()); +// //everything start with an op +// myHeaderWriter.Write(op); + +// /*go through each of the rows we're about the serialize, and populate the lookup table +// * that way, when we serialize the lookup tables, well have something to report*/ +// foreach (IBlueRowCollection collection in rows) +// foreach (IBlueRow r in collection) +// myRowSerializer.updateLookupTables(r); + +// if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change) +// mySerializeSingleRow(op, BlueDataUtility.extractRow(rows[0], 0)); +// else if (op == StreamOpCode.create) +// mySerializeCreate(rows[0]); +// else if (op == StreamOpCode.multipleChange) +// mySerializeMultipleChange(rows[0]); +// else if (op == StreamOpCode.multipleRemoveAdd) +// mySerializeMultipleRemoveAdd(rows[0], rows[1]); +// else +// { +// serializedData = null; +// // BlueEventLog.logAssert(false, String.Format("{0} is not a valid op", op)); +// } - } - #endregion - void mySerializeSingleRow(byte op, IBlueRow row) - { - mySerializeLookupTableUpdates(myBodyWriter); - myRowSerializer.serializeRow(row, op, myBodyWriter); - } +// //copy the data from the header and the body of the message +// myHeaderWriter.BaseStream.Position = 0; +// myBodyWriter.BaseStream.Position = 0; +// int headerLength = (int) myHeaderWriter.BaseStream.Length; - private void mySerializeCreate (IBlueRowCollection rows) - { - myHeaderWriter.Write(myPubSubDeser); - myHeaderWriter.Write(myClientDeser); - mySchema.serialize(myBodyWriter); - mySerializeLookupTableUpdates(myBodyWriter); - myBodyWriter.Write(rows.Count); - foreach (IBlueRow r in rows) - myRowSerializer.serializeRow(r, StreamOpCode.create, myBodyWriter); - } - private void mySerializeMultipleChange(IBlueRowCollection rows) - { - mySerializeLookupTableUpdates(myBodyWriter); - myBodyWriter.Write(rows.Count); - foreach (IBlueRow r in rows) - myRowSerializer.serializeRow(r,StreamOpCode.change,myBodyWriter); - - } - private void mySerializeMultipleRemoveAdd(IBlueRowCollection deletedRows, IBlueRowCollection addedRows) - { - mySerializeLookupTableUpdates(myBodyWriter); - myBodyWriter.Write(deletedRows.Count); - foreach (IBlueRow r in deletedRows) - myRowSerializer.serializeRow(r, StreamOpCode.change, myBodyWriter); - myBodyWriter.Write(addedRows.Count); - foreach (IBlueRow r in addedRows) - myRowSerializer.serializeRow(r, StreamOpCode.add, myBodyWriter); - - } +// MemoryStream zippedMemStream = new MemoryStream(); +// SharpZipCompressor Kompressor = new SharpZipCompressor(); - private void mySerializeLookupTableUpdates(BinaryWriter bw) - { - for (int i = 0; i < myLookupTables.Length; ++i) - myLookupTables[i].serializeUpdates(bw); - } +// Kompressor.compress(myBodyWriter.BaseStream,zippedMemStream); +// byte[] compressedBytes = zippedMemStream.ToArray(); - } -} + +// serializedData = new byte[headerLength + zippedMemStream.Length]; + +// Console.WriteLine(String.Format("Lookuptable Serializer: Header is {0}. Body compresesd from {1} to {2} bytes.", headerLength , myBodyWriter.BaseStream.Length, zippedMemStream.Length)); + + +// myHeaderWriter.BaseStream.Read(serializedData, 0, headerLength); +// BinaryWriter sizeWriter = new BinaryWriter(new MemoryStream(serializedData)); +// zippedMemStream.Position = 0; +// zippedMemStream.Read(serializedData, headerLength, (int) zippedMemStream.Length); + + +// } +// #endregion + +// void mySerializeSingleRow(byte op, IBlueRow row) +// { +// mySerializeLookupTableUpdates(myBodyWriter); +// myRowSerializer.serializeRow(row, op, myBodyWriter); +// } + +// private void mySerializeCreate (IBlueRowCollection rows) +// { +// myHeaderWriter.Write(myPubSubDeser); +// myHeaderWriter.Write(myClientDeser); +// mySchema.serialize(myBodyWriter); +// mySerializeLookupTableUpdates(myBodyWriter); +// myBodyWriter.Write(rows.Count); +// foreach (IBlueRow r in rows) +// myRowSerializer.serializeRow(r, StreamOpCode.create, myBodyWriter); +// } + +// private void mySerializeMultipleChange(IBlueRowCollection rows) +// { +// mySerializeLookupTableUpdates(myBodyWriter); +// myBodyWriter.Write(rows.Count); +// foreach (IBlueRow r in rows) +// myRowSerializer.serializeRow(r,StreamOpCode.change,myBodyWriter); + +// } + +// private void mySerializeMultipleRemoveAdd(IBlueRowCollection deletedRows, IBlueRowCollection addedRows) +// { +// mySerializeLookupTableUpdates(myBodyWriter); +// myBodyWriter.Write(deletedRows.Count); +// foreach (IBlueRow r in deletedRows) +// myRowSerializer.serializeRow(r, StreamOpCode.change, myBodyWriter); +// myBodyWriter.Write(addedRows.Count); +// foreach (IBlueRow r in addedRows) +// myRowSerializer.serializeRow(r, StreamOpCode.add, myBodyWriter); + +// } + +// private void mySerializeLookupTableUpdates(BinaryWriter bw) +// { +// for (int i = 0; i < myLookupTables.Length; ++i) +// myLookupTables[i].serializeUpdates(bw); +// } + +// } +// } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamIOCommon.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamIOCommon.cs new file mode 100644 index 000000000..6f93ea702 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamIOCommon.cs @@ -0,0 +1,137 @@ +using System; +using BCG; +using System.Collections; +using System.IO; + +// common place for all serialization interfaces + +namespace bts { + // this is the data that is sent across the write for stream IO + public class StreamIOMessage { + public enum StreamOp { + CREATE = 0, + // removes and adds multiple rows + // other operations can be expressed with this + MULTIPLE_REMOVE_ADD = 1, + } + public class CreateMessage { + public IBlueRowCollection rows; + public CreateMessage(IBlueRowCollection rows_) { + rows = rows_; + } + public override string ToString() { + string rv = "Create message "; + if (rows == null) { + rv += " rows: null"; + return rv; + } + foreach (IBlueRow r in rows) { + rv += $"id: {r["id"]} "; + } + return rv; + } + public override bool Equals(object obj) { + if (obj != null && obj is CreateMessage) { + var other = obj as CreateMessage; + return BlueDataUtility.compute_equality(rows, other.rows); + } + return false; + } + } + public class MultipleRemoveAddMessage { + public IBlueRowCollection removed_rows; + public IBlueRowCollection added_rows; + public MultipleRemoveAddMessage(IBlueRowCollection removed_rows_, IBlueRowCollection added_rows_) { + removed_rows = removed_rows_; + added_rows = added_rows_; + } + public override string ToString() { + string rv = "MultipleRemoveAdd removed: "; + foreach (IBlueRow r in removed_rows) { + rv += $"id: {r["id"]} "; + } + rv += "added: "; + foreach (IBlueRow r in added_rows) { + rv += $"id: {r["id"]} "; + } + return rv; + } + public override bool Equals(object obj) { + if (obj != null && obj is MultipleRemoveAddMessage) { + MultipleRemoveAddMessage other = obj as MultipleRemoveAddMessage; + return BlueDataUtility.compute_equality(removed_rows, other.removed_rows) && + BlueDataUtility.compute_equality(added_rows, other.added_rows); + } + return false; + } + } + public StreamOp op; + public readonly CreateMessage create_message = null; + public readonly MultipleRemoveAddMessage multiple_remove_add_message = null; + + public StreamIOMessage(CreateMessage create_message_) { + op = StreamOp.CREATE; + create_message = create_message_; + multiple_remove_add_message = null; + } + + public StreamIOMessage(MultipleRemoveAddMessage multiple_remove_add_message_) { + op = StreamOp.MULTIPLE_REMOVE_ADD; + create_message = null; + multiple_remove_add_message = multiple_remove_add_message_; + } + + public override string ToString() { + switch (op) { + case StreamIOMessage.StreamOp.CREATE: + return create_message.ToString(); + case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: + return multiple_remove_add_message.ToString(); + } + bts.Log.assert(false); + return String.Empty; + } + public override bool Equals(object obj) { + if (obj != null && obj is StreamIOMessage) { + StreamIOMessage other = obj as StreamIOMessage; + if (op == other.op) { + switch (op) { + case StreamOp.CREATE: + return create_message.Equals(other.create_message); + case StreamOp.MULTIPLE_REMOVE_ADD: + return multiple_remove_add_message.Equals(other.multiple_remove_add_message); + } + } + } + return false; + } + + } + #region serialization + // This helps putting stream callbacks (op + data) on the wire, to send it to the pubsubserver. + public interface IStreamIOMsgSerializer { + public void serialize(StreamIOMessage op_msg, ref BinaryWriter bw); + } + // Responsible for serializing one row + public interface IStreamRowSerializer { + /// + /// Serializes a row and writes it into the BinaryWriter. Since its only one row, the ops should be appropriate + /// for one row; the ones allowed are StreamopCode.add, remove and change only. + /// + public void serialize_row(IBlueRow r, StreamIOMessage.StreamOp op, ref BinaryWriter bw); + } + + #endregion + + #region Deser + // this is how stream messages received from the wire are deserialized + public interface IStreamMsgDeserilizer { + public StreamIOMessage deserialize(ref BinaryReader br); + } + + public interface IStreamRowDeserializer { + IBlueRow deserialize_row(ref BinaryReader br); + } + #endregion + +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamIOSerializationController.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamIOSerializationController.cs new file mode 100644 index 000000000..f45192de9 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamIOSerializationController.cs @@ -0,0 +1,44 @@ +using BCG; + +using System.IO; +using System.Net.WebSockets; + +namespace bts { + public class StreamIOSerializationController : IStreamIOMsgSerializer { + private IStreamIOMsgSerializer _real_serializer = null; + public StreamIOSerializationController() { } + + public void serialize(StreamIOMessage op_msg, ref BinaryWriter bw) { + switch (op_msg.op) { + // + case StreamIOMessage.StreamOp.CREATE: + var table_name = op_msg.create_message.rows.schema.tableName; + _real_serializer = _get_serializer(table_name); + string client_deser = _get_client_deser_name(table_name); + string pubsub_deser = _get_pubsub_deser_name(table_name); + bw.Write((int)op_msg.op); + bw.write_string_cpp(client_deser); + bw.write_string_cpp(pubsub_deser); + _real_serializer.serialize(op_msg, ref bw); + break; + case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: + bts.Log.assert(_real_serializer != null); + _real_serializer.serialize(op_msg, ref bw); + break; + } + } + + private IStreamIOMsgSerializer _get_serializer(string table_name) { + // TODO read config + return new UniversalStreamIOSerializer(); + } + + private string _get_client_deser_name(string table_name) { + return "UniversalStreamIODeserializer"; + } + + private string _get_pubsub_deser_name(string table_name) { + return string.Empty; + } + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs new file mode 100644 index 000000000..d9901e44f --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs @@ -0,0 +1,162 @@ +namespace bts { + using BCG; + + using System; + using System.Collections.Generic; + using System.IO; + using System.Runtime.Serialization.Formatters.Binary; + using System.Text; + + public static class StreamSerializationUtils { + // extension method to write a string C++ style (len prefixed) + // public static void write_string_cpp(this BinaryWriter bw, string str) { + // str = str ?? String.Empty; + // int len = str.Length; + // bw.Write(len); + // foreach (char c in str) { + // bw.Write(c); + // } + // } + + // public static string read_string_cpp(this BinaryReader br) { + // int len = br.ReadInt32(); + // string rv = String.Empty; + // char[] c_arr = br.ReadChars(len); + // return new string(c_arr); + // } + + public static byte row_null_value_flag = 1; + public static byte row_nonnull_value_flag = 0; + + public static byte[] get_row_nullmap(IBlueRow r) { + byte[] rv = new byte[r.schema.columnInfo.Length]; + for (int i = 0; i < r.schema.columnInfo.Length; i++) { + var v = r[i]; + bool is_dbnull = v == null || DBNull.Value.Equals(v); + rv[i] = is_dbnull ? row_null_value_flag : row_nonnull_value_flag; + } + return rv; + } + private static Dictionary> _col_ser_map = + new Dictionary>(){ + {typeof(bool), (bw, r, col_no) => {bw.Write(r.get(col_no)); }}, + {typeof(byte), (bw, r, col_no) => {bw.Write(r.get(col_no)); }}, + {typeof(char), (bw, r, col_no) => {bw.Write(r.get(col_no)); }}, + // TODO + //{typeof(DateTime), (bw, r, col_no) => {bw.Write(r.get(col_no)); }}, + {typeof(Decimal), (bw, r, col_no) => {bw.Write(r.get(col_no)); }}, + {typeof(Double), (bw, r, col_no) => {bw.Write(r.get(col_no)); }}, + {typeof(Int16), (bw, r, col_no) => {bw.Write(r.get(col_no)); }}, + {typeof(Int32), (bw, r, col_no) => {bw.Write(r.get(col_no)); }}, + {typeof(Int64), (bw, r, col_no) => {bw.Write(r.get(col_no)); }}, + {typeof(SByte), (bw, r, col_no) => {bw.Write(r.get(col_no)); }}, + {typeof(Single), (bw, r, col_no) => {bw.Write(r.get(col_no)); }}, + {typeof(String), (bw, r, col_no) => {bw.write_string_cpp(r.get(col_no)); }}, + {typeof(UInt16), (bw, r, col_no) => {bw.Write(r.get(col_no)); }}, + {typeof(UInt32), (bw, r, col_no) => {bw.Write(r.get(col_no)); }}, + {typeof(UInt64), (bw, r, col_no) => {bw.Write(r.get(col_no)); }} + }; + private static Dictionary> _col_deser_map = + new Dictionary>(){ + {typeof(bool), (br) => {return br.ReadBoolean();}}, + {typeof(byte), (br) => {return br.ReadByte();}}, + {typeof(char), (br) => {return br.ReadChar();}}, + // TODO DateTime + //{typeof(byte), (br) => {return br.ReadByte()} + {typeof(Decimal), (br) => {return br.ReadDecimal();}}, + {typeof(Double), (br) => {return br.ReadDouble();}}, + {typeof(Int16), (br) => {return br.ReadInt16();}}, + {typeof(Int32), (br) => {return br.ReadInt32();}}, + {typeof(Int64), (br) => {return br.ReadInt64();}}, + {typeof(SByte), (br) => {return br.ReadSByte();}}, + {typeof(Single), (br) => {return br.ReadSingle();}}, + {typeof(String), (br) => {return br.read_string_cpp();}}, + {typeof(UInt16), (br) => {return br.ReadUInt16();}}, + {typeof(UInt32), (br) => {return br.ReadUInt32();}}, + {typeof(UInt64), (br) => {return br.ReadUInt64();}}, + + }; + public static void serialize_schema(IBlueTableSchema schema, ref BinaryWriter bw) { + // + bw.write_string_cpp(schema.tableName); + bw.Write(schema.columnInfo.Length); + foreach (ColumnInfo ci in schema.columnInfo) + serialize_column_info(ci, ref bw); + } + + public static IBlueTableSchema deserialize_schema(ref BinaryReader br) { + var hex_txt = to_hex(br.BaseStream); + bts.Log.Gen.info($"StreamSerializationUtils.deserialize_schema bytes: {hex_txt}"); + string table_name = br.read_string_cpp(); + int n_cols = br.ReadInt32(); // doing a br.Read was only advancing the data 1 byte!! + List cis = new List(); + for (int i = 0; i < n_cols; i++) { + var ci = deserialize_column_info(ref br); + cis.Add(ci); + } + return new BlueTableSchema(table_name, cis.ToArray()); + } + + public static void serialize_column_info(ColumnInfo ci, ref BinaryWriter bw) { + // + bw.Write(ci.colNo); + bw.write_string_cpp(ci.colName); + TypeCode tc = ci.type_code(); + int tc_int = (int)tc; + bw.Write(tc_int); + } + + public static object deserialize_column(ref BinaryReader br, Type t) { + Func reader_func; + Log.assert(_col_deser_map.TryGetValue(t, out reader_func)); + return reader_func(br); + } + + public static void serialize_row_column(BinaryWriter bw, IBlueRow r, int col_no) { + var ci = r.schema.columnInfo[col_no]; + var type = ci.colType; + Action ser_action; + Log.assert(_col_ser_map.TryGetValue(type, out ser_action)); + ser_action(bw, r, col_no); + } + + public static ColumnInfo deserialize_column_info(ref BinaryReader br) { + //col_no, name, type> + var hex_txt = to_hex(br.BaseStream); + bts.Log.Gen.info($"StreamSerializationUtils.deserialize_column_info bytes: {hex_txt}"); + int col_no = br.ReadInt32(); + string col_name = br.read_string_cpp(); + int tc_int = br.ReadInt32(); + TypeCode type_code = (TypeCode)tc_int; + return new ColumnInfo(col_name, col_no, type_code); + } + + public static void write_column(IBlueRow r, int col_no, ref BinaryWriter bw) { + //bw.Write() + } + + public static string to_hex(byte[] byte_arr) { + StringBuilder hex_builder = new StringBuilder(byte_arr.Length * 2); + + foreach (byte b in byte_arr) { + hex_builder.Append($"{b:X2} "); + } + return hex_builder.ToString().Trim(); + } + + public static byte[] read_bytes_from_stream(Stream str) { + var init_pos = str.Position; + BinaryReader br = new BinaryReader(str); + var sz = Convert.ToInt32(str.Length - str.Position); + var rv = br.ReadBytes(sz); + str.Position = init_pos; + return rv; + } + + public static string to_hex(Stream str) { + var byte_arr = read_bytes_from_stream(str); + return to_hex(byte_arr); + } + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/UniversalRowSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/UniversalRowSerializer.cs new file mode 100644 index 000000000..a1d1b84c6 --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/UniversalRowSerializer.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections; +using System.IO; +using System.Runtime.Serialization.Formatters.Binary; + +using BCG; + +namespace bts { + + internal class UniversalRowSerializer : IStreamRowSerializer { + private readonly IBlueTableSchema _schema; + + public UniversalRowSerializer(IBlueTableSchema schema) { + _schema = schema; + } + + public void serialize_row(IBlueRow r, StreamIOMessage.StreamOp __, ref BinaryWriter bw) { + //< null_map, row_values > + byte[] null_map = StreamSerializationUtils.get_row_nullmap(r); + //null_map + bw.Write(null_map); + foreach (var ci in _schema.columnInfo) { + if (null_map[ci.colNo] != StreamSerializationUtils.row_null_value_flag) + StreamSerializationUtils.serialize_row_column(bw, r, ci.colNo); + } + } + + } +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/UniversalStreamIOSerializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/UniversalStreamIOSerializer.cs new file mode 100644 index 000000000..014e4666d --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/UniversalStreamIOSerializer.cs @@ -0,0 +1,45 @@ +using BCG; + +using System.IO; + +namespace bts { + + public class UniversalStreamIOSerializer : IStreamIOMsgSerializer { + private IStreamRowSerializer _row_serializer = null; + public UniversalStreamIOSerializer() { + } + + public void serialize(StreamIOMessage op_msg, ref BinaryWriter bw) { + // TODO + // just need create and multiple remove add + switch (op_msg.op) { + case StreamIOMessage.StreamOp.CREATE: + var cr_msg = op_msg.create_message; + // + //IBlueRowCollection createdRows = rows[0]; + bw.Write((int)op_msg.op); + StreamSerializationUtils.serialize_schema(cr_msg.rows.schema, ref bw); + _row_serializer = new UniversalRowSerializer(cr_msg.rows.schema); + var rows = op_msg.create_message.rows; + bw.Write(rows.Count); + foreach (IBlueRow r in rows) + _row_serializer.serialize_row(r, op_msg.op, ref bw); + break; + case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: + // + // TODO + bw.Write((int)op_msg.op); + var del_rows = op_msg.multiple_remove_add_message.removed_rows; + var added_rows = op_msg.multiple_remove_add_message.added_rows; + bw.Write(del_rows.Count); + foreach (IBlueRow dr in del_rows) + _row_serializer.serialize_row(dr, StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD, ref bw); + bw.Write(added_rows.Count); + foreach (IBlueRow ar in added_rows) + _row_serializer.serialize_row(ar, StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD, ref bw); + break; + } + } + } + +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/Deserialization/StreamIODeserializationController.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/Deserialization/StreamIODeserializationController.cs new file mode 100644 index 000000000..d283101db --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/Deserialization/StreamIODeserializationController.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections; +using System.ComponentModel; +using System.IO; +using System.Runtime.InteropServices; + +using BCG; + +namespace bts { + // This is the equivelant of StreamIOSerializationController on the deser side. The IStreamMsgDeserilizer + // thats tsreams deal with is this, it hides the specific serialization inside of it + public class StreamIODeserializationController : IStreamMsgDeserilizer { + protected IStreamMsgDeserilizer _real_deserializer = null; + private IBlueTableSchema _schema = null; + + public StreamIODeserializationController() { + + } + + public StreamIOMessage deserialize(ref BinaryReader br) { + if (_real_deserializer != null) { + return _real_deserializer.deserialize(ref br); + } + StreamIOMessage.StreamOp op = (StreamIOMessage.StreamOp)br.ReadInt32(); + switch (op) { + case StreamIOMessage.StreamOp.CREATE: + Log.assert(_real_deserializer == null); + string client_deser_name = br.read_string_cpp(); + _create_real_deser(client_deser_name); + string pubsubserver_deser_name = br.read_string_cpp(); + var rv = _real_deserializer.deserialize(ref br); + return rv; + case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: + Log.assert(false); + return _real_deserializer.deserialize(ref br); + } + Log.assert(false); + return null; + } + + private void _create_real_deser(string name) { + _real_deserializer = new UniversalStreamIODeserializer(); + } + private bool _received_create => _schema != null; + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/Deserialization/UniversalRowDeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/Deserialization/UniversalRowDeserializer.cs new file mode 100644 index 000000000..58fa6a44a --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/Deserialization/UniversalRowDeserializer.cs @@ -0,0 +1,43 @@ +using BCG; + +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; + +namespace bts { + /// + /// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process + /// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations) + /// + internal class UniversalRowDeserializer : IStreamRowDeserializer { + private IBlueTableSchema _schema; + + + public UniversalRowDeserializer(IBlueTableSchema schema) { + _schema = schema; + } + + #region IClientRowDeserializer Members + + public IBlueRow deserialize_row(ref BinaryReader br) { + byte[] null_map = br.ReadBytes(_schema.columnInfo.Length); + List row_vals = new List(); + foreach (ColumnInfo ci in _schema.columnInfo) { + var idx = ci.colNo; + bool is_dbnull = null_map[ci.colNo] == StreamSerializationUtils.row_null_value_flag; + if (is_dbnull) { + row_vals.Add(DBNull.Value); + } else { + var val = StreamSerializationUtils.deserialize_column(ref br, ci.colType); + row_vals.Add(val); + } + } + return new BlueRow(_schema, row_vals.ToArray()); + } + + + #endregion + } + +} \ No newline at end of file diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/Deserialization/UniversalStreamIODeserializer.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/Deserialization/UniversalStreamIODeserializer.cs new file mode 100644 index 000000000..3d0a6410a --- /dev/null +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/Deserialization/UniversalStreamIODeserializer.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections; +using System.IO; + +using BCG; + +namespace bts { + public class UniversalStreamIODeserializer : IStreamMsgDeserilizer { + protected IStreamRowDeserializer _row_deserializer = null; + private IBlueTableSchema _schema = null; + + public UniversalStreamIODeserializer() { + + } + + public StreamIOMessage deserialize(ref BinaryReader br) { + StreamIOMessage.StreamOp op = (StreamIOMessage.StreamOp)br.ReadInt32(); + switch (op) { + case StreamIOMessage.StreamOp.CREATE: + // + _schema = StreamSerializationUtils.deserialize_schema(ref br); + _row_deserializer = new UniversalRowDeserializer(_schema); + var rows = _get_length_prefix_rows(ref br); + StreamIOMessage rv = new StreamIOMessage(new StreamIOMessage.CreateMessage(rows)); + return rv; + case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: + Log.assert(_received_create); + var del_rows = _get_length_prefix_rows(ref br); + var added_rows = _get_length_prefix_rows(ref br); + StreamIOMessage.MultipleRemoveAddMessage mram = new StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows); + return new StreamIOMessage(mram); + } + return null; + } + + private IBlueRowCollection _get_length_prefix_rows(ref BinaryReader br) { + BlueTable rv = new BlueTable(_schema); + int n_rows = br.ReadInt32(); + for (int i = 0; i < n_rows; i++) { + var r = _row_deserializer.deserialize_row(ref br); + rv.addRow(r); + } + return rv; + } + + private bool _received_create => _schema != null; + + } +} \ No newline at end of file diff --git a/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs b/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs index 93b6077cd..5a5b9790d 100644 --- a/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs +++ b/src/DataStreamSuite/PubSub/Client/Communication/ManagedTCP/PubSubServerManagedTCPConnection.cs @@ -97,10 +97,10 @@ namespace BCG { // send(msg.to_stream()); // } - public void send_with_confirmation(PubSubMessage msg, ref string conf_id) { - var bytes = msg.to_bytes(); - send_with_confirmation(bytes, ref conf_id); - } + // public void send_with_confirmation(PubSubMessage msg, ref string conf_id) { + // var bytes = msg.to_bytes(); + // send_with_confirmation(bytes, ref conf_id); + // } public void send_with_confirmation(bts.PubSubClientMessage msg, ref string conf_id, SendConfirmationDelegate del) { if (_connection == null) { diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs deleted file mode 100644 index ac4d7c952..000000000 --- a/src/DataStreamSuite/PubSub/Client/MetaData/MetaData.cs +++ /dev/null @@ -1,150 +0,0 @@ -using System; -using System.Runtime.Serialization; -using System.Collections; -using System.Reflection; -using System.IO; -using System.Text; - - -using BCG; -using System.Collections.Generic; - -namespace BCG { - // Thhis "wraps" around the following 4 PubSub meta objects, serializes it on the client and - // deserialize it on the pubsub - // PubMetaData - // SubMetaData - // TransactionMetaData - // UnSubMetaData - // - public class MetaData : ICloneable { - private IDictionary _name_value_map = new Dictionary(); - int mySize = -1; - - public MetaData() { } - - public MetaData(object o) { copyFrom(o); } - - public MetaData(Stream s) { - long initPos = s.Position; - load(s); - mySize = Convert.ToInt32(s.Position - initPos); - } - - public string this[string name] { - get { return (string)_name_value_map[name]; } - set { - if (value != null) - _name_value_map[name] = value; - } - } - - private void _add_string_array(string name, string[] values) { - StringBuilder sb = new StringBuilder(); - foreach (string s in values) { - if (sb.Length != 0) - sb.Append(" "); - sb.Append(s); - } - _name_value_map[name] = sb.ToString(); - } - - private int getAsInt(string name) { return Int32.Parse(this[name]); } - - private string[] _get_as_string_array(string name) { - if (this[name] != "" && this[name] != null) - return this[name].Split(' '); - else - return new string[0]; - } - - public int size() { - if (mySize < 0) { - MemoryStream ms = new MemoryStream(); - this.save(ms); - return mySize = (int)ms.Position; - } else - return mySize; - } - - private bool isEqual(MetaData md) { - if (md._name_value_map.Keys.Count != _name_value_map.Keys.Count) - return false; - - foreach (string k in _name_value_map.Keys) - if (!_name_value_map[k].Equals(md._name_value_map[k])) - return false; - return true; - } - - // loads the content of the stream into this instance's map - private void load(Stream s) { - _name_value_map = new Hashtable(); - string[] state = - new BinaryReader(s).ReadString().Split('\n'); - - for (int i = 0; i < state.Length - 1; i += 2) - _name_value_map[state[i]] = state[i + 1]; - } - - // writes the content of this instance to stream s - public void save(Stream s) { - new BinaryWriter(s).Write(this.ToString()); - } - - public override string ToString() { - StringBuilder sb = new StringBuilder(); - foreach (string key in _name_value_map.Keys) - sb.AppendFormat("{0}\n{1}\n", key, _name_value_map[key]); - return sb.ToString(); - } - - // iterates through the fields and enters the name, values into _hash - public void copyFrom(object o) { - this["op-code"] = o.GetType().Name; - foreach (FieldInfo fi in o.GetType().GetFields()) { - if (fi.GetValue(o) == null) - continue; - else if (fi.GetValue(o) is String[]) - _add_string_array(fi.Name, (string[])fi.GetValue(o)); - else - _name_value_map[fi.Name] = fi.GetValue(o).ToString(); - - } - } - // copies the name value map into object o - public void copyTo(object o) { - foreach (FieldInfo fi in o.GetType().GetFields()) { - string value = this[fi.Name]; - if (value == null) { - } else { - if (fi.FieldType.Equals(typeof(string))) - fi.SetValue(o, value); - else if (fi.FieldType.Equals(typeof(int))) - fi.SetValue(o, Int32.Parse(value)); - else if (fi.FieldType.Equals(typeof(long))) - fi.SetValue(o, long.Parse(value)); - else if (fi.FieldType.Equals(typeof(double))) - fi.SetValue(o, double.Parse(value)); - else if (fi.FieldType.Equals(typeof(DateTime))) - fi.SetValue(o, DateTime.Parse(value)); - else if (fi.FieldType.Equals(typeof(Boolean))) - fi.SetValue(o, Boolean.Parse(value)); - else if (fi.FieldType.Equals(typeof(TimeSpan))) - fi.SetValue(o, TimeSpan.Parse(value)); - else if (fi.FieldType.Equals(typeof(String[]))) - fi.SetValue(o, value.Split(' ')); - else - throw new Exception("Field type not supported"); - } - } - } - - public object Clone() { - MemoryStream ms = new MemoryStream(); - this.save(ms); - ms.Seek(0, SeekOrigin.Begin); - return new MetaData(ms); - } - } -} diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/PubMetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/PubMetaData.cs deleted file mode 100644 index 13ab4bbdf..000000000 --- a/src/DataStreamSuite/PubSub/Client/MetaData/PubMetaData.cs +++ /dev/null @@ -1,91 +0,0 @@ -using System; -//using BCG.Utilities.General; -//using BCG.Utilities.BlueConfiguration; -//using BCG.Utilities; - -namespace BCG -{ - - [Serializable] - public class PubMetaData - { - /// - /// tag to be subscribed - /// - public string tag; - /// - /// type of tag - /// - public string type; - /// - /// size of the data published - /// - public int size; - - /// - /// How big is the data when it's decompressed - /// - public int decompressedSize = 0; - - /// - /// is the data compressed? - /// - public bool isCompressed = false; - /// - /// identification of the publisher and the connection to the server to send the data - /// machine; ip; pid; serverURL - /// - public string connection; - - public string publishingApp; - - public long publishTimeTicks; - - public PubMetaData() {} - - public PubMetaData(string theTagType, string theTag, int theSize, string theConnection, string thePublishingApp) - : this(theTagType,theTag,theSize,theSize,theConnection,thePublishingApp) - { - isCompressed = false; - } - - public PubMetaData(string theTagType, string theTag, int theSize, int theDecompressedSize, string theConnection, string thePublishingApp) - { - type = theTagType; - tag = theTag; - size = theSize; - connection = theConnection; - publishingApp = thePublishingApp; - decompressedSize = theDecompressedSize; - isCompressed = true; - - publishTimeTicks = DateTime.Now.ToUniversalTime().Ticks; - myValidateTag(); - } - - //make sure the tag doesn't contain any newlines - private void myValidateTag() - { - if (tag.Contains("\n")) - throw new ArgumentException("Error: tag " + tag + " is not valid; tags cannot contain newlines."); - - } - - /// - /// Returns true if the ticks on this message are greater than the latency report threshold in config - /// - /// - public bool warnIfLatencyExceedsThreshold() - { - DateTime publishTime = new DateTime(publishTimeTicks); - double elapsedSeconds = (DateTime.Now.ToUniversalTime() - publishTime).TotalMilliseconds; - if (elapsedSeconds > PubSub3Config.instance.PS3_LatencyReportThresholdMS) - { - //BlueEventLog.logEvent(string.Concat("Recieved pub message on tag ", tag, " ", elapsedSeconds, " milliseconds after the publisher ",publishingApp," sent it."), PS3LogLevels.latency); - return true; - } - return false; - } - - } -} diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs b/src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs index 22b4b1b64..d7bc20f22 100644 --- a/src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs +++ b/src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs @@ -5,10 +5,9 @@ using System.Collections.Generic; namespace bts { using BCG; - using System.Collections; using System.IO; using System.Linq; - using System.Security.Cryptography; + public interface IPubSubMessage { void serialize_into_str(Stream str); @@ -40,28 +39,31 @@ namespace bts { public class SubscribeMsg : IPubSubMessage { public string connection; - public string subscribing_app; + public string subscribing_app = ClientInfo.thisClient.app; public string[] tags; public void serialize_into_str(Stream str) { BinaryWriter bw = new BinaryWriter(str); - bw.Write(connection); - bw.Write(subscribing_app); + bw.write_string_cpp(connection); + bw.write_string_cpp(subscribing_app); bw.Write(tags.Length); foreach (var t in tags) - bw.Write(t); + bw.write_string_cpp(t); } public void consruct_deserialize(Stream str) { BinaryReader br = new BinaryReader(str); - this.connection = br.ReadString(); - this.subscribing_app = br.ReadString(); + this.connection = br.read_string_cpp(); + this.subscribing_app = br.read_string_cpp(); var n_tags = br.ReadInt32(); tags = new string[n_tags]; for (var i = 0; i < n_tags; i++) - tags[i] = br.ReadString(); + tags[i] = br.read_string_cpp(); } public override string ToString() { - return $"{GetType().Name} connection: {connection} subscribing_app: {subscribing_app} tags: {tags.Length}"; + var tgs_str = String.Empty; + foreach (var t in tags) + tgs_str += t; + return $"{GetType().Name} connection: {connection} subscribing_app: {subscribing_app} tags: {tgs_str}"; } public override bool Equals(object? obj) { if (obj != null && obj is SubscribeMsg) { @@ -73,6 +75,11 @@ namespace bts { } return false; } + + public override int GetHashCode() { + var str = ToString(); + return str.GetHashCode(); + } } // Unsubscribe //////////////////////////////////////////////////////////// @@ -112,17 +119,17 @@ namespace bts { public string publishing_app = ClientInfo.thisClient.app; public string tag; public string tag_type; - public bool is_compressed; + public bool is_compressed = false; // this is the size that of the msg, regarless of it being serialized or not public long publish_time_epoch_ms; public byte[] data_to_publish; public void serialize_into_str(Stream str) { BinaryWriter bw = new BinaryWriter(str); - bw.Write(connection); - bw.Write(publishing_app); - bw.Write(tag); - bw.Write(tag_type); + bw.write_string_cpp(connection); + bw.write_string_cpp(publishing_app); + bw.write_string_cpp(tag); + bw.write_string_cpp(tag_type); bw.Write(is_compressed); bw.Write(publish_time_epoch_ms); var sz = data_to_publish.Length; @@ -131,10 +138,10 @@ namespace bts { } public void consruct_deserialize(Stream str) { BinaryReader br = new BinaryReader(str); - connection = br.ReadString(); - publishing_app = br.ReadString(); - tag = br.ReadString(); - tag_type = br.ReadString(); + connection = br.read_string_cpp(); + publishing_app = br.read_string_cpp(); + tag = br.read_string_cpp(); + tag_type = br.read_string_cpp(); is_compressed = br.ReadBoolean(); publish_time_epoch_ms = br.ReadInt64(); var size = br.ReadInt32(); @@ -277,8 +284,8 @@ namespace bts { public byte[] data; public void serialize_into_str(Stream str) { BinaryWriter bw = new BinaryWriter(str); - bw.Write(tag); - bw.Write(tag_type); + bw.write_string_cpp(tag); + bw.write_string_cpp(tag_type); bw.Write(data.Length); bw.Write(data); } @@ -292,8 +299,8 @@ namespace bts { } public void consruct_deserialize(Stream str) { BinaryReader br = new BinaryReader(str); - tag = br.ReadString(); - tag_type = br.ReadString(); + tag = br.read_string_cpp(); + tag_type = br.read_string_cpp(); var data_sz = br.ReadInt32(); data = br.ReadBytes(data_sz); } diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/RenewMetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/RenewMetaData.cs deleted file mode 100644 index a55b4a111..000000000 --- a/src/DataStreamSuite/PubSub/Client/MetaData/RenewMetaData.cs +++ /dev/null @@ -1,25 +0,0 @@ -// using System; - -// namespace BCG -// { -// [Serializable] -// public class RenewMetaData -// { -// public string subscriber; -// public int lease; -// public bool c; - -// public SubAddress getSubAddress() -// {return SubAddress.decode(subscriber);} - -// public RenewMetaData() -// {} - -// public RenewMetaData(string s, -// int lease) -// { -// this.subscriber = s; -// this.lease = lease; -// } -// } -// } diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs deleted file mode 100644 index 6c09a4c55..000000000 --- a/src/DataStreamSuite/PubSub/Client/MetaData/SubMetaData.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System; - -namespace BCG { - //[Serializable] - public class SubMetaData { - /// - /// connection URL on which the connection will receive data - /// - public string connection; - /// - /// identification of the client and the connection on which to receive data - /// machine; ip; port; clientURL - /// - public readonly string subscribing_app = ClientInfo.thisClient.processString; - /// - /// tags that are to be subscribed - /// - public string[] tags; - - public SubMetaData() { } - - public SubMetaData(string the_connection, params string[] the_tags) { - connection = the_connection; - tags = the_tags; - //subscribing_app = ClientInfo.thisClient.processString; - } - } -} diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/TransactionMetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/TransactionMetaData.cs deleted file mode 100644 index 722c6dfa0..000000000 --- a/src/DataStreamSuite/PubSub/Client/MetaData/TransactionMetaData.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace BCG -{ - /// - /// when a publisher sends transactional data to the pubsub server, - /// this is the first meta data, followed by pub messages. this indicated - /// the number of messages that are part of the transaction - /// - [Serializable] - public class TransactionMetaData - { - public int nMessages; - - public TransactionMetaData() { } - - public TransactionMetaData(int theNMessages) - { - nMessages = theNMessages; - } - } -} diff --git a/src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs b/src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs deleted file mode 100644 index deb98d8d8..000000000 --- a/src/DataStreamSuite/PubSub/Client/MetaData/UnSubMetaData.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; - -namespace BCG { - //[Serializable] - public class UnSubMetaData { - public string connection; - public readonly string[] tags; - public readonly string unsubscribing_app = ClientInfo.thisClient.processString; - - public UnSubMetaData() { } - - public UnSubMetaData(string the_connection, params string[] the_tags) { - connection = the_connection; - tags = the_tags; - } - } -} diff --git a/src/DataStreamSuite/PubSub/Misc/StreamUtilityMethods.cs b/src/DataStreamSuite/PubSub/Misc/StreamUtilityMethods.cs index c4b607606..637d54365 100644 --- a/src/DataStreamSuite/PubSub/Misc/StreamUtilityMethods.cs +++ b/src/DataStreamSuite/PubSub/Misc/StreamUtilityMethods.cs @@ -6,10 +6,11 @@ namespace BCG { /// /// Collection of utility methods used throughout /// - public static class StreamUtilityMethods { + public static class StreamUtilityMethods { // extension method to write a string C++ style (len prefixed) public static void write_string_cpp(this BinaryWriter bw, string str) { + str = str ?? String.Empty; int len = str.Length; bw.Write(len); foreach (char c in str) { diff --git a/src/DataStreamSuite/PubSub/PubSubMessage.cs b/src/DataStreamSuite/PubSub/PubSubMessage.cs deleted file mode 100644 index db0aaa5a5..000000000 --- a/src/DataStreamSuite/PubSub/PubSubMessage.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System; -using System.IO; - -using BCG; -using BCG.Utilities; - -namespace BCG { - // PubItems are published to the pubsub wrapped in this class. It contains PubMetaData, wrapped in the MetaData - // object and the actualy published payload in the body stream - public class PubSubMessage { - public PubSubMessage(MetaData metaData, Stream body) { this.meta_data = metaData; this.body = body; } - - public PubSubMessage(PubMetaData pub_meta_data, Stream body) - : this(new MetaData(pub_meta_data), body) { } - - public PubSubMessage(MetaData metaData) { this.meta_data = metaData; } - - public MetaData meta_data; - public Stream body = new MemoryStream(0); - - public Stream to_stream() { - MemoryStream ms = new MemoryStream(); - meta_data.save(ms); - if (body.Length == 0 || body.Length == body.Position) { - bts.Log.Gen.error("Error: Trying to publishing message with no content."); - Console.WriteLine("Error: Trying to publishing message with no content."); - } - StreamUtilityMethods.copyTo(new Stream[] { body }, ms); - ms.Position = 0; - return ms; - } - - public byte[] to_bytes() { - return StreamUtilityMethods.getBytesFromStream(to_stream()); - } - - public override string ToString() { - return $"PubSubMessage meta:data: {meta_data.ToString()} body.Posiiton: {body.Position} body.Length: {body.Length}"; - } - - } -} diff --git a/src/DataStreamSuite/PubSub/Server/Governors/IPubSubGovernor.cs b/src/DataStreamSuite/PubSub/Server/Governors/IPubSubGovernor.cs deleted file mode 100644 index 8c9bcd131..000000000 --- a/src/DataStreamSuite/PubSub/Server/Governors/IPubSubGovernor.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.IO; - -namespace BCG -{ - /* - * The PubSubGovernor accepts or rejects publishes to the pubsub - * if the governor rejects a publish attempt, the publish fails. - * Clients will not get the update and the state of the pubsub will not change. - * - */ - public enum PubSubGovernorResponse {Accepted, Rejected_ShouldBeImported, Rejected_ThisIsNotARootPubsub}; - public interface IPubSubGovernor - { - PubSubGovernorResponse requestToPublish(PubMetaData metaData, Stream pubData); - } -} diff --git a/src/DataStreamSuite/PubSub/Server/Governors/PermissiveGovernor.cs b/src/DataStreamSuite/PubSub/Server/Governors/PermissiveGovernor.cs deleted file mode 100644 index 0fc7477bf..000000000 --- a/src/DataStreamSuite/PubSub/Server/Governors/PermissiveGovernor.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace BCG -{ - public class PermissiveGovernor : IPubSubGovernor - { - public PermissiveGovernor() - { - - } - - public PubSubGovernorResponse requestToPublish(PubMetaData md, System.IO.Stream message) - { - return PubSubGovernorResponse.Accepted; - } - } -} diff --git a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs index c47d2ce25..08ec4bb5d 100644 --- a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs +++ b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemStatsPublisher.cs @@ -67,9 +67,11 @@ namespace BCG { _dirty_tags.Clear(); var bytes = TagStatsInfo.serialize(dirty_ts_info); PublishMsg pub_msg = new PublishMsg() { - connection = "InternalPublish", + connection = "Internal", + publishing_app = "PubSubServer", tag = SystemStatsPublisher.tag_name, tag_type = SystemStatsPublisher.tag_type, + publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), data_to_publish = bytes }; PubSubMessageReactor.internal_publish(pub_msg); diff --git a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs index da841c564..35dfe72ac 100644 --- a/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs +++ b/src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemSubscriptionMapPublisher.cs @@ -50,7 +50,15 @@ namespace BCG { } private void _on_tag_state_changes(string[] tags) { - bts.Log.Gen.info($"SystemSubscriptionMapPublisher._on_tag_state_changes len: {tags.Length}"); + Func tags_to_string = () => { + var rv = String.Empty; + foreach (var t in tags) { + rv += t + " "; + } + rv.Trim(); + return rv; + }; + bts.Log.Gen.info($"SystemSubscriptionMapPublisher._on_tag_state_changes tags: {tags_to_string()}"); _dirty_tags.AddRange(tags); } diff --git a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs index e70e8d780..d0295d14e 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs @@ -86,7 +86,7 @@ namespace BCG { var internal_pubs = _internal_pub_queue.dequeueMultipleElements(1, 64000); foreach (var msg in internal_pubs) { try { - _handle_pub("Internal", msg); + _handle_pub("Internal", msg, true); } catch (Exception ex) { bts.Log.Gen.error($"PubSubMessage>reactor._polling_thread_func ex: {ex}"); } @@ -129,7 +129,7 @@ namespace BCG { _handle_transaction(sender, ps_client_msg.transaction_msg); break; case PubSubMessageType.PUBLISH: - _handle_pub(sender, ps_client_msg.pub_msg); + _handle_pub(sender, ps_client_msg.pub_msg, false); break; } } @@ -158,9 +158,9 @@ namespace BCG { } } - private void _handle_pub(string sender, PublishMsg pub_msg) { + private void _handle_pub(string sender, PublishMsg pub_msg, bool is_internal_pub) { bts.Log.assert(!pub_msg.is_compressed); - bts.Log.Gen.info($"PubSubMessageReactor._handle_pub sender: {sender} pub_msg: {pub_msg}"); + bts.Log.Gen.info($"PubSubMessageReactor._handle_pub sender: {sender} pub_msg: {pub_msg} is_internal_pub: {is_internal_pub}"); // this is where pubsub's data state is updated _state.update_pub_data(pub_msg); var sub_connections_for_tag = _state.get_subscriber_connections_for_tag(pub_msg.tag); diff --git a/src/DataStreamSuite/PubSub/Server/PubSubState.cs b/src/DataStreamSuite/PubSub/Server/PubSubState.cs index 527dfd1fb..75e74be39 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubState.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubState.cs @@ -180,30 +180,32 @@ namespace BCG { // } public void update_pub_data(bts.PublishMsg pub_msg) { - // TODO - // int sz = (int)(msg.Length - msg.Position); - bts.Log.Gen.info($"PubSubData.update_pub_data pub_msg: {pub_msg}"); - ///this is the only place where SingleTagData should be created - if (!_tag_to_data_map.ContainsKey(pub_msg.tag)) { - ///first time data is published for this tag - _tag_to_data_map.Add(pub_msg.tag, new SingleTagData(pub_msg.tag, pub_msg.tag_type)); - } - _tag_to_data_map[pub_msg.tag].handle_publish(pub_msg); + try { + bts.Log.Gen.info($"PubSubData.update_pub_data pub_msg: {pub_msg}"); + ///this is the only place where SingleTagData should be created + if (!_tag_to_data_map.ContainsKey(pub_msg.tag)) { + ///first time data is published for this tag + _tag_to_data_map.Add(pub_msg.tag, new SingleTagData(pub_msg.tag, pub_msg.tag_type)); + } + _tag_to_data_map[pub_msg.tag].handle_publish(pub_msg); - if (!_tag_to_connection_map.ContainsKey(pub_msg.tag)) { - _tag_to_connection_map.Add(pub_msg.tag, new SingleTagConnectionData(pub_msg.tag)); - } - bool pub_changed = false; - _tag_to_connection_map[pub_msg.tag].handle_pub(pub_msg, out pub_changed); - if (pub_changed) { - if (onTagMapChanged != null) - onTagMapChanged(new string[] { pub_msg.tag }); + if (!_tag_to_connection_map.ContainsKey(pub_msg.tag)) { + _tag_to_connection_map.Add(pub_msg.tag, new SingleTagConnectionData(pub_msg.tag)); + } + bool pub_changed = false; + _tag_to_connection_map[pub_msg.tag].handle_pub(pub_msg, out pub_changed); + if (pub_changed) { + if (onTagMapChanged != null) + onTagMapChanged(new string[] { pub_msg.tag }); - if (onSingleTagChange != null) - onSingleTagChange( - new PubSubSingleTagStateEvent(pub_msg.tag, - PubSubSingleTagStateEvent.EventType.publish, - pub_msg.connection, pub_msg.publishing_app)); + if (onSingleTagChange != null) + onSingleTagChange( + new PubSubSingleTagStateEvent(pub_msg.tag, + PubSubSingleTagStateEvent.EventType.publish, + pub_msg.connection, pub_msg.publishing_app)); + } + } catch (Exception ex) { + bts.Log.Gen.error($"PubSubState.update_pub_data ex: {ex} pub_msg: {pub_msg}"); } if (onPublishStats != null) onPublishStats(pub_msg.tag, pub_msg.data_to_publish.Length); diff --git a/src/DataStreamSuite/PubSub/Server/SingleTagConnectionData.cs b/src/DataStreamSuite/PubSub/Server/SingleTagConnectionData.cs index e4afc991f..a64288fff 100644 --- a/src/DataStreamSuite/PubSub/Server/SingleTagConnectionData.cs +++ b/src/DataStreamSuite/PubSub/Server/SingleTagConnectionData.cs @@ -77,25 +77,6 @@ namespace BCG { get { return _dead_pub_conn_to_app_map; } } - /// - /// Called by PubsubState when a subscription is made to this tag. - /// Returns a true, is the subscriber was not present, else logs - /// a warning and returns a false - /// - /// - /// - public bool onSub(SubMetaData sub) { - if (_sub_conn_to_app_map.ContainsKey(sub.connection)) { - string warningMsg = String.Format("tag {0} has already been subscribed by app {0}, subscriber {1}" - + "it should unsubscribe before resubscribing", _tag, sub.connection, sub.subscribing_app); - //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning); - return false; - } else { - _sub_conn_to_app_map.Add(sub.connection, sub.subscribing_app); - return true; - } - } - public bool handle_sub(SubscribeMsg sub_msg) { if (_sub_conn_to_app_map.ContainsKey(sub_msg.connection)) { bts.Log.Gen.info($"tag: {_tag} has already been subscribed by connection: {sub_msg.connection}"); @@ -106,25 +87,6 @@ namespace BCG { } } - /// - /// Called by PubsubState when a subscriber unsubscribes to this tag. - /// Returns a true when a subscriber exists, else logs a warning and - /// returns a false - /// - /// - /// - public bool onUnsub(UnSubMetaData unsub) { - if (_sub_conn_to_app_map.ContainsKey(unsub.connection)) { - _sub_conn_to_app_map.Remove(unsub.connection); - return true; - } else { - string warningMsg = String.Format("tag {0} has already was either unsubscribed or never subscribed by app {0}, subscriber {1}", - _tag, unsub.connection, unsub.unsubscribing_app); - //BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning); - return false; - } - } - public bool handle_unsub(UnsubscribeMsg unsub_msg) { if (_sub_conn_to_app_map.ContainsKey(unsub_msg.connection)) { _sub_conn_to_app_map.Remove(unsub_msg.connection); @@ -159,21 +121,6 @@ namespace BCG { return rv; } - /// - /// Called by pubsubState, when this tag is published. if publisher - /// has changed, the out paramater is set to true - /// - /// - /// - public void onPub(PubMetaData pub, out bool newPublisher) { - if (_active_pub_conn_to_app_map.ContainsKey(pub.connection)) { - newPublisher = false; - } else { - _active_pub_conn_to_app_map.Add(pub.connection, pub.publishingApp); - newPublisher = true; - } - } - public void handle_pub(PublishMsg pub_msg, out bool new_publisher) { if (_active_pub_conn_to_app_map.ContainsKey(pub_msg.connection)) { new_publisher = false; diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs index b4c929b3e..6ec046d62 100644 --- a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs +++ b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs @@ -27,6 +27,7 @@ namespace BCG { // TODO, construct schema _schema = BlueDataUtility.form_schema("pubsub_server_state", Tuple.Create("id", TypeCode.Int32), + Tuple.Create("tag", TypeCode.String), Tuple.Create("ps_server", TypeCode.String), Tuple.Create("subscibers", TypeCode.String), Tuple.Create("publisher", TypeCode.String), @@ -71,7 +72,7 @@ namespace BCG { var sorted_tags = _tag_map.Keys.OrderBy(t => t); foreach (var kv in _tag_map) { var tsi = kv.Value; - IBlueRow r = new BlueRow(_schema, bt.Count, tsi.pubsub_server, tsi.subscribers_as_string, + IBlueRow r = new BlueRow(_schema, bt.Count, tsi.tag, tsi.pubsub_server, tsi.subscribers_as_string, tsi.publishers_as_string, tsi.dead_publishers_as_string); bt.addRow(r); }