Adding tests for stream serialization

This commit is contained in:
Taha committed 2024-04-22 13:29:50 -04:00
commit e2f5e6717d
46 files changed
+2822 -2518

No files matched your search

@@ -1,17 +1,14 @@
using System;
using System.Collections;
namespace BCG
{
internal class AddColumnsRow : IBlueRow
{
namespace BCG {
internal class AddColumnsRow : IBlueRow {
private IBlueTableSchema mySchema;
private IBlueRow myRow;
//private object[] myCachedValues;
#region Constructors
public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
{
public AddColumnsRow(IBlueTableSchema schema, IBlueRow row) {
mySchema = schema;
myRow = row;
//myCachedValues = new object[schema.columnNames.Length];
@@ -19,85 +16,78 @@ namespace BCG
#endregion
#region IBlueRow members
public IBlueTableSchema schema{get{return mySchema;}}
public IBlueTableSchema schema { get { return mySchema; } }
#endregion
#region ICollection members
public int Count {get{return mySchema.columnNames.Length;}}
public int Count { get { return mySchema.columnNames.Length; } }
public bool IsSynchronized {get{return false;}}
public bool IsSynchronized { get { return false; } }
public object SyncRoot {get{return this;}}
public object SyncRoot { get { return this; } }
public void CopyTo (Array array, int index)
{
public void CopyTo(Array array, int index) {
int i = index;
foreach (object o in this as IEnumerable)
array.SetValue (o, i++);
array.SetValue(o, i++);
}
#endregion
#region IDictionary members
public bool IsFixedSize {get{return true;}}
public bool IsFixedSize { get { return true; } }
public bool IsReadOnly {get{return true;}}
public bool IsReadOnly { get { return true; } }
/// <summary>
/// The key can either be the string corrsponding to the column name or
/// the order of the column
/// </summary>
public object this [object key]
{
get
{
public object this[object key] {
get {
if (key is string)
return this[(string)key];
return this[(int)key];
}
set {throw new NotSupportedException ("The IBlueRow is readonly");}
set { throw new NotSupportedException("The IBlueRow is readonly"); }
}
public object this[int index]
{
get
{
public object this[int index] {
get {
//if (myCachedValues[index] == null)
// myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
//return myCachedValues[index];
return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
return mySchema[index].isConstant ? DBNull.Value : myRow[index];
}
}
public object this[string colName] {get{return this[mySchema[colName].colNo];}}
public object this[string colName] { get { return this[mySchema[colName].colNo]; } }
public ICollection Keys{get{return mySchema.columnNames;}}
public ICollection Keys { get { return mySchema.columnNames; } }
public ICollection Values
{
get
{
public ICollection Values {
get {
object[] retVal = new object[mySchema.columnNames.Length];
CopyTo (retVal, 0);
CopyTo(retVal, 0);
return retVal;
}
}
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
public bool Contains (object key){return mySchema[(string)key] != null;}
public bool Contains(object key) { return mySchema[(string)key] != null; }
/// <summary>
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
/// iterate through the values, typecast this into an IEnumerable
/// </summary>
/// <returns></returns>
public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
#endregion
}
@@ -4,18 +4,14 @@ using BCG.Utilities;
using System.Collections.Generic;
using System.Linq;
namespace BCG
{
public class GroupByStream : MethodDrivenStream
{
private class MyFuncGroupByEvaluator : IGroupByEvaluator
{
namespace BCG {
public class GroupByStream : MethodDrivenStream {
private class MyFuncGroupByEvaluator : IGroupByEvaluator {
private string[] _groupby_cols;
private Func<DataPacket, IBlueRow> _groupby_func;
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
{
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
_groupby_cols = groupby_cols.ToArray();
_groupby_func = groupby_func;
_groupby_func = groupby_func;
}
public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
@@ -29,7 +25,7 @@ namespace BCG
/// Serves as a quick way to access all rows belonging to the same group
/// </summary>
private bool myIdGuard = true;
private RowDictionaryTable myBookKeepingTable ;
private RowDictionaryTable myBookKeepingTable;
private IGroupByEvaluator myGroupByEvaluator;
private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
//can map a schema to another
@@ -37,112 +33,98 @@ namespace BCG
private int myIdGenerator = 0;
public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
: this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func))
{ }
: this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
public GroupByStream (ViewStream stream, IGroupByEvaluator groupByEvaluator)
: this (stream, groupByEvaluator, true)
{}
public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
: this(stream, groupByEvaluator, true) { }
public GroupByStream (ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
: base (stream)
{
public GroupByStream(ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
: base(stream) {
myGroupByEvaluator = groupbyEvaluator;
myIdGuard = idGuard;
}
protected override void myHandleFirstCall ()
{
protected override void myHandleFirstCall() {
mySchema = myLastArrivedPacket.schema;
myOutputSchema = myGroupByEvaluator.getOutputSchema (mySchema);
myBookKeepingTable = new RowDictionaryTable (myLastArrivedPacket.newRows,
myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
myGroupByEvaluator.groupByColumns);
GroupByOutputTable temp = new GroupByOutputTable (myOutputSchema);
foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups())
{
IBlueRow newRow = myEvaluate (new DataPacket (rowGroup,
rowGroup, new BlueTable (mySchema)));
GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
rowGroup, new BlueTable(mySchema)));
if (newRow != null)
temp.addRow (newRow, myGetGroupString (rowGroup));
temp.addRow(newRow, myGetGroupString(rowGroup));
}
myLastFiredPacket = new DataPacket (temp, new BlueTable(temp), new BlueTable (temp.schema));
myLastFiredPacket = new DataPacket(temp, new BlueTable(temp), new BlueTable(temp.schema));
}
/// <summary>
/// Adding need not to anything as deletion already takes care
/// of addition
/// </summary>
protected override void myHandleIncrementalAdd() {}
protected override void myHandleIncrementalAdd() { }
/// <summary>
/// Both deletes and adds are handled here to make the intermediate results not
/// appear in the spitted fired DataPacket (note this is done only in GroupByStream
/// because of the nature of it)
/// </summary>
protected override void myHandleIncrementalDelete()
{
protected override void myHandleIncrementalDelete() {
myFireSwitch = false;
RowDictionaryTable delGroups, addGroups;
//form groups for deleted and added rows
myInitIncrementalChangeData (out delGroups, out addGroups);
ArrayList handledAddGroups = new ArrayList ();
myInitIncrementalChangeData(out delGroups, out addGroups);
ArrayList handledAddGroups = new ArrayList();
string groupRepString;
foreach (IBlueRowCollection delGroup in delGroups.getGroups ())
{
foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
// BlueEventLog.logAssert (delGroup.Count > 0);
groupRepString = myGetGroupString (delGroup);
IBlueRowCollection addGroup = addGroups.getGroup (groupRepString) == null ?
new BlueTable (mySchema) : addGroups.getGroup (groupRepString);
handledAddGroups.Add (addGroup);
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString) == null ?
new BlueTable (mySchema) : myBookKeepingTable.getGroup (groupRepString);
IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, delGroup));
myFireSwitch |= myMergeRow (newRow, groupRepString);
groupRepString = myGetGroupString(delGroup);
IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
handledAddGroups.Add(addGroup);
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
myFireSwitch |= myMergeRow(newRow, groupRepString);
}
foreach (IBlueRowCollection addGroup in addGroups.getGroups ())
{
foreach (IBlueRowCollection addGroup in addGroups.getGroups()) {
// BlueEventLog.logAssert (addGroup.Count > 0);
if (!handledAddGroups.Contains (addGroup))
{
if (!handledAddGroups.Contains(addGroup)) {
//this is the case when there are rows added to a group and none
//have been removed, hence it 'escapes' from being dealt with in
//the loop above
groupRepString = myGetGroupString (addGroup);
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString);
groupRepString = myGetGroupString(addGroup);
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
// BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup,
new BlueTable (aggGroup.schema)));
myFireSwitch |= myMergeRow (newRow, groupRepString);
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
new BlueTable(aggGroup.schema)));
myFireSwitch |= myMergeRow(newRow, groupRepString);
}
}
}
private bool myMergeRow (IBlueRow newRow, string groupRepString)
{
IBlueRow existingRow = ((GroupByOutputTable) myLastFiredPacket.newRows).getRow (groupRepString);
private bool myMergeRow(IBlueRow newRow, string groupRepString) {
IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
//BlueEventLog.logAssert (newRow != null || existingRow != null);
if (existingRow == null && newRow == null)
return false;
else if (existingRow != null && newRow != null && BlueDataUtility.computeRowEquality (existingRow, newRow))
else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
return false;
else if (existingRow == null)
{
else if (existingRow == null) {
//row added for the first time
((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
return true;
}
else
{
((GroupByOutputTable) myLastFiredPacket.newRows).removeRow (groupRepString);
((BlueTable) myLastFiredPacket.deletedRows).addRow (existingRow);
if (newRow != null)
{
((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
} else {
((GroupByOutputTable)myLastFiredPacket.newRows).removeRow(groupRepString);
((BlueTable)myLastFiredPacket.deletedRows).addRow(existingRow);
if (newRow != null) {
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
}
return true;
}
@@ -154,77 +136,68 @@ namespace BCG
/// </summary>
/// <param name="delGroups"></param>
/// <param name="addGroups"></param>
private void myInitIncrementalChangeData (out RowDictionaryTable delGroups, out RowDictionaryTable addGroups)
{
private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
delGroups =
new RowDictionaryTable (myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
addGroups =
new RowDictionaryTable (myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
new RowDictionaryTable(myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
myBookKeepingTable.removeRow (deletedRow);
myBookKeepingTable.removeRow(deletedRow);
foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
myBookKeepingTable.addRow (addedRow);
myLastFiredPacket.addedRows = new BlueTable (myOutputSchema);
myLastFiredPacket.deletedRows = new BlueTable (myOutputSchema);
myBookKeepingTable.addRow(addedRow);
myLastFiredPacket.addedRows = new BlueTable(myOutputSchema);
myLastFiredPacket.deletedRows = new BlueTable(myOutputSchema);
}
protected override void myOnTableClear ()
{
protected override void myOnTableClear() {
if (myBookKeepingTable.Count == 0)
return; //ie there arent any rows, so no need to optimize the 'cleansing' process
myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows);
myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
(myLastFiredPacket.newRows as GroupByOutputTable).clear ();
myBookKeepingTable.clear ();
myGroupByEvaluator.clear ();
myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
(myLastFiredPacket.newRows as GroupByOutputTable).clear();
myBookKeepingTable.clear();
myGroupByEvaluator.clear();
}
private IBlueRow myEvaluate (DataPacket dp)
{
IBlueRow evalRow = myGroupByEvaluator.evaluate (dp);
return evalRow == null ? null :
private IBlueRow myEvaluate(DataPacket dp) {
IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
return evalRow == null ? null :
myIdGuard ?
BlueDataUtility.getNewReplacedRow (evalRow, myIdGenerator++, "id") : evalRow;
BlueDataUtility.getNewReplacedRow(evalRow, myIdGenerator++, "id") : evalRow;
}
private string myGetGroupString (IBlueRowCollection group)
{
return BlueDataUtility.rowToString(BlueDataUtility.extractRow (group, 0), myGroupByEvaluator.groupByColumns);
private string myGetGroupString(IBlueRowCollection group) {
return BlueDataUtility.rowToString(BlueDataUtility.extractRow(group, 0), myGroupByEvaluator.groupByColumns);
}
protected override void myOnLastSubscriber ()
{
protected override void myOnLastSubscriber() {
//the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream
//gets resubscribed, we start the process of building data again, and treat the callback as the first
//callback.
myInit ();
myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
myInit();
myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
}
protected override void myInit ()
{
base.myInit ();
if (myBookKeepingTable != null)
{
protected override void myInit() {
base.myInit();
if (myBookKeepingTable != null) {
//to avoid calling init before the first callback
myBookKeepingTable.clear ();
myGroupByEvaluator.clear ();
myBookKeepingTable.clear();
myGroupByEvaluator.clear();
}
}
public static string getRepString (DataPacket dp, string[] colNames)
{
return BlueDataUtility.rowToString (getRepRow (dp), colNames);
public static string getRepString(DataPacket dp, string[] colNames) {
return BlueDataUtility.rowToString(getRepRow(dp), colNames);
}
public static IBlueRow getRepRow (DataPacket dp)
{
public static IBlueRow getRepRow(DataPacket dp) {
if (dp.newRows.Count > 0)
return BlueDataUtility.extractRow (dp.newRows, 0);
return BlueDataUtility.extractRow(dp.newRows, 0);
else if (dp.addedRows.Count > 0)
return BlueDataUtility.extractRow (dp.addedRows, 0);
return BlueDataUtility.extractRow(dp.addedRows, 0);
else if (dp.deletedRows.Count > 0)
return BlueDataUtility.extractRow (dp.deletedRows, 0);
return BlueDataUtility.extractRow(dp.deletedRows, 0);
// BlueEventLog.logAssert (false, "Illegal Data Packet");
return null;
}
@@ -237,22 +210,21 @@ namespace BCG
/// <param name="newValue"></param>
/// <param name="isAdd"></param>
/// <returns></returns>
public static object incrementalAdd (object lastValue, object newValue, bool isAdd)
{
if (lastValue == null || lastValue.Equals (DBNull.Value))
public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
if (lastValue == null || lastValue.Equals(DBNull.Value))
return newValue;
// BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
// lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
// lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
// lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
// lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
Object retVal = null;
if (newValue.GetType().Equals (typeof(int)))
if (newValue.GetType().Equals(typeof(int)))
retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
else if (newValue.GetType().Equals (typeof(Double)))
else if (newValue.GetType().Equals(typeof(Double)))
retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
else if (lastValue.GetType().Equals (typeof(Decimal)))
else if (lastValue.GetType().Equals(typeof(Decimal)))
retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
else{}
// BlueEventLog.logAssert (false, "Illegal type");
else { }
// BlueEventLog.logAssert (false, "Illegal type");
return retVal;
}
@@ -1,24 +1,24 @@
using System;
// using System;
namespace BCG
{
public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
{
IBlueTableSchema mySchema;
// namespace BCG
// {
// public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
// {
// IBlueTableSchema mySchema;
public IBlueRow map (IBlueRow row)
{
if (mySchema == null)
{
ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
int i = 0;
foreach (ColumnInfo ci in row.schema.columnInfo)
cis[i++] = ci.colType == null ?
new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
mySchema = new BlueTableSchema (row.schema.tableName, cis);
}
return new BlueRow (mySchema, row);
}
}
}
// public IBlueRow map (IBlueRow row)
// {
// if (mySchema == null)
// {
// ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
// int i = 0;
// foreach (ColumnInfo ci in row.schema.columnInfo)
// cis[i++] = ci.colType == null ?
// new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
// new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
// mySchema = new BlueTableSchema (row.schema.tableName, cis);
// }
// return new BlueRow (mySchema, row);
// }
// }
// }
@@ -73,7 +73,7 @@ namespace bts {
private readonly int _payload_sz;
private const byte _payload_byte = (byte)'A';
private readonly string _tag_name;
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
_vsd = vsd;
_payload_sz = payload_size;
var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -87,7 +87,7 @@ namespace bts {
Tuple.Create("match", TypeCode.Boolean),
Tuple.Create("echo_time", TypeCode.Int32));
_table = new BlueTable(schema);
_publish_period = TimeSpan.FromSeconds(publish_period_seconds);
_publish_period = TimeSpan.FromMilliseconds(publish_period_msecs);
_msg_size = payload_size;
_max_pubs = max_pubs;
_tag_name = tag_name;
@@ -1,15 +1,21 @@
using Amazon.Runtime.Internal;
using bts;
using DnsClient.Protocol;
using System;
using System.Collections.Generic;
using System.IO;
using System;
using Microsoft.VisualBasic;
namespace BCG {
/// <summary>
/// This stream (not for prod) provies a way to run tests - that could otherwise be run in NUnits.
/// This stream (not for prod) provides a way to run tests - that could otherwise be run in NUnits.
/// (truth is stream rpovide a RDE for quickly coding and running, so it was eaiser to run tests here)
/// </summary>
[DataStreamUserRespresentation("tests")]
@@ -39,21 +45,38 @@ namespace BCG {
_schema = BlueDataUtility.form_schema("Tests",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("test", TypeCode.String),
Tuple.Create("result", TypeCode.Boolean)
Tuple.Create("result", TypeCode.Boolean),
Tuple.Create("exception", TypeCode.String)
);
// resgiter tests here
_tests.Add(new Func<TestResult>(_run_sub_msg_test));
_tests.Add(new Func<TestResult>(_run_pub_msg_test));
_tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
_tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
_tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
_tests.Add(new Func<TestResult>(_run_schema_serialization_test));
_tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls));
_tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls));
_tests.Add(new Func<TestResult>(_row_serialization_tests_all_types));
_tests.Add(new Func<TestResult>(_run_full_stream_serialization_test));
_tests.Add(new Func<TestResult>(_run_full_stream_serialization_with_controller_test));
}
protected override void myOnFirstSubscriber() {
BlueTable rows = new BlueTable(_schema);
// u the tests here
foreach (var test in _tests) {
var tr = test();
IBlueRow r = new BlueRow(_schema, rows.Count, tr.test_name, tr.test_result);
string ex_msg = String.Empty;
TestResult tr = new TestResult() {
test_name = test.Method.Name,
test_result = false
};
try {
tr = test();
} catch (Exception ex) {
ex_msg = ex.Message + "\n" + ex.StackTrace;
}
IBlueRow r = new BlueRow(_schema, rows.Count, tr.test_name, tr.test_result, ex_msg);
rows.addRow(r);
}
var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
@@ -141,5 +164,303 @@ namespace BCG {
test_result = res
};
}
private TestResult _run_columninfo_serialization_test() {
Func<ColumnInfo, bool> test_ci = (ci) => {
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
StreamSerializationUtils.serialize_column_info(ci, ref bw);
str.Position = 0;
//var ci_txt = BlueDataUtility.to_string(ci);
//var ci_ser =
//bts.Log.Gen.info("ci: {}")
BinaryReader br = new BinaryReader(str);
var ci_prime = StreamSerializationUtils.deserialize_column_info(ref br);
return BlueDataUtility.compute_equality(ci, ci_prime);
};
int i = 0;
foreach (Type t in BlueDataUtility.valid_column_types()) {
ColumnInfo ci = new ColumnInfo(Guid.NewGuid().ToString(),
i++, t);
var res = test_ci(ci);
if (!res) {
return new TestResult {
test_name = "ColumnInfoSerialization",
test_result = false
};
}
}
return new TestResult {
test_name = "ColumnInfoSerialization",
test_result = true
};
}
private TestResult _run_schema_serialization_test() {
var table_name = "test_table";//Guid.NewGuid().ToString();
BlueTableSchema schema = new BlueTableSchema();
List<ColumnInfo> cis = new List<ColumnInfo>();
foreach (Type t in BlueDataUtility.valid_column_types()) {
if (t == null)
continue;
var col_name = t.ToString() + "_" + cis.Count.ToString(); // Guid.NewGuid().ToString();
ColumnInfo ci = new ColumnInfo(col_name, cis.Count, t);
cis.Add(ci);
}
schema = new BlueTableSchema(table_name, cis.ToArray());
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
StreamSerializationUtils.serialize_schema(schema, ref bw);
var hex_txt = StreamSerializationUtils.to_hex(str);
bts.Log.Gen.info($"_run_schema_serialization_test ser: {hex_txt}");
str.Position = 0;
BinaryReader br = new BinaryReader(str);
var schema_prime = StreamSerializationUtils.deserialize_schema(ref br);
var res = BlueDataUtility.compute_equality(schema, schema_prime);
return new TestResult {
test_name = "SchemaSerialization",
test_result = res
};
}
private TestResult _run_universal_row_serialization_test() {
return new TestResult {
test_name = "UniversalRowwSerialization",
test_result = false
};
}
private TestResult _run_full_stream_serialization_test() {
// create message
var schema = create_schema_with_all_types();
BlueTable cr_rows = new BlueTable(schema);
// 100 rows
int n_rows = 100;
for (int i = 0; i < n_rows; i++) {
var nr = generate_random_row(schema, i);
cr_rows.addRow(nr);
}
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
UniversalStreamIOSerializer ser = new UniversalStreamIOSerializer();
ser.serialize(stream_io_msg, ref bw);
str.Position = 0;
BinaryReader br = new System.IO.BinaryReader(str);
UniversalStreamIODeserializer deser = new bts.UniversalStreamIODeserializer();
var stream_io_msg_prime = deser.deserialize(ref br);
bool passes = stream_io_msg.Equals(stream_io_msg_prime);
var n_dels = 50;
BlueTable del_rows = new BlueTable(cr_rows.schema);
foreach (IBlueRow r in cr_rows) {
del_rows.addRow(r);
if (del_rows.Count == n_dels)
break;
}
IBlueRowCollection added_rows = del_rows;
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
str = new MemoryStream();
bw = new BinaryWriter(str);
stream_io_msg = new bts.StreamIOMessage(mra_msg);
ser.serialize(stream_io_msg, ref bw);
bw.BaseStream.Position = 0;
br = new BinaryReader(str);
//deser = new UniversalStreamIODeserializer();
stream_io_msg_prime = deser.deserialize(ref br);
passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
return new TestResult {
test_name = "UniversalSerialization",
test_result = passes
};
}
private TestResult _run_full_stream_serialization_with_controller_test() {
// create message
var schema = create_schema_with_all_types();
BlueTable cr_rows = new BlueTable(schema);
// 100 rows
int n_rows = 100;
for (int i = 0; i < n_rows; i++) {
var nr = generate_random_row(schema, i);
cr_rows.addRow(nr);
}
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
IStreamIOMsgSerializer ser = new StreamIOSerializationController();
ser.serialize(stream_io_msg, ref bw);
str.Position = 0;
BinaryReader br = new System.IO.BinaryReader(str);
IStreamMsgDeserilizer deser = new bts.StreamIODeserializationController();
var stream_io_msg_prime = deser.deserialize(ref br);
bool passes = stream_io_msg.Equals(stream_io_msg_prime);
//mult remove adds
var n_dels = 50;
BlueTable del_rows = new BlueTable(cr_rows.schema);
foreach (IBlueRow r in cr_rows) {
del_rows.addRow(r);
if (del_rows.Count == n_dels)
break;
}
IBlueRowCollection added_rows = del_rows;
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
str = new MemoryStream();
bw = new BinaryWriter(str);
stream_io_msg = new bts.StreamIOMessage(mra_msg);
ser.serialize(stream_io_msg, ref bw);
bw.BaseStream.Position = 0;
br = new BinaryReader(str);
stream_io_msg_prime = deser.deserialize(ref br);
passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
return new TestResult {
test_name = "UniversalSerializationController",
test_result = passes
};
}
private TestResult _row_serialization_tests_no_dbnulls() {
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
new Tuple<string, TypeCode>("id", TypeCode.Int32),
new Tuple<string, TypeCode>("i1", TypeCode.Int32),
new Tuple<string, TypeCode>("d1", TypeCode.Double),
new Tuple<string, TypeCode>("string_1", TypeCode.String));
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
IBlueRow r = new BlueRow(schema, 0, 100, 123.45, "test_string");
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
row_ser.serialize_row(r, 0, ref bw);
str.Position = 0;
BinaryReader br = new BinaryReader(str);
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
IBlueRow r_prime = des.deserialize_row(ref br);
var passes = BlueDataUtility.compute_equality(r, r_prime);
return new TestResult {
test_name = "UniversalRowSerialization_NonNull",
test_result = passes
};
}
private TestResult _row_serialization_tests_with_dbnulls() {
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
new Tuple<string, TypeCode>("id", TypeCode.Int32),
new Tuple<string, TypeCode>("i1", TypeCode.Int32),
new Tuple<string, TypeCode>("d1", TypeCode.Double),
new Tuple<string, TypeCode>("string_1", TypeCode.String));
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
IBlueRow r = new BlueRow(schema, 0, DBNull.Value, 123.45, "test_string");
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
row_ser.serialize_row(r, 0, ref bw);
str.Position = 0;
BinaryReader br = new BinaryReader(str);
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
IBlueRow r_prime = des.deserialize_row(ref br);
var passes = BlueDataUtility.compute_equality(r, r_prime);
return new TestResult {
test_name = "UniversalRowSerialization_WithDBNulls",
test_result = passes
};
}
private TestResult _row_serialization_tests_all_types() {
var schema = create_schema_with_all_types();
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
IBlueRow r = generate_random_row(schema, 0);
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
row_ser.serialize_row(r, 0, ref bw);
str.Position = 0;
BinaryReader br = new BinaryReader(str);
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
IBlueRow r_prime = des.deserialize_row(ref br);
var passes = BlueDataUtility.compute_equality(r, r_prime);
return new TestResult {
test_name = "UniversalRowSerialization_AllTypes",
test_result = passes
};
}
#region Helper functions
public static IBlueTableSchema create_schema_with_all_types() {
var table_name = "test_table";
BlueTableSchema schema = new BlueTableSchema();
List<ColumnInfo> cis = new List<ColumnInfo>();
foreach (Type t in BlueDataUtility.valid_column_types()) {
if (t == null || t == typeof(DateTime))
continue;
var col_name = t.ToString() + "_" + cis.Count.ToString(); // Guid.NewGuid().ToString();
ColumnInfo ci = new ColumnInfo(col_name, cis.Count, t);
cis.Add(ci);
}
cis.Add(new ColumnInfo("id", cis.Count, typeof(int)));
schema = new BlueTableSchema(table_name, cis.ToArray());
return schema;
}
public static IBlueRow generate_random_row(IBlueTableSchema schema, int id) {
Random rnd = new Random();
object[] row_vals = new object[schema.columnInfo.Length];
foreach (ColumnInfo ci in schema.columnInfo) {
if (ci.colName == "id") {
row_vals[ci.colNo] = id;
continue;
}
object v = get_random_object(ci.type_code());
if (rnd.NextSingle() % 4 == 0)
v = DBNull.Value;
row_vals[ci.colNo] = v;
}
return new BlueRow(schema, row_vals);
}
public static object get_random_object(TypeCode type_code) {
Random random = new Random();
switch (type_code) {
case TypeCode.Boolean:
return random.Next(2) == 0; // Random boolean value (true or false)
case TypeCode.Byte:
return (byte)random.Next(byte.MinValue, byte.MaxValue + 1); // Random byte value
case TypeCode.Char:
return (char)random.Next(65, 65 + 27); // Random char value
case TypeCode.Decimal:
Decimal dec = 100.123M;
return dec;
// object obj_d = get_random_object(TypeCode.Double);
// return Convert.ToDecimal(obj_d);
case TypeCode.Double:
return random.NextDouble() * (double.MaxValue - double.MinValue) + double.MinValue; // Random double value
case TypeCode.Int16:
return (short)random.Next(short.MinValue, short.MaxValue + 1); // Random short value
case TypeCode.Int32:
return random.Next(int.MinValue, int.MaxValue); // Random int value
case TypeCode.Int64:
return random.NextInt64(long.MinValue, long.MaxValue); // Random long value
case TypeCode.SByte:
return (sbyte)random.Next(sbyte.MinValue, sbyte.MaxValue + 1); // Random sbyte value
case TypeCode.Single:
return (float)(random.NextDouble() * (float.MaxValue - float.MinValue) + float.MinValue); // Random float value
case TypeCode.String:
return Guid.NewGuid().ToString();
case TypeCode.UInt16:
return (ushort)random.Next(ushort.MinValue, ushort.MaxValue + 1); // Random ushort value
case TypeCode.UInt32:
return (uint)random.Next(int.MinValue, int.MaxValue) + (uint)int.MaxValue; // Random uint value
case TypeCode.UInt64:
byte[] ulongBuffer = new byte[8];
random.NextBytes(ulongBuffer);
return BitConverter.ToUInt64(ulongBuffer, 0);
default:
throw new ArgumentException($"Unsupported TypeCode: {type_code}");
return null;
}
}
#endregion
}
}
@@ -8,38 +8,36 @@ namespace BCG {
private ViewStream mySrc;
private IBlueTableSchema mySchema = null;
public AlphabetizeColumnsStream (ViewStream src) {
public AlphabetizeColumnsStream(ViewStream src) {
mySrc = src;
}
protected override void myOnFirstSubscriber()
{
protected override void myOnFirstSubscriber() {
mySrc.onChange += new ChangeDelegate(mySrc_onChange);
}
protected override void myOnLastSubscriber ()
{
mySrc.onChange -= new ChangeDelegate (mySrc_onChange);
protected override void myOnLastSubscriber() {
mySrc.onChange -= new ChangeDelegate(mySrc_onChange);
}
private void mySrc_onChange (object newValue) {
private void mySrc_onChange(object newValue) {
DataPacket dp = (DataPacket)newValue;
if (mySchema == null) {
SortedList alphabetical = new SortedList ();
SortedList alphabetical = new SortedList();
foreach (ColumnInfo cinfo in dp.newRows.schema.columnInfo)
alphabetical.Add (cinfo.colName,cinfo);
alphabetical.Add(cinfo.colName, cinfo);
ColumnInfo[] cinfos = new ColumnInfo[alphabetical.Count];
alphabetical.Values.CopyTo (cinfos,0);
alphabetical.Values.CopyTo(cinfos, 0);
for (int i = 0; i < cinfos.Length; ++i)
cinfos[i] = new ColumnInfo (cinfos[i].colName,i,cinfos[i].colType,cinfos[i].colValue);
mySchema = new BlueTableSchema (cinfos,dp.schema.tableName);
cinfos[i] = new ColumnInfo(cinfos[i].colName, i, cinfos[i].colType);
mySchema = new BlueTableSchema(cinfos, dp.schema.tableName);
}
BlueTable
deleted = new BlueTable (mySchema),
added = new BlueTable (mySchema),
newrows = new BlueTable (mySchema);
deleted.addRows (dp.deletedRows);
added.addRows (dp.addedRows);
newrows.addRows (dp.newRows);
myFireEvent (new DataPacket (newrows,added,deleted));
deleted = new BlueTable(mySchema),
added = new BlueTable(mySchema),
newrows = new BlueTable(mySchema);
deleted.addRows(dp.deletedRows);
added.addRows(dp.addedRows);
newrows.addRows(dp.newRows);
myFireEvent(new DataPacket(newrows, added, deleted));
}
}
}
@@ -201,20 +201,21 @@ namespace BCG {
}
return rv;
}
#region Schema types
// these are the only types allowed in schemas
private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
{
{ TypeCode.Boolean, typeof(bool) },
{ TypeCode.Byte, typeof(byte) },
{ TypeCode.Char, typeof(char) },
{ TypeCode.DateTime, typeof(DateTime) },
{ TypeCode.DBNull, typeof(DBNull) },
// TODO
// { TypeCode.DateTime, typeof(DateTime) },
{ TypeCode.Decimal, typeof(decimal) },
{ TypeCode.Double, typeof(double) },
{ TypeCode.Empty, null },
//{ TypeCode.Empty, null },
{ TypeCode.Int16, typeof(short) },
{ TypeCode.Int32, typeof(int) },
{ TypeCode.Int64, typeof(long) },
{ TypeCode.Object, typeof(object) },
{ TypeCode.SByte, typeof(sbyte) },
{ TypeCode.Single, typeof(Single) },
{ TypeCode.String, typeof(string) },
@@ -223,15 +224,42 @@ namespace BCG {
{ TypeCode.UInt64, typeof(UInt64) }
};
public static IEnumerable<TypeCode> valid_column_typecodes() {
return _typecode_to_type.Keys;
}
public static IEnumerable<Type> valid_column_types() {
return _typecode_to_type.Values;
}
public static bool is_valid_col_type(Type t) {
TypeCode type_code = get_typecode(t);
return type_code != TypeCode.Empty;
}
public static TypeCode get_typecode(Type t) {
foreach (var kv in _typecode_to_type) {
if (kv.Value == t)
return kv.Key;
}
bts.Log.assert_d(false, $"type: {t} not found");
//bts.Log.assert_d(false, $"type: {t} not found");
return TypeCode.Empty;
}
public static Type get_type(TypeCode type_code) {
Type rv = null;
if (_typecode_to_type.TryGetValue(type_code, out rv)) {
return rv;
}
return null;
}
public static TypeCode type_code(this ColumnInfo ci) {
return get_typecode(ci.colType);
}
#endregion
public static IBlueTableSchema form_schema(string table_name, params Tuple<string, TypeCode>[] cols) {
List<ColumnInfo> cis = new List<ColumnInfo>();
foreach (Tuple<string, TypeCode> c in cols)
@@ -375,12 +403,12 @@ namespace BCG {
foreach (PropertyInfo pi in t.GetProperties())
if (myIsActiveProperty(pi))
colInfos.Add(new ColumnInfo(pi.Name, i++,
ColumnInfo.isAllowedType(pi.PropertyType) ?
is_valid_col_type(pi.PropertyType) ?
pi.PropertyType : typeof(String)));
foreach (FieldInfo fi in t.GetFields())
if (myIsActiveField(fi))
colInfos.Add(new ColumnInfo(fi.Name, i++,
ColumnInfo.isAllowedType(fi.FieldType) ? fi.FieldType : typeof(String)));
is_valid_col_type(fi.FieldType) ? fi.FieldType : typeof(String)));
if (createIdField)
colInfos.Add(new ColumnInfo("id", i++, typeof(int)));
@@ -405,12 +433,12 @@ namespace BCG {
foreach (PropertyInfo pi in t.GetProperties())
if (myIsActiveProperty(pi))
rowArray[schema[pi.Name].colNo] =
ColumnInfo.isAllowedType(pi.PropertyType) ?
is_valid_col_type(pi.PropertyType) ?
pi.GetValue(o, null) : pi.GetValue(o, null).ToString();
foreach (FieldInfo fi in t.GetFields())
if (myIsActiveField(fi))
rowArray[schema[fi.Name].colNo] =
ColumnInfo.isAllowedType(fi.FieldType) ?
is_valid_col_type(fi.FieldType) ?
fi.GetValue(o) : fi.GetValue(o).ToString();
if (idVal != null)
rowArray[schema["id"].colNo] = idVal;
@@ -424,12 +452,12 @@ namespace BCG {
foreach (PropertyInfo pi in t.GetProperties())
if (myIsActiveProperty(pi))
rowArray[schema[pi.Name].colNo] =
ColumnInfo.isAllowedType(pi.PropertyType) ?
is_valid_col_type(pi.PropertyType) ?
pi.GetValue(o, null) : type_converter(pi.PropertyType, pi.GetValue(o, null));
foreach (FieldInfo fi in t.GetFields())
if (myIsActiveField(fi))
rowArray[schema[fi.Name].colNo] =
ColumnInfo.isAllowedType(fi.FieldType) ?
is_valid_col_type(fi.FieldType) ?
fi.GetValue(o) : type_converter(fi.FieldType, fi.GetValue(o));
if (idVal != null)
rowArray[schema["id"].colNo] = idVal;
@@ -618,15 +646,7 @@ namespace BCG {
/// <param name="r1"></param>
/// <param name="r2"></param>
/// <returns></returns>
public static bool computeRowEquality(IBlueRow r1, IBlueRow r2) {
return computeRowEquality(r1, r2, r1.schema.columnNames);
}
public static bool computeRowEquality(IBlueRow r1,
IBlueRow r2,
params string[] colNames) {
return compareRows(r1, r2, colNames) == 0 ? true : false;
}
/// <summary>
/// Computes a string equivelant for the row, using the columns provided
@@ -664,7 +684,7 @@ namespace BCG {
/// <param name="r2"></param>
/// <param name="colNames"></param>
/// <returns></returns>
public static int compareRows(IBlueRow r1,
public static int compare_rows(IBlueRow r1,
IBlueRow r2,
params string[] colNames) {
int retVal = 0;
@@ -815,7 +835,7 @@ namespace BCG {
public static IBlueRow computeMaxRow(IBlueRowCollection rows, params string[] cols) {
IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null;
foreach (IBlueRow r in rows)
retVal = compareRows(r, retVal, cols) > 0 ? r : retVal;
retVal = compare_rows(r, retVal, cols) > 0 ? r : retVal;
return retVal;
}
@@ -828,7 +848,7 @@ namespace BCG {
public static IBlueRow computeMinRow(IBlueRowCollection rows, params string[] cols) {
IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null;
foreach (IBlueRow r in rows)
retVal = compareRows(r, retVal, cols) < 0 ? r : retVal;
retVal = compare_rows(r, retVal, cols) < 0 ? r : retVal;
return retVal;
}
@@ -907,6 +927,43 @@ namespace BCG {
out aRows);
}
public static bool compute_equality(IBlueRowCollection rows1, IBlueRowCollection rows2) {
IBlueRowCollection d_rows, a_rows;
compute_differences_fast(rows2, rows1, out d_rows, out a_rows);
return d_rows.Count == 0 && a_rows.Count == 0;
}
public static bool compute_equality(ColumnInfo ci1, ColumnInfo ci2) {
return (ci1.colNo == ci2.colNo) && (ci1.colName == ci2.colName) && (ci1.colType == ci2.colType);
}
public static bool compute_equality(IBlueTableSchema s1, IBlueTableSchema s2) {
if (s1.tableName != s1.tableName)
return false;
if (s1.columnInfo.Length != s2.columnInfo.Length)
return false;
for (int i = 0; i < s1.columnInfo.Length; i++) {
ColumnInfo ci1 = s2.columnInfo[i], ci2 = s2.columnInfo[i];
if (!compute_equality(ci1, ci2))
return false;
}
return true;
}
public static bool compute_equality(IBlueRow r1, IBlueRow r2) {
return compute_row_equality(r1, r2, r1.schema.columnNames);
}
public static bool compute_row_equality(IBlueRow r1,
IBlueRow r2,
params string[] col_names) {
return compare_rows(r1, r2, col_names) == 0 ? true : false;
}
public static string to_string(ColumnInfo ci) {
return ci.colNo + ";" + ci.colName + ";" + ci.colType;
}
public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
int[] col_indices = new int[new_rows.schema.columnNames.Length];
@@ -960,7 +1017,7 @@ namespace BCG {
foreach (IBlueRow nr in c1)
if (!BlueDataUtility.rowPresent(c2, nr))
(aRows as BlueTable).addRow(nr);
else if (BlueDataUtility.compareRows(nr, c2[(int)
else if (BlueDataUtility.compare_rows(nr, c2[(int)
nr["id"]], columns) != 0) {
(aRows as BlueTable).addRow(nr);
(dRows as BlueTable).addRow(c2[(int)nr["id"]]);
@@ -1040,7 +1097,7 @@ namespace BCG {
truelyAdds = new BlueTable(adds);
foreach (IBlueRow r in dels) {
if (adds.containsID((int)r["id"]) &&
BlueDataUtility.computeRowEquality(adds[(int)r["id"]], r, cols)) {
BlueDataUtility.compute_row_equality(adds[(int)r["id"]], r, cols)) {
(truelyDels as BlueTable).deleteRow((int)r["id"]);
(truelyAdds as BlueTable).deleteRow((int)r["id"]);
}
@@ -1,45 +1,42 @@
using System;
namespace BCG
{
/// <summary>
/// Allowed types
/// </summary>
public class BlueDataValidTypes
{
public static readonly Type BOOL = typeof(bool);
public static readonly Type BYTE = typeof(byte);
public static readonly Type CHAR = typeof(char);
public static readonly Type DECIMAL = typeof(decimal);
public static readonly Type DOUBLE = typeof(double);
public static readonly Type SHORT = typeof(short);
public static readonly Type INT = typeof(int);
public static readonly Type LONG = typeof(long);
public static readonly Type SBYTE = typeof(sbyte);
//public static readonly Type FLOAT = typeof(float);
public static readonly Type STRING = typeof(string);
//public static readonly Type USHORT = typeof(ushort);
public static readonly Type UINT = typeof(uint);
public static readonly Type ULONG = typeof(ulong);
public static readonly Type DATETIME = typeof(DateTime);
public static readonly Type ISERIALIZEDTYPE = typeof(ISerializedType);
public static readonly Type SINGLE = typeof(Single);
public static readonly Type BYTEARRAY = typeof(byte[]);
namespace BCG {
/// <summary>
/// Allowed types
/// </summary>
public class BlueDataValidTypes {
public static readonly Type BOOL = typeof(bool);
public static readonly Type BYTE = typeof(byte);
public static readonly Type CHAR = typeof(char);
public static readonly Type DECIMAL = typeof(decimal);
public static readonly Type DOUBLE = typeof(double);
public static readonly Type SHORT = typeof(short);
public static readonly Type INT = typeof(int);
public static readonly Type LONG = typeof(long);
public static readonly Type SBYTE = typeof(sbyte);
//public static readonly Type FLOAT = typeof(float);
public static readonly Type STRING = typeof(string);
//public static readonly Type USHORT = typeof(ushort);
public static readonly Type UINT = typeof(uint);
public static readonly Type ULONG = typeof(ulong);
public static readonly Type DATETIME = typeof(DateTime);
public static readonly Type ISERIALIZEDTYPE = typeof(ISerializedType);
public static readonly Type SINGLE = typeof(Single);
public static readonly Type BYTEARRAY = typeof(byte[]);
public static int getValueTypeSize (Type t)
{
if (t == BOOL || t == BYTE || t == SBYTE)
return 1;
else if (t == SHORT || t == CHAR)
return 2;
else if (t == INT || t == UINT || t == SINGLE)
return 4;
else if (t == DOUBLE || t == LONG || t == ULONG || t == DATETIME)
return 8;
else if (t == DECIMAL)
return 16;
else
throw new ApplicationException("Unsupported Data Type");
}
}
public static int getValueTypeSize(Type t) {
if (t == BOOL || t == BYTE || t == SBYTE)
return 1;
else if (t == SHORT || t == CHAR)
return 2;
else if (t == INT || t == UINT || t == SINGLE)
return 4;
else if (t == DOUBLE || t == LONG || t == ULONG || t == DATETIME)
return 8;
else if (t == DECIMAL)
return 16;
else
throw new ApplicationException("Unsupported Data Type");
}
}
}
@@ -1,116 +1,117 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
// using System;
// using System.Collections.Generic;
// using System.Linq;
// using System.Text;
/*
* This class reprsents one lookup group that is stored together in a lookup table because it contains repeated data patterns.
* if groupNo is zero, then the group is not stord in a lookup table at all.
*
*
*
*/
namespace BCG
{
public class BlueRowLookupGroup
{
BlueTableSchema myEntrySchema;
public IBlueTableSchema entrySchema
{
get { return myEntrySchema; }
//cant' set this value, have to build it from the row's schema
//TODORemove
// /*
// * This class reprsents one lookup group that is stored together in a lookup table because it contains repeated data patterns.
// * if groupNo is zero, then the group is not stord in a lookup table at all.
// *
// *
// *
// */
// namespace BCG
// {
// public class BlueRowLookupGroup
// {
// BlueTableSchema myEntrySchema;
// public IBlueTableSchema entrySchema
// {
// get { return myEntrySchema; }
// //cant' set this value, have to build it from the row's schema
}
// }
//for faster access and such
private int[] myGroupColNos;
public int[] groupColNos
{
get { return myGroupColNos; }
// //for faster access and such
// private int[] myGroupColNos;
// public int[] groupColNos
// {
// get { return myGroupColNos; }
}
private string[] myGroupColNames;
public string[] groupColNames
{
get { return myGroupColNames; }
}
// }
// private string[] myGroupColNames;
// public string[] groupColNames
// {
// get { return myGroupColNames; }
// }
private short myGroupNumber;
public short groupNumber
{
get { return myGroupNumber; }
// private short myGroupNumber;
// public short groupNumber
// {
// get { return myGroupNumber; }
}
/* how many bits are the ids into the table?
* In c# everything is stored as integers, but when it goes across the wire
* the data can be represented as a byte, a short, or an int
*/
private short myTableSize;
public short tableSize
{
get { return myTableSize; }
// }
}
// /* how many bits are the ids into the table?
// * In c# everything is stored as integers, but when it goes across the wire
// * the data can be represented as a byte, a short, or an int
// */
// private short myTableSize;
// public short tableSize
// {
// get { return myTableSize; }
//if the group number is 0, the group is not stored in tables for serialization
public bool isStoredInTable
{
get { return myGroupNumber != 0; }
// }
}
public BlueRowLookupGroup(short num, short sizeInBits, params string[] cols)
{
// //if the group number is 0, the group is not stored in tables for serialization
// public bool isStoredInTable
// {
// get { return myGroupNumber != 0; }
myGroupNumber = num;
myTableSize = sizeInBits;
myGroupColNames = cols ;
}
// }
// public BlueRowLookupGroup(short num, short sizeInBits, params string[] cols)
// {
/*Takes a LookupTableSchema for the rows this group caches for, and builds the schema representing
each of these entries in the lookup table. This function should be called by the LookupTableSchema
during its constructor.
This function also caches the column numbers for this group, for quicker access*/
public void buildEntrySchema(LookupTableSchema theSchema)
{
ColumnInfo[] columns = new ColumnInfo[myGroupColNames.Length + 1];
myGroupColNos = new int[myGroupColNames.Length];
//the first column in our table is always the ID
int idColumn = 0;
columns[idColumn] = new ColumnInfo("id", idColumn, typeof(int));
//go through each of the columns
for (int col = 1; col < columns.Length; ++col)
{
ColumnInfo columnInInputTable = theSchema[groupColNames[col - 1]];
if (columnInInputTable.isConstant)
columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colValue);
else
columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colType);
// myGroupNumber = num;
// myTableSize = sizeInBits;
// myGroupColNames = cols ;
// }
myGroupColNos[col - 1] = columnInInputTable.colNo;
}
// /*Takes a LookupTableSchema for the rows this group caches for, and builds the schema representing
// each of these entries in the lookup table. This function should be called by the LookupTableSchema
// during its constructor.
// This function also caches the column numbers for this group, for quicker access*/
// public void buildEntrySchema(LookupTableSchema theSchema)
// {
// ColumnInfo[] columns = new ColumnInfo[myGroupColNames.Length + 1];
// myGroupColNos = new int[myGroupColNames.Length];
// //the first column in our table is always the ID
// int idColumn = 0;
// columns[idColumn] = new ColumnInfo("id", idColumn, typeof(int));
// //go through each of the columns
// for (int col = 1; col < columns.Length; ++col)
// {
// ColumnInfo columnInInputTable = theSchema[groupColNames[col - 1]];
// if (columnInInputTable.isConstant)
// columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colValue);
// else
// columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colType);
myEntrySchema = new BlueTableSchema(columns, "Lookup Group " + theSchema.tableName + ":" + myGroupNumber);
}
/*takes a row as input, and extracts the members of the group from the row */
public object[] extractGroupFromRow(IBlueRow input)
{
int numCols = myGroupColNames.Length;
object[] results = new object[numCols];
for (int col = 0; col < numCols; ++col)
results[col] = input[myGroupColNos[col]];
return results;
}
// myGroupColNos[col - 1] = columnInInputTable.colNo;
// }
}
}
// myEntrySchema = new BlueTableSchema(columns, "Lookup Group " + theSchema.tableName + ":" + myGroupNumber);
// }
// /*takes a row as input, and extracts the members of the group from the row */
// public object[] extractGroupFromRow(IBlueRow input)
// {
// int numCols = myGroupColNames.Length;
// object[] results = new object[numCols];
// for (int col = 0; col < numCols; ++col)
// results[col] = input[myGroupColNos[col]];
// return results;
// }
// }
// }
@@ -8,8 +8,7 @@ using System.Linq;
using BCG.Utilities.General;
namespace BCG
{
namespace BCG {
/// <summary>
/// The BlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
/// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
@@ -17,112 +16,105 @@ namespace BCG
/// the respective data, sorted according to the ColumnOrder.
/// </summary>
[Serializable]
public class BlueTableSchema : IBlueTableSchema
{
public class BlueTableSchemaSerializationHelper
{
public static object parse(Type t, string data)
{
if (t == BlueDataValidTypes.BOOL)
return Boolean.Parse(data);
else if (t == BlueDataValidTypes.BYTE)
return Byte.Parse(data);
else if (t == BlueDataValidTypes.CHAR)
return char.Parse(data);
else if (t == BlueDataValidTypes.DATETIME)
return DateTime.Parse(data);
else if (t == BlueDataValidTypes.DOUBLE)
return Double.Parse(data);
else if (t == BlueDataValidTypes.DECIMAL)
return Decimal.Parse(data);
else if (t == BlueDataValidTypes.INT)
return int.Parse(data);
else if (t == BlueDataValidTypes.LONG)
return long.Parse(data);
else if (t == BlueDataValidTypes.SBYTE)
return SByte.Parse(data);
else if (t == BlueDataValidTypes.STRING)
return data;
else if (t == BlueDataValidTypes.UINT)
return uint.Parse(data);
else if (t == BlueDataValidTypes.ULONG)
return ulong.Parse(data);
else if (t == BlueDataValidTypes.SINGLE)
return Single.Parse(data);
else
throw new ApplicationException("Can not be parsed");
}
public static void serialize(BinaryWriter bw, Type t, object o)
{
if (t == BlueDataValidTypes.BOOL)
bw.Write((bool)o);
else if (t == BlueDataValidTypes.BYTE)
bw.Write((byte)o);
else if (t == BlueDataValidTypes.CHAR)
bw.Write((char)o);
else if (t == BlueDataValidTypes.DATETIME)
bw.Write(((DateTime)o).Ticks);
else if (t == BlueDataValidTypes.DOUBLE)
bw.Write((double)o);
else if (t == BlueDataValidTypes.DECIMAL)
bw.Write((decimal)o);
else if (t == BlueDataValidTypes.INT)
bw.Write((int)o);
else if (t == BlueDataValidTypes.LONG)
bw.Write((long)o);
else if (t == BlueDataValidTypes.SBYTE)
bw.Write((sbyte)o);
else if (t == BlueDataValidTypes.STRING)
bw.Write(o as string);
else if (t == BlueDataValidTypes.SINGLE)
bw.Write((Single)o);
else if (t == BlueDataValidTypes.BYTEARRAY)
{
byte[] bs = o as byte[];
bw.Write(bs.Length);
bw.Write(bs);
}
else if (t == BlueDataValidTypes.UINT)
bw.Write((uint)o);
else if (t == BlueDataValidTypes.ULONG)
bw.Write((ulong)o);
else
throw new ApplicationException("Unsupported type");
}
public class BlueTableSchema : IBlueTableSchema {
public class BlueTableSchemaSerializationHelper {
// public static object parse(Type t, string data) {
// if (t == BlueDataValidTypes.BOOL)
// return Boolean.Parse(data);
// else if (t == BlueDataValidTypes.BYTE)
// return Byte.Parse(data);
// else if (t == BlueDataValidTypes.CHAR)
// return char.Parse(data);
// else if (t == BlueDataValidTypes.DATETIME)
// return DateTime.Parse(data);
// else if (t == BlueDataValidTypes.DOUBLE)
// return Double.Parse(data);
// else if (t == BlueDataValidTypes.DECIMAL)
// return Decimal.Parse(data);
// else if (t == BlueDataValidTypes.INT)
// return int.Parse(data);
// else if (t == BlueDataValidTypes.LONG)
// return long.Parse(data);
// else if (t == BlueDataValidTypes.SBYTE)
// return SByte.Parse(data);
// else if (t == BlueDataValidTypes.STRING)
// return data;
// else if (t == BlueDataValidTypes.UINT)
// return uint.Parse(data);
// else if (t == BlueDataValidTypes.ULONG)
// return ulong.Parse(data);
// else if (t == BlueDataValidTypes.SINGLE)
// return Single.Parse(data);
// else
// throw new ApplicationException("Can not be parsed");
// }
// public static void serialize(BinaryWriter bw, Type t, object o) {
// if (t == BlueDataValidTypes.BOOL)
// bw.Write((bool)o);
// else if (t == BlueDataValidTypes.BYTE)
// bw.Write((byte)o);
// else if (t == BlueDataValidTypes.CHAR)
// bw.Write((char)o);
// else if (t == BlueDataValidTypes.DATETIME)
// bw.Write(((DateTime)o).Ticks);
// else if (t == BlueDataValidTypes.DOUBLE)
// bw.Write((double)o);
// else if (t == BlueDataValidTypes.DECIMAL)
// bw.Write((decimal)o);
// else if (t == BlueDataValidTypes.INT)
// bw.Write((int)o);
// else if (t == BlueDataValidTypes.LONG)
// bw.Write((long)o);
// else if (t == BlueDataValidTypes.SBYTE)
// bw.Write((sbyte)o);
// else if (t == BlueDataValidTypes.STRING)
// bw.Write(o as string);
// else if (t == BlueDataValidTypes.SINGLE)
// bw.Write((Single)o);
// else if (t == BlueDataValidTypes.BYTEARRAY) {
// byte[] bs = o as byte[];
// bw.Write(bs.Length);
// bw.Write(bs);
// } else if (t == BlueDataValidTypes.UINT)
// bw.Write((uint)o);
// else if (t == BlueDataValidTypes.ULONG)
// bw.Write((ulong)o);
// else
// throw new ApplicationException("Unsupported type");
// }
public static object deserialize(BinaryReader br, Type t)
{
if (t == BlueDataValidTypes.BOOL)
return br.ReadBoolean();
else if (t == BlueDataValidTypes.BYTE)
return br.ReadByte();
else if (t == BlueDataValidTypes.CHAR)
return br.ReadChar();
else if (t == BlueDataValidTypes.DATETIME)
return new DateTime(br.ReadInt64());
else if (t == BlueDataValidTypes.DOUBLE)
return br.ReadDouble();
else if (t == BlueDataValidTypes.DECIMAL)
return br.ReadDecimal();
else if (t == BlueDataValidTypes.INT)
return br.ReadInt32();
else if (t == BlueDataValidTypes.LONG)
return br.ReadInt64();
else if (t == BlueDataValidTypes.SBYTE)
return br.ReadSByte();
else if (t == BlueDataValidTypes.STRING)
return br.ReadString();
else if (t == BlueDataValidTypes.BYTEARRAY)
return br.ReadBytes(br.ReadInt32());
else if (t == BlueDataValidTypes.UINT)
return br.ReadUInt32();
else if (t == BlueDataValidTypes.ULONG)
return br.ReadUInt64();
else if (t == BlueDataValidTypes.SINGLE)
return br.ReadSingle();
else
throw new ApplicationException("Unsupported type");
}
// public static object deserialize(BinaryReader br, Type t) {
// if (t == BlueDataValidTypes.BOOL)
// return br.ReadBoolean();
// else if (t == BlueDataValidTypes.BYTE)
// return br.ReadByte();
// else if (t == BlueDataValidTypes.CHAR)
// return br.ReadChar();
// else if (t == BlueDataValidTypes.DATETIME)
// return new DateTime(br.ReadInt64());
// else if (t == BlueDataValidTypes.DOUBLE)
// return br.ReadDouble();
// else if (t == BlueDataValidTypes.DECIMAL)
// return br.ReadDecimal();
// else if (t == BlueDataValidTypes.INT)
// return br.ReadInt32();
// else if (t == BlueDataValidTypes.LONG)
// return br.ReadInt64();
// else if (t == BlueDataValidTypes.SBYTE)
// return br.ReadSByte();
// else if (t == BlueDataValidTypes.STRING)
// return br.ReadString();
// else if (t == BlueDataValidTypes.BYTEARRAY)
// return br.ReadBytes(br.ReadInt32());
// else if (t == BlueDataValidTypes.UINT)
// return br.ReadUInt32();
// else if (t == BlueDataValidTypes.ULONG)
// return br.ReadUInt64();
// else if (t == BlueDataValidTypes.SINGLE)
// return br.ReadSingle();
// else
// throw new ApplicationException("Unsupported type");
// }
}
protected Hashtable myIndexByColName = new Hashtable();
@@ -132,29 +124,24 @@ namespace BCG
public BlueTableSchema() { }
public BlueTableSchema(DataTable table)
{
public BlueTableSchema(DataTable table) {
myIndexByColOrder = new ColumnInfo[table.Columns.Count];
myTableName = table.TableName;
foreach (DataColumn col in table.Columns)
{
foreach (DataColumn col in table.Columns) {
ColumnInfo ci = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
myIndexByColName[ci.colName] = ci;
myIndexByColOrder[ci.colNo] = ci;
}
}
public BlueTableSchema(ColumnInfo[] cis, string tableName)
{
public BlueTableSchema(ColumnInfo[] cis, string tableName) {
myConstruct(cis, tableName);
}
private void myConstruct(ColumnInfo[] cis, string tableName)
{
private void myConstruct(ColumnInfo[] cis, string tableName) {
myIndexByColOrder = new ColumnInfo[cis.Length];
myTableName = tableName;
foreach (ColumnInfo ci in cis)
{
foreach (ColumnInfo ci in cis) {
myIndexByColOrder[ci.colNo] = ci;
myIndexByColName[ci.colName] = ci;
}
@@ -169,15 +156,13 @@ namespace BCG
/// </summary>
/// <param name="tableName">the table name</param>
/// <param name="fromType">the plain old data type whose fields will be mirrored in the schema</param>
public BlueTableSchema(string tableName, Type fromType)
{
public BlueTableSchema(string tableName, Type fromType) {
List<ColumnInfo> cis = new List<ColumnInfo>();
int i = 0;
// we sort the field name so that we have a deterministic order given
// the same type.
bool foundId = false;
foreach (FieldInfo fi in fromType.GetFields().OrderBy((f) => { return f.Name; }))
{
foreach (FieldInfo fi in fromType.GetFields().OrderBy((f) => { return f.Name; })) {
if (fi.Name.ToLower().Equals("id"))
foundId = true;
cis.Add(new ColumnInfo(fi.Name, i++, fi.FieldType));
@@ -188,32 +173,25 @@ namespace BCG
myConstruct(cis.ToArray(), tableName);
}
public ColumnInfo this[string colName]
{
get
{
public ColumnInfo this[string colName] {
get {
object retVal = myIndexByColName[colName];
return retVal != null ? retVal as ColumnInfo : null;
}
}
public ColumnInfo this[int colNo]
{
public ColumnInfo this[int colNo] {
get { return myIndexByColOrder[colNo]; }
}
/// <summary>
/// The returned Array is sorted according to the Column No
/// </summary>
public string[] columnNames
{
get
{
if (myTableColNames == null)
{
public string[] columnNames {
get {
if (myTableColNames == null) {
myTableColNames = new string[myIndexByColName.Count];
foreach (DictionaryEntry de in myIndexByColName)
{
foreach (DictionaryEntry de in myIndexByColName) {
ColumnInfo temp = (ColumnInfo)de.Value;
myTableColNames[temp.colNo] = temp.colName;
}
@@ -225,8 +203,7 @@ namespace BCG
/// <summary>
/// The returned Array is sorted according to the Column No
/// </summary>
public ColumnInfo[] columnInfo
{
public ColumnInfo[] columnInfo {
get { return myIndexByColOrder; }
}
@@ -234,49 +211,30 @@ namespace BCG
#region ISerializedType methods
public void serialize(BinaryWriter bw)
{
public void serialize(BinaryWriter bw) {
bw.Write(columnNames.Length);
bw.Write(tableName);
foreach (ColumnInfo ci in columnInfo)
{
foreach (ColumnInfo ci in columnInfo) {
bw.Write(ci.colName);
bw.Write(ci.colType.ToString());
if (ci.isConstant)
{
bw.Write(true);
BlueTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
}
else
{
bw.Write(false);
}
}
}
public void deserialze(BinaryReader br)
{
public void deserialze(BinaryReader br) {
ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()];
string name = br.ReadString();
for (int i = 0; i < cis.Length; i++)
{
for (int i = 0; i < cis.Length; i++) {
string colName = br.ReadString();
Type t = TypeInstanceLoader.loadType(br.ReadString(), true);
bool isConstant = br.ReadBoolean();
if (isConstant)
{
cis[i] = new ColumnInfo(colName, i, t, BlueTableSchemaSerializationHelper.deserialize(br, t));
}
else
{
cis[i] = new ColumnInfo(colName, i, t);
}
cis[i] = new ColumnInfo(colName, i, t);
}
myConstruct(cis, name);
}
public int getSerializedSize()
{
public int getSerializedSize() {
throw new NotImplementedException("Feature for provided constant values to" +
"non-primitive types in schema, not provided as of now");
}
@@ -1,56 +1,25 @@
using System;
namespace BCG
{
namespace BCG {
/// <summary>
/// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
/// </summary>
[Serializable]
public class ColumnInfo : IComparable
{
public static bool isAllowedType (Type t)
{
return
t.Equals (typeof (System.Int32)) ||
t.Equals (typeof (System.Int64)) ||
t.Equals (typeof (System.Double)) ||
t.Equals (typeof (System.String)) ||
t.Equals (typeof (System.DateTime)) ? true :
false;
}
public class ColumnInfo : IComparable {
//the default column info does not cach values in a lookup table, therefore it is in group 0
public ColumnInfo(string colName, int colNo, Type colType)
: this(colName, colNo, colType, 0)
{
}
public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
{
public ColumnInfo(string colName, int colNo, Type colType) {
myColName = colName;
myColNo = colNo;
myColType = colType;
myLookupTableGroup = colGroup;
myColValue = null;
}
public ColumnInfo(string col_name, int col_no, TypeCode type_code)
: this(col_name, col_no, BlueDataUtility.get_type(type_code)) {
}
public ColumnInfo (string colName, int colNo, object colValue)
: this (colName, colNo, colValue.GetType()) {myColValue = colValue == null ? DBNull.Value : colValue;}
public ColumnInfo (string colName, int colNo, Type colType, object colValue)
: this (colName, colNo, colType)
{
myColValue = colValue == null ? DBNull.Value : colValue;
}
public int CompareTo (object obj)
{
public int CompareTo(object obj) {
int otherColNo = ((ColumnInfo)(obj)).colNo;
if (myColNo < otherColNo)
return -1;
@@ -59,37 +28,29 @@ namespace BCG
return 0;
}
//public bool isCalculated{get{return myExpression != null;}}
public override bool Equals(object obj) {
if (obj is ColumnInfo) {
ColumnInfo other = obj as ColumnInfo;
return (colNo == other.colNo) && (colName == other.colName) && (colType == other.colType);
}
return false;
}
public bool isConstant {get{return myColValue != null;}}
public override string ToString() {
return colName + " " + colNo + " " + colType;
}
// for backwards compatibility to allow build
public bool isConstant { get { return false; } }
private int myColNo;
private string myColName;
private Type myColType;
//private string myExpression;
private object myColValue;
//no table will ever have more than 65,536 columns
//if it does, we are in serious trouble
private short myLookupTableGroup;
public int colNo { get { return myColNo; } }
public string colName { get { return myColName; } }
public Type colType { get { return myColType; } }
public int colNo{get{return myColNo;}}
public string colName {get{return myColName;}}
public Type colType {get{return myColType;}}
//this value corresponds to how the data in the column is serialized.
//if the group number is 0, the values are not stored in a table
//if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
//are stored together in a lookup table
public short lookupTableGroup
{
get { return myLookupTableGroup; }
set { myLookupTableGroup = value; }
}
//public string expression {get{return myExpression;}}
public object colValue {get{return myColValue;}}
}
}
@@ -1,324 +1,324 @@
using System;
using System.Data;
using System.Collections;
using System.IO;
using System.Reflection;
using System.Collections.Generic;
using System.Linq;
// using System;
// using System.Data;
// using System.Collections;
// using System.IO;
// using System.Reflection;
// using System.Collections.Generic;
// using System.Linq;
using BCG.Utilities.General;
// using BCG.Utilities.General;
namespace BCG
{
/// <summary>
/// The LookupTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
/// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
/// are the indexers (both order and name based), and the property columnInfo and columnNames, that return
/// the respective data, sorted according to the ColumnOrder.
/// </summary>
[Serializable]
public class LookupTableSchema : IBlueTableSchema
{
public class LookupTableSchemaSerializationHelper
{
public static object parse(Type t, string data)
{
if (t == BlueDataValidTypes.BOOL)
return Boolean.Parse(data);
else if (t == BlueDataValidTypes.BYTE)
return Byte.Parse(data);
else if (t == BlueDataValidTypes.CHAR)
return char.Parse(data);
else if (t == BlueDataValidTypes.DATETIME)
return DateTime.Parse(data);
else if (t == BlueDataValidTypes.DOUBLE)
return Double.Parse(data);
else if (t == BlueDataValidTypes.DECIMAL)
return Decimal.Parse(data);
else if (t == BlueDataValidTypes.INT)
return int.Parse(data);
else if (t == BlueDataValidTypes.LONG)
return long.Parse(data);
else if (t == BlueDataValidTypes.SBYTE)
return SByte.Parse(data);
else if (t == BlueDataValidTypes.STRING)
return data;
else if (t == BlueDataValidTypes.UINT)
return uint.Parse(data);
else if (t == BlueDataValidTypes.ULONG)
return ulong.Parse(data);
else if (t == BlueDataValidTypes.SINGLE)
return Single.Parse(data);
else
throw new ApplicationException("Can not be parsed");
}
public static void serialize(BinaryWriter bw, Type t, object o)
{
if (t == BlueDataValidTypes.BOOL)
bw.Write((bool)o);
else if (t == BlueDataValidTypes.BYTE)
bw.Write((byte)o);
else if (t == BlueDataValidTypes.CHAR)
bw.Write((char)o);
else if (t == BlueDataValidTypes.DATETIME)
bw.Write(((DateTime)o).Ticks);
else if (t == BlueDataValidTypes.DOUBLE)
bw.Write((double)o);
else if (t == BlueDataValidTypes.DECIMAL)
bw.Write((decimal)o);
else if (t == BlueDataValidTypes.INT)
bw.Write((int)o);
else if (t == BlueDataValidTypes.LONG)
bw.Write((long)o);
else if (t == BlueDataValidTypes.SBYTE)
bw.Write((sbyte)o);
else if (t == BlueDataValidTypes.STRING)
bw.Write(o as string);
else if (t == BlueDataValidTypes.SINGLE)
bw.Write((Single)o);
else if (t == BlueDataValidTypes.BYTEARRAY)
{
byte[] bs = o as byte[];
bw.Write(bs.Length);
bw.Write(bs);
}
else if (t == BlueDataValidTypes.UINT)
bw.Write((uint)o);
else if (t == BlueDataValidTypes.ULONG)
bw.Write((ulong)o);
else
throw new ApplicationException("Unsupported type");
}
// namespace BCG
// {
// /// <summary>
// /// The LookupTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
// /// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
// /// are the indexers (both order and name based), and the property columnInfo and columnNames, that return
// /// the respective data, sorted according to the ColumnOrder.
// /// </summary>
// [Serializable]
// public class LookupTableSchema : IBlueTableSchema
// {
// public class LookupTableSchemaSerializationHelper
// {
// public static object parse(Type t, string data)
// {
// if (t == BlueDataValidTypes.BOOL)
// return Boolean.Parse(data);
// else if (t == BlueDataValidTypes.BYTE)
// return Byte.Parse(data);
// else if (t == BlueDataValidTypes.CHAR)
// return char.Parse(data);
// else if (t == BlueDataValidTypes.DATETIME)
// return DateTime.Parse(data);
// else if (t == BlueDataValidTypes.DOUBLE)
// return Double.Parse(data);
// else if (t == BlueDataValidTypes.DECIMAL)
// return Decimal.Parse(data);
// else if (t == BlueDataValidTypes.INT)
// return int.Parse(data);
// else if (t == BlueDataValidTypes.LONG)
// return long.Parse(data);
// else if (t == BlueDataValidTypes.SBYTE)
// return SByte.Parse(data);
// else if (t == BlueDataValidTypes.STRING)
// return data;
// else if (t == BlueDataValidTypes.UINT)
// return uint.Parse(data);
// else if (t == BlueDataValidTypes.ULONG)
// return ulong.Parse(data);
// else if (t == BlueDataValidTypes.SINGLE)
// return Single.Parse(data);
// else
// throw new ApplicationException("Can not be parsed");
// }
// public static void serialize(BinaryWriter bw, Type t, object o)
// {
// if (t == BlueDataValidTypes.BOOL)
// bw.Write((bool)o);
// else if (t == BlueDataValidTypes.BYTE)
// bw.Write((byte)o);
// else if (t == BlueDataValidTypes.CHAR)
// bw.Write((char)o);
// else if (t == BlueDataValidTypes.DATETIME)
// bw.Write(((DateTime)o).Ticks);
// else if (t == BlueDataValidTypes.DOUBLE)
// bw.Write((double)o);
// else if (t == BlueDataValidTypes.DECIMAL)
// bw.Write((decimal)o);
// else if (t == BlueDataValidTypes.INT)
// bw.Write((int)o);
// else if (t == BlueDataValidTypes.LONG)
// bw.Write((long)o);
// else if (t == BlueDataValidTypes.SBYTE)
// bw.Write((sbyte)o);
// else if (t == BlueDataValidTypes.STRING)
// bw.Write(o as string);
// else if (t == BlueDataValidTypes.SINGLE)
// bw.Write((Single)o);
// else if (t == BlueDataValidTypes.BYTEARRAY)
// {
// byte[] bs = o as byte[];
// bw.Write(bs.Length);
// bw.Write(bs);
// }
// else if (t == BlueDataValidTypes.UINT)
// bw.Write((uint)o);
// else if (t == BlueDataValidTypes.ULONG)
// bw.Write((ulong)o);
// else
// throw new ApplicationException("Unsupported type");
// }
public static object deserialize(BinaryReader br, Type t)
{
if (t == BlueDataValidTypes.BOOL)
return br.ReadBoolean();
else if (t == BlueDataValidTypes.BYTE)
return br.ReadByte();
else if (t == BlueDataValidTypes.CHAR)
return br.ReadChar();
else if (t == BlueDataValidTypes.DATETIME)
return new DateTime(br.ReadInt64());
else if (t == BlueDataValidTypes.DOUBLE)
return br.ReadDouble();
else if (t == BlueDataValidTypes.DECIMAL)
return br.ReadDecimal();
else if (t == BlueDataValidTypes.INT)
return br.ReadInt32();
else if (t == BlueDataValidTypes.LONG)
return br.ReadInt64();
else if (t == BlueDataValidTypes.SBYTE)
return br.ReadSByte();
else if (t == BlueDataValidTypes.STRING)
return br.ReadString();
else if (t == BlueDataValidTypes.BYTEARRAY)
return br.ReadBytes(br.ReadInt32());
else if (t == BlueDataValidTypes.UINT)
return br.ReadUInt32();
else if (t == BlueDataValidTypes.ULONG)
return br.ReadUInt64();
else if (t == BlueDataValidTypes.SINGLE)
return br.ReadSingle();
else
throw new ApplicationException("Unsupported type");
}
}
// public static object deserialize(BinaryReader br, Type t)
// {
// if (t == BlueDataValidTypes.BOOL)
// return br.ReadBoolean();
// else if (t == BlueDataValidTypes.BYTE)
// return br.ReadByte();
// else if (t == BlueDataValidTypes.CHAR)
// return br.ReadChar();
// else if (t == BlueDataValidTypes.DATETIME)
// return new DateTime(br.ReadInt64());
// else if (t == BlueDataValidTypes.DOUBLE)
// return br.ReadDouble();
// else if (t == BlueDataValidTypes.DECIMAL)
// return br.ReadDecimal();
// else if (t == BlueDataValidTypes.INT)
// return br.ReadInt32();
// else if (t == BlueDataValidTypes.LONG)
// return br.ReadInt64();
// else if (t == BlueDataValidTypes.SBYTE)
// return br.ReadSByte();
// else if (t == BlueDataValidTypes.STRING)
// return br.ReadString();
// else if (t == BlueDataValidTypes.BYTEARRAY)
// return br.ReadBytes(br.ReadInt32());
// else if (t == BlueDataValidTypes.UINT)
// return br.ReadUInt32();
// else if (t == BlueDataValidTypes.ULONG)
// return br.ReadUInt64();
// else if (t == BlueDataValidTypes.SINGLE)
// return br.ReadSingle();
// else
// throw new ApplicationException("Unsupported type");
// }
// }
protected Hashtable myIndexByColName = new Hashtable();
protected ColumnInfo[] myIndexByColOrder;
private string[] myTableColNames;
private BlueRowLookupGroup[] myLookupGroups;
protected string myTableName;
// protected Hashtable myIndexByColName = new Hashtable();
// protected ColumnInfo[] myIndexByColOrder;
// private string[] myTableColNames;
// private BlueRowLookupGroup[] myLookupGroups;
// protected string myTableName;
public LookupTableSchema() { }
// public LookupTableSchema() { }
public LookupTableSchema(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
{
myConstruct(tableName, cis, groups);
}
// public LookupTableSchema(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
// {
// myConstruct(tableName, cis, groups);
// }
private void myConstruct(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
{
myIndexByColOrder = new ColumnInfo[cis.Length];
myTableName = tableName;
foreach (ColumnInfo ci in cis)
{
myIndexByColOrder[ci.colNo] = ci;
myIndexByColName[ci.colName] = ci;
}
myLookupGroups = groups;
// private void myConstruct(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
// {
// myIndexByColOrder = new ColumnInfo[cis.Length];
// myTableName = tableName;
// foreach (ColumnInfo ci in cis)
// {
// myIndexByColOrder[ci.colNo] = ci;
// myIndexByColName[ci.colName] = ci;
// }
// myLookupGroups = groups;
foreach (BlueRowLookupGroup group in groups)
{
group.buildEntrySchema(this);
// foreach (BlueRowLookupGroup group in groups)
// {
// group.buildEntrySchema(this);
}
}
// }
// }
public ColumnInfo this[string colName]
{
get
{
object retVal = myIndexByColName[colName];
return retVal != null ? retVal as ColumnInfo : null;
}
}
// public ColumnInfo this[string colName]
// {
// get
// {
// object retVal = myIndexByColName[colName];
// return retVal != null ? retVal as ColumnInfo : null;
// }
// }
public BlueRowLookupGroup[] lookupGroups
{
get { return myLookupGroups; }
// public BlueRowLookupGroup[] lookupGroups
// {
// get { return myLookupGroups; }
}
public ColumnInfo this[int colNo]
{
get { return myIndexByColOrder[colNo]; }
}
// }
// public ColumnInfo this[int colNo]
// {
// get { return myIndexByColOrder[colNo]; }
// }
/// <summary>
/// The returned Array is sorted according to the Column No
/// </summary>
public string[] columnNames
{
get
{
if (myTableColNames == null)
{
myTableColNames = new string[myIndexByColName.Count];
foreach (DictionaryEntry de in myIndexByColName)
{
ColumnInfo temp = (ColumnInfo)de.Value;
myTableColNames[temp.colNo] = temp.colName;
}
}
return myTableColNames;
}
}
// /// <summary>
// /// The returned Array is sorted according to the Column No
// /// </summary>
// public string[] columnNames
// {
// get
// {
// if (myTableColNames == null)
// {
// myTableColNames = new string[myIndexByColName.Count];
// foreach (DictionaryEntry de in myIndexByColName)
// {
// ColumnInfo temp = (ColumnInfo)de.Value;
// myTableColNames[temp.colNo] = temp.colName;
// }
// }
// return myTableColNames;
// }
// }
/// <summary>
/// The returned Array is sorted according to the Column No
/// </summary>
public ColumnInfo[] columnInfo
{
get { return myIndexByColOrder; }
}
// /// <summary>
// /// The returned Array is sorted according to the Column No
// /// </summary>
// public ColumnInfo[] columnInfo
// {
// get { return myIndexByColOrder; }
// }
public string tableName { get { return myTableName; } }
// public string tableName { get { return myTableName; } }
#region ISerializedType methods
public void serialize(BinaryWriter bw)
{
bw.Write(columnNames.Length);
bw.Write(tableName);
// #region ISerializedType methods
// public void serialize(BinaryWriter bw)
// {
// bw.Write(columnNames.Length);
// bw.Write(tableName);
//first, write the column names
foreach (ColumnInfo ci in columnInfo)
{
bw.Write(ci.colName);
bw.Write(ci.colType.ToString());
if (ci.isConstant)
{
bw.Write(true);
LookupTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
}
else
{
bw.Write(false);
bw.Write(ci.lookupTableGroup);
}
// //first, write the column names
// foreach (ColumnInfo ci in columnInfo)
// {
// bw.Write(ci.colName);
// bw.Write(ci.colType.ToString());
// if (ci.isConstant)
// {
// bw.Write(true);
// LookupTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
// }
// else
// {
// bw.Write(false);
// bw.Write(ci.lookupTableGroup);
// }
}
// }
//this is here for error checking
bw.Write(myLookupGroups.Length);
//now, write the size of each lookup table group
foreach (BlueRowLookupGroup group in myLookupGroups)
{
bw.Write(group.groupNumber);
bw.Write(group.tableSize);
bw.Write(group.groupColNames.Length);
foreach (string colName in group.groupColNames)
{
//write the index of this column in the table
bw.Write(this[colName].colNo);
// //this is here for error checking
// bw.Write(myLookupGroups.Length);
// //now, write the size of each lookup table group
// foreach (BlueRowLookupGroup group in myLookupGroups)
// {
// bw.Write(group.groupNumber);
// bw.Write(group.tableSize);
// bw.Write(group.groupColNames.Length);
// foreach (string colName in group.groupColNames)
// {
// //write the index of this column in the table
// bw.Write(this[colName].colNo);
}
// }
}
// }
}
// }
public void deserialze(BinaryReader br)
{
int numCols = br.ReadInt32();
//these are the final output columns that will be created
ColumnInfo[] columnInfos = new ColumnInfo[numCols];
// public void deserialze(BinaryReader br)
// {
// int numCols = br.ReadInt32();
// //these are the final output columns that will be created
// ColumnInfo[] columnInfos = new ColumnInfo[numCols];
Dictionary<int, List<string>> colLookupGroupMap = new Dictionary<int, List<string>>();
// Dictionary<int, List<string>> colLookupGroupMap = new Dictionary<int, List<string>>();
string tableName = br.ReadString();
for (int colNo = 0; colNo < numCols; ++colNo)
{
string colName = br.ReadString();
Type colType = TypeInstanceLoader.loadType(br.ReadString(), true);
bool colIsConst = br.ReadBoolean();
if (colIsConst)
{
columnInfos[colNo] = new ColumnInfo(colName, colNo, LookupTableSchemaSerializationHelper.deserialize(br, colType));
// string tableName = br.ReadString();
// for (int colNo = 0; colNo < numCols; ++colNo)
// {
// string colName = br.ReadString();
// Type colType = TypeInstanceLoader.loadType(br.ReadString(), true);
// bool colIsConst = br.ReadBoolean();
// if (colIsConst)
// {
// columnInfos[colNo] = new ColumnInfo(colName, colNo, LookupTableSchemaSerializationHelper.deserialize(br, colType));
}
else
{
short thisColGroup = br.ReadInt16();
columnInfos[colNo] = new ColumnInfo(colName, colNo, colType, thisColGroup);
}
}
// }
// else
// {
// short thisColGroup = br.ReadInt16();
// columnInfos[colNo] = new ColumnInfo(colName, colNo, colType, thisColGroup);
// }
// }
int numGroups = br.ReadInt32();
BlueRowLookupGroup[] theGroups = new BlueRowLookupGroup[numGroups];
// int numGroups = br.ReadInt32();
// BlueRowLookupGroup[] theGroups = new BlueRowLookupGroup[numGroups];
for (int group = 0; group < numGroups; ++group)
{
short groupNo = br.ReadInt16();
short groupSizeInBits = br.ReadInt16();
int numColsInGroup = br.ReadInt32();
int[] groupCols = new int[numColsInGroup];
string[] groupColNames = new string[numColsInGroup];
for (int col = 0; col < numColsInGroup; ++col)
{
groupCols[col] = br.ReadInt32();
groupColNames[col] = columnInfos[groupCols[col]].colName;
}
// for (int group = 0; group < numGroups; ++group)
// {
// short groupNo = br.ReadInt16();
// short groupSizeInBits = br.ReadInt16();
// int numColsInGroup = br.ReadInt32();
// int[] groupCols = new int[numColsInGroup];
// string[] groupColNames = new string[numColsInGroup];
// for (int col = 0; col < numColsInGroup; ++col)
// {
// groupCols[col] = br.ReadInt32();
// groupColNames[col] = columnInfos[groupCols[col]].colName;
// }
//create the group
theGroups[groupNo] = new BlueRowLookupGroup(groupNo, groupSizeInBits, groupColNames);
// //create the group
// theGroups[groupNo] = new BlueRowLookupGroup(groupNo, groupSizeInBits, groupColNames);
}
// }
myConstruct(tableName, columnInfos, theGroups);
// myConstruct(tableName, columnInfos, theGroups);
}
// }
public int getSerializedSize()
{
throw new NotImplementedException("Feature for provided constant values to" +
"non-primitive types in schema, not provided as of now");
}
// public int getSerializedSize()
// {
// throw new NotImplementedException("Feature for provided constant values to" +
// "non-primitive types in schema, not provided as of now");
// }
#endregion
}
}
// #endregion
// }
// }
@@ -5,8 +5,7 @@ using System.IO;
using BCG.Utilities.General;
//using BCG.Utilities;
namespace BCG
{
namespace BCG {
/// <summary>
/// The SerializedBlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
/// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
@@ -17,12 +16,9 @@ namespace BCG
/// memory streams, and uses this schema to unpack the values when they are needed.
/// </summary>
[Serializable]
public class SerializedBlueTableSchema : IBlueTableSchema
{
public class SerializedBlueTableSchemaSerializationHelper
{
public static object parse(Type t, string data)
{
public class SerializedBlueTableSchema : IBlueTableSchema {
public class SerializedBlueTableSchemaSerializationHelper {
public static object parse(Type t, string data) {
if (t == BlueDataValidTypes.BOOL)
return Boolean.Parse(data);
else if (t == BlueDataValidTypes.BYTE)
@@ -52,8 +48,7 @@ namespace BCG
else
throw new ApplicationException("Can not be parsed");
}
public static void serialize(BinaryWriter bw, Type t, object o)
{
public static void serialize(BinaryWriter bw, Type t, object o) {
if (t == BlueDataValidTypes.BOOL)
bw.Write((bool)o);
else if (t == BlueDataValidTypes.BYTE)
@@ -76,13 +71,11 @@ namespace BCG
bw.Write(o as string);
else if (t == BlueDataValidTypes.SINGLE)
bw.Write((Single)o);
else if (t == BlueDataValidTypes.BYTEARRAY)
{
else if (t == BlueDataValidTypes.BYTEARRAY) {
byte[] bs = o as byte[];
bw.Write(bs.Length);
bw.Write(bs);
}
else if (t == BlueDataValidTypes.UINT)
} else if (t == BlueDataValidTypes.UINT)
bw.Write((uint)o);
else if (t == BlueDataValidTypes.ULONG)
bw.Write((ulong)o);
@@ -90,8 +83,7 @@ namespace BCG
throw new ApplicationException("Unsupported type");
}
public static object deserialize(BinaryReader br, Type t)
{
public static object deserialize(BinaryReader br, Type t) {
if (t == BlueDataValidTypes.BOOL)
return br.ReadBoolean();
else if (t == BlueDataValidTypes.BYTE)
@@ -132,45 +124,37 @@ namespace BCG
public SerializedBlueTableSchema() { }
//copies the schema from an existing schema, building the dispatch table
public SerializedBlueTableSchema(IBlueTableSchema toCopy)
{
public SerializedBlueTableSchema(IBlueTableSchema toCopy) {
int numCols = toCopy.columnInfo.Length;
ColumnInfo[] theColumns = new ColumnInfo[toCopy.columnInfo.Length];
foreach (ColumnInfo toCopyCI in toCopy.columnInfo)
{
foreach (ColumnInfo toCopyCI in toCopy.columnInfo) {
theColumns[toCopyCI.colNo] = toCopyCI;
}
myConstruct(theColumns, toCopy.tableName);
}
public SerializedBlueTableSchema(DataTable table)
{
public SerializedBlueTableSchema(DataTable table) {
ColumnInfo[] theseCols = new ColumnInfo[table.Columns.Count];
myTableName = table.TableName;
foreach (DataColumn col in table.Columns)
{
foreach (DataColumn col in table.Columns) {
theseCols[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
}
myConstruct(theseCols, myTableName);
}
public SerializedBlueTableSchema(ColumnInfo[] cis, string tableName)
{
public SerializedBlueTableSchema(ColumnInfo[] cis, string tableName) {
myConstruct(cis, tableName);
}
public void serializeObject(object data, BinaryWriter bw, int colNo)
{
public void serializeObject(object data, BinaryWriter bw, int colNo) {
mySerializeDispatchTable[colNo](data, bw);
}
private void myConstruct(ColumnInfo[] cis, string tableName)
{
private void myConstruct(ColumnInfo[] cis, string tableName) {
myIndexByColOrder = new ColumnInfo[cis.Length];
myTableName = tableName;
foreach (ColumnInfo ci in cis)
{
foreach (ColumnInfo ci in cis) {
myIndexByColOrder[ci.colNo] = ci;
myIndexByColName[ci.colName] = ci;
}
@@ -179,32 +163,25 @@ namespace BCG
public SerializedBlueTableSchema(string tableName, params ColumnInfo[] cis) : this(cis, tableName) { }
public ColumnInfo this[string colName]
{
get
{
public ColumnInfo this[string colName] {
get {
object retVal = myIndexByColName[colName];
return retVal != null ? retVal as ColumnInfo : null;
}
}
public ColumnInfo this[int colNo]
{
public ColumnInfo this[int colNo] {
get { return myIndexByColOrder[colNo]; }
}
/// <summary>
/// The returned Array is sorted according to the Column No
/// </summary>
public string[] columnNames
{
get
{
if (myTableColNames == null)
{
public string[] columnNames {
get {
if (myTableColNames == null) {
myTableColNames = new string[myIndexByColName.Count];
foreach (DictionaryEntry de in myIndexByColName)
{
foreach (DictionaryEntry de in myIndexByColName) {
ColumnInfo temp = (ColumnInfo)de.Value;
myTableColNames[temp.colNo] = temp.colName;
}
@@ -216,64 +193,46 @@ namespace BCG
/// <summary>
/// The returned Array is sorted according to the Column No
/// </summary>
public ColumnInfo[] columnInfo
{
public ColumnInfo[] columnInfo {
get { return myIndexByColOrder; }
}
public string tableName { get { return myTableName; } }
public int IdColumn
{
public int IdColumn {
get { return myIdCol; }
}
#region ISerializedType methods
public void serialize(BinaryWriter bw)
{
public void serialize(BinaryWriter bw) {
bw.Write(columnNames.Length);
bw.Write(tableName);
foreach (ColumnInfo ci in columnInfo)
{
foreach (ColumnInfo ci in columnInfo) {
bw.Write(ci.colName);
bw.Write(ci.colType.ToString());
if (ci.isConstant)
{
bw.Write(true);
SerializedBlueTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
}
else
{
bw.Write(false);
}
bw.Write(false);
}
}
public void deserialze(BinaryReader br)
{
public void deserialze(BinaryReader br) {
ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()];
string name = br.ReadString();
for (int i = 0; i < cis.Length; i++)
{
for (int i = 0; i < cis.Length; i++) {
string colName = br.ReadString();
Type t = TypeInstanceLoader.loadType(br.ReadString(), true);
bool isConstant = br.ReadBoolean();
if (isConstant)
{
cis[i] = new ColumnInfo(colName, i, t, SerializedBlueTableSchemaSerializationHelper.deserialize(br, t));
}
else
{
if (isConstant) {
cis[i] = new ColumnInfo(colName, i, t);
} else {
cis[i] = new ColumnInfo(colName, i, t);
}
}
myConstruct(cis, name);
}
public int getSerializedSize()
{
public int getSerializedSize() {
throw new NotImplementedException("Feature for provided constant values to" +
"non-primitive types in schema, not provided as of now");
}
@@ -282,8 +241,7 @@ namespace BCG
//builds a dispatch table used to pull values out of the memory streams in the rows.
private void myBuildDispatchTable()
{
private void myBuildDispatchTable() {
int numCols = myIndexByColOrder.Length;
mySerializeDispatchTable = new SerializeDelegate[numCols];
myPullFromByteBufferDispatchTable = new PullFromByteBufferDelegate[numCols];
@@ -292,18 +250,14 @@ namespace BCG
myIdCol = idColumn.colNo;
for (int col = 0; col < numCols; ++col)
{
for (int col = 0; col < numCols; ++col) {
Type thisColType = myIndexByColOrder[col].colType;
//we don't store serializers for the id column, or string columns.
if (col == myIdCol || thisColType == typeof(string))
{
if (col == myIdCol || thisColType == typeof(string)) {
mySerializeDispatchTable[col] = null;
myColumnSizesTable[col] = 0;
myPullFromByteBufferDispatchTable[col] = null;
}
else
{
} else {
mySerializeDispatchTable[col] = myGetDispatchMethodSerialize(thisColType);
myPullFromByteBufferDispatchTable[col] = myGetDispatchMethodPullFromBuffer(thisColType);
myColumnSizesTable[col] = BlueDataValidTypes.getValueTypeSize(thisColType);
@@ -315,16 +269,14 @@ namespace BCG
}
public int getNullMapSize()
{
public int getNullMapSize() {
int numCols = myIndexByColOrder.Length;
return Convert.ToInt32(Math.Ceiling((Convert.ToDouble(numCols) / 8.0)));
}
//this function reads the null array from inside the byte array.
//it has to copy the bytes over.
public BitArray getNullMap(byte[] theBytes)
{
public BitArray getNullMap(byte[] theBytes) {
int nullMapSize = getNullMapSize();
//the null map starts after the id, so that we have instant access to the id.
int byteCursor = BlueDataValidTypes.getValueTypeSize(typeof(Int32));
@@ -336,8 +288,7 @@ namespace BCG
}
//gets the value either from the bytes stream but only if the value is not a string.
//doesn't work if you're getting the id.
public unsafe object getValueNonString(byte[] theBytes, int colNo)
{
public unsafe object getValueNonString(byte[] theBytes, int colNo) {
if (colNo == myIdCol)
throw new BlueDataException("Error: do not use getValueNonString to try to fetch the id of a row");
BitArray nullMap = getNullMap(theBytes);
@@ -345,8 +296,7 @@ namespace BCG
return DBNull.Value;
//the rest of the row starts after the id and the null map
int thisOffset = BlueDataValidTypes.getValueTypeSize(typeof(Int32)) + getNullMapSize();
for (int currentColumn = 0; currentColumn < colNo; ++currentColumn)
{
for (int currentColumn = 0; currentColumn < colNo; ++currentColumn) {
//pull the data from the binary reader, but only if this type is not null or a string
//if the type wer'e looking for is a string, it would have been returned already by the accesor
//in the bluerow.
@@ -357,16 +307,14 @@ namespace BCG
thisOffset += myColumnSizesTable[currentColumn];
}
fixed (byte* thisBytePtr = &theBytes[thisOffset])
{
fixed (byte* thisBytePtr = &theBytes[thisOffset]) {
return myPullFromByteBufferDispatchTable[colNo](thisBytePtr);
}
}
public unsafe object[] getAllValues(byte[] theBytes, string[] stringArray)
{
public unsafe object[] getAllValues(byte[] theBytes, string[] stringArray) {
int numCols = myColumnSizesTable.Length;
object[] theObjs = new Object[numCols];
@@ -374,8 +322,7 @@ namespace BCG
int thisOffset = BlueDataValidTypes.getValueTypeSize(typeof(Int32)) + getNullMapSize();
int thisStringNo = 0;
for (int currentColumn = 0; currentColumn < numCols; ++currentColumn)
{
for (int currentColumn = 0; currentColumn < numCols; ++currentColumn) {
//pull the data from the binary reader, but only if this type is not null or a string
//if the type wer'e looking for is a string, it would have been returned already by the accesor
//in the bluerow.
@@ -384,10 +331,8 @@ namespace BCG
theObjs[currentColumn] = DBNull.Value;
else if (myIndexByColOrder[currentColumn].colType == typeof(string))
theObjs[currentColumn] = stringArray[thisStringNo++];
else if (currentColumn != myIdCol)
{
fixed (byte* thisGuy = &theBytes[thisOffset])
{
else if (currentColumn != myIdCol) {
fixed (byte* thisGuy = &theBytes[thisOffset]) {
theObjs[currentColumn] = myPullFromByteBufferDispatchTable[currentColumn](thisGuy);
}
thisOffset += myColumnSizesTable[currentColumn];
@@ -416,8 +361,7 @@ namespace BCG
private void mySerializeDateTime(object data, BinaryWriter bw) { bw.Write(((DateTime)data).ToOADate()); }
private void mySerializeSingle(object data, BinaryWriter bw) { bw.Write((Single)data); }
private SerializeDelegate myGetDispatchMethodSerialize(Type t)
{
private SerializeDelegate myGetDispatchMethodSerialize(Type t) {
if (t.Equals(BlueDataValidTypes.INT)) return new SerializeDelegate(mySerializeInt32);
else if (t.Equals(BlueDataValidTypes.DOUBLE)) return new SerializeDelegate(mySerializeDouble);
else if (t.Equals(BlueDataValidTypes.CHAR)) return new SerializeDelegate(mySerializeChar);
@@ -448,16 +392,14 @@ namespace BCG
private unsafe object myPullDecimal(byte* bytePtr) { return *((decimal*)bytePtr); }
private unsafe object myPullUInt(byte* bytePtr) { return *((UInt32*)bytePtr); }
private unsafe object myPullULong(byte* bytePtr) { return *((UInt64*)bytePtr); }
private unsafe object myPullDateTime(byte* bytePtr)
{
private unsafe object myPullDateTime(byte* bytePtr) {
double thisDateTime = *((double*)bytePtr);
return DateTime.FromOADate(thisDateTime);
}
private unsafe object myPullSingle(byte* bytePtr) { return *((Single*)bytePtr); ; }
private unsafe PullFromByteBufferDelegate myGetDispatchMethodPullFromBuffer(Type t)
{
private unsafe PullFromByteBufferDelegate myGetDispatchMethodPullFromBuffer(Type t) {
if (t.Equals(BlueDataValidTypes.INT)) return new PullFromByteBufferDelegate(myPullInt32);
else if (t.Equals(BlueDataValidTypes.DOUBLE)) return new PullFromByteBufferDelegate(myPullDouble);
else if (t.Equals(BlueDataValidTypes.CHAR)) return new PullFromByteBufferDelegate(myPullChar);
@@ -1,21 +1,21 @@
using System;
using BCG;
using System.Collections;
using System.IO;
namespace BCG
{
/// <summary>
/// All the data serialized to be send over to PubSub happens here. All it does is f(op, rows) = byte[].
/// </summary>
public interface IBlueDataSerializer
{
namespace BCG {
/// <summary>
/// f(op, rows) = byte[]
/// All the data serialized to be send over to PubSub happens here. All it does is f(op, rows) = byte[].
/// </summary>
/// <param name="op">Operation that can be add, delete, change etc</param>
/// <param name="rows">The data to serialize. The only case when it will be an array of more than 1 element
/// is when there are multiple deletes and adds, ie StreamOpCode.multipleRemoveAdd.
/// </param>
/// <param name="serializedData">The data serialized as bytes to be send over the wire </param>
void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData);
}
}
public interface IBlueDataSerializer {
/// <summary>
/// f(op, rows) = byte[]
/// </summary>
/// <param name="op">Operation that can be add, delete, change etc</param>
/// <param name="rows">The data to serialize. The only case when it will be an array of more than 1 element
/// is when there are multiple deletes and adds, ie StreamOpCode.multipleRemoveAdd.
/// </param>
/// <param name="serializedData">The data serialized as bytes to be send over the wire </param>
void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData);
}
}
@@ -1,171 +1,171 @@
using System;
using System.Collections;
using System.IO;
using System.IO.Compression;
using BCG.Utilities;
//using BCG.Utilities.BlueConfiguration;
// using System;
// using System.Collections;
// using System.IO;
// using System.IO.Compression;
// using BCG.Utilities;
// //using BCG.Utilities.BlueConfiguration;
namespace BCG
{
public class LookupTableClientDeserializer : IClientDeserializer
{
public LookupTableClientDeserializer() {
// namespace BCG
// {
// public class LookupTableClientDeserializer : IClientDeserializer
// {
// public LookupTableClientDeserializer() {
mySchema = new LookupTableSchema();
myLookupTables = null;
}
// mySchema = new LookupTableSchema();
// myLookupTables = null;
// }
private IClientRowDeserializer myRowDeserializer;
private LookupTableSchema mySchema;
private LookupTable[] myLookupTables;
// private IClientRowDeserializer myRowDeserializer;
// private LookupTableSchema mySchema;
// private LookupTable[] myLookupTables;
public BinaryReader myGetReaderForRestOfStream(Stream theStream)
{
//teh rest of the stream is only compressed data
long compressedBytesLength = theStream.Length - theStream.Position;
byte[] compressedBytes = new byte[compressedBytesLength];
theStream.Read(compressedBytes, 0, (int)compressedBytesLength);
// public BinaryReader myGetReaderForRestOfStream(Stream theStream)
// {
// //teh rest of the stream is only compressed data
// long compressedBytesLength = theStream.Length - theStream.Position;
// byte[] compressedBytes = new byte[compressedBytesLength];
// theStream.Read(compressedBytes, 0, (int)compressedBytesLength);
//get a stream to read the uncompressed data
SharpZipCompressor deKompressor = new SharpZipCompressor();
// //get a stream to read the uncompressed data
// SharpZipCompressor deKompressor = new SharpZipCompressor();
MemoryStream decompressed = new MemoryStream();
deKompressor.decompress(new MemoryStream(compressedBytes), decompressed);
decompressed.Position = 0;
// MemoryStream decompressed = new MemoryStream();
// deKompressor.decompress(new MemoryStream(compressedBytes), decompressed);
// decompressed.Position = 0;
return new BinaryReader(decompressed);
}
#region IClientDeserializer methods
public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
{
ArrayList ops = new ArrayList();
ArrayList rows = new ArrayList();
IBlueRowCollection[] result;
foreach (Stream theStream in streams)
{
byte op;
// return new BinaryReader(decompressed);
// }
// #region IClientDeserializer methods
BinaryReader br = new BinaryReader(theStream);
// public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
// {
// ArrayList ops = new ArrayList();
// ArrayList rows = new ArrayList();
// IBlueRowCollection[] result;
// foreach (Stream theStream in streams)
// {
// byte op;
op = br.ReadByte();
// BinaryReader br = new BinaryReader(theStream);
if (op == StreamOpCode.create)
{
br.ReadString(); //pubsub deserializer
br.ReadString(); //client deserializer
// op = br.ReadByte();
// if (op == StreamOpCode.create)
// {
// br.ReadString(); //pubsub deserializer
// br.ReadString(); //client deserializer
br = myGetReaderForRestOfStream(theStream);
// br = myGetReaderForRestOfStream(theStream);
result = new IBlueRowCollection[] { myHandleCreate(br) };
ops.Add(op);
rows.Add(result);
// result = new IBlueRowCollection[] { myHandleCreate(br) };
// ops.Add(op);
// rows.Add(result);
}
else
{
//the schema and lookup tables should already be created. We just need to update everything.
//figure out how long the rest of the compressed data is
br = myGetReaderForRestOfStream(theStream);
// }
// else
// {
// //the schema and lookup tables should already be created. We just need to update everything.
// //figure out how long the rest of the compressed data is
// br = myGetReaderForRestOfStream(theStream);
if (mySchema != null)
{
if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
{
myDeserializeLookupTableUpdates(br);
result = new IBlueRowCollection[] { myGetRows(br, op, 1) };
ops.Add(op);
rows.Add(result);
}
else if (op == StreamOpCode.multipleChange)
{
myDeserializeLookupTableUpdates(br);
int numRows = br.ReadInt32();
result = new IBlueRowCollection[] { myGetRows(br, op, numRows) };
ops.Add(op);
rows.Add(result);
// if (mySchema != null)
// {
// if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
// {
// myDeserializeLookupTableUpdates(br);
// result = new IBlueRowCollection[] { myGetRows(br, op, 1) };
// ops.Add(op);
// rows.Add(result);
// }
// else if (op == StreamOpCode.multipleChange)
// {
// myDeserializeLookupTableUpdates(br);
// int numRows = br.ReadInt32();
// result = new IBlueRowCollection[] { myGetRows(br, op, numRows) };
// ops.Add(op);
// rows.Add(result);
}
else if (op == StreamOpCode.multipleRemoveAdd)
{
myDeserializeLookupTableUpdates(br);
result = new IBlueRowCollection[2];
int numDeletedRows = br.ReadInt32();
result[0] = myGetRows(br, StreamOpCode.remove, numDeletedRows);
int numAddedRows = br.ReadInt32();
result[1] = myGetRows(br, StreamOpCode.add, numAddedRows);
ops.Add(op);
rows.Add(result);
}
else{}
// BlueEventLog.logEvent("Illegal op-code recieved", BlueEventLevel.Error);
// }
// else if (op == StreamOpCode.multipleRemoveAdd)
// {
// myDeserializeLookupTableUpdates(br);
// result = new IBlueRowCollection[2];
// int numDeletedRows = br.ReadInt32();
// result[0] = myGetRows(br, StreamOpCode.remove, numDeletedRows);
// int numAddedRows = br.ReadInt32();
// result[1] = myGetRows(br, StreamOpCode.add, numAddedRows);
// ops.Add(op);
// rows.Add(result);
// }
// else{}
// // BlueEventLog.logEvent("Illegal op-code recieved", BlueEventLevel.Error);
}
else
{
// BlueEventLog.logEvent("Received a non create operation even though the schema" +
// "has never been created", BlueEventLevel.Error);
}
// }
// else
// {
// // BlueEventLog.logEvent("Received a non create operation even though the schema" +
// // "has never been created", BlueEventLevel.Error);
// }
}
// }
}
// }
theOps = ops.ToArray(typeof(byte)) as byte[];
theRows = rows.ToArray();
// theOps = ops.ToArray(typeof(byte)) as byte[];
// theRows = rows.ToArray();
}
// }
#endregion
// #endregion
IBlueRowCollection myHandleCreate(BinaryReader br)
{
// IBlueRowCollection myHandleCreate(BinaryReader br)
// {
mySchema.deserialze(br);
// mySchema.deserialze(br);
//build the lookup tables now
// There is one less than the length of the lookupTableGroups because the first one
//is always the default group which isn't stored in a lookup table.
myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1];
// //build the lookup tables now
// // There is one less than the length of the lookupTableGroups because the first one
// //is always the default group which isn't stored in a lookup table.
// myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1];
//start with 1 because we don't store the 0th lookup table
for (int i = 0; i < myLookupTables.Length; ++i)
myLookupTables[i] = new LookupTable(mySchema, i+1);
// //start with 1 because we don't store the 0th lookup table
// for (int i = 0; i < myLookupTables.Length; ++i)
// myLookupTables[i] = new LookupTable(mySchema, i+1);
myDeserializeLookupTableUpdates(br);
//now read the rows
int numRows = br.ReadInt32();
return myGetRows(br, StreamOpCode.create, numRows);
// myDeserializeLookupTableUpdates(br);
// //now read the rows
// int numRows = br.ReadInt32();
// return myGetRows(br, StreamOpCode.create, numRows);
}
// }
private IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows)
{
BlueTable retVal = new BlueTable(mySchema);
myRowDeserializer = myRowDeserializer == null ? new LookupTableClientRowDeserializer(mySchema,myLookupTables) : myRowDeserializer;
for (int i = 0; i < nRows; i++)
{
IBlueRow row;
myRowDeserializer.deserializeRow(null, br, op, out row);
retVal.addRow(row);
}
return retVal;
}
// private IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows)
// {
// BlueTable retVal = new BlueTable(mySchema);
// myRowDeserializer = myRowDeserializer == null ? new LookupTableClientRowDeserializer(mySchema,myLookupTables) : myRowDeserializer;
// for (int i = 0; i < nRows; i++)
// {
// IBlueRow row;
// myRowDeserializer.deserializeRow(null, br, op, out row);
// retVal.addRow(row);
// }
// return retVal;
// }
private void myDeserializeLookupTableUpdates(BinaryReader br)
{
for (int i = 0; i < myLookupTables.Length; ++i)
myLookupTables[i].deserializeUpdates(br);
}
// private void myDeserializeLookupTableUpdates(BinaryReader br)
// {
// for (int i = 0; i < myLookupTables.Length; ++i)
// myLookupTables[i].deserializeUpdates(br);
// }
}
}
// }
// }
@@ -1,107 +1,107 @@
using System;
using System.Collections;
using System.IO;
// using System;
// using System.Collections;
// using System.IO;
using BCG.Utilities;
//using BCG.Utilities.BlueConfiguration;
// using BCG.Utilities;
// //using BCG.Utilities.BlueConfiguration;
namespace BCG
{
/// <summary>
/// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process
/// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations)
/// </summary>
internal class LookupTableClientRowDeserializer : IClientRowDeserializer
{
private LookupTableSchema mySchema;
private LookupTable[] myTables;
private int myNullMapLength = 0;
private PrimitiveDeserializationHelper myDeserializationHelper;
private BitArray myNullMap;
private int myIdColumn;
public LookupTableClientRowDeserializer (LookupTableSchema schema, LookupTable[] theTables)
{
mySchema = schema;
myTables = theTables;
// namespace BCG
// {
// /// <summary>
// /// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process
// /// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations)
// /// </summary>
// internal class LookupTableClientRowDeserializer : IClientRowDeserializer
// {
// private LookupTableSchema mySchema;
// private LookupTable[] myTables;
// private int myNullMapLength = 0;
// private PrimitiveDeserializationHelper myDeserializationHelper;
// private BitArray myNullMap;
// private int myIdColumn;
// public LookupTableClientRowDeserializer (LookupTableSchema schema, LookupTable[] theTables)
// {
// mySchema = schema;
// myTables = theTables;
myDeserializationHelper = new PrimitiveDeserializationHelper (mySchema);
myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8)));
myIdColumn = schema["id"].colNo;
// myDeserializationHelper = new PrimitiveDeserializationHelper (mySchema);
// myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8)));
// myIdColumn = schema["id"].colNo;
}
// }
#region IClientRowDeserializer Members
// #region IClientRowDeserializer Members
public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
{
int rowLength = br.ReadInt32 (); //length information for the row not needed
//rad the null map
myNullMap = new BitArray(br.ReadBytes (myNullMapLength));
object[] values = new object [mySchema.columnNames.Length];
//now read the id
values[myIdColumn] = br.ReadInt32();
//now read the lookup groups
for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo)
{
BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo];
LookupTable thisTable = myTables[groupNo - 1];
//read the id in, depending upon the size
int idRead = -1;
switch (thisGroup.tableSize)
{
case 8:
idRead = br.ReadByte();
break;
case 16:
idRead = br.ReadInt16();
break;
case 32:
idRead = br.ReadInt32();
break;
default:
throw new BlueDataException("Error: invalid size " + thisGroup.tableSize.ToString() + " specific for lookup group.");
}
// public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
// {
// int rowLength = br.ReadInt32 (); //length information for the row not needed
// //rad the null map
// myNullMap = new BitArray(br.ReadBytes (myNullMapLength));
// object[] values = new object [mySchema.columnNames.Length];
// //now read the id
// values[myIdColumn] = br.ReadInt32();
// //now read the lookup groups
// for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo)
// {
// BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo];
// LookupTable thisTable = myTables[groupNo - 1];
// //read the id in, depending upon the size
// int idRead = -1;
// switch (thisGroup.tableSize)
// {
// case 8:
// idRead = br.ReadByte();
// break;
// case 16:
// idRead = br.ReadInt16();
// break;
// case 32:
// idRead = br.ReadInt32();
// break;
// default:
// throw new BlueDataException("Error: invalid size " + thisGroup.tableSize.ToString() + " specific for lookup group.");
// }
object[] thisGroupValues = thisTable.getValuesForIndex(idRead);
// object[] thisGroupValues = thisTable.getValuesForIndex(idRead);
if (thisGroupValues == null)
throw new BlueDataException("Error: Couldnt' find values for index " + idRead.ToString());
if (thisGroupValues.Length != thisGroup.groupColNos.Length)
throw new BlueDataException("Error: Values extracted for index " + idRead.ToString() + " had length " + thisGroupValues.Length.ToString() + " but length " + thisGroup.groupColNos.Length.ToString() + " was expected.");
// if (thisGroupValues == null)
// throw new BlueDataException("Error: Couldnt' find values for index " + idRead.ToString());
// if (thisGroupValues.Length != thisGroup.groupColNos.Length)
// throw new BlueDataException("Error: Values extracted for index " + idRead.ToString() + " had length " + thisGroupValues.Length.ToString() + " but length " + thisGroup.groupColNos.Length.ToString() + " was expected.");
//insert those values into our object array
// //insert those values into our object array
for (int groupCol = 0; groupCol < thisGroup.groupColNos.Length; ++groupCol)
{
int thisColInOutputRow = thisGroup.groupColNos[groupCol];
values[thisColInOutputRow] = thisGroupValues[groupCol];
}
}
for (int colNo = 0; colNo < values.Length; colNo++)
{
// for (int groupCol = 0; groupCol < thisGroup.groupColNos.Length; ++groupCol)
// {
// int thisColInOutputRow = thisGroup.groupColNos[groupCol];
// values[thisColInOutputRow] = thisGroupValues[groupCol];
// }
// }
// for (int colNo = 0; colNo < values.Length; colNo++)
// {
if (myNullMap[colNo])
{
values[colNo] = DBNull.Value;
}
//if it's non null, not the id column, and not part of a lookup group, get it here.
else if (mySchema[colNo].lookupTableGroup == 0 && colNo != myIdColumn)
{
// if (myNullMap[colNo])
// {
// values[colNo] = DBNull.Value;
// }
// //if it's non null, not the id column, and not part of a lookup group, get it here.
// else if (mySchema[colNo].lookupTableGroup == 0 && colNo != myIdColumn)
// {
values[colNo] = myDeserializationHelper.deserializeColumn(colNo, br);
// values[colNo] = myDeserializationHelper.deserializeColumn(colNo, br);
}
}
// }
// }
r = new BlueRow(mySchema, values);
// r = new BlueRow(mySchema, values);
}
// }
#endregion
}
// #endregion
// }
}
// }
@@ -1,194 +1,194 @@
using System;
using System.Collections.Generic;
using System.Collections;
using System.IO;
using System.Linq;
using System.Text;
// using System;
// using System.Collections.Generic;
// using System.Collections;
// using System.IO;
// using System.Linq;
// using System.Text;
namespace BCG
{
// namespace BCG
// {
/*
*
* LookupTable
*
* This class represents a lookup table used to store values inside data streams. This allows indexes into the table
* to be passed over the wire, instead of actual values of the rows.
*
* Each table can have multiple lookup table groups. This class represents just one of those groups. This means that all the entries
* contained in the table will have the same schema.
*
*
*
* */
public class LookupTable
{
//each of the entries has the same schema - that is, each of the entries contains the same number of values, with the same
IBlueTableSchema myEntrySchema;
UniversalRowSerializer mySerializer;
ClientUniversalRowDeserializer myDeserializer;
private const int myIdColumn = 0;
// /*
// *
// * LookupTable
// *
// * This class represents a lookup table used to store values inside data streams. This allows indexes into the table
// * to be passed over the wire, instead of actual values of the rows.
// *
// * Each table can have multiple lookup table groups. This class represents just one of those groups. This means that all the entries
// * contained in the table will have the same schema.
// *
// *
// *
// * */
// public class LookupTable
// {
// //each of the entries has the same schema - that is, each of the entries contains the same number of values, with the same
// IBlueTableSchema myEntrySchema;
// UniversalRowSerializer mySerializer;
// ClientUniversalRowDeserializer myDeserializer;
// private const int myIdColumn = 0;
//the table is only allowed to reach a maximum size based upon the number of bits you give to it.
//it throws an error if you try to put too much data into it.
private long myMaxSize;
private int mySizeInBits;
// //the table is only allowed to reach a maximum size based upon the number of bits you give to it.
// //it throws an error if you try to put too much data into it.
// private long myMaxSize;
// private int mySizeInBits;
public int sizeInBits
{
get { return mySizeInBits; }
}
// public int sizeInBits
// {
// get { return mySizeInBits; }
// }
KeysBasedTable2 myValuesToEntryMap;
// KeysBasedTable2 myValuesToEntryMap;
//keep track of the last time somebody published you, so you know what values to publish.
int myIdOfLastPublish;
// //keep track of the last time somebody published you, so you know what values to publish.
// int myIdOfLastPublish;
int myIdToUse;
// int myIdToUse;
/* builds the local schema, serializer, and desrializer
* initializes to an empty table
*/
public LookupTable(LookupTableSchema theSchema, int lookupGroupNumber)
{
BlueRowLookupGroup group = theSchema.lookupGroups[lookupGroupNumber];
// /* builds the local schema, serializer, and desrializer
// * initializes to an empty table
// */
// public LookupTable(LookupTableSchema theSchema, int lookupGroupNumber)
// {
// BlueRowLookupGroup group = theSchema.lookupGroups[lookupGroupNumber];
myEntrySchema = group.entrySchema;
// myEntrySchema = group.entrySchema;
//compute how much room you have in the table
mySizeInBits = group.tableSize;
myMaxSize = (long)(Math.Pow(2, group.tableSize) - 1);
// //compute how much room you have in the table
// mySizeInBits = group.tableSize;
// myMaxSize = (long)(Math.Pow(2, group.tableSize) - 1);
myDeserializer = new ClientUniversalRowDeserializer(myEntrySchema);
mySerializer = new UniversalRowSerializer(myEntrySchema);
// myDeserializer = new ClientUniversalRowDeserializer(myEntrySchema);
// mySerializer = new UniversalRowSerializer(myEntrySchema);
//create the map
myValuesToEntryMap = new KeysBasedTable2(myEntrySchema, group.groupColNames);
myIdOfLastPublish = 0;
// //create the map
// myValuesToEntryMap = new KeysBasedTable2(myEntrySchema, group.groupColNames);
// myIdOfLastPublish = 0;
}
// }
/*Checks to see if the given values are in the table. IF they are in there, it returns the index to those values.
If they are not present, it adds them to teh table and returns the new index. */
public int getIndexForValues(params object[] values)
{
// /*Checks to see if the given values are in the table. IF they are in there, it returns the index to those values.
// If they are not present, it adds them to teh table and returns the new index. */
// public int getIndexForValues(params object[] values)
// {
int index = 0;
//see if the key is present
IBlueRow thisRowEntry = myValuesToEntryMap.getRow(values);
// int index = 0;
// //see if the key is present
// IBlueRow thisRowEntry = myValuesToEntryMap.getRow(values);
if (thisRowEntry == null)
{
//check to see if we even have room in the table
if (myIdToUse > myMaxSize)
throw new BlueDataException("Error: LookupTable has reached maximum size. Make a bigger table.");
//we need to entrer this new value into the table. Create a blue row using the next ID
//and the values they supplied to us
object[] thisRowValues = new object[myEntrySchema.columnInfo.Length];
index = myIdToUse;
thisRowValues[myIdColumn] = index;
myIdToUse ++;
//now add in the values they supplied
for (int colNo = myIdColumn+1; colNo < myEntrySchema.columnInfo.Length; ++colNo)
{
thisRowValues[colNo] = values[colNo - 1];
// if (thisRowEntry == null)
// {
// //check to see if we even have room in the table
// if (myIdToUse > myMaxSize)
// throw new BlueDataException("Error: LookupTable has reached maximum size. Make a bigger table.");
// //we need to entrer this new value into the table. Create a blue row using the next ID
// //and the values they supplied to us
// object[] thisRowValues = new object[myEntrySchema.columnInfo.Length];
// index = myIdToUse;
// thisRowValues[myIdColumn] = index;
// myIdToUse ++;
// //now add in the values they supplied
// for (int colNo = myIdColumn+1; colNo < myEntrySchema.columnInfo.Length; ++colNo)
// {
// thisRowValues[colNo] = values[colNo - 1];
}
IBlueRow newEntryRow = new BlueRow(myEntrySchema, thisRowValues);
// }
// IBlueRow newEntryRow = new BlueRow(myEntrySchema, thisRowValues);
myValuesToEntryMap.update(newEntryRow);
}
else
index = (int) thisRowEntry["id"];
// myValuesToEntryMap.update(newEntryRow);
// }
// else
// index = (int) thisRowEntry["id"];
return index;
// return index;
}
// }
//queries the table to get the values that were stored on a given index.
//returns null if the values are not found
public object[] getValuesForIndex(int index)
{
object[] results = null;
// //queries the table to get the values that were stored on a given index.
// //returns null if the values are not found
// public object[] getValuesForIndex(int index)
// {
// object[] results = null;
//get the row from the table
if (myValuesToEntryMap.containsID(index))
{
IBlueRow thisRow = myValuesToEntryMap[index];
results = new object[myEntrySchema.columnInfo.Length - 1];
// //get the row from the table
// if (myValuesToEntryMap.containsID(index))
// {
// IBlueRow thisRow = myValuesToEntryMap[index];
// results = new object[myEntrySchema.columnInfo.Length - 1];
//copy the values from the row into the array
for (int colNo = myIdColumn+1; colNo < myEntrySchema.columnInfo.Length; ++ colNo)
{
results[colNo - 1] = thisRow[colNo];
// //copy the values from the row into the array
// for (int colNo = myIdColumn+1; colNo < myEntrySchema.columnInfo.Length; ++ colNo)
// {
// results[colNo - 1] = thisRow[colNo];
}
// }
}
// }
//if the index isn't in the tablem, can just return the null value
// //if the index isn't in the tablem, can just return the null value
return results;
// return results;
}
// }
public void deserializeUpdates(BinaryReader br)
{
// public void deserializeUpdates(BinaryReader br)
// {
//get the number of updates that are being published.
int numUpdates = br.ReadInt32();
// //get the number of updates that are being published.
// int numUpdates = br.ReadInt32();
for (int i = 0; i < numUpdates; ++i)
{
// for (int i = 0; i < numUpdates; ++i)
// {
IBlueRow theNewUpdate = null;
myDeserializer.deserializeRow(null, br, StreamOpCode.create, out theNewUpdate);
myValuesToEntryMap.update(theNewUpdate);
// IBlueRow theNewUpdate = null;
// myDeserializer.deserializeRow(null, br, StreamOpCode.create, out theNewUpdate);
// myValuesToEntryMap.update(theNewUpdate);
int thisID = (int) theNewUpdate[myIdColumn];
if (thisID != myIdToUse)
throw new BlueDataException("Error: bad serialized update given to lookup table. Expected id " + myIdToUse.ToString() + ", found " + thisID.ToString());
++myIdToUse;
// int thisID = (int) theNewUpdate[myIdColumn];
// if (thisID != myIdToUse)
// throw new BlueDataException("Error: bad serialized update given to lookup table. Expected id " + myIdToUse.ToString() + ", found " + thisID.ToString());
// ++myIdToUse;
}
// }
}
// }
public void serializeUpdates(BinaryWriter bw)
{
int numUpdates = myIdToUse - myIdOfLastPublish;
// public void serializeUpdates(BinaryWriter bw)
// {
// int numUpdates = myIdToUse - myIdOfLastPublish;
//serialize the number of updates
bw.Write(numUpdates);
// //serialize the number of updates
// bw.Write(numUpdates);
//serialize each update
for (int i = myIdOfLastPublish; i < myIdToUse; ++i)
mySerializer.serializeRow(myValuesToEntryMap[i], StreamOpCode.add, bw);
// //serialize each update
// for (int i = myIdOfLastPublish; i < myIdToUse; ++i)
// mySerializer.serializeRow(myValuesToEntryMap[i], StreamOpCode.add, bw);
//store the last id you published so that next time we don't serialize the same updates
myIdOfLastPublish = myIdToUse;
}
// //store the last id you published so that next time we don't serialize the same updates
// myIdOfLastPublish = myIdToUse;
// }
/*this function serializes EVERYTHING in the table into the writer. This is
used by the pubsub deserializer, which needs to be able to supply the new state
to any clients connection. */
public void serializeWholeTable(BinaryWriter bw)
{
//serialize the number of updates
bw.Write(myIdToUse);
// /*this function serializes EVERYTHING in the table into the writer. This is
// used by the pubsub deserializer, which needs to be able to supply the new state
// to any clients connection. */
// public void serializeWholeTable(BinaryWriter bw)
// {
// //serialize the number of updates
// bw.Write(myIdToUse);
//serialize each update
for (int i = 0; i < myIdToUse; ++i)
mySerializer.serializeRow(myValuesToEntryMap[i], StreamOpCode.add, bw);
// //serialize each update
// for (int i = 0; i < myIdToUse; ++i)
// mySerializer.serializeRow(myValuesToEntryMap[i], StreamOpCode.add, bw);
}
}
}
// }
// }
// }
@@ -1,226 +1,226 @@
using System;
using System.Collections;
using System.IO;
using BCG.Utilities;
using System.IO.Compression;
// using System;
// using System.Collections;
// using System.IO;
// using BCG.Utilities;
// using System.IO.Compression;
namespace BCG
{
// namespace BCG
// {
public class LookupTablePubSubDeserializer : IPubsubDeserializer
{
protected IDictionary myState = new Hashtable();
protected LookupTableSchema mySchema;
protected string myClientDeserializerName; //for the getState method
protected IPubsubRowDeserializer myDeserializer;
LookupTable[] myLookupTables;
// public class LookupTablePubSubDeserializer : IPubsubDeserializer
// {
// protected IDictionary myState = new Hashtable();
// protected LookupTableSchema mySchema;
// protected string myClientDeserializerName; //for the getState method
// protected IPubsubRowDeserializer myDeserializer;
// LookupTable[] myLookupTables;
public LookupTablePubSubDeserializer() {
// public LookupTablePubSubDeserializer() {
mySchema = new LookupTableSchema();
}
// mySchema = new LookupTableSchema();
// }
public BinaryReader myGetReaderForCompressedPortion(Stream theStream)
{
//teh rest of the stream is only compressed data
long compressedBytesLength = theStream.Length - theStream.Position;
byte[] compressedBytes = new byte[compressedBytesLength];
theStream.Read(compressedBytes, 0, (int)compressedBytesLength);
// public BinaryReader myGetReaderForCompressedPortion(Stream theStream)
// {
// //teh rest of the stream is only compressed data
// long compressedBytesLength = theStream.Length - theStream.Position;
// byte[] compressedBytes = new byte[compressedBytesLength];
// theStream.Read(compressedBytes, 0, (int)compressedBytesLength);
//get a stream to read the uncompressed data
SharpZipCompressor deKompressor = new SharpZipCompressor();
// //get a stream to read the uncompressed data
// SharpZipCompressor deKompressor = new SharpZipCompressor();
MemoryStream decompressed = new MemoryStream();
deKompressor.decompress(new MemoryStream(compressedBytes), decompressed);
decompressed.Position = 0;
// MemoryStream decompressed = new MemoryStream();
// deKompressor.decompress(new MemoryStream(compressedBytes), decompressed);
// decompressed.Position = 0;
return new BinaryReader(decompressed);
}
// return new BinaryReader(decompressed);
// }
#region IPubsubDeserializer Members
public virtual void deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
{
ArrayList ops = new ArrayList(), rows = new ArrayList();
foreach (Stream theStream in streams)
{
BinaryReader br = new BinaryReader(theStream);
IDictionary[] result;
byte op;
op = br.ReadByte();
if (op == StreamOpCode.create)
{
myState.Clear();
br.ReadString(); //the pubsub-deserializer name, should be the same as this's name
myClientDeserializerName = br.ReadString(); //the client-deserializer name
//at this point, the rest of the bytes in the stream are from a GZIP stream
//so we need to read into a GZipStream and decompress
//figure out how long the rest of the compressed data is
br = myGetReaderForCompressedPortion(theStream);
// #region IPubsubDeserializer Members
// public virtual void deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
// {
// ArrayList ops = new ArrayList(), rows = new ArrayList();
// foreach (Stream theStream in streams)
// {
// BinaryReader br = new BinaryReader(theStream);
// IDictionary[] result;
// byte op;
// op = br.ReadByte();
mySchema.deserialze(br);
//now build the lookup tables from the knowledge gleaned at the schema
myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1];
// if (op == StreamOpCode.create)
// {
// myState.Clear();
// br.ReadString(); //the pubsub-deserializer name, should be the same as this's name
// myClientDeserializerName = br.ReadString(); //the client-deserializer name
//start with 1 because we don't store the 0th lookup table
for (int i = 0; i < myLookupTables.Length; ++i)
myLookupTables[i] = new LookupTable(mySchema, i + 1);
myDeserializeLookupTableUpdates(br);
int numRows = br.ReadInt32();
result = new IDictionary[] { myGetRows(br, op, numRows) };
myEncorporatRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else
{
//figure out how long the rest of the compressed data is
br = myGetReaderForCompressedPortion(theStream);
if (mySchema != null)
{
if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
{
myDeserializeLookupTableUpdates(br);
result = new IDictionary[] { myGetRows(br, op, 1) };
myEncorporatRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else if (op == StreamOpCode.multipleChange)
{
myDeserializeLookupTableUpdates(br);
int numRows = br.ReadInt32();
result = new IDictionary[] { myGetRows(br, op, numRows) };
myEncorporatRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else if (op == StreamOpCode.multipleRemoveAdd)
{
myDeserializeLookupTableUpdates(br);
int numRemoved = br.ReadInt32();
result = new IDictionary[2];
result[0] = myGetRows(br, StreamOpCode.remove, numRemoved);
int numAdded = br.ReadInt32();
result[1] = myGetRows (br, StreamOpCode.add, numAdded);
myEncorporatRows(result[0], StreamOpCode.remove);
myEncorporatRows(result[1], StreamOpCode.add);
ops.Add(op);
rows.Add(result);
}
else
{
// BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error);
}
}
else
{
// BlueEventLog.logEvent("Received a non create operation even though the schema" +
// "has never been created", BlueEventLevel.Warning);
}
}
}
theOps = ops.ToArray(typeof(byte)) as byte[];
theRows = rows.ToArray();
}
/// <summary>
/// Sends in a create signal, so that new clients can get a create everytime and start
/// afresh from there
/// </summary>
/// <returns></returns>
public virtual byte[] getState()
{
if (mySchema == null)
return new byte[] { };
else
{
BinaryWriter bw = new BinaryWriter(new MemoryStream());
bw.Write(StreamOpCode.create);
bw.Write(GetType().FullName);
bw.Write(myClientDeserializerName);
//the rest of this data has to be serialized.
BinaryWriter compressedBw = new BinaryWriter(new MemoryStream());
mySchema.serialize(compressedBw);
//serialize all the tables
for (int i = 0; i < myLookupTables.Length; ++i)
myLookupTables[i].serializeWholeTable(compressedBw);
compressedBw.Write(myState.Count);
foreach (byte[] row in myState.Values)
compressedBw.Write(row);
//now compress the bytes
MemoryStream compressedStream = new MemoryStream();
SharpZipCompressor Kompressor = new SharpZipCompressor();
compressedBw.BaseStream.Position = 0;
Kompressor.compress(compressedBw.BaseStream, compressedStream);
//append the two streams together
byte[] output = new byte[bw.BaseStream.Length + compressedStream.Length];
bw.BaseStream.Position = 0;
compressedStream.Position = 0;
// //at this point, the rest of the bytes in the stream are from a GZIP stream
// //so we need to read into a GZipStream and decompress
// //figure out how long the rest of the compressed data is
// br = myGetReaderForCompressedPortion(theStream);
bw.BaseStream.Read(output, 0, (int) bw.BaseStream.Length);
compressedStream.Read(output, (int) bw.BaseStream.Length, (int) compressedStream.Length);
// mySchema.deserialze(br);
// //now build the lookup tables from the knowledge gleaned at the schema
// myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1];
return output;
}
}
#endregion
// //start with 1 because we don't store the 0th lookup table
// for (int i = 0; i < myLookupTables.Length; ++i)
// myLookupTables[i] = new LookupTable(mySchema, i + 1);
private void myDeserializeLookupTableUpdates(BinaryReader br)
{
for (int i = 0; i < myLookupTables.Length; ++i)
myLookupTables[i].deserializeUpdates(br);
}
/// <summary>
/// Every row that has been deserialized passes through this method, so that it can update the state.
/// </summary>
/// <param name="rows"></param>
/// <param name="op"></param>
protected virtual void myEncorporatRows(IDictionary rows, byte op)
{
if (op == StreamOpCode.add || op == StreamOpCode.create)
foreach (DictionaryEntry de in rows)
myState.Add(de.Key, de.Value);
else if (op == StreamOpCode.remove)
foreach (DictionaryEntry de in rows)
myState.Remove(de.Key);
else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange)
foreach (DictionaryEntry de in rows)
myState[de.Key] = de.Value;
}
// myDeserializeLookupTableUpdates(br);
private IDictionary myGetRows(BinaryReader br, byte op, int noOfRows)
{
//the row deserializer doesn't need to know anything about the lookup tables
myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer(mySchema) : myDeserializer;
Hashtable retVal = new Hashtable();
for (int i = 0; i < noOfRows; i++)
{
int id; object row;
myDeserializer.deserializeRow(myState, br, op, out id, out row);
retVal.Add(id, row);
}
return retVal;
}
// int numRows = br.ReadInt32();
// result = new IDictionary[] { myGetRows(br, op, numRows) };
// myEncorporatRows(result[0], op);
// ops.Add(op);
// rows.Add(result);
// }
// else
// {
// //figure out how long the rest of the compressed data is
// br = myGetReaderForCompressedPortion(theStream);
// if (mySchema != null)
// {
// if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
// {
// myDeserializeLookupTableUpdates(br);
// result = new IDictionary[] { myGetRows(br, op, 1) };
// myEncorporatRows(result[0], op);
// ops.Add(op);
// rows.Add(result);
// }
// else if (op == StreamOpCode.multipleChange)
// {
// myDeserializeLookupTableUpdates(br);
// int numRows = br.ReadInt32();
// result = new IDictionary[] { myGetRows(br, op, numRows) };
// myEncorporatRows(result[0], op);
// ops.Add(op);
// rows.Add(result);
// }
// else if (op == StreamOpCode.multipleRemoveAdd)
// {
// myDeserializeLookupTableUpdates(br);
}
}
// int numRemoved = br.ReadInt32();
// result = new IDictionary[2];
// result[0] = myGetRows(br, StreamOpCode.remove, numRemoved);
// int numAdded = br.ReadInt32();
// result[1] = myGetRows (br, StreamOpCode.add, numAdded);
// myEncorporatRows(result[0], StreamOpCode.remove);
// myEncorporatRows(result[1], StreamOpCode.add);
// ops.Add(op);
// rows.Add(result);
// }
// else
// {
// // BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error);
// }
// }
// else
// {
// // BlueEventLog.logEvent("Received a non create operation even though the schema" +
// // "has never been created", BlueEventLevel.Warning);
// }
// }
// }
// theOps = ops.ToArray(typeof(byte)) as byte[];
// theRows = rows.ToArray();
// }
// /// <summary>
// /// Sends in a create signal, so that new clients can get a create everytime and start
// /// afresh from there
// /// </summary>
// /// <returns></returns>
// public virtual byte[] getState()
// {
// if (mySchema == null)
// return new byte[] { };
// else
// {
// BinaryWriter bw = new BinaryWriter(new MemoryStream());
// bw.Write(StreamOpCode.create);
// bw.Write(GetType().FullName);
// bw.Write(myClientDeserializerName);
// //the rest of this data has to be serialized.
// BinaryWriter compressedBw = new BinaryWriter(new MemoryStream());
// mySchema.serialize(compressedBw);
// //serialize all the tables
// for (int i = 0; i < myLookupTables.Length; ++i)
// myLookupTables[i].serializeWholeTable(compressedBw);
// compressedBw.Write(myState.Count);
// foreach (byte[] row in myState.Values)
// compressedBw.Write(row);
// //now compress the bytes
// MemoryStream compressedStream = new MemoryStream();
// SharpZipCompressor Kompressor = new SharpZipCompressor();
// compressedBw.BaseStream.Position = 0;
// Kompressor.compress(compressedBw.BaseStream, compressedStream);
// //append the two streams together
// byte[] output = new byte[bw.BaseStream.Length + compressedStream.Length];
// bw.BaseStream.Position = 0;
// compressedStream.Position = 0;
// bw.BaseStream.Read(output, 0, (int) bw.BaseStream.Length);
// compressedStream.Read(output, (int) bw.BaseStream.Length, (int) compressedStream.Length);
// return output;
// }
// }
// #endregion
// private void myDeserializeLookupTableUpdates(BinaryReader br)
// {
// for (int i = 0; i < myLookupTables.Length; ++i)
// myLookupTables[i].deserializeUpdates(br);
// }
// /// <summary>
// /// Every row that has been deserialized passes through this method, so that it can update the state.
// /// </summary>
// /// <param name="rows"></param>
// /// <param name="op"></param>
// protected virtual void myEncorporatRows(IDictionary rows, byte op)
// {
// if (op == StreamOpCode.add || op == StreamOpCode.create)
// foreach (DictionaryEntry de in rows)
// myState.Add(de.Key, de.Value);
// else if (op == StreamOpCode.remove)
// foreach (DictionaryEntry de in rows)
// myState.Remove(de.Key);
// else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange)
// foreach (DictionaryEntry de in rows)
// myState[de.Key] = de.Value;
// }
// private IDictionary myGetRows(BinaryReader br, byte op, int noOfRows)
// {
// //the row deserializer doesn't need to know anything about the lookup tables
// myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer(mySchema) : myDeserializer;
// Hashtable retVal = new Hashtable();
// for (int i = 0; i < noOfRows; i++)
// {
// int id; object row;
// myDeserializer.deserializeRow(myState, br, op, out id, out row);
// retVal.Add(id, row);
// }
// return retVal;
// }
// }
// }
@@ -1,211 +1,211 @@
using System;
using System.Collections;
using System.IO;
using BCG.Utilities;
using System.Collections.Generic;
namespace BCG
{
/// <summary>
/// Helps the LookupTableRowSerializer by serializing one row.
///
/// It does NOT sort the schema, even though the sorted schema doesn't do anything.
/// That is because the schema sorting utility is agnostic of schema type, and so it would destory the grouping information
/// which is VITAL
///
/// It performs a mapping : (row)->byte[]
/// </summary
internal class LookupTableRowSerializer : IBlueRowSerializer
{
private PrimitiveSerializationHelper myPrimitiveSerializationHelper;
private int myFixedSize;
private int[] myNonFixedColIndexes;
private int myIdColumn;
private LookupTableSchema mySchema;
private LookupTable[] myTables;
//this null map is the null map that will be written to the wire
//therefore, its values represent the nulls in row, ordered by the-rearranged schema.
private BitArray myNullMap;
private byte[] myNullMapArray; //just so that a new instance is not created everytime
// using System;
// using System.Collections;
// using System.IO;
// using BCG.Utilities;
// using System.Collections.Generic;
// namespace BCG
// {
// /// <summary>
// /// Helps the LookupTableRowSerializer by serializing one row.
// ///
// /// It does NOT sort the schema, even though the sorted schema doesn't do anything.
// /// That is because the schema sorting utility is agnostic of schema type, and so it would destory the grouping information
// /// which is VITAL
// ///
// /// It performs a mapping : (row)->byte[]
// /// </summary
// internal class LookupTableRowSerializer : IBlueRowSerializer
// {
// private PrimitiveSerializationHelper myPrimitiveSerializationHelper;
// private int myFixedSize;
// private int[] myNonFixedColIndexes;
// private int myIdColumn;
// private LookupTableSchema mySchema;
// private LookupTable[] myTables;
// //this null map is the null map that will be written to the wire
// //therefore, its values represent the nulls in row, ordered by the-rearranged schema.
// private BitArray myNullMap;
// private byte[] myNullMapArray; //just so that a new instance is not created everytime
public LookupTableRowSerializer(LookupTableSchema schema, LookupTable[] theTables)
{
// public LookupTableRowSerializer(LookupTableSchema schema, LookupTable[] theTables)
// {
mySchema = schema;
myTables = theTables;
myPrimitiveSerializationHelper = new PrimitiveSerializationHelper(mySchema);
// mySchema = schema;
// myTables = theTables;
// myPrimitiveSerializationHelper = new PrimitiveSerializationHelper(mySchema);
myIdColumn = schema["id"].colNo;
// myIdColumn = schema["id"].colNo;
myNullMapArray =
new byte[Convert.ToInt32(Math.Ceiling(Convert.ToDouble(mySchema.columnInfo.Length) / 8))];
myNullMap =
new BitArray(myNullMapArray);
myInitSizeCalculations();
}
// myNullMapArray =
// new byte[Convert.ToInt32(Math.Ceiling(Convert.ToDouble(mySchema.columnInfo.Length) / 8))];
// myNullMap =
// new BitArray(myNullMapArray);
// myInitSizeCalculations();
// }
//the LookupTableSerializer calls this function on all rows to be serialized,
//before it serializes any of them. It does this to update the lookup tables
//that's becuase it writes the updates to the lookup tables to the byte stream
//before it writes the rows.
public void updateLookupTables(IBlueRow r)
{
for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo)
{
BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo];
LookupTable thisTable = myTables[groupNo - 1];
thisTable.getIndexForValues(thisGroup.extractGroupFromRow(r));
// //the LookupTableSerializer calls this function on all rows to be serialized,
// //before it serializes any of them. It does this to update the lookup tables
// //that's becuase it writes the updates to the lookup tables to the byte stream
// //before it writes the rows.
// public void updateLookupTables(IBlueRow r)
// {
// for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo)
// {
// BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo];
// LookupTable thisTable = myTables[groupNo - 1];
// thisTable.getIndexForValues(thisGroup.extractGroupFromRow(r));
}
// }
}
#region IBlueRowSerializer methods
/// <summary>
/// Row serialized in the following order
/// row size (nullBitMap size + actual row size)
/// null bit map
/// row id
/// row lookup group ids
/// rest of row values
/// </summary>
/// <param name="r"></param>
/// <param name="bw"></param>
public void serializeRow(IBlueRow r, byte op, BinaryWriter bw)
{
myNullMap.SetAll(false);
int sz = myFixedSize;
//first, build the null map
for (int i = 0; i < mySchema.columnNames.Length; i++)
{
if (r[i] == DBNull.Value || r[i] == null)
{
myNullMap[i] = true;
// }
// #region IBlueRowSerializer methods
// /// <summary>
// /// Row serialized in the following order
// /// row size (nullBitMap size + actual row size)
// /// null bit map
// /// row id
// /// row lookup group ids
// /// rest of row values
// /// </summary>
// /// <param name="r"></param>
// /// <param name="bw"></param>
// public void serializeRow(IBlueRow r, byte op, BinaryWriter bw)
// {
// myNullMap.SetAll(false);
// int sz = myFixedSize;
// //first, build the null map
// for (int i = 0; i < mySchema.columnNames.Length; i++)
// {
// if (r[i] == DBNull.Value || r[i] == null)
// {
// myNullMap[i] = true;
//if this thing isn't in a lookup group, it's no longer going to be accounted for in the fixed size of the table
//therefore we need to decrement the size of this row by the size of the type
if (mySchema[i].lookupTableGroup == 0)
{
if (SerializationUtilityMethods.isFixedLengthType(mySchema[i].colType))
sz -= SerializationUtilityMethods.getFixedLengthTypeSize(mySchema[i].colType);
}
}
// //if this thing isn't in a lookup group, it's no longer going to be accounted for in the fixed size of the table
// //therefore we need to decrement the size of this row by the size of the type
// if (mySchema[i].lookupTableGroup == 0)
// {
// if (SerializationUtilityMethods.isFixedLengthType(mySchema[i].colType))
// sz -= SerializationUtilityMethods.getFixedLengthTypeSize(mySchema[i].colType);
// }
// }
}
//now serialize the null map
myNullMap.CopyTo(myNullMapArray, 0);
int thisRowSize = myGetRowSizeInBytes(sz, r);
bw.Write(thisRowSize);
//tehcnically the size calculatinos are all off by 4 becuase tehy don't account for integer telling how big the row is
long startPos = bw.BaseStream.Position;
bw.Write(myNullMapArray);
//write the ID first, for easy access
bw.Write((int)r[myIdColumn]);
//now write all the value stored in lookup table groups
//we don't do this with the first lookup group becuase it's the default one that isn't stored in lookup tables
for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo)
{
BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo];
LookupTable thisTable = myTables[groupNo - 1];
int thisLookupID = thisTable.getIndexForValues(thisGroup.extractGroupFromRow(r));
// }
// //now serialize the null map
// myNullMap.CopyTo(myNullMapArray, 0);
// int thisRowSize = myGetRowSizeInBytes(sz, r);
// bw.Write(thisRowSize);
// //tehcnically the size calculatinos are all off by 4 becuase tehy don't account for integer telling how big the row is
// long startPos = bw.BaseStream.Position;
// bw.Write(myNullMapArray);
// //write the ID first, for easy access
// bw.Write((int)r[myIdColumn]);
// //now write all the value stored in lookup table groups
// //we don't do this with the first lookup group becuase it's the default one that isn't stored in lookup tables
// for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo)
// {
// BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo];
// LookupTable thisTable = myTables[groupNo - 1];
// int thisLookupID = thisTable.getIndexForValues(thisGroup.extractGroupFromRow(r));
//now, depending on the group size, write this value as either a byte, a short, or an int.
// //now, depending on the group size, write this value as either a byte, a short, or an int.
switch (thisGroup.tableSize)
{
case 8:
bw.Write((byte)thisLookupID);
break;
case 16:
bw.Write((short)thisLookupID);
break;
case 32:
bw.Write(thisLookupID);
break;
default:
throw new BlueDataException("Error: invalid size " + thisGroup.tableSize.ToString() + " specified for lookup group.");
// switch (thisGroup.tableSize)
// {
// case 8:
// bw.Write((byte)thisLookupID);
// break;
// case 16:
// bw.Write((short)thisLookupID);
// break;
// case 32:
// bw.Write(thisLookupID);
// break;
// default:
// throw new BlueDataException("Error: invalid size " + thisGroup.tableSize.ToString() + " specified for lookup group.");
}
// }
}
//now write the rest of the individual columns
foreach (ColumnInfo ci in mySchema.columnInfo)
//we have already serialized the id column, as well as all the column groups, and we don't serialize any columns that are null
if ( (!myNullMap[ci.colNo])
&& ci.colNo != myIdColumn
&& ci.lookupTableGroup == 0)
{
// }
// //now write the rest of the individual columns
// foreach (ColumnInfo ci in mySchema.columnInfo)
// //we have already serialized the id column, as well as all the column groups, and we don't serialize any columns that are null
// if ( (!myNullMap[ci.colNo])
// && ci.colNo != myIdColumn
// && ci.lookupTableGroup == 0)
// {
myPrimitiveSerializationHelper.serializeColumn(r, ci.colNo, bw);
// myPrimitiveSerializationHelper.serializeColumn(r, ci.colNo, bw);
}
// }
long stopPos = bw.BaseStream.Position;
// long stopPos = bw.BaseStream.Position;
if (stopPos- startPos != thisRowSize)
throw new BlueDataException("Error: Bad Size Calculation in LookupTableRowSerializer. Expected size was "+thisRowSize.ToString()+", actual size was "+(stopPos-startPos).ToString());
}
// if (stopPos- startPos != thisRowSize)
// throw new BlueDataException("Error: Bad Size Calculation in LookupTableRowSerializer. Expected size was "+thisRowSize.ToString()+", actual size was "+(stopPos-startPos).ToString());
// }
public void serializeSchema(BinaryWriter bw)
{
mySchema.serialize(bw);
// public void serializeSchema(BinaryWriter bw)
// {
// mySchema.serialize(bw);
}
#endregion
// }
// #endregion
#region size calculations
/// <summary>
/// Sets up the fixed length (size in bytes occupied by each of the primitive type)
/// and an index for the non primitive type
/// </summary>
private void myInitSizeCalculations()
{
ArrayList temp = new ArrayList();
myFixedSize = myNullMapArray.Length;
//go through all the columns that aren't in lookup groups, and add up their sizes
foreach (ColumnInfo ci in mySchema.columnInfo)
{
//if you're in a lookup table group, your size is always fixed.
if (ci.lookupTableGroup == 0)
{
if (SerializationUtilityMethods.isFixedLengthType(ci.colType))
myFixedSize += SerializationUtilityMethods.getFixedLengthTypeSize(ci.colType);
else
temp.Add(ci.colNo);
}
}
//now add on the size of the lookup groups
foreach (BlueRowLookupGroup group in mySchema.lookupGroups)
myFixedSize += (group.tableSize / 8);
// #region size calculations
// /// <summary>
// /// Sets up the fixed length (size in bytes occupied by each of the primitive type)
// /// and an index for the non primitive type
// /// </summary>
// private void myInitSizeCalculations()
// {
// ArrayList temp = new ArrayList();
// myFixedSize = myNullMapArray.Length;
// //go through all the columns that aren't in lookup groups, and add up their sizes
// foreach (ColumnInfo ci in mySchema.columnInfo)
// {
// //if you're in a lookup table group, your size is always fixed.
// if (ci.lookupTableGroup == 0)
// {
// if (SerializationUtilityMethods.isFixedLengthType(ci.colType))
// myFixedSize += SerializationUtilityMethods.getFixedLengthTypeSize(ci.colType);
// else
// temp.Add(ci.colNo);
// }
// }
// //now add on the size of the lookup groups
// foreach (BlueRowLookupGroup group in mySchema.lookupGroups)
// myFixedSize += (group.tableSize / 8);
myNonFixedColIndexes = temp.ToArray(typeof(int)) as int[];
}
// myNonFixedColIndexes = temp.ToArray(typeof(int)) as int[];
// }
/// <summary>
/// Find the size of the serialized row, assumes that nullMap has already
/// been built
/// </summary>
/// <param name="fixedSize"></param>
/// <param name="r"></param>
/// <returns></returns>
private int myGetRowSizeInBytes(int fixedSize, IBlueRow r)
{
foreach (int colNo in myNonFixedColIndexes)
{
if (!myNullMap[colNo])
fixedSize += //myGetStringSizeInBytes (r[colNo] as string);
SerializationUtilityMethods.getObjectSerializedSize(r[colNo]);
}
return fixedSize;
}
#endregion
}
}
// /// <summary>
// /// Find the size of the serialized row, assumes that nullMap has already
// /// been built
// /// </summary>
// /// <param name="fixedSize"></param>
// /// <param name="r"></param>
// /// <returns></returns>
// private int myGetRowSizeInBytes(int fixedSize, IBlueRow r)
// {
// foreach (int colNo in myNonFixedColIndexes)
// {
// if (!myNullMap[colNo])
// fixedSize += //myGetStringSizeInBytes (r[colNo] as string);
// SerializationUtilityMethods.getObjectSerializedSize(r[colNo]);
// }
// return fixedSize;
// }
// #endregion
// }
// }
@@ -1,279 +1,279 @@
using System;
using System.IO;
using BCG.Utilities;
using System.Collections.Generic;
using System.IO.Compression;
// using System;
// using System.IO;
// using BCG.Utilities;
// using System.Collections.Generic;
// using System.IO.Compression;
namespace BCG
{
/// <summary>
/// Serializes data stored in all IBlueRowCollections that use a LookupTableSchema.
/// </summary>
public class LookupTableSerializer : IBlueDataSerializer
{
private const string myPubSubDeser = "BCG.LookupTablePubSubDeserializer",
myClientDeser = "BCG.LookupTableClientDeserializer";
// namespace BCG
// {
// /// <summary>
// /// Serializes data stored in all IBlueRowCollections that use a LookupTableSchema.
// /// </summary>
// public class LookupTableSerializer : IBlueDataSerializer
// {
// private const string myPubSubDeser = "BCG.LookupTablePubSubDeserializer",
// myClientDeser = "BCG.LookupTableClientDeserializer";
private LookupTable[] myLookupTables;
private LookupTableSchema mySchema;
private LookupTableRowSerializer myRowSerializer;
// private LookupTable[] myLookupTables;
// private LookupTableSchema mySchema;
// private LookupTableRowSerializer myRowSerializer;
//the header writer writes into a regular stream
private BinaryWriter myHeaderWriter;
//the body writer writes into a compressed binary stream
//that way, we shrink the size of the data that we send over the wire
private BinaryWriter myBodyWriter;
public LookupTableSerializer (IBlueTableSchema inSchema, object[] groups)
{
//get the column infos from the input schema
ColumnInfo[] inputColumns = inSchema.columnInfo;
// //the header writer writes into a regular stream
// private BinaryWriter myHeaderWriter;
// //the body writer writes into a compressed binary stream
// //that way, we shrink the size of the data that we send over the wire
// private BinaryWriter myBodyWriter;
// public LookupTableSerializer (IBlueTableSchema inSchema, object[] groups)
// {
// //get the column infos from the input schema
// ColumnInfo[] inputColumns = inSchema.columnInfo;
//build the lookup table groups
//there is one more group than input string because we have the 'default' group, which isn't stored in a table
int numGroups = groups.Length + 1;
BlueRowLookupGroup[] lookupGroups = new BlueRowLookupGroup[numGroups];
//we start the groups with group no 1, becuase groups with 0 are not stored in lookup tables
short groupNo = 1;
Dictionary<string, short> colNameToGroupNo = new Dictionary<string, short>();
// //build the lookup table groups
// //there is one more group than input string because we have the 'default' group, which isn't stored in a table
// int numGroups = groups.Length + 1;
// BlueRowLookupGroup[] lookupGroups = new BlueRowLookupGroup[numGroups];
// //we start the groups with group no 1, becuase groups with 0 are not stored in lookup tables
// short groupNo = 1;
// Dictionary<string, short> colNameToGroupNo = new Dictionary<string, short>();
foreach (string group in groups)
{
string[] sizeAndColumns = group.Split(';');
// foreach (string group in groups)
// {
// string[] sizeAndColumns = group.Split(';');
if (sizeAndColumns.Length != 2)
throw new ArgumentException("Error, bad Lookup Table Group found: " + group);
// if (sizeAndColumns.Length != 2)
// throw new ArgumentException("Error, bad Lookup Table Group found: " + group);
short tableSize = short.Parse(sizeAndColumns[0]);
// short tableSize = short.Parse(sizeAndColumns[0]);
string[] columns = sizeAndColumns[1].Split(',');
// string[] columns = sizeAndColumns[1].Split(',');
BlueRowLookupGroup thisGroup = new BlueRowLookupGroup(groupNo, tableSize, columns);
lookupGroups[groupNo] = thisGroup;
foreach (string column in columns)
colNameToGroupNo[column] = groupNo;
// BlueRowLookupGroup thisGroup = new BlueRowLookupGroup(groupNo, tableSize, columns);
// lookupGroups[groupNo] = thisGroup;
// foreach (string column in columns)
// colNameToGroupNo[column] = groupNo;
++groupNo;
// ++groupNo;
}
// }
//make the lookup group correpsonding to columns that aren't stored in the lookup table
List<string> nonLookupTableColumns = new List<string>();
foreach (ColumnInfo ci in inputColumns)
{
if (!colNameToGroupNo.ContainsKey(ci.colName))
{
nonLookupTableColumns.Add(ci.colName);
}
}
// //make the lookup group correpsonding to columns that aren't stored in the lookup table
// List<string> nonLookupTableColumns = new List<string>();
// foreach (ColumnInfo ci in inputColumns)
// {
// if (!colNameToGroupNo.ContainsKey(ci.colName))
// {
// nonLookupTableColumns.Add(ci.colName);
// }
// }
lookupGroups[0] = new BlueRowLookupGroup(0, 0, nonLookupTableColumns.ToArray());
// lookupGroups[0] = new BlueRowLookupGroup(0, 0, nonLookupTableColumns.ToArray());
//now build new column infos to replace the old ones
int numCols = inputColumns.Length;
ColumnInfo[] newColumns = new ColumnInfo[numCols];
// //now build new column infos to replace the old ones
// int numCols = inputColumns.Length;
// ColumnInfo[] newColumns = new ColumnInfo[numCols];
for (int col = 0; col < numCols; ++col)
{
ColumnInfo inCol = inputColumns[col];
if (inCol.isConstant)
newColumns[col] = new ColumnInfo(inCol.colName, col, inCol.colType, inCol.colValue);
else
{
string colName = inCol.colName;
if (!colNameToGroupNo.ContainsKey(inCol.colName))
newColumns[col] = new ColumnInfo(colName, col, inCol.colType);
else
newColumns[col] = new ColumnInfo(colName, col, inCol.colType, colNameToGroupNo[colName]);
}
// for (int col = 0; col < numCols; ++col)
// {
// ColumnInfo inCol = inputColumns[col];
// if (inCol.isConstant)
// newColumns[col] = new ColumnInfo(inCol.colName, col, inCol.colType, inCol.colValue);
// else
// {
// string colName = inCol.colName;
// if (!colNameToGroupNo.ContainsKey(inCol.colName))
// newColumns[col] = new ColumnInfo(colName, col, inCol.colType);
// else
// newColumns[col] = new ColumnInfo(colName, col, inCol.colType, colNameToGroupNo[colName]);
// }
}
// }
//now build the schema
mySchema = new LookupTableSchema(inSchema.tableName, newColumns, lookupGroups);
//build the lookup tables. There is one less than the length of the lookupTableGroups because the first one
//is always the default group which isn't stored in a lookup table.
myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1];
//start with 1 because we don't store the 0th lookup table
for (int i = 0; i < myLookupTables.Length; ++i)
{
myLookupTables[i] = new LookupTable(mySchema, i+1);
}
//now create the row serializer
myRowSerializer = new LookupTableRowSerializer(mySchema, myLookupTables);
}
#region IBlueDataSerializer Members
/// <summary>
/// Serializes the data related to a single op.
/// Supports All Ops
///
/// First, a LookupTableUpdate, which is contained in each op, looks like this:
///
///
/// -LookupTableUpdate:
/// group 1 values size \
/// group 1 value |- all serialized by a single LookupTableObject
/// group 1 value /
/// ....
/// group 2 values size
/// group 2 value key
/// ....
///
///
/// Next, the stream formats of the different op codes
///
///
/// -create :
/// op
/// clientDeser name
/// pubsubsDeser name
/// schema (which is a LookupTableSchema)
/// LookupTableUpdate
/// num rows
/// row 1
/// row 2
/// ....
///
///
/// - add, remove, change:
/// op
/// LookupTableUpdate
/// the row
///
/// -multipleRemoveAdd :
/// op
/// LookupTableUpdate
/// num deleted rows
/// deleted row 1
/// deleted row 2
/// ....
/// num added rows
/// added row 1
/// added row 2
/// ...
///
/// -multipleChange
/// op
/// LookupTableUpdate
/// num changed rows
/// changd row 1
/// changed row 2
/// ...
/// changd row n
///
/// </summary>
/// <param name="op"></param>
/// <param name="rows"></param>
/// <param name="serializedData"></param>
public void serialize(byte op, BCG.IBlueRowCollection[] rows, out byte[] serializedData)
{
myHeaderWriter = new BinaryWriter(new MemoryStream());
myBodyWriter = new BinaryWriter(new MemoryStream());
//everything start with an op
myHeaderWriter.Write(op);
/*go through each of the rows we're about the serialize, and populate the lookup table
* that way, when we serialize the lookup tables, well have something to report*/
foreach (IBlueRowCollection collection in rows)
foreach (IBlueRow r in collection)
myRowSerializer.updateLookupTables(r);
if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change)
mySerializeSingleRow(op, BlueDataUtility.extractRow(rows[0], 0));
else if (op == StreamOpCode.create)
mySerializeCreate(rows[0]);
else if (op == StreamOpCode.multipleChange)
mySerializeMultipleChange(rows[0]);
else if (op == StreamOpCode.multipleRemoveAdd)
mySerializeMultipleRemoveAdd(rows[0], rows[1]);
else
{
serializedData = null;
// BlueEventLog.logAssert(false, String.Format("{0} is not a valid op", op));
}
// //now build the schema
// mySchema = new LookupTableSchema(inSchema.tableName, newColumns, lookupGroups);
//copy the data from the header and the body of the message
myHeaderWriter.BaseStream.Position = 0;
myBodyWriter.BaseStream.Position = 0;
int headerLength = (int) myHeaderWriter.BaseStream.Length;
// //build the lookup tables. There is one less than the length of the lookupTableGroups because the first one
// //is always the default group which isn't stored in a lookup table.
// myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1];
MemoryStream zippedMemStream = new MemoryStream();
SharpZipCompressor Kompressor = new SharpZipCompressor();
// //start with 1 because we don't store the 0th lookup table
// for (int i = 0; i < myLookupTables.Length; ++i)
// {
// myLookupTables[i] = new LookupTable(mySchema, i+1);
// }
Kompressor.compress(myBodyWriter.BaseStream,zippedMemStream);
byte[] compressedBytes = zippedMemStream.ToArray();
serializedData = new byte[headerLength + zippedMemStream.Length];
// //now create the row serializer
// myRowSerializer = new LookupTableRowSerializer(mySchema, myLookupTables);
Console.WriteLine(String.Format("Lookuptable Serializer: Header is {0}. Body compresesd from {1} to {2} bytes.", headerLength , myBodyWriter.BaseStream.Length, zippedMemStream.Length));
// }
myHeaderWriter.BaseStream.Read(serializedData, 0, headerLength);
BinaryWriter sizeWriter = new BinaryWriter(new MemoryStream(serializedData));
zippedMemStream.Position = 0;
zippedMemStream.Read(serializedData, headerLength, (int) zippedMemStream.Length);
// #region IBlueDataSerializer Members
// /// <summary>
// /// Serializes the data related to a single op.
// /// Supports All Ops
// ///
// /// First, a LookupTableUpdate, which is contained in each op, looks like this:
// ///
// ///
// /// -LookupTableUpdate:
// /// group 1 values size \
// /// group 1 value |- all serialized by a single LookupTableObject
// /// group 1 value /
// /// ....
// /// group 2 values size
// /// group 2 value key
// /// ....
// ///
// ///
// /// Next, the stream formats of the different op codes
// ///
// ///
// /// -create :
// /// op
// /// clientDeser name
// /// pubsubsDeser name
// /// schema (which is a LookupTableSchema)
// /// LookupTableUpdate
// /// num rows
// /// row 1
// /// row 2
// /// ....
// ///
// ///
// /// - add, remove, change:
// /// op
// /// LookupTableUpdate
// /// the row
// ///
// /// -multipleRemoveAdd :
// /// op
// /// LookupTableUpdate
// /// num deleted rows
// /// deleted row 1
// /// deleted row 2
// /// ....
// /// num added rows
// /// added row 1
// /// added row 2
// /// ...
// ///
// /// -multipleChange
// /// op
// /// LookupTableUpdate
// /// num changed rows
// /// changd row 1
// /// changed row 2
// /// ...
// /// changd row n
// ///
// /// </summary>
// /// <param name="op"></param>
// /// <param name="rows"></param>
// /// <param name="serializedData"></param>
// public void serialize(byte op, BCG.IBlueRowCollection[] rows, out byte[] serializedData)
// {
// myHeaderWriter = new BinaryWriter(new MemoryStream());
// myBodyWriter = new BinaryWriter(new MemoryStream());
// //everything start with an op
// myHeaderWriter.Write(op);
// /*go through each of the rows we're about the serialize, and populate the lookup table
// * that way, when we serialize the lookup tables, well have something to report*/
// foreach (IBlueRowCollection collection in rows)
// foreach (IBlueRow r in collection)
// myRowSerializer.updateLookupTables(r);
// if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change)
// mySerializeSingleRow(op, BlueDataUtility.extractRow(rows[0], 0));
// else if (op == StreamOpCode.create)
// mySerializeCreate(rows[0]);
// else if (op == StreamOpCode.multipleChange)
// mySerializeMultipleChange(rows[0]);
// else if (op == StreamOpCode.multipleRemoveAdd)
// mySerializeMultipleRemoveAdd(rows[0], rows[1]);
// else
// {
// serializedData = null;
// // BlueEventLog.logAssert(false, String.Format("{0} is not a valid op", op));
// }
}
#endregion
void mySerializeSingleRow(byte op, IBlueRow row)
{
mySerializeLookupTableUpdates(myBodyWriter);
myRowSerializer.serializeRow(row, op, myBodyWriter);
}
// //copy the data from the header and the body of the message
// myHeaderWriter.BaseStream.Position = 0;
// myBodyWriter.BaseStream.Position = 0;
// int headerLength = (int) myHeaderWriter.BaseStream.Length;
private void mySerializeCreate (IBlueRowCollection rows)
{
myHeaderWriter.Write(myPubSubDeser);
myHeaderWriter.Write(myClientDeser);
mySchema.serialize(myBodyWriter);
mySerializeLookupTableUpdates(myBodyWriter);
myBodyWriter.Write(rows.Count);
foreach (IBlueRow r in rows)
myRowSerializer.serializeRow(r, StreamOpCode.create, myBodyWriter);
}
private void mySerializeMultipleChange(IBlueRowCollection rows)
{
mySerializeLookupTableUpdates(myBodyWriter);
myBodyWriter.Write(rows.Count);
foreach (IBlueRow r in rows)
myRowSerializer.serializeRow(r,StreamOpCode.change,myBodyWriter);
}
private void mySerializeMultipleRemoveAdd(IBlueRowCollection deletedRows, IBlueRowCollection addedRows)
{
mySerializeLookupTableUpdates(myBodyWriter);
myBodyWriter.Write(deletedRows.Count);
foreach (IBlueRow r in deletedRows)
myRowSerializer.serializeRow(r, StreamOpCode.change, myBodyWriter);
myBodyWriter.Write(addedRows.Count);
foreach (IBlueRow r in addedRows)
myRowSerializer.serializeRow(r, StreamOpCode.add, myBodyWriter);
}
// MemoryStream zippedMemStream = new MemoryStream();
// SharpZipCompressor Kompressor = new SharpZipCompressor();
private void mySerializeLookupTableUpdates(BinaryWriter bw)
{
for (int i = 0; i < myLookupTables.Length; ++i)
myLookupTables[i].serializeUpdates(bw);
}
// Kompressor.compress(myBodyWriter.BaseStream,zippedMemStream);
// byte[] compressedBytes = zippedMemStream.ToArray();
}
}
// serializedData = new byte[headerLength + zippedMemStream.Length];
// Console.WriteLine(String.Format("Lookuptable Serializer: Header is {0}. Body compresesd from {1} to {2} bytes.", headerLength , myBodyWriter.BaseStream.Length, zippedMemStream.Length));
// myHeaderWriter.BaseStream.Read(serializedData, 0, headerLength);
// BinaryWriter sizeWriter = new BinaryWriter(new MemoryStream(serializedData));
// zippedMemStream.Position = 0;
// zippedMemStream.Read(serializedData, headerLength, (int) zippedMemStream.Length);
// }
// #endregion
// void mySerializeSingleRow(byte op, IBlueRow row)
// {
// mySerializeLookupTableUpdates(myBodyWriter);
// myRowSerializer.serializeRow(row, op, myBodyWriter);
// }
// private void mySerializeCreate (IBlueRowCollection rows)
// {
// myHeaderWriter.Write(myPubSubDeser);
// myHeaderWriter.Write(myClientDeser);
// mySchema.serialize(myBodyWriter);
// mySerializeLookupTableUpdates(myBodyWriter);
// myBodyWriter.Write(rows.Count);
// foreach (IBlueRow r in rows)
// myRowSerializer.serializeRow(r, StreamOpCode.create, myBodyWriter);
// }
// private void mySerializeMultipleChange(IBlueRowCollection rows)
// {
// mySerializeLookupTableUpdates(myBodyWriter);
// myBodyWriter.Write(rows.Count);
// foreach (IBlueRow r in rows)
// myRowSerializer.serializeRow(r,StreamOpCode.change,myBodyWriter);
// }
// private void mySerializeMultipleRemoveAdd(IBlueRowCollection deletedRows, IBlueRowCollection addedRows)
// {
// mySerializeLookupTableUpdates(myBodyWriter);
// myBodyWriter.Write(deletedRows.Count);
// foreach (IBlueRow r in deletedRows)
// myRowSerializer.serializeRow(r, StreamOpCode.change, myBodyWriter);
// myBodyWriter.Write(addedRows.Count);
// foreach (IBlueRow r in addedRows)
// myRowSerializer.serializeRow(r, StreamOpCode.add, myBodyWriter);
// }
// private void mySerializeLookupTableUpdates(BinaryWriter bw)
// {
// for (int i = 0; i < myLookupTables.Length; ++i)
// myLookupTables[i].serializeUpdates(bw);
// }
// }
// }
@@ -0,0 +1,137 @@
using System;
using BCG;
using System.Collections;
using System.IO;
// common place for all serialization interfaces
namespace bts {
// this is the data that is sent across the write for stream IO
public class StreamIOMessage {
public enum StreamOp {
CREATE = 0,
// removes and adds multiple rows
// other operations can be expressed with this
MULTIPLE_REMOVE_ADD = 1,
}
public class CreateMessage {
public IBlueRowCollection rows;
public CreateMessage(IBlueRowCollection rows_) {
rows = rows_;
}
public override string ToString() {
string rv = "Create message ";
if (rows == null) {
rv += " rows: null";
return rv;
}
foreach (IBlueRow r in rows) {
rv += $"id: {r["id"]} ";
}
return rv;
}
public override bool Equals(object obj) {
if (obj != null && obj is CreateMessage) {
var other = obj as CreateMessage;
return BlueDataUtility.compute_equality(rows, other.rows);
}
return false;
}
}
public class MultipleRemoveAddMessage {
public IBlueRowCollection removed_rows;
public IBlueRowCollection added_rows;
public MultipleRemoveAddMessage(IBlueRowCollection removed_rows_, IBlueRowCollection added_rows_) {
removed_rows = removed_rows_;
added_rows = added_rows_;
}
public override string ToString() {
string rv = "MultipleRemoveAdd removed: ";
foreach (IBlueRow r in removed_rows) {
rv += $"id: {r["id"]} ";
}
rv += "added: ";
foreach (IBlueRow r in added_rows) {
rv += $"id: {r["id"]} ";
}
return rv;
}
public override bool Equals(object obj) {
if (obj != null && obj is MultipleRemoveAddMessage) {
MultipleRemoveAddMessage other = obj as MultipleRemoveAddMessage;
return BlueDataUtility.compute_equality(removed_rows, other.removed_rows) &&
BlueDataUtility.compute_equality(added_rows, other.added_rows);
}
return false;
}
}
public StreamOp op;
public readonly CreateMessage create_message = null;
public readonly MultipleRemoveAddMessage multiple_remove_add_message = null;
public StreamIOMessage(CreateMessage create_message_) {
op = StreamOp.CREATE;
create_message = create_message_;
multiple_remove_add_message = null;
}
public StreamIOMessage(MultipleRemoveAddMessage multiple_remove_add_message_) {
op = StreamOp.MULTIPLE_REMOVE_ADD;
create_message = null;
multiple_remove_add_message = multiple_remove_add_message_;
}
public override string ToString() {
switch (op) {
case StreamIOMessage.StreamOp.CREATE:
return create_message.ToString();
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD:
return multiple_remove_add_message.ToString();
}
bts.Log.assert(false);
return String.Empty;
}
public override bool Equals(object obj) {
if (obj != null && obj is StreamIOMessage) {
StreamIOMessage other = obj as StreamIOMessage;
if (op == other.op) {
switch (op) {
case StreamOp.CREATE:
return create_message.Equals(other.create_message);
case StreamOp.MULTIPLE_REMOVE_ADD:
return multiple_remove_add_message.Equals(other.multiple_remove_add_message);
}
}
}
return false;
}
}
#region serialization
// This helps putting stream callbacks (op + data) on the wire, to send it to the pubsubserver.
public interface IStreamIOMsgSerializer {
public void serialize(StreamIOMessage op_msg, ref BinaryWriter bw);
}
// Responsible for serializing one row
public interface IStreamRowSerializer {
/// <summary>
/// Serializes a row and writes it into the BinaryWriter. Since its only one row, the ops should be appropriate
/// for one row; the ones allowed are StreamopCode.add, remove and change only.
/// </summary>
public void serialize_row(IBlueRow r, StreamIOMessage.StreamOp op, ref BinaryWriter bw);
}
#endregion
#region Deser
// this is how stream messages received from the wire are deserialized
public interface IStreamMsgDeserilizer {
public StreamIOMessage deserialize(ref BinaryReader br);
}
public interface IStreamRowDeserializer {
IBlueRow deserialize_row(ref BinaryReader br);
}
#endregion
}
@@ -0,0 +1,44 @@
using BCG;
using System.IO;
using System.Net.WebSockets;
namespace bts {
public class StreamIOSerializationController : IStreamIOMsgSerializer {
private IStreamIOMsgSerializer _real_serializer = null;
public StreamIOSerializationController() { }
public void serialize(StreamIOMessage op_msg, ref BinaryWriter bw) {
switch (op_msg.op) {
// <op, client deser, pubsub deser, real_ser>
case StreamIOMessage.StreamOp.CREATE:
var table_name = op_msg.create_message.rows.schema.tableName;
_real_serializer = _get_serializer(table_name);
string client_deser = _get_client_deser_name(table_name);
string pubsub_deser = _get_pubsub_deser_name(table_name);
bw.Write((int)op_msg.op);
bw.write_string_cpp(client_deser);
bw.write_string_cpp(pubsub_deser);
_real_serializer.serialize(op_msg, ref bw);
break;
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD:
bts.Log.assert(_real_serializer != null);
_real_serializer.serialize(op_msg, ref bw);
break;
}
}
private IStreamIOMsgSerializer _get_serializer(string table_name) {
// TODO read config
return new UniversalStreamIOSerializer();
}
private string _get_client_deser_name(string table_name) {
return "UniversalStreamIODeserializer";
}
private string _get_pubsub_deser_name(string table_name) {
return string.Empty;
}
}
}
@@ -0,0 +1,162 @@
namespace bts {
using BCG;
using System;
using System.Collections.Generic;
using System.IO;
using System.Runtime.Serialization.Formatters.Binary;
using System.Text;
public static class StreamSerializationUtils {
// extension method to write a string C++ style (len prefixed)
// public static void write_string_cpp(this BinaryWriter bw, string str) {
// str = str ?? String.Empty;
// int len = str.Length;
// bw.Write(len);
// foreach (char c in str) {
// bw.Write(c);
// }
// }
// public static string read_string_cpp(this BinaryReader br) {
// int len = br.ReadInt32();
// string rv = String.Empty;
// char[] c_arr = br.ReadChars(len);
// return new string(c_arr);
// }
public static byte row_null_value_flag = 1;
public static byte row_nonnull_value_flag = 0;
public static byte[] get_row_nullmap(IBlueRow r) {
byte[] rv = new byte[r.schema.columnInfo.Length];
for (int i = 0; i < r.schema.columnInfo.Length; i++) {
var v = r[i];
bool is_dbnull = v == null || DBNull.Value.Equals(v);
rv[i] = is_dbnull ? row_null_value_flag : row_nonnull_value_flag;
}
return rv;
}
private static Dictionary<Type, Action<BinaryWriter, IBlueRow, int>> _col_ser_map =
new Dictionary<Type, Action<BinaryWriter, IBlueRow, int>>(){
{typeof(bool), (bw, r, col_no) => {bw.Write(r.get<bool>(col_no)); }},
{typeof(byte), (bw, r, col_no) => {bw.Write(r.get<byte>(col_no)); }},
{typeof(char), (bw, r, col_no) => {bw.Write(r.get<char>(col_no)); }},
// TODO
//{typeof(DateTime), (bw, r, col_no) => {bw.Write(r.get<DateTime>(col_no)); }},
{typeof(Decimal), (bw, r, col_no) => {bw.Write(r.get<Decimal>(col_no)); }},
{typeof(Double), (bw, r, col_no) => {bw.Write(r.get<Double>(col_no)); }},
{typeof(Int16), (bw, r, col_no) => {bw.Write(r.get<Int16>(col_no)); }},
{typeof(Int32), (bw, r, col_no) => {bw.Write(r.get<Int32>(col_no)); }},
{typeof(Int64), (bw, r, col_no) => {bw.Write(r.get<Int64>(col_no)); }},
{typeof(SByte), (bw, r, col_no) => {bw.Write(r.get<SByte>(col_no)); }},
{typeof(Single), (bw, r, col_no) => {bw.Write(r.get<Single>(col_no)); }},
{typeof(String), (bw, r, col_no) => {bw.write_string_cpp(r.get<String>(col_no)); }},
{typeof(UInt16), (bw, r, col_no) => {bw.Write(r.get<UInt16>(col_no)); }},
{typeof(UInt32), (bw, r, col_no) => {bw.Write(r.get<UInt32>(col_no)); }},
{typeof(UInt64), (bw, r, col_no) => {bw.Write(r.get<UInt64>(col_no)); }}
};
private static Dictionary<Type, Func<BinaryReader, object>> _col_deser_map =
new Dictionary<Type, Func<BinaryReader, object>>(){
{typeof(bool), (br) => {return br.ReadBoolean();}},
{typeof(byte), (br) => {return br.ReadByte();}},
{typeof(char), (br) => {return br.ReadChar();}},
// TODO DateTime
//{typeof(byte), (br) => {return br.ReadByte()}
{typeof(Decimal), (br) => {return br.ReadDecimal();}},
{typeof(Double), (br) => {return br.ReadDouble();}},
{typeof(Int16), (br) => {return br.ReadInt16();}},
{typeof(Int32), (br) => {return br.ReadInt32();}},
{typeof(Int64), (br) => {return br.ReadInt64();}},
{typeof(SByte), (br) => {return br.ReadSByte();}},
{typeof(Single), (br) => {return br.ReadSingle();}},
{typeof(String), (br) => {return br.read_string_cpp();}},
{typeof(UInt16), (br) => {return br.ReadUInt16();}},
{typeof(UInt32), (br) => {return br.ReadUInt32();}},
{typeof(UInt64), (br) => {return br.ReadUInt64();}},
};
public static void serialize_schema(IBlueTableSchema schema, ref BinaryWriter bw) {
//<name, n_cols, cols>
bw.write_string_cpp(schema.tableName);
bw.Write(schema.columnInfo.Length);
foreach (ColumnInfo ci in schema.columnInfo)
serialize_column_info(ci, ref bw);
}
public static IBlueTableSchema deserialize_schema(ref BinaryReader br) {
var hex_txt = to_hex(br.BaseStream);
bts.Log.Gen.info($"StreamSerializationUtils.deserialize_schema bytes: {hex_txt}");
string table_name = br.read_string_cpp();
int n_cols = br.ReadInt32(); // doing a br.Read was only advancing the data 1 byte!!
List<ColumnInfo> cis = new List<ColumnInfo>();
for (int i = 0; i < n_cols; i++) {
var ci = deserialize_column_info(ref br);
cis.Add(ci);
}
return new BlueTableSchema(table_name, cis.ToArray());
}
public static void serialize_column_info(ColumnInfo ci, ref BinaryWriter bw) {
// <col_no, name, type>
bw.Write(ci.colNo);
bw.write_string_cpp(ci.colName);
TypeCode tc = ci.type_code();
int tc_int = (int)tc;
bw.Write(tc_int);
}
public static object deserialize_column(ref BinaryReader br, Type t) {
Func<BinaryReader, object> reader_func;
Log.assert(_col_deser_map.TryGetValue(t, out reader_func));
return reader_func(br);
}
public static void serialize_row_column(BinaryWriter bw, IBlueRow r, int col_no) {
var ci = r.schema.columnInfo[col_no];
var type = ci.colType;
Action<BinaryWriter, IBlueRow, int> ser_action;
Log.assert(_col_ser_map.TryGetValue(type, out ser_action));
ser_action(bw, r, col_no);
}
public static ColumnInfo deserialize_column_info(ref BinaryReader br) {
//col_no, name, type>
var hex_txt = to_hex(br.BaseStream);
bts.Log.Gen.info($"StreamSerializationUtils.deserialize_column_info bytes: {hex_txt}");
int col_no = br.ReadInt32();
string col_name = br.read_string_cpp();
int tc_int = br.ReadInt32();
TypeCode type_code = (TypeCode)tc_int;
return new ColumnInfo(col_name, col_no, type_code);
}
public static void write_column(IBlueRow r, int col_no, ref BinaryWriter bw) {
//bw.Write()
}
public static string to_hex(byte[] byte_arr) {
StringBuilder hex_builder = new StringBuilder(byte_arr.Length * 2);
foreach (byte b in byte_arr) {
hex_builder.Append($"{b:X2} ");
}
return hex_builder.ToString().Trim();
}
public static byte[] read_bytes_from_stream(Stream str) {
var init_pos = str.Position;
BinaryReader br = new BinaryReader(str);
var sz = Convert.ToInt32(str.Length - str.Position);
var rv = br.ReadBytes(sz);
str.Position = init_pos;
return rv;
}
public static string to_hex(Stream str) {
var byte_arr = read_bytes_from_stream(str);
return to_hex(byte_arr);
}
}
}
@@ -0,0 +1,29 @@
using System;
using System.Collections;
using System.IO;
using System.Runtime.Serialization.Formatters.Binary;
using BCG;
namespace bts {
internal class UniversalRowSerializer : IStreamRowSerializer {
private readonly IBlueTableSchema _schema;
public UniversalRowSerializer(IBlueTableSchema schema) {
_schema = schema;
}
public void serialize_row(IBlueRow r, StreamIOMessage.StreamOp __, ref BinaryWriter bw) {
//< null_map, row_values >
byte[] null_map = StreamSerializationUtils.get_row_nullmap(r);
//null_map
bw.Write(null_map);
foreach (var ci in _schema.columnInfo) {
if (null_map[ci.colNo] != StreamSerializationUtils.row_null_value_flag)
StreamSerializationUtils.serialize_row_column(bw, r, ci.colNo);
}
}
}
}
@@ -0,0 +1,45 @@
using BCG;
using System.IO;
namespace bts {
public class UniversalStreamIOSerializer : IStreamIOMsgSerializer {
private IStreamRowSerializer _row_serializer = null;
public UniversalStreamIOSerializer() {
}
public void serialize(StreamIOMessage op_msg, ref BinaryWriter bw) {
// TODO
// just need create and multiple remove add
switch (op_msg.op) {
case StreamIOMessage.StreamOp.CREATE:
var cr_msg = op_msg.create_message;
// <op, schema, n_rows, rows>
//IBlueRowCollection createdRows = rows[0];
bw.Write((int)op_msg.op);
StreamSerializationUtils.serialize_schema(cr_msg.rows.schema, ref bw);
_row_serializer = new UniversalRowSerializer(cr_msg.rows.schema);
var rows = op_msg.create_message.rows;
bw.Write(rows.Count);
foreach (IBlueRow r in rows)
_row_serializer.serialize_row(r, op_msg.op, ref bw);
break;
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD:
// <op, n_dels, deleted_rows, n_adds, added_rows>
// TODO
bw.Write((int)op_msg.op);
var del_rows = op_msg.multiple_remove_add_message.removed_rows;
var added_rows = op_msg.multiple_remove_add_message.added_rows;
bw.Write(del_rows.Count);
foreach (IBlueRow dr in del_rows)
_row_serializer.serialize_row(dr, StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD, ref bw);
bw.Write(added_rows.Count);
foreach (IBlueRow ar in added_rows)
_row_serializer.serialize_row(ar, StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD, ref bw);
break;
}
}
}
}
@@ -0,0 +1,47 @@
using System;
using System.Collections;
using System.ComponentModel;
using System.IO;
using System.Runtime.InteropServices;
using BCG;
namespace bts {
// This is the equivelant of StreamIOSerializationController on the deser side. The IStreamMsgDeserilizer
// thats tsreams deal with is this, it hides the specific serialization inside of it
public class StreamIODeserializationController : IStreamMsgDeserilizer {
protected IStreamMsgDeserilizer _real_deserializer = null;
private IBlueTableSchema _schema = null;
public StreamIODeserializationController() {
}
public StreamIOMessage deserialize(ref BinaryReader br) {
if (_real_deserializer != null) {
return _real_deserializer.deserialize(ref br);
}
StreamIOMessage.StreamOp op = (StreamIOMessage.StreamOp)br.ReadInt32();
switch (op) {
case StreamIOMessage.StreamOp.CREATE:
Log.assert(_real_deserializer == null);
string client_deser_name = br.read_string_cpp();
_create_real_deser(client_deser_name);
string pubsubserver_deser_name = br.read_string_cpp();
var rv = _real_deserializer.deserialize(ref br);
return rv;
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD:
Log.assert(false);
return _real_deserializer.deserialize(ref br);
}
Log.assert(false);
return null;
}
private void _create_real_deser(string name) {
_real_deserializer = new UniversalStreamIODeserializer();
}
private bool _received_create => _schema != null;
}
}
@@ -0,0 +1,43 @@
using BCG;
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
namespace bts {
/// <summary>
/// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process
/// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations)
/// </summary>
internal class UniversalRowDeserializer : IStreamRowDeserializer {
private IBlueTableSchema _schema;
public UniversalRowDeserializer(IBlueTableSchema schema) {
_schema = schema;
}
#region IClientRowDeserializer Members
public IBlueRow deserialize_row(ref BinaryReader br) {
byte[] null_map = br.ReadBytes(_schema.columnInfo.Length);
List<object> row_vals = new List<object>();
foreach (ColumnInfo ci in _schema.columnInfo) {
var idx = ci.colNo;
bool is_dbnull = null_map[ci.colNo] == StreamSerializationUtils.row_null_value_flag;
if (is_dbnull) {
row_vals.Add(DBNull.Value);
} else {
var val = StreamSerializationUtils.deserialize_column(ref br, ci.colType);
row_vals.Add(val);
}
}
return new BlueRow(_schema, row_vals.ToArray());
}
#endregion
}
}
@@ -0,0 +1,49 @@
using System;
using System.Collections;
using System.IO;
using BCG;
namespace bts {
public class UniversalStreamIODeserializer : IStreamMsgDeserilizer {
protected IStreamRowDeserializer _row_deserializer = null;
private IBlueTableSchema _schema = null;
public UniversalStreamIODeserializer() {
}
public StreamIOMessage deserialize(ref BinaryReader br) {
StreamIOMessage.StreamOp op = (StreamIOMessage.StreamOp)br.ReadInt32();
switch (op) {
case StreamIOMessage.StreamOp.CREATE:
// <op, schema, n_rows, rows>
_schema = StreamSerializationUtils.deserialize_schema(ref br);
_row_deserializer = new UniversalRowDeserializer(_schema);
var rows = _get_length_prefix_rows(ref br);
StreamIOMessage rv = new StreamIOMessage(new StreamIOMessage.CreateMessage(rows));
return rv;
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD:
Log.assert(_received_create);
var del_rows = _get_length_prefix_rows(ref br);
var added_rows = _get_length_prefix_rows(ref br);
StreamIOMessage.MultipleRemoveAddMessage mram = new StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
return new StreamIOMessage(mram);
}
return null;
}
private IBlueRowCollection _get_length_prefix_rows(ref BinaryReader br) {
BlueTable rv = new BlueTable(_schema);
int n_rows = br.ReadInt32();
for (int i = 0; i < n_rows; i++) {
var r = _row_deserializer.deserialize_row(ref br);
rv.addRow(r);
}
return rv;
}
private bool _received_create => _schema != null;
}
}
@@ -97,10 +97,10 @@ namespace BCG {
// send(msg.to_stream());
// }
public void send_with_confirmation(PubSubMessage msg, ref string conf_id) {
var bytes = msg.to_bytes();
send_with_confirmation(bytes, ref conf_id);
}
// public void send_with_confirmation(PubSubMessage msg, ref string conf_id) {
// var bytes = msg.to_bytes();
// send_with_confirmation(bytes, ref conf_id);
// }
public void send_with_confirmation(bts.PubSubClientMessage msg, ref string conf_id, SendConfirmationDelegate del) {
if (_connection == null) {
@@ -1,150 +0,0 @@
using System;
using System.Runtime.Serialization;
using System.Collections;
using System.Reflection;
using System.IO;
using System.Text;
using BCG;
using System.Collections.Generic;
namespace BCG {
// Thhis "wraps" around the following 4 PubSub meta objects, serializes it on the client and
// deserialize it on the pubsub
// PubMetaData
// SubMetaData
// TransactionMetaData
// UnSubMetaData
//
public class MetaData : ICloneable {
private IDictionary _name_value_map = new Dictionary<string, string>();
int mySize = -1;
public MetaData() { }
public MetaData(object o) { copyFrom(o); }
public MetaData(Stream s) {
long initPos = s.Position;
load(s);
mySize = Convert.ToInt32(s.Position - initPos);
}
public string this[string name] {
get { return (string)_name_value_map[name]; }
set {
if (value != null)
_name_value_map[name] = value;
}
}
private void _add_string_array(string name, string[] values) {
StringBuilder sb = new StringBuilder();
foreach (string s in values) {
if (sb.Length != 0)
sb.Append(" ");
sb.Append(s);
}
_name_value_map[name] = sb.ToString();
}
private int getAsInt(string name) { return Int32.Parse(this[name]); }
private string[] _get_as_string_array(string name) {
if (this[name] != "" && this[name] != null)
return this[name].Split(' ');
else
return new string[0];
}
public int size() {
if (mySize < 0) {
MemoryStream ms = new MemoryStream();
this.save(ms);
return mySize = (int)ms.Position;
} else
return mySize;
}
private bool isEqual(MetaData md) {
if (md._name_value_map.Keys.Count != _name_value_map.Keys.Count)
return false;
foreach (string k in _name_value_map.Keys)
if (!_name_value_map[k].Equals(md._name_value_map[k]))
return false;
return true;
}
// loads the content of the stream into this instance's map
private void load(Stream s) {
_name_value_map = new Hashtable();
string[] state =
new BinaryReader(s).ReadString().Split('\n');
for (int i = 0; i < state.Length - 1; i += 2)
_name_value_map[state[i]] = state[i + 1];
}
// writes the content of this instance to stream s
public void save(Stream s) {
new BinaryWriter(s).Write(this.ToString());
}
public override string ToString() {
StringBuilder sb = new StringBuilder();
foreach (string key in _name_value_map.Keys)
sb.AppendFormat("{0}\n{1}\n", key, _name_value_map[key]);
return sb.ToString();
}
// iterates through the fields and enters the name, values into _hash
public void copyFrom(object o) {
this["op-code"] = o.GetType().Name;
foreach (FieldInfo fi in o.GetType().GetFields()) {
if (fi.GetValue(o) == null)
continue;
else if (fi.GetValue(o) is String[])
_add_string_array(fi.Name, (string[])fi.GetValue(o));
else
_name_value_map[fi.Name] = fi.GetValue(o).ToString();
}
}
// copies the name value map into object o
public void copyTo(object o) {
foreach (FieldInfo fi in o.GetType().GetFields()) {
string value = this[fi.Name];
if (value == null) {
} else {
if (fi.FieldType.Equals(typeof(string)))
fi.SetValue(o, value);
else if (fi.FieldType.Equals(typeof(int)))
fi.SetValue(o, Int32.Parse(value));
else if (fi.FieldType.Equals(typeof(long)))
fi.SetValue(o, long.Parse(value));
else if (fi.FieldType.Equals(typeof(double)))
fi.SetValue(o, double.Parse(value));
else if (fi.FieldType.Equals(typeof(DateTime)))
fi.SetValue(o, DateTime.Parse(value));
else if (fi.FieldType.Equals(typeof(Boolean)))
fi.SetValue(o, Boolean.Parse(value));
else if (fi.FieldType.Equals(typeof(TimeSpan)))
fi.SetValue(o, TimeSpan.Parse(value));
else if (fi.FieldType.Equals(typeof(String[])))
fi.SetValue(o, value.Split(' '));
else
throw new Exception("Field type not supported");
}
}
}
public object Clone() {
MemoryStream ms = new MemoryStream();
this.save(ms);
ms.Seek(0, SeekOrigin.Begin);
return new MetaData(ms);
}
}
}
@@ -1,91 +0,0 @@
using System;
//using BCG.Utilities.General;
//using BCG.Utilities.BlueConfiguration;
//using BCG.Utilities;
namespace BCG
{
[Serializable]
public class PubMetaData
{
/// <summary>
/// tag to be subscribed
/// </summary>
public string tag;
/// <summary>
/// type of tag
/// </summary>
public string type;
/// <summary>
/// size of the data published
/// </summary>
public int size;
/// <summary>
/// How big is the data when it's decompressed
/// </summary>
public int decompressedSize = 0;
/// <summary>
/// is the data compressed?
/// </summary>
public bool isCompressed = false;
/// <summary>
/// identification of the publisher and the connection to the server to send the data
/// machine; ip; pid; serverURL
/// </summary>
public string connection;
public string publishingApp;
public long publishTimeTicks;
public PubMetaData() {}
public PubMetaData(string theTagType, string theTag, int theSize, string theConnection, string thePublishingApp)
: this(theTagType,theTag,theSize,theSize,theConnection,thePublishingApp)
{
isCompressed = false;
}
public PubMetaData(string theTagType, string theTag, int theSize, int theDecompressedSize, string theConnection, string thePublishingApp)
{
type = theTagType;
tag = theTag;
size = theSize;
connection = theConnection;
publishingApp = thePublishingApp;
decompressedSize = theDecompressedSize;
isCompressed = true;
publishTimeTicks = DateTime.Now.ToUniversalTime().Ticks;
myValidateTag();
}
//make sure the tag doesn't contain any newlines
private void myValidateTag()
{
if (tag.Contains("\n"))
throw new ArgumentException("Error: tag " + tag + " is not valid; tags cannot contain newlines.");
}
/// <summary>
/// Returns true if the ticks on this message are greater than the latency report threshold in config
/// </summary>
/// <returns></returns>
public bool warnIfLatencyExceedsThreshold()
{
DateTime publishTime = new DateTime(publishTimeTicks);
double elapsedSeconds = (DateTime.Now.ToUniversalTime() - publishTime).TotalMilliseconds;
if (elapsedSeconds > PubSub3Config.instance.PS3_LatencyReportThresholdMS)
{
//BlueEventLog.logEvent(string.Concat("Recieved pub message on tag ", tag, " ", elapsedSeconds, " milliseconds after the publisher ",publishingApp," sent it."), PS3LogLevels.latency);
return true;
}
return false;
}
}
}
@@ -5,10 +5,9 @@ using System.Collections.Generic;
namespace bts {
using BCG;
using System.Collections;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
public interface IPubSubMessage {
void serialize_into_str(Stream str);
@@ -40,28 +39,31 @@ namespace bts {
public class SubscribeMsg : IPubSubMessage {
public string connection;
public string subscribing_app;
public string subscribing_app = ClientInfo.thisClient.app;
public string[] tags;
public void serialize_into_str(Stream str) {
BinaryWriter bw = new BinaryWriter(str);
bw.Write(connection);
bw.Write(subscribing_app);
bw.write_string_cpp(connection);
bw.write_string_cpp(subscribing_app);
bw.Write(tags.Length);
foreach (var t in tags)
bw.Write(t);
bw.write_string_cpp(t);
}
public void consruct_deserialize(Stream str) {
BinaryReader br = new BinaryReader(str);
this.connection = br.ReadString();
this.subscribing_app = br.ReadString();
this.connection = br.read_string_cpp();
this.subscribing_app = br.read_string_cpp();
var n_tags = br.ReadInt32();
tags = new string[n_tags];
for (var i = 0; i < n_tags; i++)
tags[i] = br.ReadString();
tags[i] = br.read_string_cpp();
}
public override string ToString() {
return $"{GetType().Name} connection: {connection} subscribing_app: {subscribing_app} tags: {tags.Length}";
var tgs_str = String.Empty;
foreach (var t in tags)
tgs_str += t;
return $"{GetType().Name} connection: {connection} subscribing_app: {subscribing_app} tags: {tgs_str}";
}
public override bool Equals(object? obj) {
if (obj != null && obj is SubscribeMsg) {
@@ -73,6 +75,11 @@ namespace bts {
}
return false;
}
public override int GetHashCode() {
var str = ToString();
return str.GetHashCode();
}
}
// Unsubscribe ////////////////////////////////////////////////////////////
@@ -112,17 +119,17 @@ namespace bts {
public string publishing_app = ClientInfo.thisClient.app;
public string tag;
public string tag_type;
public bool is_compressed;
public bool is_compressed = false;
// this is the size that of the msg, regarless of it being serialized or not
public long publish_time_epoch_ms;
public byte[] data_to_publish;
public void serialize_into_str(Stream str) {
BinaryWriter bw = new BinaryWriter(str);
bw.Write(connection);
bw.Write(publishing_app);
bw.Write(tag);
bw.Write(tag_type);
bw.write_string_cpp(connection);
bw.write_string_cpp(publishing_app);
bw.write_string_cpp(tag);
bw.write_string_cpp(tag_type);
bw.Write(is_compressed);
bw.Write(publish_time_epoch_ms);
var sz = data_to_publish.Length;
@@ -131,10 +138,10 @@ namespace bts {
}
public void consruct_deserialize(Stream str) {
BinaryReader br = new BinaryReader(str);
connection = br.ReadString();
publishing_app = br.ReadString();
tag = br.ReadString();
tag_type = br.ReadString();
connection = br.read_string_cpp();
publishing_app = br.read_string_cpp();
tag = br.read_string_cpp();
tag_type = br.read_string_cpp();
is_compressed = br.ReadBoolean();
publish_time_epoch_ms = br.ReadInt64();
var size = br.ReadInt32();
@@ -277,8 +284,8 @@ namespace bts {
public byte[] data;
public void serialize_into_str(Stream str) {
BinaryWriter bw = new BinaryWriter(str);
bw.Write(tag);
bw.Write(tag_type);
bw.write_string_cpp(tag);
bw.write_string_cpp(tag_type);
bw.Write(data.Length);
bw.Write(data);
}
@@ -292,8 +299,8 @@ namespace bts {
}
public void consruct_deserialize(Stream str) {
BinaryReader br = new BinaryReader(str);
tag = br.ReadString();
tag_type = br.ReadString();
tag = br.read_string_cpp();
tag_type = br.read_string_cpp();
var data_sz = br.ReadInt32();
data = br.ReadBytes(data_sz);
}
@@ -1,25 +0,0 @@
// using System;
// namespace BCG
// {
// [Serializable]
// public class RenewMetaData
// {
// public string subscriber;
// public int lease;
// public bool c;
// public SubAddress getSubAddress()
// {return SubAddress.decode(subscriber);}
// public RenewMetaData()
// {}
// public RenewMetaData(string s,
// int lease)
// {
// this.subscriber = s;
// this.lease = lease;
// }
// }
// }
@@ -1,28 +0,0 @@
using System;
namespace BCG {
//[Serializable]
public class SubMetaData {
/// <summary>
/// connection URL on which the connection will receive data
/// </summary>
public string connection;
/// <summary>
/// identification of the client and the connection on which to receive data
/// machine; ip; port; clientURL
/// </summary>
public readonly string subscribing_app = ClientInfo.thisClient.processString;
/// <summary>
/// tags that are to be subscribed
/// </summary>
public string[] tags;
public SubMetaData() { }
public SubMetaData(string the_connection, params string[] the_tags) {
connection = the_connection;
tags = the_tags;
//subscribing_app = ClientInfo.thisClient.processString;
}
}
}
@@ -1,24 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BCG
{
/// <summary>
/// when a publisher sends transactional data to the pubsub server,
/// this is the first meta data, followed by pub messages. this indicated
/// the number of messages that are part of the transaction
/// </summary>
[Serializable]
public class TransactionMetaData
{
public int nMessages;
public TransactionMetaData() { }
public TransactionMetaData(int theNMessages)
{
nMessages = theNMessages;
}
}
}
@@ -1,17 +0,0 @@
using System;
namespace BCG {
//[Serializable]
public class UnSubMetaData {
public string connection;
public readonly string[] tags;
public readonly string unsubscribing_app = ClientInfo.thisClient.processString;
public UnSubMetaData() { }
public UnSubMetaData(string the_connection, params string[] the_tags) {
connection = the_connection;
tags = the_tags;
}
}
}
@@ -6,10 +6,11 @@ namespace BCG {
/// <summary>
/// Collection of utility methods used throughout
/// </summary>
public static class StreamUtilityMethods {
public static class StreamUtilityMethods {
// extension method to write a string C++ style (len prefixed)
public static void write_string_cpp(this BinaryWriter bw, string str) {
str = str ?? String.Empty;
int len = str.Length;
bw.Write(len);
foreach (char c in str) {
@@ -1,42 +0,0 @@
using System;
using System.IO;
using BCG;
using BCG.Utilities;
namespace BCG {
// PubItems are published to the pubsub wrapped in this class. It contains PubMetaData, wrapped in the MetaData
// object and the actualy published payload in the body stream
public class PubSubMessage {
public PubSubMessage(MetaData metaData, Stream body) { this.meta_data = metaData; this.body = body; }
public PubSubMessage(PubMetaData pub_meta_data, Stream body)
: this(new MetaData(pub_meta_data), body) { }
public PubSubMessage(MetaData metaData) { this.meta_data = metaData; }
public MetaData meta_data;
public Stream body = new MemoryStream(0);
public Stream to_stream() {
MemoryStream ms = new MemoryStream();
meta_data.save(ms);
if (body.Length == 0 || body.Length == body.Position) {
bts.Log.Gen.error("Error: Trying to publishing message with no content.");
Console.WriteLine("Error: Trying to publishing message with no content.");
}
StreamUtilityMethods.copyTo(new Stream[] { body }, ms);
ms.Position = 0;
return ms;
}
public byte[] to_bytes() {
return StreamUtilityMethods.getBytesFromStream(to_stream());
}
public override string ToString() {
return $"PubSubMessage meta:data: {meta_data.ToString()} body.Posiiton: {body.Position} body.Length: {body.Length}";
}
}
}
@@ -1,20 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
namespace BCG
{
/*
* The PubSubGovernor accepts or rejects publishes to the pubsub
* if the governor rejects a publish attempt, the publish fails.
* Clients will not get the update and the state of the pubsub will not change.
*
*/
public enum PubSubGovernorResponse {Accepted, Rejected_ShouldBeImported, Rejected_ThisIsNotARootPubsub};
public interface IPubSubGovernor
{
PubSubGovernorResponse requestToPublish(PubMetaData metaData, Stream pubData);
}
}
@@ -1,20 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BCG
{
public class PermissiveGovernor : IPubSubGovernor
{
public PermissiveGovernor()
{
}
public PubSubGovernorResponse requestToPublish(PubMetaData md, System.IO.Stream message)
{
return PubSubGovernorResponse.Accepted;
}
}
}
@@ -67,9 +67,11 @@ namespace BCG {
_dirty_tags.Clear();
var bytes = TagStatsInfo.serialize(dirty_ts_info);
PublishMsg pub_msg = new PublishMsg() {
connection = "InternalPublish",
connection = "Internal",
publishing_app = "PubSubServer",
tag = SystemStatsPublisher.tag_name,
tag_type = SystemStatsPublisher.tag_type,
publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
data_to_publish = bytes
};
PubSubMessageReactor.internal_publish(pub_msg);
@@ -50,7 +50,15 @@ namespace BCG {
}
private void _on_tag_state_changes(string[] tags) {
bts.Log.Gen.info($"SystemSubscriptionMapPublisher._on_tag_state_changes len: {tags.Length}");
Func<string> tags_to_string = () => {
var rv = String.Empty;
foreach (var t in tags) {
rv += t + " ";
}
rv.Trim();
return rv;
};
bts.Log.Gen.info($"SystemSubscriptionMapPublisher._on_tag_state_changes tags: {tags_to_string()}");
_dirty_tags.AddRange(tags);
}
@@ -86,7 +86,7 @@ namespace BCG {
var internal_pubs = _internal_pub_queue.dequeueMultipleElements(1, 64000);
foreach (var msg in internal_pubs) {
try {
_handle_pub("Internal", msg);
_handle_pub("Internal", msg, true);
} catch (Exception ex) {
bts.Log.Gen.error($"PubSubMessage>reactor._polling_thread_func ex: {ex}");
}
@@ -129,7 +129,7 @@ namespace BCG {
_handle_transaction(sender, ps_client_msg.transaction_msg);
break;
case PubSubMessageType.PUBLISH:
_handle_pub(sender, ps_client_msg.pub_msg);
_handle_pub(sender, ps_client_msg.pub_msg, false);
break;
}
}
@@ -158,9 +158,9 @@ namespace BCG {
}
}
private void _handle_pub(string sender, PublishMsg pub_msg) {
private void _handle_pub(string sender, PublishMsg pub_msg, bool is_internal_pub) {
bts.Log.assert(!pub_msg.is_compressed);
bts.Log.Gen.info($"PubSubMessageReactor._handle_pub sender: {sender} pub_msg: {pub_msg}");
bts.Log.Gen.info($"PubSubMessageReactor._handle_pub sender: {sender} pub_msg: {pub_msg} is_internal_pub: {is_internal_pub}");
// this is where pubsub's data state is updated
_state.update_pub_data(pub_msg);
var sub_connections_for_tag = _state.get_subscriber_connections_for_tag(pub_msg.tag);
@@ -180,30 +180,32 @@ namespace BCG {
// }
public void update_pub_data(bts.PublishMsg pub_msg) {
// TODO
// int sz = (int)(msg.Length - msg.Position);
bts.Log.Gen.info($"PubSubData.update_pub_data pub_msg: {pub_msg}");
///this is the only place where SingleTagData should be created
if (!_tag_to_data_map.ContainsKey(pub_msg.tag)) {
///first time data is published for this tag
_tag_to_data_map.Add(pub_msg.tag, new SingleTagData(pub_msg.tag, pub_msg.tag_type));
}
_tag_to_data_map[pub_msg.tag].handle_publish(pub_msg);
try {
bts.Log.Gen.info($"PubSubData.update_pub_data pub_msg: {pub_msg}");
///this is the only place where SingleTagData should be created
if (!_tag_to_data_map.ContainsKey(pub_msg.tag)) {
///first time data is published for this tag
_tag_to_data_map.Add(pub_msg.tag, new SingleTagData(pub_msg.tag, pub_msg.tag_type));
}
_tag_to_data_map[pub_msg.tag].handle_publish(pub_msg);
if (!_tag_to_connection_map.ContainsKey(pub_msg.tag)) {
_tag_to_connection_map.Add(pub_msg.tag, new SingleTagConnectionData(pub_msg.tag));
}
bool pub_changed = false;
_tag_to_connection_map[pub_msg.tag].handle_pub(pub_msg, out pub_changed);
if (pub_changed) {
if (onTagMapChanged != null)
onTagMapChanged(new string[] { pub_msg.tag });
if (!_tag_to_connection_map.ContainsKey(pub_msg.tag)) {
_tag_to_connection_map.Add(pub_msg.tag, new SingleTagConnectionData(pub_msg.tag));
}
bool pub_changed = false;
_tag_to_connection_map[pub_msg.tag].handle_pub(pub_msg, out pub_changed);
if (pub_changed) {
if (onTagMapChanged != null)
onTagMapChanged(new string[] { pub_msg.tag });
if (onSingleTagChange != null)
onSingleTagChange(
new PubSubSingleTagStateEvent(pub_msg.tag,
PubSubSingleTagStateEvent.EventType.publish,
pub_msg.connection, pub_msg.publishing_app));
if (onSingleTagChange != null)
onSingleTagChange(
new PubSubSingleTagStateEvent(pub_msg.tag,
PubSubSingleTagStateEvent.EventType.publish,
pub_msg.connection, pub_msg.publishing_app));
}
} catch (Exception ex) {
bts.Log.Gen.error($"PubSubState.update_pub_data ex: {ex} pub_msg: {pub_msg}");
}
if (onPublishStats != null)
onPublishStats(pub_msg.tag, pub_msg.data_to_publish.Length);
@@ -77,25 +77,6 @@ namespace BCG {
get { return _dead_pub_conn_to_app_map; }
}
/// <summary>
/// Called by PubsubState when a subscription is made to this tag.
/// Returns a true, is the subscriber was not present, else logs
/// a warning and returns a false
/// </summary>
/// <param name="sub"></param>
/// <returns></returns>
public bool onSub(SubMetaData sub) {
if (_sub_conn_to_app_map.ContainsKey(sub.connection)) {
string warningMsg = String.Format("tag {0} has already been subscribed by app {0}, subscriber {1}"
+ "it should unsubscribe before resubscribing", _tag, sub.connection, sub.subscribing_app);
//BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
return false;
} else {
_sub_conn_to_app_map.Add(sub.connection, sub.subscribing_app);
return true;
}
}
public bool handle_sub(SubscribeMsg sub_msg) {
if (_sub_conn_to_app_map.ContainsKey(sub_msg.connection)) {
bts.Log.Gen.info($"tag: {_tag} has already been subscribed by connection: {sub_msg.connection}");
@@ -106,25 +87,6 @@ namespace BCG {
}
}
/// <summary>
/// Called by PubsubState when a subscriber unsubscribes to this tag.
/// Returns a true when a subscriber exists, else logs a warning and
/// returns a false
/// </summary>
/// <param name="unsub"></param>
/// <returns></returns>
public bool onUnsub(UnSubMetaData unsub) {
if (_sub_conn_to_app_map.ContainsKey(unsub.connection)) {
_sub_conn_to_app_map.Remove(unsub.connection);
return true;
} else {
string warningMsg = String.Format("tag {0} has already was either unsubscribed or never subscribed by app {0}, subscriber {1}",
_tag, unsub.connection, unsub.unsubscribing_app);
//BlueEventLog.logEvent(warningMsg, BlueEventLevel.Warning);
return false;
}
}
public bool handle_unsub(UnsubscribeMsg unsub_msg) {
if (_sub_conn_to_app_map.ContainsKey(unsub_msg.connection)) {
_sub_conn_to_app_map.Remove(unsub_msg.connection);
@@ -159,21 +121,6 @@ namespace BCG {
return rv;
}
/// <summary>
/// Called by pubsubState, when this tag is published. if publisher
/// has changed, the out paramater is set to true
/// </summary>
/// <param name="pub"></param>
/// <param name="publisherChanged"></param>
public void onPub(PubMetaData pub, out bool newPublisher) {
if (_active_pub_conn_to_app_map.ContainsKey(pub.connection)) {
newPublisher = false;
} else {
_active_pub_conn_to_app_map.Add(pub.connection, pub.publishingApp);
newPublisher = true;
}
}
public void handle_pub(PublishMsg pub_msg, out bool new_publisher) {
if (_active_pub_conn_to_app_map.ContainsKey(pub_msg.connection)) {
new_publisher = false;
@@ -27,6 +27,7 @@ namespace BCG {
// TODO, construct schema
_schema = BlueDataUtility.form_schema("pubsub_server_state",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("tag", TypeCode.String),
Tuple.Create("ps_server", TypeCode.String),
Tuple.Create("subscibers", TypeCode.String),
Tuple.Create("publisher", TypeCode.String),
@@ -71,7 +72,7 @@ namespace BCG {
var sorted_tags = _tag_map.Keys.OrderBy(t => t);
foreach (var kv in _tag_map) {
var tsi = kv.Value;
IBlueRow r = new BlueRow(_schema, bt.Count, tsi.pubsub_server, tsi.subscribers_as_string,
IBlueRow r = new BlueRow(_schema, bt.Count, tsi.tag, tsi.pubsub_server, tsi.subscribers_as_string,
tsi.publishers_as_string, tsi.dead_publishers_as_string);
bt.addRow(r);
}