deleting old serializers

This commit is contained in:
Taha Afzal authored and John Colvin committed 2024-05-25 15:37:33 +00:00
1 parent 061481419e
commit 90ec64d25c
74 files changed
+941 -6080

No files matched your search

@@ -221,23 +221,23 @@ namespace BCG {
// these are the only types allowed in schemas
private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
{
{ TypeCode.Boolean, typeof(bool) },
{ TypeCode.Byte, typeof(byte) },
{ TypeCode.Char, typeof(char) },
{ TypeCode.Boolean, typeof(bool) }, //3
{ TypeCode.Byte, typeof(byte) }, //6
{ TypeCode.Char, typeof(char) }, //4
// TODO
{ TypeCode.DateTime, typeof(DateTime) },
{ TypeCode.Decimal, typeof(decimal) },
{ TypeCode.Double, typeof(double) },
{ TypeCode.DateTime, typeof(DateTime) },//16
{ TypeCode.Decimal, typeof(decimal) }, //15
{ TypeCode.Double, typeof(double) }, //14
//{ TypeCode.Empty, null },
{ TypeCode.Int16, typeof(short) },
{ TypeCode.Int32, typeof(int) },
{ TypeCode.Int64, typeof(long) },
{ TypeCode.SByte, typeof(sbyte) },
{ TypeCode.Single, typeof(Single) },
{ TypeCode.String, typeof(string) },
{ TypeCode.UInt16, typeof(UInt16) },
{ TypeCode.UInt32, typeof(UInt32) },
{ TypeCode.UInt64, typeof(UInt64) }
{ TypeCode.Int16, typeof(short) }, //7
{ TypeCode.Int32, typeof(int) }, //9
{ TypeCode.Int64, typeof(long) }, //11
{ TypeCode.SByte, typeof(sbyte) }, //5
{ TypeCode.Single, typeof(Single) }, //13
{ TypeCode.String, typeof(string) }, //18
{ TypeCode.UInt16, typeof(UInt16) }, //8
{ TypeCode.UInt32, typeof(UInt32) }, //10
{ TypeCode.UInt64, typeof(UInt64) } //12
};
public static IEnumerable<TypeCode> valid_column_typecodes() {
@@ -1139,6 +1139,14 @@ namespace BCG {
return ci.colNo + ";" + ci.colName + ";" + ci.colType;
}
public static string to_string(this IBlueRow r) {
string rv = String.Empty;
foreach (ColumnInfo c in r.schema.columnInfo) {
rv += $"{c.colName}: {r[c.colName]}\n";
}
return rv;
}
public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
if (old_rows == null || old_rows.Count == 0) {
@@ -1,252 +1,252 @@
using System;
using System.Collections;
//using BCG.Utilities;
// using System;
// using System.Collections;
// //using BCG.Utilities;
namespace BCG
{
/// <summary>
/// Provides a multiple indexing mechanism, required to increase the peformance of the streams
/// </summary>
public class BlueTableIndexException : Exception
{
public BlueTableIndexException(string msg) : base(msg) { }
}
// namespace BCG
// {
// /// <summary>
// /// Provides a multiple indexing mechanism, required to increase the peformance of the streams
// /// </summary>
// public class BlueTableIndexException : Exception
// {
// public BlueTableIndexException(string msg) : base(msg) { }
// }
public interface IBlueTableIndex
{
void addRow(IBlueRow row);
void deleteRowOnPrimaryKey(object pkValue);
void deleteRowOnPrimaryKey(IBlueRow row);
void deleteRowOnBothKeys(object pkValue, object skValue);
void deleteRowOnBothKeys(IBlueRow row);
ArrayList searchRowOnPrimaryKey(object pkValue);
ArrayList searchRowOnPrimaryKey(IBlueRow row);
ArrayList searchRowOnBothKeys(object pkValue, object skValue);
ArrayList searchRowOnBothKeys(IBlueRow row);
// public interface IBlueTableIndex
// {
// void addRow(IBlueRow row);
// void deleteRowOnPrimaryKey(object pkValue);
// void deleteRowOnPrimaryKey(IBlueRow row);
// void deleteRowOnBothKeys(object pkValue, object skValue);
// void deleteRowOnBothKeys(IBlueRow row);
// ArrayList searchRowOnPrimaryKey(object pkValue);
// ArrayList searchRowOnPrimaryKey(IBlueRow row);
// ArrayList searchRowOnBothKeys(object pkValue, object skValue);
// ArrayList searchRowOnBothKeys(IBlueRow row);
}
// }
/*
public class BlueTableDoubleIndex : IBlueTableIndex
{
private IBlueRowCollection myTable;
private string myPrimaryIndex;
private string mySecondaryIndex;
private Hashtable myHt = new Hashtable ();
/// <summary>
/// Builds an index with 2 columns of the table acting as primary index and secondary index
/// respectively
/// </summary>
/// <param name="table">
/// table to index
/// </param>
/// <param name="primaryIndex">
/// primary key to index upon
/// </param>
/// <param name="secondaryIndex">
/// column serving as the secondary key
/// </param>
public BlueTableDoubleIndex (IBlueRowCollection table, string primaryIndex, string secondaryIndex)
{
myTable = table;
myPrimaryIndex = primaryIndex;
mySecondaryIndex = secondaryIndex;
myInit ();
}
private void myInit()
{
foreach (IBlueRow row in myTable)
addRow (row);
}
// /*
// public class BlueTableDoubleIndex : IBlueTableIndex
// {
// private IBlueRowCollection myTable;
// private string myPrimaryIndex;
// private string mySecondaryIndex;
// private Hashtable myHt = new Hashtable ();
/// <summary>
/// To add a row to the index
/// </summary>
/// <param name="row">
/// row to be added
/// </param>
public void addRow (IBlueRow row)
{
if (myHt[row[myPrimaryIndex]] != null)
((Hashtable)(myHt[row[myPrimaryIndex]]))[row[mySecondaryIndex]] = row;
else
{
TimeManager.start ();
Hashtable ht = new Hashtable();
TimeManager.stop ();
ht[row[mySecondaryIndex]] = row;
myHt[row[myPrimaryIndex]] = ht;
}
}
// /// <summary>
// /// Builds an index with 2 columns of the table acting as primary index and secondary index
// /// respectively
// /// </summary>
// /// <param name="table">
// /// table to index
// /// </param>
// /// <param name="primaryIndex">
// /// primary key to index upon
// /// </param>
// /// <param name="secondaryIndex">
// /// column serving as the secondary key
// /// </param>
// public BlueTableDoubleIndex (IBlueRowCollection table, string primaryIndex, string secondaryIndex)
// {
// myTable = table;
// myPrimaryIndex = primaryIndex;
// mySecondaryIndex = secondaryIndex;
// myInit ();
// }
/// <summary>
/// If index or the primary key has already been deleted appropriate exceptions are thrown
/// </summary>
/// <param name="row"></param>
public void deleteRow (IBlueRow row)
{
if (myHt[row[myPrimaryIndex]] == null)
{
throw new BlueTableIndexException ("Theres no index (primary hash table key) for this index value");
}
((Hashtable)(myHt[row[myPrimaryIndex]])).Remove(row[mySecondaryIndex]);
if (((Hashtable)(myHt[row[myPrimaryIndex]])).Keys.Count == 0)
myHt.Remove(row[myPrimaryIndex]);
}
// private void myInit()
// {
// foreach (IBlueRow row in myTable)
// addRow (row);
// }
/// <summary>
/// Returns an array of DataRow that match the primary key being passed as an argument
/// </summary>
/// <param name="o">
/// Primary Key of the table
/// </param>
/// <returns>
/// Array of rows matching the criteria
/// </returns>
public IBlueRow[] findValueFromPrimaryKey (object o)
{
object oht = myHt[o];
if (oht == null)
return new IBlueRow[0];
Hashtable ht = (Hashtable)oht;
IBlueRow[] rows = new IBlueRow[ht.Keys.Count];
ht.Values.CopyTo(rows, 0);
return rows;
}
// /// <summary>
// /// To add a row to the index
// /// </summary>
// /// <param name="row">
// /// row to be added
// /// </param>
// public void addRow (IBlueRow row)
// {
// if (myHt[row[myPrimaryIndex]] != null)
// ((Hashtable)(myHt[row[myPrimaryIndex]]))[row[mySecondaryIndex]] = row;
// else
// {
// TimeManager.start ();
// Hashtable ht = new Hashtable();
// TimeManager.stop ();
// ht[row[mySecondaryIndex]] = row;
// myHt[row[myPrimaryIndex]] = ht;
// }
// }
// /// <summary>
// /// If index or the primary key has already been deleted appropriate exceptions are thrown
// /// </summary>
// /// <param name="row"></param>
// public void deleteRow (IBlueRow row)
// {
// if (myHt[row[myPrimaryIndex]] == null)
// {
// throw new BlueTableIndexException ("Theres no index (primary hash table key) for this index value");
// }
// ((Hashtable)(myHt[row[myPrimaryIndex]])).Remove(row[mySecondaryIndex]);
// if (((Hashtable)(myHt[row[myPrimaryIndex]])).Keys.Count == 0)
// myHt.Remove(row[myPrimaryIndex]);
// }
// /// <summary>
// /// Returns an array of DataRow that match the primary key being passed as an argument
// /// </summary>
// /// <param name="o">
// /// Primary Key of the table
// /// </param>
// /// <returns>
// /// Array of rows matching the criteria
// /// </returns>
// public IBlueRow[] findValueFromPrimaryKey (object o)
// {
// object oht = myHt[o];
// if (oht == null)
// return new IBlueRow[0];
// Hashtable ht = (Hashtable)oht;
// IBlueRow[] rows = new IBlueRow[ht.Keys.Count];
// ht.Values.CopyTo(rows, 0);
// return rows;
// }
/// <summary>
/// returns as array of the primary keys
/// </summary>
/// <returns>
/// object[] of primary keys
/// </returns>
public Array getPrimaryKeys()
{
object[] retval = new object [myHt.Keys.Count];
myHt.Keys.CopyTo (retval, 0);
return retval;
}
}
*/
// /// <summary>
// /// returns as array of the primary keys
// /// </summary>
// /// <returns>
// /// object[] of primary keys
// /// </returns>
// public Array getPrimaryKeys()
// {
// object[] retval = new object [myHt.Keys.Count];
// myHt.Keys.CopyTo (retval, 0);
// return retval;
// }
// }
// */
/// <summary>
/// Provides a 1-Level indexing structure.
/// </summary>
public class BlueTableSingleIndex : IBlueTableIndex
{
string myPrimaryKey;
string mySecondaryKey;
Hashtable myHt = new Hashtable();
// /// <summary>
// /// Provides a 1-Level indexing structure.
// /// </summary>
// public class BlueTableSingleIndex : IBlueTableIndex
// {
// string myPrimaryKey;
// string mySecondaryKey;
// Hashtable myHt = new Hashtable();
public BlueTableSingleIndex(string primaryKey, string secondaryKey)
{
myPrimaryKey = primaryKey;
mySecondaryKey = secondaryKey;
}
// public BlueTableSingleIndex(string primaryKey, string secondaryKey)
// {
// myPrimaryKey = primaryKey;
// mySecondaryKey = secondaryKey;
// }
/// <summary>
/// The primary key is the indexing key, and the secondary key is used as the key for
/// searching and hence deletion
/// </summary>
/// <param name="table"></param>
/// <param name="primaryKey"></param>
/// <param name="secondaryKey"></param>
public BlueTableSingleIndex(IBlueRowCollection table, string primaryKey, string secondaryKey)
: this(primaryKey, secondaryKey)
{
foreach (IBlueRow row in table)
addRow(row);
}
// /// <summary>
// /// The primary key is the indexing key, and the secondary key is used as the key for
// /// searching and hence deletion
// /// </summary>
// /// <param name="table"></param>
// /// <param name="primaryKey"></param>
// /// <param name="secondaryKey"></param>
// public BlueTableSingleIndex(IBlueRowCollection table, string primaryKey, string secondaryKey)
// : this(primaryKey, secondaryKey)
// {
// foreach (IBlueRow row in table)
// addRow(row);
// }
/// <summary>
/// Adds row to the indexing new structure
/// </summary>
/// <param name="row"></param>
public void addRow(IBlueRow row)
{
if (myHt[row[myPrimaryKey]] == null)
myHt[row[myPrimaryKey]] = new ArrayList();
((ArrayList)(myHt[row[myPrimaryKey]])).Add(row);
}
// /// <summary>
// /// Adds row to the indexing new structure
// /// </summary>
// /// <param name="row"></param>
// public void addRow(IBlueRow row)
// {
// if (myHt[row[myPrimaryKey]] == null)
// myHt[row[myPrimaryKey]] = new ArrayList();
// ((ArrayList)(myHt[row[myPrimaryKey]])).Add(row);
// }
/// <summary>
/// Getts rows based on the Primary Key. Matching of the primary key is the only criteria
/// </summary>
/// <param name="row"></param>
/// <returns></returns>
public ArrayList searchRowOnPrimaryKey(IBlueRow row)
{
return searchRowOnPrimaryKey(row[myPrimaryKey]);
}
// /// <summary>
// /// Getts rows based on the Primary Key. Matching of the primary key is the only criteria
// /// </summary>
// /// <param name="row"></param>
// /// <returns></returns>
// public ArrayList searchRowOnPrimaryKey(IBlueRow row)
// {
// return searchRowOnPrimaryKey(row[myPrimaryKey]);
// }
public ArrayList searchRowOnPrimaryKey(object pkValue)
{
if (myHt[pkValue] == null)
return new ArrayList();
return (ArrayList)myHt[pkValue];
}
// public ArrayList searchRowOnPrimaryKey(object pkValue)
// {
// if (myHt[pkValue] == null)
// return new ArrayList();
// return (ArrayList)myHt[pkValue];
// }
/// <summary>
/// Gets rows based on the primary AND the secondary key
/// </summary>
/// <param name="row"></param>
/// <returns></returns>
public ArrayList searchRowOnBothKeys(IBlueRow row)
{
return searchRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
}
// /// <summary>
// /// Gets rows based on the primary AND the secondary key
// /// </summary>
// /// <param name="row"></param>
// /// <returns></returns>
// public ArrayList searchRowOnBothKeys(IBlueRow row)
// {
// return searchRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
// }
public ArrayList searchRowOnBothKeys(object pkValue, object skValue)
{
if (myHt[pkValue] == null)
return new ArrayList();
ArrayList matchingList = (ArrayList)myHt[pkValue];
ArrayList retVal = new ArrayList();
foreach (IBlueRow r in matchingList)
if (r[mySecondaryKey].Equals(skValue))
retVal.Add(r);
return retVal;
}
// public ArrayList searchRowOnBothKeys(object pkValue, object skValue)
// {
// if (myHt[pkValue] == null)
// return new ArrayList();
// ArrayList matchingList = (ArrayList)myHt[pkValue];
// ArrayList retVal = new ArrayList();
// foreach (IBlueRow r in matchingList)
// if (r[mySecondaryKey].Equals(skValue))
// retVal.Add(r);
// return retVal;
// }
/// <summary>
/// Deletes rows based on the primary key only
/// </summary>
/// <param name="row"></param>
public void deleteRowOnPrimaryKey(IBlueRow row)
{
deleteRowOnPrimaryKey(row[myPrimaryKey]);
}
// /// <summary>
// /// Deletes rows based on the primary key only
// /// </summary>
// /// <param name="row"></param>
// public void deleteRowOnPrimaryKey(IBlueRow row)
// {
// deleteRowOnPrimaryKey(row[myPrimaryKey]);
// }
public void deleteRowOnPrimaryKey(object pkValue)
{
if (myHt[pkValue] == null)
throw new BlueTableIndexException("Key to be deleted was not found in the index");
myHt.Remove(pkValue);
}
// public void deleteRowOnPrimaryKey(object pkValue)
// {
// if (myHt[pkValue] == null)
// throw new BlueTableIndexException("Key to be deleted was not found in the index");
// myHt.Remove(pkValue);
// }
public void deleteRowOnBothKeys(IBlueRow row)
{
deleteRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
}
// public void deleteRowOnBothKeys(IBlueRow row)
// {
// deleteRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
// }
public void deleteRowOnBothKeys(object pkValue, object skValue)
{
if (myHt[pkValue] == null)
throw new BlueTableIndexException("Key to be deleted was not found in the index");
ArrayList matchingList = (ArrayList)myHt[pkValue];
ArrayList aux = new ArrayList();
foreach (IBlueRow r in matchingList)
if (r[mySecondaryKey].Equals(skValue))
aux.Add(r);
foreach (IBlueRow r in aux)
matchingList.Remove(r);
}
}
}
// public void deleteRowOnBothKeys(object pkValue, object skValue)
// {
// if (myHt[pkValue] == null)
// throw new BlueTableIndexException("Key to be deleted was not found in the index");
// ArrayList matchingList = (ArrayList)myHt[pkValue];
// ArrayList aux = new ArrayList();
// foreach (IBlueRow r in matchingList)
// if (r[mySecondaryKey].Equals(skValue))
// aux.Add(r);
// foreach (IBlueRow r in aux)
// matchingList.Remove(r);
// }
// }
// }
@@ -1,285 +0,0 @@
using System;
using System.Collections;
using System.IO;
using BCG.Utilities;
using BCG.Utilities.General;
//using BCG.Utilities.BlueConfiguration;
using System.Collections.Generic;
using System.Threading;
using mbts;
using System.Linq;
using mbts.spark;
namespace BCG {
public class TableAccumulator : ITableAccumulator {
private HighPerformanceTimer _timer = new HighPerformanceTimer();
private IClientDeserializer myDeserializer = new ClientDeserializationController();
private BlueTable myStateTable;
private ISubscriberCore mySubscriptionCore;
private TableUpdateDelegate myOnUpdateDelegate;
private string myTag;
private bool myUseIncrementsForMultipleOps = DataStreamGatewaySettings.instance.useIncrementsForMultipleOps;
public TableAccumulator(string tag, ISubscriberCore subCore) {
myTag = tag;
mySubscriptionCore = subCore;
}
#region ITableAccumulator Members
public bool is_active() { return myOnUpdateDelegate != null; }
public event TableUpdateDelegate on_update {
add {
// BlueEventLog.logEvent ("TableSubscriber being Subscribed for tag :" + myTag, BlueEventLevel.Information);
if (myOnUpdateDelegate == null) {
//ie first subscription
// BlueEventLog.logEvent ("Subscription made to PubSub for tag : " + myTag, BlueEventLevel.Information);
mySubscriptionCore.subscribe(myTag);
}
myOnUpdateDelegate =
Delegate.Combine(myOnUpdateDelegate, value) as TableUpdateDelegate;
if (myStateTable != null)
value(myStateTable, myStateTable, new BlueTable(myStateTable.schema));
}
remove {
// BlueEventLog.logEvent ("TableSubscriber being Unsubscribed for tag :" + myTag, BlueEventLevel.Information);
if (myOnUpdateDelegate != null) {
myOnUpdateDelegate =
Delegate.Remove(myOnUpdateDelegate, value) as TableUpdateDelegate;
if (myOnUpdateDelegate == null) {
//all subscribers gone
// BlueEventLog.logEvent ("Unubscription made to PubSub for tag : " + myTag, BlueEventLevel.Information);
mySubscriptionCore.unsubscribe(myTag);
myStateTable = null;
}
}
}
}
public void handle_updates(IEnumerable<byte[]> msgs) {
update(msgs.Select(t => new MemoryStream(t)).ToArray());
}
public void update(Stream[] streams) {
bts.Log.Gen.info($"TableAccumulator.update tag: {myTag} n_streams: {streams.Length} stream[0].len: {streams[0].Length - streams[0].Position}");
//checking the condition as a subscriber might unsubscribe in a callback and there might be more than one
//data element for that tag in the ViewStreamDatabase
if (myOnUpdateDelegate != null) {
byte[] ops;
Array rows;
_timer.start();
myDeserializer.deserialize(streams, out ops, out rows);
_timer.stop();
// BlueEventLog.logEvent(String.Format("deserializing {0} took {1} msecs", myTag, _timer.lastIntervalTime), BlueEventLevel.Performance);
_timer.start();
myAccumulateAndFire(ops, rows);
_timer.stop();
// BlueEventLog.logEvent(String.Format("myAccumulateAndFire for {0} took {1} msecs", myTag, _timer.lastIntervalTime), BlueEventLevel.Performance);
}
}
#endregion
private void myAccumulateAndFire(byte[] ops, Array rows) {
//if a non create comes before the create, the deserializer returns 0 sized arrays
if (ops.Length == 0 || rows.Length == 0) {
//ie if one is zero sized so should be the other one
// BlueEventLog.logAssert (ops.Length + rows.Length == 0);
} else {
if (ops.Length > 1) {
int index = 0;
if (myStateTable == null) {
for (index = 0; index < ops.Length; index++)
if (ops[index] == StreamOpCode.create)
break;
if (index == ops.Length)
return;
}
if (myUseIncrementsForMultipleOps)
myHandleMultipleOpsIncrementally(ops, rows, index);
else
myHandleMultipleOps(ops, rows, index);
} else {
if (myStateTable != null || ops[0] == StreamOpCode.create)
myHandleSingleOp(ops, rows);
}
}
}
private void myHandleMultipleOps(byte[] ops, Array rows, int offset) {
HighPerformanceTimer timer = new HighPerformanceTimer();
timer.start();
IBlueRowCollection stateAtStart = myStateTable == null ? null : new BlueTable(myStateTable);
for (int i = offset; i < ops.Length; i++) {
byte op = ops[i];
IBlueRowCollection[] tableArray = rows.GetValue(i) as IBlueRowCollection[];
if (op == StreamOpCode.add)
myStateTable.addRows(tableArray[0]);
else if (op == StreamOpCode.remove)
myStateTable.deleteRows(tableArray[0]);
else if (op == StreamOpCode.change)
myStateTable.changeRows(tableArray[0]);
else if (op == StreamOpCode.create)
myStateTable = new BlueTable(tableArray[0]);
else if (op == StreamOpCode.multipleChange)
myStateTable.changeRows(tableArray[0]);
else if (op == StreamOpCode.multipleRemoveAdd) {
myStateTable.deleteRows(tableArray[0]);
myStateTable.addRows(tableArray[1]);
}
}
IBlueRowCollection dRows, aRows;
if (stateAtStart == null) {
dRows = new BlueTable(myStateTable.schema);
aRows = myStateTable;
} else {
BlueDataUtility.computeDifferences(myStateTable, stateAtStart, out dRows, out aRows);
}
timer.stop();
//Console.WriteLine("multiple ops for table " + myTag + " took " + timer.lastIntervalTime + " msecs");
myOnUpdateDelegate(myStateTable, aRows, dRows);
}
#region handling mutliple ops in single update call, incrementally
private void myHandleMultipleOpsIncrementally(byte[] ops, Array rows, int offset) {
BlueTable dels = null, adds = null;
if (myStateTable != null) {
dels = new BlueTable(myStateTable.schema);
adds = new BlueTable(myStateTable.schema);
}
for (int i = offset; i < ops.Length; i++) {
byte op = ops[i];
IBlueRowCollection[] tableArray = rows.GetValue(i) as IBlueRowCollection[];
if (op == StreamOpCode.add) {
myHandleAdd(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds);
} else if (op == StreamOpCode.remove) {
myHandleRemove(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds);
} else if (op == StreamOpCode.change) {
myHandleChange(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds);
} else if (op == StreamOpCode.create) {
myHandleCreate(tableArray[0], ref dels, ref adds);
} else if (op == StreamOpCode.multipleChange) {
myHandleMultipleChanges(tableArray[0], dels, adds);
} else if (op == StreamOpCode.multipleRemoveAdd) {
myHandleMultipleRemovesAdds(tableArray[0], tableArray[1], dels, adds);
}
}
myOnUpdateDelegate(myStateTable, adds, dels);
}
private void myHandleAdd(IBlueRow r, BlueTable dels, BlueTable adds) {
if (dels != null) {
myStateTable.addRow(r);
adds.addRow(r);
}
}
private void myHandleRemove(IBlueRow r, BlueTable dels, BlueTable adds) {
if (dels != null) {
myStateTable.deleteRow(r);
if (adds.containsID((int)r["id"]))
adds.deleteRow(r);
else
dels.addRow(r);
}
}
private void myHandleChange(IBlueRow r, BlueTable dels, BlueTable adds) {
if (dels != null) {
IBlueRow er = myStateTable[(int)r["id"]];
myHandleRemove(er, dels, adds);
myHandleAdd(r, dels, adds);
}
}
private void myHandleCreate(IBlueRowCollection newTable, ref BlueTable dels, ref BlueTable adds) {
if (dels == null) {
// BlueEventLog.logAssert(adds == null);
dels = new BlueTable(newTable.schema);
adds = new BlueTable(newTable);
myStateTable = new BlueTable(newTable);
} else {
dels = new BlueTable(myStateTable);
myStateTable = new BlueTable(newTable);
adds = new BlueTable(newTable);
}
}
private void myHandleMultipleRemovesAdds(IBlueRowCollection deletedRows, IBlueRowCollection addedRows,
BlueTable dels, BlueTable adds) {
if (dels != null) {
foreach (IBlueRow r in deletedRows)
myHandleRemove(r, dels, adds);
foreach (IBlueRow r in addedRows)
myHandleAdd(r, dels, adds);
}
}
private void myHandleMultipleChanges(IBlueRowCollection changedRows, BlueTable dels, BlueTable adds) {
if (dels != null)
foreach (IBlueRow r in changedRows)
myHandleChange(r, dels, adds);
}
#endregion
private void myHandleSingleOp(byte[] ops, Array rows) {
byte op = ops[0];
IBlueRowCollection[] tableArray = rows.GetValue(0) as IBlueRowCollection[];
IBlueRowCollection dRows, aRows;
if (op == StreamOpCode.create) {
if (myStateTable == null) {
aRows = myStateTable = new BlueTable(tableArray[0]);
dRows = new BlueTable(myStateTable.schema);
myOnUpdateDelegate(myStateTable, aRows, dRows);
} else {
IBlueRowCollection newState = new BlueTable(tableArray[0]);
BlueDataUtility.computeDifferences(newState, myStateTable, out dRows, out aRows);
if (aRows.Count > 0 || dRows.Count > 0) {
myStateTable = new BlueTable(newState);
myOnUpdateDelegate(myStateTable, aRows, dRows);
}
}
} else {
if (op == StreamOpCode.add) {
myStateTable.addRows(aRows = tableArray[0]);
dRows = new BlueTable(myStateTable.schema);
} else if (op == StreamOpCode.remove) {
myStateTable.deleteRows(dRows = tableArray[0]);
aRows = new BlueTable(myStateTable.schema);
} else if (op == StreamOpCode.change) {
IBlueRow cRow = BlueDataUtility.extractRow(tableArray[0], 0);
aRows = new BlueTable(myStateTable.schema);
dRows = new BlueTable(myStateTable.schema);
(aRows as BlueTable).addRow(cRow);
(dRows as BlueTable).addRow(myStateTable[(int)cRow["id"]]);
myStateTable.changeRow(cRow);
} else if (op == StreamOpCode.multipleChange) {
if (myStateTable.Count == tableArray[0].Count) {
//all rows are changed, so we can optimize this
dRows = myStateTable;
aRows = myStateTable = new BlueTable(tableArray[0]);
} else {
dRows = new BlueTable(myStateTable.schema);
aRows = tableArray[0];
foreach (IBlueRow r in tableArray[0]) {
(dRows as BlueTable).addRow(myStateTable[(int)r["id"]]);
myStateTable.changeRow(r);
}
}
} else if (op == StreamOpCode.multipleRemoveAdd) {
BlueDataUtility.absorbChanges(myStateTable, dRows = tableArray[0], aRows = tableArray[1]);
} else {
aRows = dRows = null;
// BlueEventLog.logAssert(false, "unknown opcode");
}
myOnUpdateDelegate(myStateTable, aRows, dRows);
}
}
}
}
@@ -1,67 +0,0 @@
using System;
using System.IO;
using System.Collections;
using BCG.Utilities;
namespace BCG {
/// <summary>
/// Controls (ie creates dynamically) an IClientDeserializer and then passes data to it.
/// </summary>
public class ClientDeserializationController : IClientDeserializer {
private IClientDeserializer myDeserializer;
private int _n_desers = 0;
public ClientDeserializationController() { }
#region IClientDeserializer Members
/// <summary>
/// Deserializes bytes into operation and data. If it gets a non create before a create, both
/// the out arrays get initialized to empty arrays
/// </summary>
/// <param name="streams"></param>
/// <param name="theOps"></param>
/// <param name="theRows"></param>
public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows) {
bts.Log.Gen.info($"ClientDeserializationController.deserialize len: {streams[0].Length - streams[0].Position}");
if (myDeserializer != null) {
myDeserializer.deserialize(streams, out theOps, out theRows);
} else {
Stream totalStream = UtilityMethods.getAggregatedStream(streams);
long initialPos = totalStream.Position;
BinaryReader br = new BinaryReader(totalStream);
byte op;
try {
op = br.ReadByte();
} catch (EndOfStreamException) {
op = Byte.MaxValue;
}
if (op == StreamOpCode.create) {
br.ReadString(); //pubsubclientdeserializer
string deserName = br.ReadString();
myDeserializer = Activator.CreateInstance(Type.GetType(deserName))
as IClientDeserializer;
totalStream.Position = initialPos;
myDeserializer.deserialize(new Stream[] { totalStream }, out theOps, out theRows);
} else {
// BlueEventLog.logEvent ("Received a non create before a create", BlueEventLevel.Warning);
theOps = new byte[] { };
theRows = new object[] { };
}
}
var msgs = StreamIOMessage.assemble_msgs(theOps, theRows);
int i = 0;
foreach (var m in msgs) {
bts.Log.Gen.info($"ClientDeserializationController n_desers: {_n_desers} msg no: {i++}: {m}");
}
_n_desers++;
// log it here
}
#endregion
}
}
@@ -1,124 +0,0 @@
using System;
using System.Collections;
using System.IO;
using BCG.Utilities;
//using BCG.Utilities.BlueConfiguration;
namespace BCG
{
/// <summary>
/// Pairs up with the UniversalSerializer to deserialize the corresponding data. Everythign anologous
/// to UniversalSerializer.
/// </summary>
public abstract class ClientDeserializer : IClientDeserializer
{
protected IBlueTableSchema mySchema;
protected IBlueTableSchema myFirstSchema;
protected IClientRowDeserializer myRowDeserializer;
public ClientDeserializer()
{
myFirstSchema = null;
mySchema = null;
}
#region IClientDeserializer Members
void BCG.IClientDeserializer.deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
{
BinaryReader br = new BinaryReader(UtilityMethods.getAggregatedStream(streams));
// BlueTrace.WriteDeserializeStart(streams.Length, br.BaseStream.Length);
// emit # of streams and total length
ArrayList ops = new ArrayList(), rows = new ArrayList();
IBlueRowCollection[] result;
while (true)
{
byte op;
try
{
int value = br.BaseStream.ReadByte();
if (value == -1) break;
op = (byte)value;
// BlueTrace.WriteDeserializeOperation(op);
}
catch (EndOfStreamException)
{
break;
}
if (op == StreamOpCode.create)
{
br.ReadString(); //pubsubDeserializer
br.ReadString(); //clientDeserializer
mySchema = new BlueTableSchema();
mySchema.deserialze(br);
//keep track of the first schema you've ever seen, so you know how to reorder columns
if (myFirstSchema == null)
myFirstSchema = mySchema;
result = new IBlueRowCollection[] { myGetRows(br, op, br.ReadInt32()) };
myEncorporateRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else
{
if (mySchema != null)
{
if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
{
result = new IBlueRowCollection[] { myGetRows(br, op, 1) };
myEncorporateRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else if (op == StreamOpCode.multipleChange)
{
result = new IBlueRowCollection[] { myGetRows(br, op, br.ReadInt32()) };
myEncorporateRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else if (op == StreamOpCode.multipleRemoveAdd)
{
result =
new IBlueRowCollection[] {
myGetRows (br, StreamOpCode.remove, br.ReadInt32 ()),
myGetRows (br, StreamOpCode.add, br.ReadInt32 ())
};
myEncorporateRows(result[0], StreamOpCode.remove);
myEncorporateRows(result[1], StreamOpCode.add);
ops.Add(op);
rows.Add(result);
}
else
{
// BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error);
}
}
else
{
// BlueEventLog.logEvent("Received a non create operation even though the schema" +
// "has never been created", BlueEventLevel.Warning);
}
}
}
theOps = ops.ToArray(typeof(byte)) as byte[];
theRows = rows.ToArray();
// BlueTrace.WriteDeserializeStop(rows.Count);
}
#endregion
protected abstract IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows);
protected abstract void myEncorporateRows(IBlueRowCollection rows, byte op);
}
}
@@ -1,34 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
namespace BCG
{
class ClientReorderingRowDeserializer : IClientRowDeserializer
{
private IBlueTableSchema myOriginalSchema;
private IBlueTableSchema myOutputSchema;
private ClientUniversalRowDeserializer myDeserializer;
public ClientReorderingRowDeserializer(IBlueTableSchema originalSchema, IBlueTableSchema newSchema)
{
myOriginalSchema = originalSchema;
myOutputSchema = newSchema;
myDeserializer = new ClientUniversalRowDeserializer(newSchema);
}
#region IClientRowDeserializer Members
public void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
{
IBlueRow temp = null;
myDeserializer.deserializeRow(currentState, br, op, out temp);
r = BlueDataUtility.changeSchema(temp, myOutputSchema);
}
#endregion
}
}
@@ -1,35 +0,0 @@
using System;
using System.Collections;
using System.IO;
namespace BCG
{
public class ClientUniversalDeserializer : ClientDeserializer
{
public ClientUniversalDeserializer () {}
protected override IBlueRowCollection myGetRows (BinaryReader br, byte op, int nRows)
{
BlueTable retVal = new BlueTable (mySchema);
if (myRowDeserializer == null)
myRowDeserializer = new ClientUniversalRowDeserializer(mySchema);
else if (op == StreamOpCode.create)
myRowDeserializer = new ClientReorderingRowDeserializer(myFirstSchema, mySchema);
for (int i = 0; i < nRows; i++)
{
IBlueRow row;
myRowDeserializer.deserializeRow (null, br, op, out row);
retVal.addRow (row);
}
return retVal;
}
protected override void myEncorporateRows (IBlueRowCollection rows, byte op)
{
//this does not keep state so ignore calls
}
}
}
@@ -1,69 +0,0 @@
using System;
using System.Collections;
using System.IO;
using BCG.Utilities;
//using BCG.Utilities.BlueConfiguration;
namespace BCG {
/// <summary>
/// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process
/// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations)
/// </summary>
internal class ClientUniversalRowDeserializer : IClientRowDeserializer {
private IBlueTableSchema mySchema;
private int myNullMapLength = 0;
private PrimitiveDeserializationHelper myDeserializationHelper;
private BitArray myNullMap;
private int[] mySerIndexToSchemaIndexMap;
public ClientUniversalRowDeserializer(IBlueTableSchema schema) {
myDeserializationHelper = new PrimitiveDeserializationHelper(mySchema = schema);
myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length) / 8)));
myInitReorderSchema();
}
#region Schema Reordering initialization
private void myInitReorderSchema() {
try {
int idColIndex = mySchema["id"].colNo;
int[] temp = new int[mySchema.columnInfo.Length];
foreach (ColumnInfo ci in mySchema.columnInfo) {
temp[ci.colNo] =
ci.colNo == idColIndex ? 0 :
ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo;
}
mySerIndexToSchemaIndexMap = new int[temp.Length];
for (int i = 0; i < temp.Length; i++)
mySerIndexToSchemaIndexMap[temp[i]] = i;
} catch (Exception ex) {
string schema_name = mySchema == null ? "NULL" : mySchema.tableName;
string err_msg = $"ClientUniversalRowDeserializer.myInitReorderSchema table: {schema_name} ex: {ex}";
bts.Log.Gen.error(err_msg);
throw new ApplicationException(err_msg);
}
}
#endregion
#region IClientRowDeserializer Members
public void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r) {
br.ReadInt32(); //length information for the row not needed
myNullMap = new BitArray(br.ReadBytes(myNullMapLength));
object[] values = new object[mySchema.columnNames.Length];
for (int i = 0; i < values.Length; i++) {
int colNo = mySerIndexToSchemaIndexMap[i];
values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn(colNo, br);
}
r = new BlueRow(mySchema, values);
}
#endregion
}
}
@@ -1,149 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace BCG {
public class StreamIOMessage {
public enum StreamOp {
Create = 0,
Add = 1, //adds a single row
Remove = 2, //removes a single row
Change = 3, //changes a single row
MultipleRemoveAdd = 4,//removes and adds multiple rows
MultipleChange = 5 //changes multiples rows
}
public class CreateMessage {
public IBlueRowCollection rows;
public override string ToString() {
string rv = "Create message ";
if (rows == null) {
rv += " rows: null";
return rv;
}
foreach (IBlueRow r in rows) {
rv += $"id: {r["id"]} ";
}
return rv;
}
}
public class AddMessage {
public IBlueRow added_row;
}
public class RemoveMessage {
public IBlueRow removed_row;
}
public class ChangeMessage {
public IBlueRow changed_row;
}
public class MultipleRemoveAddMessage {
public IBlueRowCollection removed;
public IBlueRowCollection added;
public override string ToString() {
string rv = "MultipleRemoveAdd removed: ";
foreach (IBlueRow r in removed) {
rv += $"id: {r["id"]} ";
}
rv += "added: ";
foreach (IBlueRow r in added) {
rv += $"id: {r["id"]} ";
}
return rv;
}
}
public class MultipleChangeMessage {
public IBlueRowCollection changed;
}
public StreamOp op;
public CreateMessage create_message = null;
public AddMessage add_message = null;
public RemoveMessage remove_message = null;
public ChangeMessage change_message = null;
public MultipleRemoveAddMessage multiple_remove_add_message = null;
public MultipleChangeMessage multiple_change_message = null;
public StreamIOMessage(byte op_, object data) {
op = (StreamIOMessage.StreamOp)(op_);
switch (op) {
case StreamIOMessage.StreamOp.Create:
IBlueRowCollection state = null;
// TODO, not sure why it can be an array and by itself
if (data is IBlueRowCollection) state = data as IBlueRowCollection;
else if (data is IBlueRowCollection[]) state = (data as IBlueRowCollection[]).ElementAt(0);
create_message = new CreateMessage() { rows = state };
break;
case StreamIOMessage.StreamOp.Add:
bts.Log.assert(false);
break;
case StreamIOMessage.StreamOp.Remove:
bts.Log.assert(false);
break;
case StreamIOMessage.StreamOp.Change:
bts.Log.assert(false);
break;
case StreamIOMessage.StreamOp.MultipleRemoveAdd:
IBlueRowCollection[] row_args = data as IBlueRowCollection[];
multiple_remove_add_message = new MultipleRemoveAddMessage() { removed = row_args[0], added = row_args[1] };
break;
case StreamIOMessage.StreamOp.MultipleChange:
//IBlueRowCollection[] row_args = data as IBlueRowCollection[];
multiple_change_message = new MultipleChangeMessage() { changed = (data as IBlueRowCollection[])[0] };
break;
}
}
public override string ToString() {
switch (op) {
case StreamIOMessage.StreamOp.Create:
return create_message.ToString();
break;
case StreamIOMessage.StreamOp.Add:
bts.Log.assert(false);
return String.Empty;
break;
case StreamIOMessage.StreamOp.Remove:
bts.Log.assert(false);
return String.Empty;
break;
case StreamIOMessage.StreamOp.Change:
bts.Log.assert(false);
return String.Empty;
break;
case StreamIOMessage.StreamOp.MultipleRemoveAdd:
return multiple_remove_add_message.ToString();
break;
case StreamIOMessage.StreamOp.MultipleChange:
return multiple_change_message.ToString();
break;
}
bts.Log.assert(false);
return String.Empty;
}
public static IEnumerable<StreamIOMessage> assemble_msgs(byte[] ops, Array op_data) {
int i = 0;
List<StreamIOMessage> rv = new List<StreamIOMessage>();
foreach (byte op in ops) {
object data = op_data.GetValue(i++);
rv.Add(new StreamIOMessage(op, data));
}
return rv;
}
}
/// <summary>
/// Performs deserialization inside the client app. The client gets either the complete data set or the
/// increments from PubSub and this interface makes sense out of them from a tabular/datastream stand point.
/// </summary>
public interface IClientDeserializer {
/// <summary>
/// deserializes the data received by the client from the pubsub
/// </summary>
/// <param name="serializedData">the data in byte format received from pubsub</param>
/// <param name="theOps">the operations that the data maps onto</param>
/// <param name="theRows">the data corresponding to the operations, is an array of IblueRowCollection[]</param>
void deserialize(Stream[] streams, out byte[] theOps, out Array theRows);
}
}
@@ -1,12 +0,0 @@
using System;
using System.IO;
using System.Collections;
namespace BCG {
/// <summary>
/// Deserializes one row. the current state is also provided as some deserialization schemes might use it.
/// </summary>
public interface IClientRowDeserializer {
void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r);
}
}
@@ -1,79 +0,0 @@
using System;
using System.IO;
namespace BCG
{
/// <summary>
/// Deserializes primitive types. Contructs a hash based scheme based on the index, to make everythign O(1)
/// rather than searching it up and making it O(n)
/// </summary>
public class PrimitiveDeserializationHelper
{
private delegate object DeserializeDelegate (int colNo, BinaryReader br);
private DeserializeDelegate[] myDispatchTable;
private ColumnInfo[] myCis;
public PrimitiveDeserializationHelper (IBlueTableSchema schema)
: this (schema.columnInfo)
{}
public PrimitiveDeserializationHelper (ColumnInfo[] cis)
{
myDispatchTable = new DeserializeDelegate [(myCis = cis).Length];
foreach (ColumnInfo ci in cis)
myDispatchTable[ci.colNo] = myGetDispatchMethod (ci.colType);
}
public object deserializeColumn (int colNo, BinaryReader br)
{
return myDispatchTable[colNo] (colNo, br);
}
private DeserializeDelegate myGetDispatchMethod (Type t)
{
if (t.Equals (BlueDataValidTypes.INT)) return new DeserializeDelegate (myDeserializeInt32);
else if (t.Equals (BlueDataValidTypes.DOUBLE)) return new DeserializeDelegate (myDeserializeDouble);
else if (t.Equals (BlueDataValidTypes.STRING)) return new DeserializeDelegate (myDeserializeString);
else if (t.Equals (BlueDataValidTypes.BYTEARRAY)) return new DeserializeDelegate (myDeserializeByteArray);
else if (t.Equals (BlueDataValidTypes.CHAR)) return new DeserializeDelegate (myDeserializeChar);
else if (t.Equals (BlueDataValidTypes.LONG)) return new DeserializeDelegate (myDeserializeLong);
else if (t.Equals (BlueDataValidTypes.SBYTE)) return new DeserializeDelegate (myDeserializeSByte);
else if (t.Equals (BlueDataValidTypes.BOOL)) return new DeserializeDelegate (myDeserializeBool);
else if (t.Equals (BlueDataValidTypes.BYTE)) return new DeserializeDelegate (myDeserializeByte);
else if (t.Equals (BlueDataValidTypes.DECIMAL)) return new DeserializeDelegate (myDeserializeDecimal);
else if (t.Equals (BlueDataValidTypes.UINT)) return new DeserializeDelegate (myDeserializeUInt);
else if (t.Equals (BlueDataValidTypes.ULONG)) return new DeserializeDelegate (myDeserializeULong);
else if (t.Equals (BlueDataValidTypes.DATETIME)) return new DeserializeDelegate (myDeserializeDateTime);
else if (t.Equals (BlueDataValidTypes.SINGLE)) return new DeserializeDelegate (myDeserializeSingle);
else if (SerializationUtilityMethods.isSerializedType (t)) return new DeserializeDelegate (myDeserializeISerializedType);
else throw new ApplicationException (t.FullName + " is not a supported type");
}
private object myDeserializeInt32 (int colNo, BinaryReader br) {return br.ReadInt32 ();}
private object myDeserializeISerializedType (int colNo, BinaryReader br)
{
ISerializedType o = Activator.CreateInstance (myCis[colNo].colType)
as ISerializedType;
o.deserialze (br);
return o;
}
private object myDeserializeDouble (int colNo, BinaryReader br) {return br.ReadDouble ();}
private object myDeserializeString (int colNo, BinaryReader br) {return br.ReadString ();}
private object myDeserializeByteArray (int colNo, BinaryReader br) {return br.ReadBytes (br.ReadInt32 ());}
private object myDeserializeChar (int colNo, BinaryReader br) {return br.ReadChar ();}
private object myDeserializeLong (int colNo, BinaryReader br) {return br.ReadInt64 ();}
private object myDeserializeSByte (int colNo, BinaryReader br) {return br.ReadSByte ();}
private object myDeserializeBool (int colNo, BinaryReader br) {return br.ReadBoolean ();}
private object myDeserializeByte (int colNo, BinaryReader br) {return br.ReadByte ();}
private object myDeserializeDecimal (int colNo, BinaryReader br) {return br.ReadDecimal ();}
private object myDeserializeUInt (int colNo, BinaryReader br) {return br.ReadUInt32 ();}
private object myDeserializeDateTime (int colNo, BinaryReader br)
{
return new DateTime(br.ReadInt64());
//return DateTime.FromOADate (br.ReadDouble ());
//return new DateTime(br.ReadInt64());
}
private object myDeserializeULong (int colNo, BinaryReader br) {return br.ReadUInt64 ();}
private object myDeserializeSingle (int colNo, BinaryReader br) {return br.ReadSingle ();}
}
}
@@ -1,155 +0,0 @@
using System;
using System.Collections;
using System.Reflection;
//using BCG.Utilities.BlueConfiguration;
using BCG.Utilities;
namespace BCG
{
public class CorePrimtiveManager
{
private IDictionary myTypeLookupTable = new Hashtable ();
private static CorePrimtiveManager mySelf;
private ArrayList myAssemblyNames = new ArrayList ();
private CorePrimtiveManager()
{
myLoadAllTypes ();
}
private void myLoadAllTypes ()
{
//we only load one type here
loadAssembly ("X:\\bcg\\blue2\\Debug\\CppToCSharpDataStreamBridge.dll");
}
private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args)
{
foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies())
if (args.Name.Equals(a.FullName))
return a;
return null;
}
/// <summary>
/// Loads all the types associated with the assembly
/// </summary>
/// <param name="assemblyName"></param>
public void loadAssembly (string assemblyName)
{
if (myAssemblyNames.Contains (assemblyName))
return;
myAssemblyNames.Add (assemblyName);
Assembly loadedAssembly = GetType ().Assembly.FullName.Equals (assemblyName) ?
GetType ().Assembly :
Assembly.LoadFrom (assemblyName);
foreach (Type type in loadedAssembly.GetTypes())
if (!type.IsInterface
&& !type.IsAbstract
&& type.IsPublic)
{
bool isVisible = true;
string userRep = type.Name;
if (isVisible)
{
// BlueEventLog.logAssert(myTypeLookupTable [userRep] == null);
myTypeLookupTable[userRep] = type;
}
}
}
public static CorePrimtiveManager instance
{
get {return mySelf = mySelf == null ? new CorePrimtiveManager () : mySelf;}
}
/// <summary>
/// Returns the Type name corresponding to the primitive name
/// </summary>
/// <param name="friendlyName"></param>
/// <returns></returns>
public string find (string friendlyName)
{
if (myTypeLookupTable[friendlyName] == null)
throw new ApplicationException (friendlyName + " is not the friendly name of any of the loaded types");
return (myTypeLookupTable[friendlyName] as Type).Name;
}
/// <summary>
/// Constructs and returns an object of type corresponding to the friendly name
/// </summary>
/// <param name="friendlyName">friendly primitive name </param>
/// <param name="args">args can be null or an empty ICollection if the paramterless constructor
/// is the desired one</param>
/// <returns></returns>
public Object build (string friendlyName, ICollection args)
{
object[] constructorArgs = null;
if (args != null && args.Count > 0)
{
constructorArgs = new object [args.Count];
args.CopyTo (constructorArgs, 0);
}
if (myTypeLookupTable[friendlyName] == null)
throw new ApplicationException (friendlyName + " is not the friendly name of any of the loaded types");
return
constructorArgs == null ?
Activator.CreateInstance (myTypeLookupTable[friendlyName] as Type):
Activator.CreateInstance (myTypeLookupTable[friendlyName] as Type, constructorArgs);
}
/// <summary>
/// Returns the name of all the primitives
/// </summary>
/// <returns></returns>
public string[] getPrimitiveNames ()
{
string[] retVal = new string [myTypeLookupTable.Count];
myTypeLookupTable.Keys.CopyTo (retVal, 0);
return retVal;
}
/// <summary>
/// Return the types of all the streams
/// </summary>
/// <returns></returns>
public Type[] getStreamTypes ()
{
Type[] retVal = new Type [myTypeLookupTable.Count];
myTypeLookupTable.Values.CopyTo (retVal, 0);
return retVal;
}
public ICollection getConstructorSigns (string primitiveName)
{
ArrayList retVal = new ArrayList ();
// BlueEventLog.logAssert (myTypeLookupTable[primitiveName] != null);
Type t = myTypeLookupTable[primitiveName] as Type;
foreach (ConstructorInfo ci in t.GetConstructors())
{
string sign = "";
sign = t.Name + " (";
foreach (ParameterInfo pi in ci.GetParameters ())
sign += pi.ParameterType.Name +
" " +
pi.Name + ", ";
sign = sign.Length == (t.Name + " (").Length ? sign + ")": sign.Remove (sign.Length - 2, 2) + ")";
retVal.Add (sign);
}
return retVal;
}
public string getAssemblyName (string primitiveName)
{
// BlueEventLog.logAssert (myTypeLookupTable[primitiveName] != null);
return (myTypeLookupTable[primitiveName] as Type).Assembly.GetName().Name;
}
}
}
@@ -1,339 +0,0 @@
using System;
using System.IO;
using System.Collections;
namespace BCG
{
public class SerializationUtilityMethods
{
private static IDictionary mySerializationDispatchTable,
myDeserializationDispatchTable;
private delegate void MySerialize (object data, BinaryWriter bw);
private delegate object MyDeserialize (Type t, BinaryReader br);
static SerializationUtilityMethods ()
{
mySerializationDispatchTable = new Hashtable ();
mySerializationDispatchTable [BlueDataValidTypes.BOOL.Name] =
new MySerialize (mySerializeBool);
mySerializationDispatchTable [BlueDataValidTypes.BYTE.Name] =
new MySerialize (mySerializeByte);
mySerializationDispatchTable [BlueDataValidTypes.CHAR.Name] =
new MySerialize (mySerializeChar);
mySerializationDispatchTable [BlueDataValidTypes.DATETIME.Name] =
new MySerialize (mySerializeDateTime);
mySerializationDispatchTable [BlueDataValidTypes.DECIMAL.Name] =
new MySerialize (mySerializeDecimal);
mySerializationDispatchTable [BlueDataValidTypes.DOUBLE.Name] =
new MySerialize (mySerializeDouble);
mySerializationDispatchTable [BlueDataValidTypes.INT.Name] =
new MySerialize (mySerializeInt32);
mySerializationDispatchTable [BlueDataValidTypes.ISERIALIZEDTYPE.Name] =
new MySerialize (mySerializeISerializedType);
mySerializationDispatchTable [BlueDataValidTypes.LONG.Name] =
new MySerialize (mySerializeLong);
mySerializationDispatchTable [BlueDataValidTypes.SBYTE.Name] =
new MySerialize (mySerializeSByte);
mySerializationDispatchTable [BlueDataValidTypes.STRING.Name] =
new MySerialize (mySerializeString);
mySerializationDispatchTable [BlueDataValidTypes.BYTEARRAY.Name] =
new MySerialize (mySerializeByteArray);
mySerializationDispatchTable [BlueDataValidTypes.UINT.Name] =
new MySerialize (mySerializeUInt);
mySerializationDispatchTable [BlueDataValidTypes.ULONG.Name] =
new MySerialize (mySerializeULong);
mySerializationDispatchTable [BlueDataValidTypes.SINGLE.Name] =
new MySerialize (mySerializeSingle);
myDeserializationDispatchTable = new Hashtable ();
myDeserializationDispatchTable [BlueDataValidTypes.BOOL.Name] =
new MyDeserialize (myDeserializeBool);
myDeserializationDispatchTable [BlueDataValidTypes.BYTE.Name] =
new MyDeserialize (myDeserializeByte);
myDeserializationDispatchTable [BlueDataValidTypes.CHAR.Name] =
new MyDeserialize (myDeserializeChar);
myDeserializationDispatchTable [BlueDataValidTypes.DATETIME.Name] =
new MyDeserialize (myDeserializeDateTime);
myDeserializationDispatchTable [BlueDataValidTypes.DECIMAL.Name] =
new MyDeserialize (myDeserializeDecimal);
myDeserializationDispatchTable [BlueDataValidTypes.DOUBLE.Name] =
new MyDeserialize (myDeserializeDouble);
myDeserializationDispatchTable [BlueDataValidTypes.INT.Name] =
new MyDeserialize (myDeserializeInt32);
myDeserializationDispatchTable [BlueDataValidTypes.ISERIALIZEDTYPE.Name] =
new MyDeserialize (myDeserializeISerializedType);
myDeserializationDispatchTable [BlueDataValidTypes.LONG.Name] =
new MyDeserialize (myDeserializeLong);
myDeserializationDispatchTable [BlueDataValidTypes.SBYTE.Name] =
new MyDeserialize (myDeserializeSByte);
myDeserializationDispatchTable [BlueDataValidTypes.STRING.Name] =
new MyDeserialize (myDeserializeString);
myDeserializationDispatchTable [BlueDataValidTypes.BYTEARRAY.Name] =
new MyDeserialize (myDeserializeByteArray);
myDeserializationDispatchTable [BlueDataValidTypes.UINT.Name] =
new MyDeserialize (myDeserializeUInt);
myDeserializationDispatchTable [BlueDataValidTypes.ULONG.Name] =
new MyDeserialize (myDeserializeULong);
myDeserializationDispatchTable [BlueDataValidTypes.SINGLE.Name] =
new MyDeserialize (myDeserializeSingle);
}
public static int getObjectSerializedSize (object o)
{
Type t = o.GetType ();
if (t == BlueDataValidTypes.STRING)
{
int l = (o as string).Length;
if (l < 128)
return l + 1;
if (l < 16384)
return l + 2;
if (l < 2097152)
return l + 3;
if (l < Math.Pow (2, 28))
return l + 4;
throw new ApplicationException ("String too big");
}
else if (t == BlueDataValidTypes.BYTEARRAY)
{
return getFixedLengthTypeSize (BlueDataValidTypes.INT) +
(o as byte[]).Length;
}
else if (isSerializedType (t))
{
return (o as ISerializedType).getSerializedSize ();
}
return getFixedLengthTypeSize (t);
}
public static int getFixedLengthTypeSize (Type t)
{
if (t == BlueDataValidTypes.BOOL ||
t == BlueDataValidTypes.BYTE ||
t == BlueDataValidTypes.CHAR ||
t == BlueDataValidTypes.SBYTE)
return 1;
else if (t == BlueDataValidTypes.INT ||
t == BlueDataValidTypes.UINT||
t == BlueDataValidTypes.SINGLE)
return 4;
else if (t == BlueDataValidTypes.DOUBLE ||
t == BlueDataValidTypes.LONG ||
t== BlueDataValidTypes.ULONG)
return 8;
else if (t == BlueDataValidTypes.DECIMAL)
return 16;
else if (t == BlueDataValidTypes.DATETIME)
//as objects of type DateTime are stored as doubles
return getFixedLengthTypeSize (typeof(long));
else
throw new ApplicationException (t.FullName + " is not a supported type");
}
public static bool isFixedLengthType (Type t)
{
return !(t.Equals (BlueDataValidTypes.STRING) || t.Equals (BlueDataValidTypes.BYTEARRAY) ||
isSerializedType (t));
}
public static bool isSerializedType (Type t)
{
return t.GetInterface ("BCG.ISerializedType") != null;
}
#region Serialization
public static void serializeValue (object data, BinaryWriter bw)
{
Type t = data.GetType ();
if (isSerializedType (t))
(mySerializationDispatchTable [typeof(ISerializedType).Name]
as MySerialize)(data, bw);
else
(mySerializationDispatchTable [t.Name] as MySerialize) (data, bw);
}
public static void serializeEnum(object data, BinaryWriter bw)
{
mySerializeInt32(data, bw);
}
private static void mySerializeInt32 (object data, BinaryWriter bw)
{
bw.Write ((Int32)data);
}
private static void mySerializeDouble (object data, BinaryWriter bw)
{
bw.Write ((double)data);
}
private static void mySerializeString (object data, BinaryWriter bw)
{
bw.Write (data as string);
}
private static void mySerializeByteArray (object data, BinaryWriter bw)
{
byte[] bs = data as byte[];
bw.Write (bs.Length);
bw.Write (bs);
}
private static void mySerializeSingle (object data, BinaryWriter bw)
{
bw.Write ((Single)data);
}
private static void mySerializeChar (object data, BinaryWriter bw)
{
bw.Write ((char)data);
}
private static void mySerializeLong (object data, BinaryWriter bw)
{
bw.Write ((long)data);
}
private static void mySerializeSByte (object data, BinaryWriter bw)
{
bw.Write ((sbyte)data);
}
private static void mySerializeBool (object data, BinaryWriter bw)
{
bw.Write ((bool)data);
}
private static void mySerializeByte (object data, BinaryWriter bw)
{
bw.Write ((byte)data);
}
private static void mySerializeDecimal (object data, BinaryWriter bw)
{
bw.Write ((decimal)data);
}
private static void mySerializeUInt (object data, BinaryWriter bw)
{
bw.Write ((uint)data);
}
private static void mySerializeULong (object data, BinaryWriter bw)
{
bw.Write ((ulong)data);
}
private static void mySerializeDateTime (object data, BinaryWriter bw)
{
//DateTime is written is serialiazed as OADATE
bw.Write (((DateTime)data).ToOADate());
}
private static void mySerializeISerializedType (object data, BinaryWriter bw)
{
(data as ISerializedType).serialize (bw);
}
#endregion
#region Deserialization
public static object deserializeValue(Type t, BinaryReader br)
{
return myDeserialize(t, br);
}
private static object myDeserialize (Type t, BinaryReader br)
{
return isSerializedType (t) ?
(myDeserializationDispatchTable [typeof(ISerializedType).Name]
as MyDeserialize)(t, br)
:
(myDeserializationDispatchTable [t.Name] as MyDeserialize) (t, br);
}
public static object deserializeEnum(Type t, BinaryReader br)
{
return myDeserializeInt32(t, br);
}
private static object myDeserializeInt32 (Type t, BinaryReader br)
{
return br.ReadInt32 ();
}
private static object myDeserializeISerializedType (Type t, BinaryReader br)
{
ISerializedType o = Activator.CreateInstance (t)
as ISerializedType;
o.deserialze (br);
return o;
}
private static object myDeserializeDouble (Type t, BinaryReader br)
{
return br.ReadDouble ();
}
private static object myDeserializeString (Type t, BinaryReader br)
{
return br.ReadString ();
}
private static object myDeserializeByteArray (Type t, BinaryReader br)
{
return br.ReadBytes (br.ReadInt32 ());
}
private static object myDeserializeSingle (Type t, BinaryReader br)
{
return br.ReadSingle ();
}
private static object myDeserializeChar (Type t, BinaryReader br)
{
return br.ReadChar ();
}
private static object myDeserializeLong (Type t, BinaryReader br)
{
return br.ReadInt64 ();
}
private static object myDeserializeSByte (Type t, BinaryReader br)
{
return br.ReadSByte ();
}
private static object myDeserializeBool (Type t, BinaryReader br)
{
return br.ReadBoolean ();
}
private static object myDeserializeByte (Type t, BinaryReader br)
{
return br.ReadByte ();
}
private static object myDeserializeDecimal (Type t, BinaryReader br)
{
return br.ReadDecimal ();
}
private static object myDeserializeUInt (Type t, BinaryReader br)
{
return br.ReadUInt32 ();
}
private static object myDeserializeDateTime (Type t, BinaryReader br)
{
return DateTime.FromOADate(br.ReadDouble());
//return new DateTime(br.ReadInt64());
}
private static object myDeserializeULong (Type t, BinaryReader br)
{
return br.ReadUInt64 ();
}
#endregion
}
}
@@ -0,0 +1,75 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq.Expressions;
using System.Xml.Serialization;
using BCG.Utilities;
using bts;
namespace BCG {
/// <summary>
/// This class will be used by the clients to publish tabular data to the pubsub
/// </summary>
public class PubTableBroker {
protected PublisherCore _publisher_core;
protected IDictionary<string, SinglePubTableBroker> _publisher_map =
new Dictionary<string, SinglePubTableBroker>();
protected bool myWaitingForFirstOp = false;
public PubTableBroker()
: this(new PublisherCore()) { }
public PubTableBroker(PublisherCore publisher) {
_publisher_core = publisher;
Log.Gen.info("PubTableBroker constructed");
}
public virtual void create_table(IBlueRowCollection table) {
string log_msg = $"PubTableBroker.createTable table: {table.schema.tableName}\n";
foreach (IBlueRow r in table) {
log_msg += $"id: {r["id"]}";
}
bts.Log.Gen.info(log_msg);
myCheckTransaction();
SinglePubTableBroker table_publisher = null;
var name_key = table.schema.tableName.ToLower();
if (!_publisher_map.TryGetValue(name_key, out table_publisher)) {
table_publisher = new SinglePubTableBroker(name_key, _publisher_core);
_publisher_map.Add(name_key, table_publisher);
}
table_publisher.create_table(table);
}
/// <summary>
/// If a table has not been created with the same instance of a PubTableBroker an ApplicationException
/// will be thrown
/// </summary>
/// <param name="r"></param>
public virtual void make_batch_changes(IBlueRowCollection new_rows,
IBlueRowCollection del_rows,
IBlueRowCollection added_rows) {
string log_msg = $"PubTableBroker.make_batch_changes table: {new_rows.schema.tableName} del: {del_rows.Count} added: {added_rows.Count}";
foreach (IBlueRow r in del_rows)
log_msg += $"id: {r["id"]} ";
log_msg += "addedRows: ";
foreach (IBlueRow r in added_rows)
log_msg += $"id: {r["id"]}";
bts.Log.Gen.info(log_msg);
SinglePubTableBroker table_publisher = null;
var name_key = new_rows.schema.tableName.ToLower();
if (!_publisher_map.TryGetValue(name_key, out table_publisher))
throw new ApplicationException("Batch changes can not be made, as the table has not been created yet");
else
table_publisher.make_batch_changes(del_rows, added_rows);
}
protected void myCheckTransaction() {
if (myWaitingForFirstOp)
_publisher_core.begin_transaction();
myWaitingForFirstOp = false;
}
}
}
@@ -1,57 +0,0 @@
using System;
namespace BCG {
/// <summary>
/// Deals with the publication of one tag. Gets the specific implementaion of IBlueDataSerializer according to
/// the table name, from the config, and wraps around it. The external code that will deal with user calls for
/// row operations (PubTablePublisher) will only deal with this class, hence will be agnostic of the specalized
/// serialization.
/// </summary>
public class BlueDataSerializationController : IBlueDataSerializer {
private IBlueDataSerializer mySerializer;
public BlueDataSerializationController() { }
#region IBlueDataSerializer Members
/// <summary>
/// Serializes an op and data; however if a non create is called before a create,
/// the serializedData is set to an empty byte array.
/// </summary>
/// <param name="op"></param>
/// <param name="rows"></param>
/// <param name="serializedData"></param>
public void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData) {
if (op == StreamOpCode.create) {
if (mySerializer == null)
mySerializer = myGetSerializerForTable(rows[0].schema);
mySerializer.serialize(op, rows, out serializedData);
} else {
if (mySerializer != null) {
mySerializer.serialize(op, rows, out serializedData);
} else {
serializedData = new byte[] { };
}
}
}
#endregion
private IBlueDataSerializer myGetSerializerForTable(IBlueTableSchema theSchema) {
// TODOds
return new UniversalSerializer(theSchema);
string serializerName = DataStreamGatewaySettings.instance.getSerializer(theSchema.tableName);
object[] serializerArgs = DataStreamGatewaySettings.instance.getSerializerArgs(theSchema.tableName);
if (serializerArgs != null)
return Activator.CreateInstance(Type.GetType(serializerName), theSchema, serializerArgs) as IBlueDataSerializer;
else
return Activator.CreateInstance(Type.GetType(serializerName), theSchema) as IBlueDataSerializer;
}
}
}
@@ -1,111 +0,0 @@
using System;
using System.Collections;
using System.IO;
namespace BCG
{
/// <summary>
/// Abstract base class that can potentially serve all implementations of IBlueDataSerializer. The IBlueRowSerializer
/// is used to serialize individual rows. The implementaions should, in the constructor, contruct the myRowSerializer
/// to do the specific row serializations. Every nonabtract implementation of the this, should have a contructor
/// taking in an IBlueTableSchema.
/// </summary>
public abstract class BlueDataSerializer : IBlueDataSerializer
{
protected IBlueRowSerializer myRowSerializer;
protected string myClientDeserializerName, myPubsubDeserializerName;
public BlueDataSerializer(string pubsubDeserializerName,
string clientDeserializerName)
{
myPubsubDeserializerName = pubsubDeserializerName;
myClientDeserializerName = clientDeserializerName;
}
#region IBlueDataSerializer Members
/// <summary>
/// Provides the following serialization
///
/// -add, remove, change:
/// op
/// serializedRow (depending on the implementation of myRowSerializer)
///
/// -create
/// op
/// pubsubDeserializer name
/// clientDeserializer name
/// schema
/// # of rows
/// serializedRows
///
/// -multipleAddRemoves
/// op
/// # of deleted rows
/// serializedRows
/// # of added rows
/// serializedRows
///
/// -multipleChanges
/// op
/// # of changed rows
/// serializedRows
/// </summary>
/// <param name="op"></param>
/// <param name="rows"></param>
/// <param name="serializedData"></param>
public void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData)
{
// BlueTrace.WriteSerializeStart(op, rows.Length);
BinaryWriter bw = new BinaryWriter(new MemoryStream());
if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change)
{
bw.Write(op);
myRowSerializer.serializeRow(BlueDataUtility.extractRow(rows[0], 0), op, bw);
}
else if (op == StreamOpCode.create)
{
IBlueRowCollection createdRows = rows[0];
bw.Write(op);
bw.Write(myPubsubDeserializerName);
bw.Write(myClientDeserializerName);
int dummyIdCol = 0;
int[] dummyIdList = new int[1];
//the schema doesn't sort anything right now, because the C++ pub sub client depends on things being in a certain order
IBlueTableSchema sortedSchema = BlueDataUtility.getSortedSchema(createdRows.schema, out dummyIdList, out dummyIdCol);
sortedSchema.serialize(bw);
bw.Write(createdRows.Count);
foreach (IBlueRow r in createdRows)
myRowSerializer.serializeRow(r, StreamOpCode.add, bw);
}
else if (op == StreamOpCode.multipleRemoveAdd)
{
bw.Write(op);
IBlueRowCollection delRows = rows[0],
addRows = rows[1];
bw.Write(delRows.Count);
foreach (IBlueRow dr in delRows)
myRowSerializer.serializeRow(dr, StreamOpCode.remove, bw);
bw.Write(addRows.Count);
foreach (IBlueRow ar in addRows)
myRowSerializer.serializeRow(ar, StreamOpCode.add, bw);
}
else if (op == StreamOpCode.multipleChange)
{
bw.Write(op);
IBlueRowCollection changedRows = rows[0];
bw.Write(changedRows.Count);
foreach (IBlueRow cr in changedRows)
myRowSerializer.serializeRow(cr, StreamOpCode.change, bw);
}
bw.BaseStream.Position = 0;
serializedData = (bw.BaseStream as MemoryStream).ToArray();
// BlueTrace.WriteSerializeStop(serializedData.Length);
}
#endregion
}
}
@@ -1,21 +0,0 @@
using System;
using BCG;
using System.Collections;
using System.IO;
namespace BCG {
/// <summary>
/// All the data serialized to be send over to PubSub happens here. All it does is f(op, rows) = byte[].
/// </summary>
public interface IBlueDataSerializer {
/// <summary>
/// f(op, rows) = byte[]
/// </summary>
/// <param name="op">Operation that can be add, delete, change etc</param>
/// <param name="rows">The data to serialize. The only case when it will be an array of more than 1 element
/// is when there are multiple deletes and adds, ie StreamOpCode.multipleRemoveAdd.
/// </param>
/// <param name="serializedData">The data serialized as bytes to be send over the wire </param>
void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData);
}
}
@@ -1,23 +0,0 @@
using System;
using System.IO;
namespace BCG
{
/// <summary>
/// Responsible for serilializing rows
/// The way the serialization scheme works is that the header is same
/// for all serialization algorithms, only the way the row is serialized is different. Abstracting it out and we
/// can have each specific serializer have a unique implementation of IBlueRowSerializer.
/// </summary>
public interface IBlueRowSerializer
{
/// <summary>
/// Serializes a row and writes it into the BinaryWriter. Since its only one row, the ops should be appropriate
/// for one row; the ones allowed are StreamopCode.add, remove and change only.
/// </summary>
/// <param name="r"></param>
/// <param name="bw"></param>
void serializeRow (IBlueRow r, byte op, BinaryWriter bw);
}
}
@@ -1,96 +0,0 @@
using System;
using System.IO;
namespace BCG
{
/// <summary>
/// Serializes primitive types. Contructs a hash based scheme based on the index, to make everythign O(1)
/// rather than searching it up and making it O(n), n = types supported.
/// </summary>
public class PrimitiveSerializationHelper
{
private delegate void SerializeDelegate (object data, BinaryWriter bw);
private SerializeDelegate[] myDispatchTable;
#region constructors
public PrimitiveSerializationHelper (IBlueTableSchema schema)
: this (schema.columnInfo) {}
public PrimitiveSerializationHelper (ColumnInfo[] cis)
{
myDispatchTable = new SerializeDelegate [cis.Length];
foreach (ColumnInfo ci in cis)
myDispatchTable[ci.colNo] = myGetDispatchMethod (ci.colType);
}
#endregion
#region public methods
/// <summary>
/// Serializes the column no in the row and writes it into the BinaryWriter
/// </summary>
/// <param name="r"></param>
/// <param name="colNo"></param>
/// <param name="bw"></param>
public void serializeColumn (IBlueRow r, int colNo, BinaryWriter bw)
{
myDispatchTable[colNo] (r[colNo], bw);
}
/// <summary>
/// Writes the data to the BinaryWriter. The ColNo is provided to get the appropriate
/// serializer.
/// </summary>
/// <param name="data"></param>
/// <param name="colNo"></param>
/// <param name="bw"></param>
public void serializeColumn (object data, int colNo, BinaryWriter bw)
{
myDispatchTable[colNo] (data, bw);
}
#endregion
#region private methods
private void mySerializeInt32 (object data, BinaryWriter bw) {bw.Write ((Int32)data);}
private void mySerializeDouble (object data, BinaryWriter bw) {bw.Write ((double)data);}
private void mySerializeString (object data, BinaryWriter bw){bw.Write (data as string);}
private void mySerializeByteArray (object data, BinaryWriter bw)
{
byte[] bs = data as byte[];
bw.Write (bs.Length);
bw.Write (bs);
}
private void mySerializeChar (object data, BinaryWriter bw) {bw.Write ((char)data);}
private void mySerializeLong (object data, BinaryWriter bw) {bw.Write ((long)data);}
private void mySerializeSByte (object data, BinaryWriter bw) {bw.Write ((sbyte)data);}
private void mySerializeBool (object data, BinaryWriter bw) {bw.Write ((bool)data);}
private void mySerializeByte (object data, BinaryWriter bw) {bw.Write ((byte)data);}
private void mySerializeDecimal (object data, BinaryWriter bw) {bw.Write ((decimal)data);}
private void mySerializeUInt (object data, BinaryWriter bw) {bw.Write ((uint)data);}
private void mySerializeULong (object data, BinaryWriter bw){bw.Write ((ulong)data);}
private void mySerializeDateTime (object data, BinaryWriter bw) {bw.Write (((DateTime)data).Ticks);}
private void mySerializeSingle (object data, BinaryWriter bw) {bw.Write ((Single)data);}
private void mySerializeISerializedType (object data, BinaryWriter bw) {(data as ISerializedType).serialize (bw);}
private SerializeDelegate myGetDispatchMethod (Type t)
{
if (t.Equals (BlueDataValidTypes.INT)) return new SerializeDelegate (mySerializeInt32);
else if (t.Equals (BlueDataValidTypes.DOUBLE)) return new SerializeDelegate (mySerializeDouble);
else if (t.Equals (BlueDataValidTypes.STRING)) return new SerializeDelegate (mySerializeString);
else if (t.Equals (BlueDataValidTypes.BYTEARRAY)) return new SerializeDelegate (mySerializeByteArray);
else if (t.Equals (BlueDataValidTypes.CHAR)) return new SerializeDelegate (mySerializeChar);
else if (t.Equals (BlueDataValidTypes.LONG)) return new SerializeDelegate (mySerializeLong);
else if (t.Equals (BlueDataValidTypes.SBYTE)) return new SerializeDelegate (mySerializeSByte);
else if (t.Equals (BlueDataValidTypes.BOOL)) return new SerializeDelegate (mySerializeBool);
else if (t.Equals (BlueDataValidTypes.BYTE)) return new SerializeDelegate (mySerializeByte);
else if (t.Equals (BlueDataValidTypes.DECIMAL)) return new SerializeDelegate (mySerializeDecimal);
else if (t.Equals (BlueDataValidTypes.UINT)) return new SerializeDelegate (mySerializeUInt);
else if (t.Equals (BlueDataValidTypes.ULONG)) return new SerializeDelegate (mySerializeULong);
else if (t.Equals (BlueDataValidTypes.DATETIME)) return new SerializeDelegate (mySerializeDateTime);
else if (t.Equals (BlueDataValidTypes.SINGLE)) return new SerializeDelegate (mySerializeSingle);
else if (SerializationUtilityMethods.isSerializedType (t)) return new SerializeDelegate (mySerializeISerializedType);
else throw new ApplicationException (t.FullName + " is not a supported type");
}
#endregion
}
}
@@ -1,120 +0,0 @@
using System;
using System.Collections;
using System.IO;
using BCG.Utilities;
using System.Collections.Generic;
namespace BCG
{
/// <summary>
/// Helps the UniversalSerializer by serializing one row. Hence this class is agnostic to the operation
/// which is figured out by UniversalSerializer.
///
/// it sorts the schema it gets, to make sure that the rows are always published the same way.
/// For now, the sorting operation doesn't do anything. The reason for this is that C++ clients
/// are depending upon the data arriving in a certain order.
///
/// It performs a mapping : (row)->byte[]
/// </summary
internal class UniversalRowSerializer : IBlueRowSerializer
{
private PrimitiveSerializationHelper myPrimitiveSerializationHelper;
private int myFixedSize;
private int[] myNonFixedColIndexes;
private IBlueTableSchema mySchema;
//this null map is the null map that will be written to the wire
//therefore, its values represent the nulls in row, ordered by the-rearranged schema.
private BitArray myNullMap;
private byte[] myNullMapArray; //just so that a new instance is not created everytime
//this thing stores a mapping between the columns in the original schema that were passed in
private int[] mySortedSchemaToOriginalSchemaMapping;
private int myIdColumnInSortedSchema; //this is used because the id column will be different in the original row and the schema
public UniversalRowSerializer(IBlueTableSchema schema)
{
mySchema = schema;
myPrimitiveSerializationHelper = new PrimitiveSerializationHelper(mySchema);
//therefore well use the original code
//sort the schema after we give it to the serialiaztion helper, because the helper constructs a dispatch table
//so it needs to know about the original schema.
mySchema = BlueDataUtility.getSortedSchema(mySchema, out mySortedSchemaToOriginalSchemaMapping, out myIdColumnInSortedSchema);
myNullMapArray =
new byte[Convert.ToInt32(Math.Ceiling(Convert.ToDouble(mySchema.columnInfo.Length) / 8))];
myNullMap =
new BitArray(myNullMapArray);
myInitSizeCalculations();
}
#region IBlueRowSerializer methods
/// <summary>
/// Row serialized in the following order
/// row size (nullBitMap size + actual row size)
/// null bit map
/// non null row members (with id as the first col)
/// </summary>
/// <param name="r"></param>
/// <param name="bw"></param>
public void serializeRow(IBlueRow r, byte op, BinaryWriter bw)
{
myNullMap.SetAll(false);
for (int i = 0; i < mySchema.columnNames.Length; i++)
if (r[mySortedSchemaToOriginalSchemaMapping[i]] == DBNull.Value || r[mySortedSchemaToOriginalSchemaMapping[i]] == null)
myNullMap[i] = true;
myNullMap.CopyTo(myNullMapArray, 0);
long initPos = bw.BaseStream.Position;
bw.Write((int)0); ///place holder for size
bw.Write(myNullMapArray);
bw.Write((int)r[mySortedSchemaToOriginalSchemaMapping[myIdColumnInSortedSchema]]);
foreach (ColumnInfo ci in mySchema.columnInfo)
if (!myNullMap[ci.colNo] && ci.colNo != myIdColumnInSortedSchema)
myPrimitiveSerializationHelper.serializeColumn(r, mySortedSchemaToOriginalSchemaMapping[ci.colNo], bw);
long endPos = bw.BaseStream.Position;
bw.BaseStream.Position = initPos;
bw.Write(Convert.ToInt32(endPos - (initPos+4))); /// size does not include the meta data hence substracting 4
bw.BaseStream.Position = endPos;
}
public void serializeSchema(BinaryWriter bw)
{
mySchema.serialize(bw);
}
#endregion
#region size calculations
/// <summary>
/// Sets up the fixed length (size in bytes occupied by each of the primitive type)
/// and an index for the non primitive type
/// </summary>
private void myInitSizeCalculations()
{
ArrayList temp = new ArrayList();
myFixedSize = myNullMapArray.Length;
foreach (ColumnInfo ci in mySchema.columnInfo)
{
if (!SerializationUtilityMethods.isFixedLengthType(ci.colType))
temp.Add(ci.colNo);
else
myFixedSize += SerializationUtilityMethods.getFixedLengthTypeSize(ci.colType);
}
myNonFixedColIndexes = temp.ToArray(typeof(int)) as int[];
}
/// <summary>
/// Find the size of the serialized row, assumes that nullMap has already
/// been built
/// </summary>
/// <param name="fixedSize"></param>
/// <param name="r"></param>
/// <returns></returns>
private int myGetRowSizeInBytes(int fixedSize, IBlueRow r)
{
foreach (int colNo in myNonFixedColIndexes)
{
if (!myNullMap[colNo])
fixedSize += //myGetStringSizeInBytes (r[colNo] as string);
SerializationUtilityMethods.getObjectSerializedSize(r[mySortedSchemaToOriginalSchemaMapping[colNo]]);
}
return fixedSize;
}
#endregion
}
}
@@ -1,16 +0,0 @@
using System;
namespace BCG
{
/// <summary>
/// Uses the UniversalRowSerializer as the IBlueRowSerializer implementation.
/// </summary>
public class UniversalSerializer : BlueDataSerializer
{
public UniversalSerializer (IBlueTableSchema schema)
: base ("BCG.PubsubUniversalDeserializer, DataStreamCore", "BCG.ClientUniversalDeserializer")
{
myRowSerializer = new UniversalRowSerializer (schema);
}
}
}
@@ -1,25 +0,0 @@
using System;
using System.IO;
using System.Collections;
namespace BCG
{
/// <summary>
/// Deserializes a row. Returns the id of the row and its full content
/// </summary>
public interface IPubsubRowDeserializer
{
/// <summary>
/// Reads the data from the BinaryReader and deserializes it in the form of a row
/// </summary>
/// <param name="currentState">Some deserializers might need the current state, ie a Dictionary of rows
/// keyed off the id</param>
/// <param name="br">BinaryReader starting at the row</param>
/// <param name="op">The operation, as this might drive the deserializations scheme in some cases. This op
/// however is one of the basics operations, add remove or change as all other operations can be resolved into this</param>
/// <param name="id">The id of the deserialized row</param>
/// <param name="row">The actual row contents</param>
void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row);
}
}
@@ -1,168 +0,0 @@
using System;
using System.Collections;
using System.IO;
using BCG.Utilities;
using System.Text;
namespace BCG
{
/// <summary>
/// Base class for all pubsubs deserializers.
/// </summary>
public abstract class PubsubDeserializer : IPubsubDeserializer
{
// this is where the stream state is stored in pubsub?
protected IDictionary myState = new Hashtable ();
protected IBlueTableSchema mySchema;
protected string myClientDeserializerName; //for the getState method
protected IPubsubRowDeserializer myDeserializer;
public PubsubDeserializer () {}
#region IPubSubDeserializer Members
public virtual void deserialize (Stream[] streams, out byte[] theOps, out Array theRows)
{
BinaryReader br = new BinaryReader (UtilityMethods.getAggregatedStream (streams));
ArrayList ops = new ArrayList (), rows = new ArrayList ();
while (true)
{
IDictionary[] result;
byte op;
try
{
int value = br.BaseStream.ReadByte();
if (value == -1) break;
op = (byte)value;
}
catch (EndOfStreamException)
{
// TODO, logically break loop
break;
}
if (op == StreamOpCode.create)
{
myState.Clear ();
br.ReadString (); //the pubsub-deserializer name, should be the same as this's name
myClientDeserializerName = br.ReadString (); //the client-deserializer name
(mySchema = new BlueTableSchema()).deserialze (br);
result = new IDictionary[] {myGetRows (br, op, br.ReadInt32 ())};
myEncorporatRows (result[0], op);
ops.Add (op);
rows.Add (result);
}
else
{
if (mySchema != null)
{
if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
{
result = new IDictionary[] {myGetRows (br, op, 1)};
myEncorporatRows (result[0], op);
ops.Add (op);
rows.Add (result);
}
else if (op == StreamOpCode.multipleChange)
{
result = new IDictionary[] {myGetRows (br, op, br.ReadInt32 ())};
myEncorporatRows (result[0], op);
ops.Add (op);
rows.Add (result);
}
else if (op == StreamOpCode.multipleRemoveAdd)
{
ops.Add (op);
result = new IDictionary [] {
myGetRows (br, StreamOpCode.remove, br.ReadInt32 ()),
myGetRows (br, StreamOpCode.add, br.ReadInt32 ())
};
myEncorporatRows (result[0], StreamOpCode.remove);
myEncorporatRows (result[1], StreamOpCode.add);
rows.Add (result);
}
else
{
// BlueEventLog.logEvent ("Illegal op-code received", BlueEventLevel.Error);
}
}
else
{
// BlueEventLog.logEvent ("Received a non create operation even though the schema" +
// "has never been created", BlueEventLevel.Warning);
}
}
}
theOps = ops.ToArray (typeof(byte)) as byte[];
theRows = rows.ToArray ();
}
/// <summary>
/// Sends in a create signal, so that new clients can get a create everytime and start
/// afresh from there
/// </summary>
/// <returns></returns>
public virtual byte[] getState()
{
if (mySchema == null)
return new byte[] {};
else
{
BinaryWriter bw = new BinaryWriter (new MemoryStream ());
bw.Write (StreamOpCode.create);
bw.Write (GetType().AssemblyQualifiedName);
bw.Write (myClientDeserializerName);
mySchema.serialize (bw);
bw.Write (myState.Count);
foreach (byte[] row in myState.Values)
bw.Write (row);
return (bw.BaseStream as MemoryStream).ToArray ();
}
}
#endregion
/// <summary>
/// Every row that has been deserialized passes through this method, so that it can update the state.
/// </summary>
/// <param name="rows"></param>
/// <param name="op"></param>
protected virtual void myEncorporatRows (IDictionary rows, byte op)
{
if (op == StreamOpCode.add || op == StreamOpCode.create)
foreach (DictionaryEntry de in rows)
{
if (myState.Contains(de.Key)) {
// BlueEventLog2.instance().logEvent(String.Format("Duplicate row creation. t: {0} id:{1}", mySchema.tableName, de.Key),BlueEventLog2Level.Fatal);
StringBuilder b = new StringBuilder();
foreach (string n in mySchema.columnNames)
b.AppendFormat(" {0}", n);
// BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()), BlueEventLog2Level.Fatal);
}
myState.Add(de.Key, de.Value);
}
else if (op == StreamOpCode.remove)
foreach (DictionaryEntry de in rows)
{
if (!myState.Contains(de.Key))
{
// BlueEventLog2.instance().logEvent(String.Format("Unknown row deletion. t: {0} id:{1}", mySchema.tableName, de.Key), BlueEventLog2Level.Fatal);
StringBuilder b = new StringBuilder();
foreach (string n in mySchema.columnNames)
b.AppendFormat(" {0}", n);
// BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()),BlueEventLog2Level.Fatal);
}
myState.Remove(de.Key);
}
else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange)
foreach (DictionaryEntry de in rows)
myState[de.Key] = de.Value;
}
/// <summary>
/// Mass deserialization happens here, also since there is no knowledge of the schema
/// </summary>
/// <param name="br"></param>
/// <param name="op"></param>
/// <param name="noOfRows"></param>
/// <returns></returns>
protected abstract IDictionary myGetRows (BinaryReader br, byte op, int noOfRows);
}
}
@@ -1,23 +0,0 @@
using System;
using System.Collections;
using System.IO;
namespace BCG {
public class PubsubUniversalDeserializer : PubsubDeserializer {
public PubsubUniversalDeserializer() {
bts.Log.Gen.info("PubsubUniversalDeserializer.cons");
}
protected override IDictionary myGetRows(BinaryReader br, byte op, int noOfRows) {
myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer(mySchema) : myDeserializer;
Hashtable retVal = new Hashtable();
for (int i = 0; i < noOfRows; i++) {
int id; object row;
myDeserializer.deserializeRow(myState, br, op, out id, out row);
retVal.Add(id, row);
}
return retVal;
}
}
}
@@ -1,34 +0,0 @@
using System;
using System.IO;
using System.Collections;
namespace BCG
{
public class PubsubUniversalRowDeserializer : IPubsubRowDeserializer
{
private int myNullMapLength;
private IBlueTableSchema mySchema;
private int myRowOverheadSize = SerializationUtilityMethods.getFixedLengthTypeSize (typeof(int));
public PubsubUniversalRowDeserializer (IBlueTableSchema schema)
{
mySchema = schema;
myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8)));
}
#region IRowDeserializer Members
public void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row)
{
//current state is ignored
long pos = br.BaseStream.Position;
int rowSize = br.ReadInt32 ();
br.BaseStream.Position = pos;
row = br.ReadBytes (rowSize + myRowOverheadSize);
id = BitConverter.ToInt32 (row as byte[], myNullMapLength + myRowOverheadSize);
}
#endregion
}
}
@@ -1,46 +0,0 @@
// using System;
// using System.IO;
// using System.Collections.Generic;
// using BCG.Utilities;
// namespace BCG {
// /// <summary>
// /// This is where the all the pubSub State is handled, and its cache kept
// /// Keeps a collection of TagStateBanks to hold the state of each tag
// /// </summary>
// public class PubsubServerTagHandler : IPubSubServerTagHandler {
// private IDictionary<string, TagStateBank> _tag_state_map =
// new Dictionary<string, TagStateBank>();
// #region IPubSubServerTagHandler Members
// public byte[] get_content(string tag) {
// TagStateBank tag_state_bank;
// if (_tag_state_map.TryGetValue(tag, out tag_state_bank)) {
// return tag_state_bank.get_content();
// } else {
// bts.Log.Gen.warn("Information requested for tag: {tag}, it does not exist now");
// return new byte[] { };
// }
// }
// public void update(string tag, Stream[] messageBodies) {
// bts.Log.Gen.info($"PubsubServerTagHandler.update tag: {tag} streams: {messageBodies.Length}");
// TagStateBank tag_state_bank;
// if (!_tag_state_map.TryGetValue(tag, out tag_state_bank)) {
// bts.Log.Gen.info("tag: {tag} received for the first time creating TagStateBank for it now");
// tag_state_bank = new TagStateBank();
// _tag_state_map.Add(tag, tag_state_bank);
// }
// tag_state_bank.update(messageBodies);
// }
// public void doWork() { }
// public string[] supportedTagTypes { get { return new string[] { "DataStreamTag" }; } }
// public void register(InternalPublishDelegate callback) { }
// #endregion
// }
// }
@@ -1,102 +0,0 @@
using System;
using System.IO;
using System.Collections;
using BCG.Utilities;
using System.Threading;
namespace BCG
{
class TagData
{
public DateTime lastUpdateTime;
public long totalSize;
public int lastUpdateSize;
public TagData(DateTime ut, long ts, int lus)
{
lastUpdateTime = ut;
totalSize = ts;
lastUpdateSize = lus;
}
public override string ToString()
{
return lastUpdateTime.ToString() + "," + totalSize.ToString() + "," + lastUpdateSize.ToString();
}
}
class StatisticsLogger
{
private IDictionary myTagData = new Hashtable();
private static StatisticsLogger mySelf;
private BlockingQueue myQueue = new BlockingQueue();
private DateTime myLastFireTime = DateTime.MinValue;
private TimeSpan myPubsubStatsUpdateInterval = new TimeSpan(0, 0, 0, 0, DataStreamGatewaySettings.instance.pubsubStatsUpdateInterval);
private TimeSpan myTagStatsUpdateInterval = new TimeSpan(0, 0, 0, 0, DataStreamGatewaySettings.instance.singleTagStatsUpdateInterval);
private StatisticsLogger()
{
Thread readThread = new Thread(new ThreadStart(myReadAndPublish));
readThread.Priority = ThreadPriority.BelowNormal;
readThread.Start();
}
public static StatisticsLogger instance
{
get
{
if (mySelf == null)
mySelf = new StatisticsLogger();
return mySelf;
}
}
public void log(string tag, Stream[] bodies)
{
long nBytes = 0;
foreach (Stream s in bodies)
nBytes += s.Length;
myQueue.enqueue(new object[] { tag, Convert.ToInt32 (nBytes) });
}
private void myReadAndPublish()
{
while (true)
{
object qe = myQueue.dequeue(50);
if (qe != null)
{
object[] args = qe as object[];
string tag = args[0].ToString();
int dataSize = (int)args[1];
if (!myTagData.Contains(tag))
{
myTagData.Add(tag, new TagData(DateTime.Now, Convert.ToInt64(dataSize), dataSize));
}
else
{
TagData tdToSend = myTagData[tag] as TagData;
tdToSend.lastUpdateSize += dataSize;
tdToSend.totalSize += dataSize;
}
}
myFireMonitors();
}
}
private void myFireMonitors()
{
if (DateTime.Now.Subtract(myLastFireTime) >= myPubsubStatsUpdateInterval)
{
foreach (string tag in myTagData.Keys)
{
TagData td = myTagData[tag] as TagData;
if (td.lastUpdateSize > 0 && DateTime.Now.Subtract(td.lastUpdateTime) >= myTagStatsUpdateInterval)
{
td.lastUpdateTime = DateTime.Now;
td.lastUpdateSize = 0;
}
}
myLastFireTime = DateTime.Now;
}
}
}
}
@@ -1,32 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
namespace BCG
{
public class TagStateBank
{
private PubsubDeserializationController myDeserializationController = new PubsubDeserializationController();
//private ClientDeserializationController _deserializer = new ClientDeserializationController();
public TagStateBank() { }
public TagStateBank(string tag)
{
bts.Log.Gen.info("TagStateBank created for tag: {tag}");
}
public byte[] get_content()
{
return myDeserializationController.getState();
}
public void update(Stream[] streams)
{
bts.Log.Gen.info($"TagStateBank.update streams: {streams.Length}");
byte[] ops;
Array rowCollection;
//myDeserializationController.deserialize(streams, out ops, out rowCollection);
}
}
}
@@ -1,146 +0,0 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using BCG.Utilities;
namespace BCG
{
public class CompressionClientDeserializer : IClientDeserializer
{
protected IBlueTableSchema mySchema;
protected IBlueTableSchema myFirstSchema;
protected IClientRowDeserializer myRowDeserializer;
public CompressionClientDeserializer()
{
myFirstSchema = null;
mySchema = null;
}
#region IClientDeserializer Members
public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
{
BinaryReader br = new BinaryReader(UtilityMethods.getAggregatedStream(streams));
// BlueTrace.WriteDeserializeStart(streams.Length, br.BaseStream.Length);
// emit # of streams and total length
ArrayList ops = new ArrayList(), rows = new ArrayList();
IBlueRowCollection[] result;
while (true)
{
byte op;
try
{
int value = br.BaseStream.ReadByte();
if (value == -1) break;
op = (byte)value;
// BlueTrace.WriteDeserializeOperation(op);
}
catch (EndOfStreamException)
{
break;
}
if (op == StreamOpCode.create)
{
br.ReadString(); //pubsubDeserializer
br.ReadString(); //clientDeserializer
mySchema = new BlueTableSchema();
mySchema.deserialze(br);
//keep track of the first schema you've ever seen, so you know how to reorder columns
if (myFirstSchema == null)
myFirstSchema = mySchema;
/// decompress here
int compressedMessageSize = br.ReadInt32();
byte[] compressedData = br.ReadBytes(compressedMessageSize);
BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(compressedData));
result = new IBlueRowCollection[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) };
myEncorporateRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else
{
if (mySchema != null)
{
if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
{
result = new IBlueRowCollection[] { myGetRows(br, op, 1) };
myEncorporateRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else if (op == StreamOpCode.multipleChange)
{
int compressedMessageSize = br.ReadInt32();
byte[] compressedData = br.ReadBytes(compressedMessageSize);
BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(compressedData));
result = new IBlueRowCollection[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) };
myEncorporateRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else if (op == StreamOpCode.multipleRemoveAdd)
{
int compressedMessageSize = br.ReadInt32();
byte[] compressedData = br.ReadBytes(compressedMessageSize);
BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(compressedData));
result =
new IBlueRowCollection[] {
myGetRows (decompressedBr, StreamOpCode.remove, decompressedBr.ReadInt32 ()),
myGetRows (decompressedBr, StreamOpCode.add, decompressedBr.ReadInt32 ())
};
myEncorporateRows(result[0], StreamOpCode.remove);
myEncorporateRows(result[1], StreamOpCode.add);
ops.Add(op);
rows.Add(result);
}
else
{
// BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error);
}
}
else
{
// BlueEventLog.logEvent("Received a non create operation even though the schema" +
// "has never been created", BlueEventLevel.Warning);
}
}
}
theOps = ops.ToArray(typeof(byte)) as byte[];
theRows = rows.ToArray();
// BlueTrace.WriteDeserializeStop(rows.Count);
}
#endregion
private IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows)
{
BlueTable retVal = new BlueTable(mySchema);
if (myRowDeserializer == null)
myRowDeserializer = new ClientUniversalRowDeserializer(mySchema);
else if (op == StreamOpCode.create)
myRowDeserializer = new ClientReorderingRowDeserializer(myFirstSchema, mySchema);
for (int i = 0; i < nRows; i++)
{
IBlueRow row;
myRowDeserializer.deserializeRow(null, br, op, out row);
retVal.addRow(row);
}
return retVal;
}
private void myEncorporateRows(IBlueRowCollection rows, byte op) { }
}
}
@@ -1,192 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Collections;
using BCG.Utilities;
namespace BCG
{
public class CompressionPubsubDeserializer : IPubsubDeserializer
{
protected IDictionary myState = new Hashtable();
protected IBlueTableSchema mySchema;
protected string myClientDeserializerName; //for the getState method
protected IPubsubRowDeserializer myRowDeserializer;
public CompressionPubsubDeserializer() { }
#region IPubSubDeserializer Members
public virtual void deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
{
BinaryReader br = new BinaryReader(UtilityMethods.getAggregatedStream(streams));
ArrayList ops = new ArrayList(), rows = new ArrayList();
while (true)
{
IDictionary[] result;
byte op;
try
{
int value = br.BaseStream.ReadByte();
if (value == -1) break;
op = (byte)value;
}
catch (EndOfStreamException)
{
break;
}
if (op == StreamOpCode.create)
{
myState.Clear();
br.ReadString(); //the pubsub-deserializer name, should be the same as this's name
myClientDeserializerName = br.ReadString(); //the client-deserializer name
(mySchema = new BlueTableSchema()).deserialze(br);
/// deserialize this part
int compressedMessageSize = br.ReadInt32();
BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(br.ReadBytes(compressedMessageSize)));
result = new IDictionary[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) };
myEncorporatRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else
{
if (mySchema != null)
{
if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
{
result = new IDictionary[] { myGetRows(br, op, 1) };
myEncorporatRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else if (op == StreamOpCode.multipleChange)
{
int compressedMessageSize = br.ReadInt32();
BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(br.ReadBytes(compressedMessageSize)));
result = new IDictionary[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) };
myEncorporatRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else if (op == StreamOpCode.multipleRemoveAdd)
{
ops.Add(op);
int compressedMessageSize = br.ReadInt32();
BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(br.ReadBytes(compressedMessageSize)));
result = new IDictionary[] {
myGetRows (decompressedBr, StreamOpCode.remove, decompressedBr.ReadInt32 ()),
myGetRows (decompressedBr, StreamOpCode.add, decompressedBr.ReadInt32 ())
};
myEncorporatRows(result[0], StreamOpCode.remove);
myEncorporatRows(result[1], StreamOpCode.add);
rows.Add(result);
}
else
{
// BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error);
}
}
else
{
// BlueEventLog.logEvent("Received a non create operation even though the schema" +
// "has never been created", BlueEventLevel.Warning);
}
}
}
theOps = ops.ToArray(typeof(byte)) as byte[];
theRows = rows.ToArray();
}
/// <summary>
/// Sends in a create signal, so that new clients can get a create everytime and start
/// afresh from there
/// </summary>
/// <returns></returns>
public virtual byte[] getState()
{
if (mySchema == null)
return new byte[] { };
else
{
BinaryWriter bw = new BinaryWriter(new MemoryStream());
bw.Write(StreamOpCode.create);
bw.Write(GetType().AssemblyQualifiedName);
bw.Write(myClientDeserializerName);
mySchema.serialize(bw);
/// compress it here
BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream());
decompressedBw.Write(myState.Count);
foreach (byte[] row in myState.Values)
decompressedBw.Write(row);
decompressedBw.BaseStream.Position = 0;
byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream);
bw.Write(compressedData.Length);
bw.Write(compressedData);
return (bw.BaseStream as MemoryStream).ToArray();
}
}
#endregion
/// <summary>
/// Every row that has been deserialized passes through this method, so that it can update the state.
/// </summary>
/// <param name="rows"></param>
/// <param name="op"></param>
protected virtual void myEncorporatRows(IDictionary rows, byte op)
{
if (op == StreamOpCode.add || op == StreamOpCode.create)
foreach (DictionaryEntry de in rows)
{
if (myState.Contains(de.Key))
{
// BlueEventLog2.instance().logEvent(String.Format("Duplicate row creation. t: {0} id:{1}", mySchema.tableName, de.Key), BlueEventLog2Level.Fatal);
StringBuilder b = new StringBuilder();
foreach (string n in mySchema.columnNames)
b.AppendFormat(" {0}", n);
// BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()), BlueEventLog2Level.Fatal);
}
myState.Add(de.Key, de.Value);
}
else if (op == StreamOpCode.remove)
foreach (DictionaryEntry de in rows)
{
if (!myState.Contains(de.Key))
{
// BlueEventLog2.instance().logEvent(String.Format("Unknown row deletion. t: {0} id:{1}", mySchema.tableName, de.Key), BlueEventLog2Level.Fatal);
StringBuilder b = new StringBuilder();
foreach (string n in mySchema.columnNames)
b.AppendFormat(" {0}", n);
// BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()), BlueEventLog2Level.Fatal);
}
myState.Remove(de.Key);
}
else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange)
foreach (DictionaryEntry de in rows)
myState[de.Key] = de.Value;
}
/// <summary>
/// Mass deserialization happens here, also since there is no knowledge of the schema
/// </summary>
/// <param name="br"></param>
/// <param name="op"></param>
/// <param name="noOfRows"></param>
/// <returns></returns>
private IDictionary myGetRows (BinaryReader br, byte op, int noOfRows)
{
myRowDeserializer = myRowDeserializer == null ? new PubsubUniversalRowDeserializer (mySchema) : myRowDeserializer;
Hashtable retVal = new Hashtable ();
for (int i = 0; i < noOfRows; i++)
{
int id; object row;
myRowDeserializer.deserializeRow (myState, br, op, out id, out row);
retVal.Add (id, row);
}
return retVal;
}
}
}
@@ -1,90 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
namespace BCG
{
public class CompressionSerializer : IBlueDataSerializer
{
private IBlueRowSerializer myRowSerializer;
private string myPubsubDeserializerName = "BCG.CompressionPubsubDeserializer, DataStreamCore", myClientDeserializerName = "BCG.CompressionClientDeserializer";
public CompressionSerializer (IBlueTableSchema schema)
{
myRowSerializer = new UniversalRowSerializer (schema);
}
public void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData)
{
// BlueTrace.WriteSerializeStart(op, rows.Length);
BinaryWriter bw = new BinaryWriter(new MemoryStream());
if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change)
{
bw.Write(op);
/// no need to compress the only row
myRowSerializer.serializeRow(BlueDataUtility.extractRow(rows[0], 0), op, bw);
}
else if (op == StreamOpCode.create)
{
IBlueRowCollection createdRows = rows[0];
bw.Write(op);
bw.Write(myPubsubDeserializerName);
bw.Write(myClientDeserializerName);
int dummyIdCol = 0;
int[] dummyIdList = new int[1];
//the schema doesn't sort anything right now, because the C++ pub sub client depends on things being in a certain order
IBlueTableSchema sortedSchema = BlueDataUtility.getSortedSchema(createdRows.schema, out dummyIdList, out dummyIdCol);
sortedSchema.serialize(bw);
/// compress it here
BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream());
decompressedBw.Write(createdRows.Count);
foreach (IBlueRow r in createdRows)
myRowSerializer.serializeRow(r, StreamOpCode.add, decompressedBw);
decompressedBw.BaseStream.Position = 0;
byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream);
bw.Write(compressedData.Length);
bw.Write(compressedData);
}
else if (op == StreamOpCode.multipleRemoveAdd)
{
bw.Write(op);
IBlueRowCollection delRows = rows[0], addRows = rows[1];
/// compress it here
BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream());
decompressedBw.Write(delRows.Count);
foreach (IBlueRow dr in delRows)
myRowSerializer.serializeRow(dr, StreamOpCode.remove, decompressedBw);
decompressedBw.Write(addRows.Count);
foreach (IBlueRow ar in addRows)
myRowSerializer.serializeRow(ar, StreamOpCode.add, decompressedBw);
decompressedBw.BaseStream.Position = 0;
byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream);
bw.Write(compressedData.Length);
bw.Write(compressedData);
}
else if (op == StreamOpCode.multipleChange)
{
bw.Write(op);
IBlueRowCollection changedRows = rows[0];
/// compress it here
BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream());
decompressedBw.Write(changedRows.Count);
foreach (IBlueRow cr in changedRows)
myRowSerializer.serializeRow(cr, StreamOpCode.change, decompressedBw);
decompressedBw.BaseStream.Position = 0;
byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream);
bw.Write(compressedData.Length);
bw.Write(compressedData);
}
bw.BaseStream.Position = 0;
serializedData = (bw.BaseStream as MemoryStream).ToArray();
// BlueTrace.WriteSerializeStop(serializedData.Length);
}
}
}
@@ -1,11 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BCG
{
public class BBOOptionDeserializer : ClientFixedDeserializer
{
public BBOOptionDeserializer() { }
}
}
@@ -1,49 +0,0 @@
using System;
using BCG.Utilities;
namespace BCG
{
public abstract class ClientFixedDeserializer : ClientDeserializer
{
private BlueTable myCurrentState;
public ClientFixedDeserializer () {}
protected override IBlueRowCollection myGetRows (System.IO.BinaryReader br, byte op, int nRows)
{
BlueTable retVal = new BlueTable (mySchema);
op = op == StreamOpCode.create ? StreamOpCode.add :
op == StreamOpCode.multipleChange ? StreamOpCode.change : op;
if (myRowDeserializer == null)
{
myRowDeserializer = new ClientFixedRowDeserializer (mySchema);
myCurrentState = new BlueTable (mySchema);
}
for (int i = 0; i < nRows; i++)
{
IBlueRow r;
myRowDeserializer.deserializeRow (myCurrentState, br, op, out r);
retVal.addRow (r);
}
return retVal;
}
protected override void myEncorporateRows (IBlueRowCollection rows, byte op)
{
if (op == StreamOpCode.create)
myCurrentState = new BlueTable (rows);
else if (op == StreamOpCode.add)
myCurrentState.addRows (rows);
else if (op == StreamOpCode.remove)
myCurrentState.deleteRows (rows);
else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange)
myCurrentState.changeRows (rows);
else
{
// BlueEventLog.logAssert (false, "Illegal operation for this context");
}
}
}
}
@@ -1,38 +0,0 @@
using System;
using System.IO;
using BCG.Utilities;
namespace BCG
{
public class ClientFixedRowDeserializer : IClientRowDeserializer
{
private PrimitiveDeserializationHelper myHelper;
private IBlueTableSchema mySchema;
public ClientFixedRowDeserializer (IBlueTableSchema schema)
{
myHelper = new PrimitiveDeserializationHelper (mySchema = schema);
}
#region IClientRowDeserializer Members
public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
{
if (op == StreamOpCode.add || op == StreamOpCode.change)
{
object[] retVal = new object[mySchema.columnInfo.Length];
foreach (ColumnInfo ci in mySchema.columnInfo)
retVal[ci.colNo] = myHelper.deserializeColumn (ci.colNo, br);
r = new BlueRow (mySchema, retVal);
}
else if (op == StreamOpCode.remove)
r = currentState[br.ReadInt32 ()];
else
{
// BlueEventLog.logAssert (false, op.ToString() + " not valid in this context");
r = null;
}
}
#endregion
}
}
@@ -1,11 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BCG
{
public class MarketOptionTheoClientDeserializer : ClientFixedDeserializer
{
public MarketOptionTheoClientDeserializer() { }
}
}
@@ -1,13 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BCG
{
public class BBOOptionSerializer : FixedSerializer
{
public BBOOptionSerializer(IBlueTableSchema schema)
: base(schema, "BCG.BBOOptionPubsubDeserializer", "BCG.BBOOptionDeserializer")
{ }
}
}
@@ -1,35 +0,0 @@
using System;
using System.IO;
namespace BCG
{
/// <summary>
/// Optimized to serialize fixed length rows, ie schemas that have only primitive types and non-nulls
/// </summary>
public class FixedRowSerializer : IBlueRowSerializer
{
private PrimitiveSerializationHelper myHelper;
private int myIdColNo;
private IBlueTableSchema mySchema;
public FixedRowSerializer (IBlueTableSchema schema)
{
myIdColNo = schema["id"].colNo;
myHelper = new PrimitiveSerializationHelper (schema);
mySchema = schema;
}
#region IBlueRowSerializer Members
public void serializeRow (IBlueRow r, byte op, BinaryWriter bw)
{
if (op == StreamOpCode.remove)
myHelper.serializeColumn (r, myIdColNo, bw);
else
foreach (ColumnInfo ci in r.schema.columnInfo)
myHelper.serializeColumn (r, ci.colNo, bw);
}
#endregion
}
}
@@ -1,20 +0,0 @@
using System;
using System.IO;
namespace BCG
{
/// <summary>
/// Tailored to serialize tables that haved fixed schema (ie no dynamic length types like strings, arrays etc and no nulls)
/// </summary>
public abstract class FixedSerializer : BlueDataSerializer
{
public FixedSerializer (IBlueTableSchema schema,
string pubsubDeserializer,
string clientDeserializer)
: base (pubsubDeserializer, clientDeserializer)
{
myRowSerializer = new FixedRowSerializer (schema);
}
}
}
@@ -1,14 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BCG
{
public class MarketOptionTheoSerializer : FixedSerializer
{
public MarketOptionTheoSerializer(IBlueTableSchema schema)
: base(schema, "BCG.MarketOptionTheoPubsubDeserializer",
"BCG.MarketOptionTheoClientDeserializer")
{ }
}
}
@@ -1,11 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BCG
{
public class BBOOptionPubsubDeserializer : PubsubFixedDeserializer
{
public BBOOptionPubsubDeserializer() { }
}
}
@@ -1,12 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BCG
{
public class MarketOptionTheoPubsubDeserializer : PubsubFixedDeserializer
{
public MarketOptionTheoPubsubDeserializer()
{ }
}
}
@@ -1,32 +0,0 @@
using System;
using System.Collections;
using System.IO;
namespace BCG
{
public abstract class PubsubFixedDeserializer : PubsubDeserializer
{
protected BlueTable myCurrentState;
public PubsubFixedDeserializer () {}
protected override IDictionary myGetRows (BinaryReader br, byte op, int noOfRows)
{
if (myDeserializer == null)
myDeserializer = new PubsubFixedRowDeserializer (mySchema);
IDictionary retVal = new Hashtable ();
op = op == StreamOpCode.create ? StreamOpCode.add :
op == StreamOpCode.multipleChange ? StreamOpCode.change : op;
for (int i = 0; i < noOfRows; i++)
{
int id;
object row;
myDeserializer.deserializeRow (myState, br, op, out id, out row);
retVal.Add (id, row);
}
return retVal;
}
}
}
@@ -1,37 +0,0 @@
using System;
using System.IO;
using System.Collections;
namespace BCG
{
public class PubsubFixedRowDeserializer : IPubsubRowDeserializer
{
private int myRowSize, myIdOffset;
public PubsubFixedRowDeserializer (IBlueTableSchema schema)
{
foreach (ColumnInfo ci in schema.columnInfo)
{
myRowSize += SerializationUtilityMethods.getFixedLengthTypeSize (ci.colType);
myIdOffset += ci.colNo < schema["id"].colNo ? SerializationUtilityMethods.getFixedLengthTypeSize (ci.colType) : 0;
}
}
#region IPubsubRowDeserializer Members
public void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row)
{
if (op == StreamOpCode.add || op == StreamOpCode.change)
{
row = br.ReadBytes (myRowSize);
id = BitConverter.ToInt32 (row as byte[], myIdOffset);
}
else //delete => only the id
{
id = br.ReadInt32 ();
row = currentState[id];
}
}
#endregion
}
}
@@ -1,56 +0,0 @@
using System;
using System.IO;
using System.Collections;
using BCG.Utilities;
namespace BCG
{
public abstract class ClientFixedChangingDeserializer : ClientDeserializer
{
private string [] myChangingCols;
protected BlueTable myCurrentState;
public ClientFixedChangingDeserializer (string[] changingCols)
{
myChangingCols = changingCols;
}
protected override IBlueRowCollection myGetRows (System.IO.BinaryReader br, byte op, int nRows)
{
BlueTable retVal = new BlueTable (mySchema);
op = op == StreamOpCode.create ? StreamOpCode.add :
op == StreamOpCode.multipleChange ? StreamOpCode.change : op;
if (myRowDeserializer == null)
{
myRowDeserializer = new ClientFixedChangingRowDeserializer (mySchema, myChangingCols);
myCurrentState = new BlueTable (mySchema);
}
for (int i = 0; i < nRows; i++)
{
IBlueRow r;
myRowDeserializer.deserializeRow (myCurrentState, br, op, out r);
retVal.addRow (r);
}
return retVal;
}
protected override void myEncorporateRows (IBlueRowCollection rows, byte op)
{
if (op == StreamOpCode.create)
myCurrentState = new BlueTable (rows);
else if (op == StreamOpCode.add)
myCurrentState.addRows (rows);
else if (op == StreamOpCode.remove)
myCurrentState.deleteRows (rows);
else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange)
myCurrentState.changeRows (rows);
else
{
// BlueEventLog.logAssert (false, "Illegal operation for this context");
}
}
}
}
@@ -1,67 +0,0 @@
using System;
using System.Collections;
using System.IO;
using BCG.Utilities;
namespace BCG
{
/// <summary>
/// Deserializes a fixed length row with no nulls on the client side.
/// </summary>
public class ClientFixedChangingRowDeserializer : IClientRowDeserializer
{
private int[] myChangingColIndices;
protected PrimitiveDeserializationHelper myHelper;
protected IBlueTableSchema mySchema;
public ClientFixedChangingRowDeserializer (IBlueTableSchema schema, string[] changingCols)
{
myHelper = new PrimitiveDeserializationHelper (mySchema = schema);
myChangingColIndices = new int [changingCols.Length];
for (int i = 0; i < changingCols.Length; i++)
myChangingColIndices[i] = schema[changingCols[i]].colNo;
Array.Sort (myChangingColIndices);
}
public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
{
if (op == StreamOpCode.add)
myDeserializeAdd (currentState, br, out r);
else if (op == StreamOpCode.remove)
myDeserializeRemove (currentState, br, out r);
else if (op == StreamOpCode.change)
myDeserializeChange (currentState, br, out r);
else
{
r = null;
// BlueEventLog.logAssert (false, "Illegal operation for this function, can only deal with single row " +
// "add, delete and change");
}
}
protected virtual void myDeserializeAdd (IBlueRowCollection currentState, BinaryReader br, out IBlueRow r)
{
object[] retVal = new object[mySchema.columnInfo.Length];
foreach (ColumnInfo ci in mySchema.columnInfo)
retVal[ci.colNo] = myHelper.deserializeColumn (ci.colNo, br);
r = new BlueRow (mySchema, retVal);
}
protected virtual void myDeserializeRemove (IBlueRowCollection currentState, BinaryReader br, out IBlueRow r)
{
r = currentState[br.ReadInt32()];
}
protected virtual void myDeserializeChange (IBlueRowCollection currentState, BinaryReader br, out IBlueRow r)
{
MutableBlueRow rv = new MutableBlueRow (currentState[br.ReadInt32 ()]);
foreach (int colNo in myChangingColIndices)
rv.setValue (colNo, myHelper.deserializeColumn (colNo, br));
r = rv;
}
}
}
@@ -1,90 +0,0 @@
using System;
using System.Collections;
using System.IO;
using BCG.Utilities;
namespace BCG
{
/// <summary>
/// Specializes in serializing rows that are sent over as a result of a change; only the changed columns are
/// sent together with the id.
/// </summary>
public class FixedChangingRowSerializer : IBlueRowSerializer
{
private int[] myChangingColIndices;
private PrimitiveSerializationHelper myHelper;
private IBlueTableSchema mySchema;
public FixedChangingRowSerializer (IBlueTableSchema schema, params string[] changingColumns)
{
myHelper = new PrimitiveSerializationHelper (schema);
myChangingColIndices = new int[changingColumns.Length];
for (int i = 0; i < changingColumns.Length; i++)
myChangingColIndices[i] = schema[changingColumns[i]].colNo;
Array.Sort (myChangingColIndices);
mySchema = schema;
}
#region IBlueRowSerializer Members
public void serializeRow (IBlueRow r, byte op, BinaryWriter bw)
{
if (op == StreamOpCode.add)
{
mySerializeAdd (r, bw);
}
else if (op == StreamOpCode.remove)
{
mySerializeRemove (r, bw);
}
else if (op == StreamOpCode.change)
{
mySerializeChange (r, bw);
}
else
{
// BlueEventLog.logAssert (false, String.Format ("{0} is not a valid op in this context", op));
}
}
#endregion
/// <summary>
/// The whole row is written, one column after the other with the order defined by the schema. There
/// are no columns headers
/// </summary>
/// <param name="r"></param>
/// <param name="bw"></param>
protected virtual void mySerializeAdd (IBlueRow r, BinaryWriter bw)
{
foreach (ColumnInfo ci in r.schema.columnInfo)
myHelper.serializeColumn (r, ci.colNo, bw);
}
/// <summary>
/// Only the id is written
/// </summary>
/// <param name="r"></param>
/// <param name="bw"></param>
protected virtual void mySerializeRemove (IBlueRow r, BinaryWriter bw)
{
myHelper.serializeColumn (r, r.schema["id"].colNo, bw);
}
/// <summary>
/// Serializes only the changed columns, alongside the id. So the format is
/// {id, changedColumns*}. The order of the changedColumns are sorted according
/// to their index in the schema.
/// </summary>
/// <param name="r"></param>
/// <param name="bw"></param>
protected virtual void mySerializeChange (IBlueRow r, BinaryWriter bw)
{
myHelper.serializeColumn (r, r.schema["id"].colNo, bw);
foreach (int colNo in myChangingColIndices)
myHelper.serializeColumn (r, colNo, bw);
}
}
}
@@ -1,110 +0,0 @@
using System;
using System.Collections;
using System.IO;
using BCG.Utilities;
namespace BCG
{
/// <summary>
/// Tailored to serialize tables that haved fixed schema (ie no dynamic length types like strings, arrays etc and no nulls)
/// and are constantly changing
/// </summary>
public abstract class FixedChangingSerializer : BlueDataSerializer
{
public FixedChangingSerializer (IBlueTableSchema schema,
string[] changingCols,
string pubsubDeserializer,
string clientDeserializer)
: base (pubsubDeserializer, clientDeserializer)
{
myRowSerializer = new FixedChangingRowSerializer (schema, changingCols);
}
}
#region BidAsk
public class BidAskSerializer : FixedChangingSerializer
{
public BidAskSerializer (IBlueTableSchema schema)
: base (schema,
new string[] {"bidAskZone"},
"BCG.BidAskPubsubDeserializer",
"BCG.BidAskClientDeserializer"
)
{}
}
public class BidAskPubsubDeserializer : PubsubFixedChangingDeserializer
{
public BidAskPubsubDeserializer () : base (new string[] {"bidAskZone"})
{}
}
public class BidAskClientDeserializer : ClientFixedChangingDeserializer
{
public BidAskClientDeserializer () : base (new string[] {"bidAskZone"})
{}
}
#endregion
#region HHMarketPrice
public class HHMarketPriceSerializer : FixedChangingSerializer
{
public HHMarketPriceSerializer(IBlueTableSchema schema)
: base(schema,
new string[] { "MarketPrice" },
"BCG.HHMarketPricePubsubDeserializer",
"BCG.HHMarketPriceClientDeserializer"
)
{ }
}
public class HHMarketPricePubsubDeserializer : PubsubFixedChangingDeserializer
{
public HHMarketPricePubsubDeserializer()
: base(new string[] { "MarketPrice" })
{ }
}
public class HHMarketPriceClientDeserializer : ClientFixedChangingDeserializer
{
public HHMarketPriceClientDeserializer()
: base(new string[] { "MarketPrice" })
{ }
}
#endregion
#region ThrottledMarketPrice
public class ThrottledPriceSerializer : FixedChangingSerializer
{
public ThrottledPriceSerializer(IBlueTableSchema schema)
: base(schema,
new string[] { "PRICE" },
"BCG.ThrottledPricePubsubDeserializer",
"BCG.ThrottledPriceClientDeserializer"
)
{ }
}
public class ThrottledPricePubsubDeserializer : PubsubFixedChangingDeserializer
{
public ThrottledPricePubsubDeserializer()
: base(new string[] { "PRICE" })
{ }
}
public class ThrottledPriceClientDeserializer : ClientFixedChangingDeserializer
{
public ThrottledPriceClientDeserializer()
: base(new string[] { "PRICE" })
{ }
}
#endregion
}
@@ -1,41 +0,0 @@
using System;
using System.IO;
using System.Collections;
using BCG.Utilities;
namespace BCG
{
/// <summary>
/// Pairs up with FixedSerializer to deserialize messages in the pubsub.
/// </summary>
public abstract class PubsubFixedChangingDeserializer : PubsubDeserializer
{
private string[] myChangingCols;
protected BlueTable myCurrentState;
public PubsubFixedChangingDeserializer (string[] changingColumns)
{
myChangingCols = changingColumns;
}
protected override IDictionary myGetRows (BinaryReader br, byte op, int noOfRows)
{
if (myDeserializer == null)
myDeserializer = new PubsubFixedChangingRowDeserializer (mySchema, myChangingCols);
IDictionary retVal = new Hashtable ();
op = op == StreamOpCode.create ? StreamOpCode.add :
op == StreamOpCode.multipleChange ? StreamOpCode.change : op;
for (int i = 0; i < noOfRows; i++)
{
int id;
object row;
myDeserializer.deserializeRow (myState, br, op, out id, out row);
retVal.Add (id, row);
}
return retVal;
}
}
}
@@ -1,84 +0,0 @@
using System;
using System.IO;
using System.Collections;
using BCG.Utilities;
namespace BCG
{
/// <summary>
/// Pairs up with FixedSerializer to deserialize messages in the pubsub.
/// </summary>
public class PubsubFixedChangingRowDeserializer : IPubsubRowDeserializer
{
private int myFullRowSize;
private PrimitiveDeserializationHelper myDeserHelper;
private PrimitiveSerializationHelper mySerHelper;
private IBlueTableSchema mySchema;
private int[] myOffsets, myChangedColIndices;
private int myIdIndex;
public PubsubFixedChangingRowDeserializer (IBlueTableSchema schema, params string[] changingColNames)
{
myDeserHelper = new PrimitiveDeserializationHelper (mySchema = schema);
mySerHelper = new PrimitiveSerializationHelper (mySchema);
myIdIndex = mySchema["id"].colNo;
myOffsets = new int[schema.columnInfo.Length];
//size calulcations
foreach (ColumnInfo ci in mySchema.columnInfo)
{
myOffsets[ci.colNo] += myFullRowSize;
myFullRowSize += SerializationUtilityMethods.getFixedLengthTypeSize (ci.colType);
}
myChangedColIndices = new int[changingColNames.Length];
for (int i = 0; i < changingColNames.Length; i++)
myChangedColIndices[i] = mySchema[changingColNames[i]].colNo;
Array.Sort (myChangedColIndices);
}
#region IPubsubRowDeserializer Members
public void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row)
{
if (op == StreamOpCode.add)
myDeserializeAdd (currentState, br, out id, out row);
else if (op == StreamOpCode.remove)
myDeserializeRemove (currentState, br, out id, out row);
else if (op == StreamOpCode.change)
myDeserializeChange (currentState, br, out id, out row);
else
{
id = int.MinValue;
row = null;
// BlueEventLog.logAssert (false, "Illegal operation for this context");
}
}
#endregion
protected virtual void myDeserializeAdd (IDictionary currentState, BinaryReader br, out int id, out object row)
{
row = br.ReadBytes (myFullRowSize);
id = BitConverter.ToInt32 (row as byte[], myOffsets[myIdIndex]);
}
protected virtual void myDeserializeRemove (IDictionary currentState, BinaryReader br, out int id, out object row)
{
id = br.ReadInt32 ();
row = currentState[id];
}
protected virtual void myDeserializeChange (IDictionary currentState, BinaryReader br, out int id, out object row)
{
id = br.ReadInt32 ();
BinaryWriter bw = new BinaryWriter (new MemoryStream (currentState[id] as byte[]));
foreach (int colIndex in myChangedColIndices)
{
object newValue = myDeserHelper.deserializeColumn (colIndex, br);
bw.BaseStream.Position = myOffsets[colIndex];
mySerHelper.serializeColumn (newValue, colIndex, bw);
}
row = (bw.BaseStream as MemoryStream).ToArray ();
}
}
}
@@ -1,171 +0,0 @@
// using System;
// using System.Collections;
// using System.IO;
// using System.IO.Compression;
// using BCG.Utilities;
// //using BCG.Utilities.BlueConfiguration;
// namespace BCG
// {
// public class LookupTableClientDeserializer : IClientDeserializer
// {
// public LookupTableClientDeserializer() {
// mySchema = new LookupTableSchema();
// myLookupTables = null;
// }
// private IClientRowDeserializer myRowDeserializer;
// private LookupTableSchema mySchema;
// private LookupTable[] myLookupTables;
// public BinaryReader myGetReaderForRestOfStream(Stream theStream)
// {
// //teh rest of the stream is only compressed data
// long compressedBytesLength = theStream.Length - theStream.Position;
// byte[] compressedBytes = new byte[compressedBytesLength];
// theStream.Read(compressedBytes, 0, (int)compressedBytesLength);
// //get a stream to read the uncompressed data
// SharpZipCompressor deKompressor = new SharpZipCompressor();
// MemoryStream decompressed = new MemoryStream();
// deKompressor.decompress(new MemoryStream(compressedBytes), decompressed);
// decompressed.Position = 0;
// return new BinaryReader(decompressed);
// }
// #region IClientDeserializer methods
// public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
// {
// ArrayList ops = new ArrayList();
// ArrayList rows = new ArrayList();
// IBlueRowCollection[] result;
// foreach (Stream theStream in streams)
// {
// byte op;
// BinaryReader br = new BinaryReader(theStream);
// op = br.ReadByte();
// if (op == StreamOpCode.create)
// {
// br.ReadString(); //pubsub deserializer
// br.ReadString(); //client deserializer
// br = myGetReaderForRestOfStream(theStream);
// result = new IBlueRowCollection[] { myHandleCreate(br) };
// ops.Add(op);
// rows.Add(result);
// }
// else
// {
// //the schema and lookup tables should already be created. We just need to update everything.
// //figure out how long the rest of the compressed data is
// br = myGetReaderForRestOfStream(theStream);
// if (mySchema != null)
// {
// if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
// {
// myDeserializeLookupTableUpdates(br);
// result = new IBlueRowCollection[] { myGetRows(br, op, 1) };
// ops.Add(op);
// rows.Add(result);
// }
// else if (op == StreamOpCode.multipleChange)
// {
// myDeserializeLookupTableUpdates(br);
// int numRows = br.ReadInt32();
// result = new IBlueRowCollection[] { myGetRows(br, op, numRows) };
// ops.Add(op);
// rows.Add(result);
// }
// else if (op == StreamOpCode.multipleRemoveAdd)
// {
// myDeserializeLookupTableUpdates(br);
// result = new IBlueRowCollection[2];
// int numDeletedRows = br.ReadInt32();
// result[0] = myGetRows(br, StreamOpCode.remove, numDeletedRows);
// int numAddedRows = br.ReadInt32();
// result[1] = myGetRows(br, StreamOpCode.add, numAddedRows);
// ops.Add(op);
// rows.Add(result);
// }
// else{}
// // BlueEventLog.logEvent("Illegal op-code recieved", BlueEventLevel.Error);
// }
// else
// {
// // BlueEventLog.logEvent("Received a non create operation even though the schema" +
// // "has never been created", BlueEventLevel.Error);
// }
// }
// }
// theOps = ops.ToArray(typeof(byte)) as byte[];
// theRows = rows.ToArray();
// }
// #endregion
// IBlueRowCollection myHandleCreate(BinaryReader br)
// {
// mySchema.deserialze(br);
// //build the lookup tables now
// // There is one less than the length of the lookupTableGroups because the first one
// //is always the default group which isn't stored in a lookup table.
// myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1];
// //start with 1 because we don't store the 0th lookup table
// for (int i = 0; i < myLookupTables.Length; ++i)
// myLookupTables[i] = new LookupTable(mySchema, i+1);
// myDeserializeLookupTableUpdates(br);
// //now read the rows
// int numRows = br.ReadInt32();
// return myGetRows(br, StreamOpCode.create, numRows);
// }
// private IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows)
// {
// BlueTable retVal = new BlueTable(mySchema);
// myRowDeserializer = myRowDeserializer == null ? new LookupTableClientRowDeserializer(mySchema,myLookupTables) : myRowDeserializer;
// for (int i = 0; i < nRows; i++)
// {
// IBlueRow row;
// myRowDeserializer.deserializeRow(null, br, op, out row);
// retVal.addRow(row);
// }
// return retVal;
// }
// private void myDeserializeLookupTableUpdates(BinaryReader br)
// {
// for (int i = 0; i < myLookupTables.Length; ++i)
// myLookupTables[i].deserializeUpdates(br);
// }
// }
// }
@@ -1,107 +0,0 @@
// using System;
// using System.Collections;
// using System.IO;
// using BCG.Utilities;
// //using BCG.Utilities.BlueConfiguration;
// namespace BCG
// {
// /// <summary>
// /// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process
// /// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations)
// /// </summary>
// internal class LookupTableClientRowDeserializer : IClientRowDeserializer
// {
// private LookupTableSchema mySchema;
// private LookupTable[] myTables;
// private int myNullMapLength = 0;
// private PrimitiveDeserializationHelper myDeserializationHelper;
// private BitArray myNullMap;
// private int myIdColumn;
// public LookupTableClientRowDeserializer (LookupTableSchema schema, LookupTable[] theTables)
// {
// mySchema = schema;
// myTables = theTables;
// myDeserializationHelper = new PrimitiveDeserializationHelper (mySchema);
// myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8)));
// myIdColumn = schema["id"].colNo;
// }
// #region IClientRowDeserializer Members
// public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
// {
// int rowLength = br.ReadInt32 (); //length information for the row not needed
// //rad the null map
// myNullMap = new BitArray(br.ReadBytes (myNullMapLength));
// object[] values = new object [mySchema.columnNames.Length];
// //now read the id
// values[myIdColumn] = br.ReadInt32();
// //now read the lookup groups
// for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo)
// {
// BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo];
// LookupTable thisTable = myTables[groupNo - 1];
// //read the id in, depending upon the size
// int idRead = -1;
// switch (thisGroup.tableSize)
// {
// case 8:
// idRead = br.ReadByte();
// break;
// case 16:
// idRead = br.ReadInt16();
// break;
// case 32:
// idRead = br.ReadInt32();
// break;
// default:
// throw new BlueDataException("Error: invalid size " + thisGroup.tableSize.ToString() + " specific for lookup group.");
// }
// object[] thisGroupValues = thisTable.getValuesForIndex(idRead);
// if (thisGroupValues == null)
// throw new BlueDataException("Error: Couldnt' find values for index " + idRead.ToString());
// if (thisGroupValues.Length != thisGroup.groupColNos.Length)
// throw new BlueDataException("Error: Values extracted for index " + idRead.ToString() + " had length " + thisGroupValues.Length.ToString() + " but length " + thisGroup.groupColNos.Length.ToString() + " was expected.");
// //insert those values into our object array
// for (int groupCol = 0; groupCol < thisGroup.groupColNos.Length; ++groupCol)
// {
// int thisColInOutputRow = thisGroup.groupColNos[groupCol];
// values[thisColInOutputRow] = thisGroupValues[groupCol];
// }
// }
// for (int colNo = 0; colNo < values.Length; colNo++)
// {
// if (myNullMap[colNo])
// {
// values[colNo] = DBNull.Value;
// }
// //if it's non null, not the id column, and not part of a lookup group, get it here.
// else if (mySchema[colNo].lookupTableGroup == 0 && colNo != myIdColumn)
// {
// values[colNo] = myDeserializationHelper.deserializeColumn(colNo, br);
// }
// }
// r = new BlueRow(mySchema, values);
// }
// #endregion
// }
// }
@@ -1,194 +0,0 @@
// using System;
// using System.Collections.Generic;
// using System.Collections;
// using System.IO;
// using System.Linq;
// using System.Text;
// namespace BCG
// {
// /*
// *
// * LookupTable
// *
// * This class represents a lookup table used to store values inside data streams. This allows indexes into the table
// * to be passed over the wire, instead of actual values of the rows.
// *
// * Each table can have multiple lookup table groups. This class represents just one of those groups. This means that all the entries
// * contained in the table will have the same schema.
// *
// *
// *
// * */
// public class LookupTable
// {
// //each of the entries has the same schema - that is, each of the entries contains the same number of values, with the same
// IBlueTableSchema myEntrySchema;
// UniversalRowSerializer mySerializer;
// ClientUniversalRowDeserializer myDeserializer;
// private const int myIdColumn = 0;
// //the table is only allowed to reach a maximum size based upon the number of bits you give to it.
// //it throws an error if you try to put too much data into it.
// private long myMaxSize;
// private int mySizeInBits;
// public int sizeInBits
// {
// get { return mySizeInBits; }
// }
// KeysBasedTable2 myValuesToEntryMap;
// //keep track of the last time somebody published you, so you know what values to publish.
// int myIdOfLastPublish;
// int myIdToUse;
// /* builds the local schema, serializer, and desrializer
// * initializes to an empty table
// */
// public LookupTable(LookupTableSchema theSchema, int lookupGroupNumber)
// {
// BlueRowLookupGroup group = theSchema.lookupGroups[lookupGroupNumber];
// myEntrySchema = group.entrySchema;
// //compute how much room you have in the table
// mySizeInBits = group.tableSize;
// myMaxSize = (long)(Math.Pow(2, group.tableSize) - 1);
// myDeserializer = new ClientUniversalRowDeserializer(myEntrySchema);
// mySerializer = new UniversalRowSerializer(myEntrySchema);
// //create the map
// myValuesToEntryMap = new KeysBasedTable2(myEntrySchema, group.groupColNames);
// myIdOfLastPublish = 0;
// }
// /*Checks to see if the given values are in the table. IF they are in there, it returns the index to those values.
// If they are not present, it adds them to teh table and returns the new index. */
// public int getIndexForValues(params object[] values)
// {
// int index = 0;
// //see if the key is present
// IBlueRow thisRowEntry = myValuesToEntryMap.getRow(values);
// if (thisRowEntry == null)
// {
// //check to see if we even have room in the table
// if (myIdToUse > myMaxSize)
// throw new BlueDataException("Error: LookupTable has reached maximum size. Make a bigger table.");
// //we need to entrer this new value into the table. Create a blue row using the next ID
// //and the values they supplied to us
// object[] thisRowValues = new object[myEntrySchema.columnInfo.Length];
// index = myIdToUse;
// thisRowValues[myIdColumn] = index;
// myIdToUse ++;
// //now add in the values they supplied
// for (int colNo = myIdColumn+1; colNo < myEntrySchema.columnInfo.Length; ++colNo)
// {
// thisRowValues[colNo] = values[colNo - 1];
// }
// IBlueRow newEntryRow = new BlueRow(myEntrySchema, thisRowValues);
// myValuesToEntryMap.update(newEntryRow);
// }
// else
// index = (int) thisRowEntry["id"];
// return index;
// }
// //queries the table to get the values that were stored on a given index.
// //returns null if the values are not found
// public object[] getValuesForIndex(int index)
// {
// object[] results = null;
// //get the row from the table
// if (myValuesToEntryMap.containsID(index))
// {
// IBlueRow thisRow = myValuesToEntryMap[index];
// results = new object[myEntrySchema.columnInfo.Length - 1];
// //copy the values from the row into the array
// for (int colNo = myIdColumn+1; colNo < myEntrySchema.columnInfo.Length; ++ colNo)
// {
// results[colNo - 1] = thisRow[colNo];
// }
// }
// //if the index isn't in the tablem, can just return the null value
// return results;
// }
// public void deserializeUpdates(BinaryReader br)
// {
// //get the number of updates that are being published.
// int numUpdates = br.ReadInt32();
// for (int i = 0; i < numUpdates; ++i)
// {
// IBlueRow theNewUpdate = null;
// myDeserializer.deserializeRow(null, br, StreamOpCode.create, out theNewUpdate);
// myValuesToEntryMap.update(theNewUpdate);
// int thisID = (int) theNewUpdate[myIdColumn];
// if (thisID != myIdToUse)
// throw new BlueDataException("Error: bad serialized update given to lookup table. Expected id " + myIdToUse.ToString() + ", found " + thisID.ToString());
// ++myIdToUse;
// }
// }
// public void serializeUpdates(BinaryWriter bw)
// {
// int numUpdates = myIdToUse - myIdOfLastPublish;
// //serialize the number of updates
// bw.Write(numUpdates);
// //serialize each update
// for (int i = myIdOfLastPublish; i < myIdToUse; ++i)
// mySerializer.serializeRow(myValuesToEntryMap[i], StreamOpCode.add, bw);
// //store the last id you published so that next time we don't serialize the same updates
// myIdOfLastPublish = myIdToUse;
// }
// /*this function serializes EVERYTHING in the table into the writer. This is
// used by the pubsub deserializer, which needs to be able to supply the new state
// to any clients connection. */
// public void serializeWholeTable(BinaryWriter bw)
// {
// //serialize the number of updates
// bw.Write(myIdToUse);
// //serialize each update
// for (int i = 0; i < myIdToUse; ++i)
// mySerializer.serializeRow(myValuesToEntryMap[i], StreamOpCode.add, bw);
// }
// }
// }
@@ -1,226 +0,0 @@
// using System;
// using System.Collections;
// using System.IO;
// using BCG.Utilities;
// using System.IO.Compression;
// namespace BCG
// {
// public class LookupTablePubSubDeserializer : IPubsubDeserializer
// {
// protected IDictionary myState = new Hashtable();
// protected LookupTableSchema mySchema;
// protected string myClientDeserializerName; //for the getState method
// protected IPubsubRowDeserializer myDeserializer;
// LookupTable[] myLookupTables;
// public LookupTablePubSubDeserializer() {
// mySchema = new LookupTableSchema();
// }
// public BinaryReader myGetReaderForCompressedPortion(Stream theStream)
// {
// //teh rest of the stream is only compressed data
// long compressedBytesLength = theStream.Length - theStream.Position;
// byte[] compressedBytes = new byte[compressedBytesLength];
// theStream.Read(compressedBytes, 0, (int)compressedBytesLength);
// //get a stream to read the uncompressed data
// SharpZipCompressor deKompressor = new SharpZipCompressor();
// MemoryStream decompressed = new MemoryStream();
// deKompressor.decompress(new MemoryStream(compressedBytes), decompressed);
// decompressed.Position = 0;
// return new BinaryReader(decompressed);
// }
// #region IPubsubDeserializer Members
// public virtual void deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
// {
// ArrayList ops = new ArrayList(), rows = new ArrayList();
// foreach (Stream theStream in streams)
// {
// BinaryReader br = new BinaryReader(theStream);
// IDictionary[] result;
// byte op;
// op = br.ReadByte();
// if (op == StreamOpCode.create)
// {
// myState.Clear();
// br.ReadString(); //the pubsub-deserializer name, should be the same as this's name
// myClientDeserializerName = br.ReadString(); //the client-deserializer name
// //at this point, the rest of the bytes in the stream are from a GZIP stream
// //so we need to read into a GZipStream and decompress
// //figure out how long the rest of the compressed data is
// br = myGetReaderForCompressedPortion(theStream);
// mySchema.deserialze(br);
// //now build the lookup tables from the knowledge gleaned at the schema
// myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1];
// //start with 1 because we don't store the 0th lookup table
// for (int i = 0; i < myLookupTables.Length; ++i)
// myLookupTables[i] = new LookupTable(mySchema, i + 1);
// myDeserializeLookupTableUpdates(br);
// int numRows = br.ReadInt32();
// result = new IDictionary[] { myGetRows(br, op, numRows) };
// myEncorporatRows(result[0], op);
// ops.Add(op);
// rows.Add(result);
// }
// else
// {
// //figure out how long the rest of the compressed data is
// br = myGetReaderForCompressedPortion(theStream);
// if (mySchema != null)
// {
// if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
// {
// myDeserializeLookupTableUpdates(br);
// result = new IDictionary[] { myGetRows(br, op, 1) };
// myEncorporatRows(result[0], op);
// ops.Add(op);
// rows.Add(result);
// }
// else if (op == StreamOpCode.multipleChange)
// {
// myDeserializeLookupTableUpdates(br);
// int numRows = br.ReadInt32();
// result = new IDictionary[] { myGetRows(br, op, numRows) };
// myEncorporatRows(result[0], op);
// ops.Add(op);
// rows.Add(result);
// }
// else if (op == StreamOpCode.multipleRemoveAdd)
// {
// myDeserializeLookupTableUpdates(br);
// int numRemoved = br.ReadInt32();
// result = new IDictionary[2];
// result[0] = myGetRows(br, StreamOpCode.remove, numRemoved);
// int numAdded = br.ReadInt32();
// result[1] = myGetRows (br, StreamOpCode.add, numAdded);
// myEncorporatRows(result[0], StreamOpCode.remove);
// myEncorporatRows(result[1], StreamOpCode.add);
// ops.Add(op);
// rows.Add(result);
// }
// else
// {
// // BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error);
// }
// }
// else
// {
// // BlueEventLog.logEvent("Received a non create operation even though the schema" +
// // "has never been created", BlueEventLevel.Warning);
// }
// }
// }
// theOps = ops.ToArray(typeof(byte)) as byte[];
// theRows = rows.ToArray();
// }
// /// <summary>
// /// Sends in a create signal, so that new clients can get a create everytime and start
// /// afresh from there
// /// </summary>
// /// <returns></returns>
// public virtual byte[] getState()
// {
// if (mySchema == null)
// return new byte[] { };
// else
// {
// BinaryWriter bw = new BinaryWriter(new MemoryStream());
// bw.Write(StreamOpCode.create);
// bw.Write(GetType().FullName);
// bw.Write(myClientDeserializerName);
// //the rest of this data has to be serialized.
// BinaryWriter compressedBw = new BinaryWriter(new MemoryStream());
// mySchema.serialize(compressedBw);
// //serialize all the tables
// for (int i = 0; i < myLookupTables.Length; ++i)
// myLookupTables[i].serializeWholeTable(compressedBw);
// compressedBw.Write(myState.Count);
// foreach (byte[] row in myState.Values)
// compressedBw.Write(row);
// //now compress the bytes
// MemoryStream compressedStream = new MemoryStream();
// SharpZipCompressor Kompressor = new SharpZipCompressor();
// compressedBw.BaseStream.Position = 0;
// Kompressor.compress(compressedBw.BaseStream, compressedStream);
// //append the two streams together
// byte[] output = new byte[bw.BaseStream.Length + compressedStream.Length];
// bw.BaseStream.Position = 0;
// compressedStream.Position = 0;
// bw.BaseStream.Read(output, 0, (int) bw.BaseStream.Length);
// compressedStream.Read(output, (int) bw.BaseStream.Length, (int) compressedStream.Length);
// return output;
// }
// }
// #endregion
// private void myDeserializeLookupTableUpdates(BinaryReader br)
// {
// for (int i = 0; i < myLookupTables.Length; ++i)
// myLookupTables[i].deserializeUpdates(br);
// }
// /// <summary>
// /// Every row that has been deserialized passes through this method, so that it can update the state.
// /// </summary>
// /// <param name="rows"></param>
// /// <param name="op"></param>
// protected virtual void myEncorporatRows(IDictionary rows, byte op)
// {
// if (op == StreamOpCode.add || op == StreamOpCode.create)
// foreach (DictionaryEntry de in rows)
// myState.Add(de.Key, de.Value);
// else if (op == StreamOpCode.remove)
// foreach (DictionaryEntry de in rows)
// myState.Remove(de.Key);
// else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange)
// foreach (DictionaryEntry de in rows)
// myState[de.Key] = de.Value;
// }
// private IDictionary myGetRows(BinaryReader br, byte op, int noOfRows)
// {
// //the row deserializer doesn't need to know anything about the lookup tables
// myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer(mySchema) : myDeserializer;
// Hashtable retVal = new Hashtable();
// for (int i = 0; i < noOfRows; i++)
// {
// int id; object row;
// myDeserializer.deserializeRow(myState, br, op, out id, out row);
// retVal.Add(id, row);
// }
// return retVal;
// }
// }
// }
@@ -1,211 +0,0 @@
// using System;
// using System.Collections;
// using System.IO;
// using BCG.Utilities;
// using System.Collections.Generic;
// namespace BCG
// {
// /// <summary>
// /// Helps the LookupTableRowSerializer by serializing one row.
// ///
// /// It does NOT sort the schema, even though the sorted schema doesn't do anything.
// /// That is because the schema sorting utility is agnostic of schema type, and so it would destory the grouping information
// /// which is VITAL
// ///
// /// It performs a mapping : (row)->byte[]
// /// </summary
// internal class LookupTableRowSerializer : IBlueRowSerializer
// {
// private PrimitiveSerializationHelper myPrimitiveSerializationHelper;
// private int myFixedSize;
// private int[] myNonFixedColIndexes;
// private int myIdColumn;
// private LookupTableSchema mySchema;
// private LookupTable[] myTables;
// //this null map is the null map that will be written to the wire
// //therefore, its values represent the nulls in row, ordered by the-rearranged schema.
// private BitArray myNullMap;
// private byte[] myNullMapArray; //just so that a new instance is not created everytime
// public LookupTableRowSerializer(LookupTableSchema schema, LookupTable[] theTables)
// {
// mySchema = schema;
// myTables = theTables;
// myPrimitiveSerializationHelper = new PrimitiveSerializationHelper(mySchema);
// myIdColumn = schema["id"].colNo;
// myNullMapArray =
// new byte[Convert.ToInt32(Math.Ceiling(Convert.ToDouble(mySchema.columnInfo.Length) / 8))];
// myNullMap =
// new BitArray(myNullMapArray);
// myInitSizeCalculations();
// }
// //the LookupTableSerializer calls this function on all rows to be serialized,
// //before it serializes any of them. It does this to update the lookup tables
// //that's becuase it writes the updates to the lookup tables to the byte stream
// //before it writes the rows.
// public void updateLookupTables(IBlueRow r)
// {
// for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo)
// {
// BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo];
// LookupTable thisTable = myTables[groupNo - 1];
// thisTable.getIndexForValues(thisGroup.extractGroupFromRow(r));
// }
// }
// #region IBlueRowSerializer methods
// /// <summary>
// /// Row serialized in the following order
// /// row size (nullBitMap size + actual row size)
// /// null bit map
// /// row id
// /// row lookup group ids
// /// rest of row values
// /// </summary>
// /// <param name="r"></param>
// /// <param name="bw"></param>
// public void serializeRow(IBlueRow r, byte op, BinaryWriter bw)
// {
// myNullMap.SetAll(false);
// int sz = myFixedSize;
// //first, build the null map
// for (int i = 0; i < mySchema.columnNames.Length; i++)
// {
// if (r[i] == DBNull.Value || r[i] == null)
// {
// myNullMap[i] = true;
// //if this thing isn't in a lookup group, it's no longer going to be accounted for in the fixed size of the table
// //therefore we need to decrement the size of this row by the size of the type
// if (mySchema[i].lookupTableGroup == 0)
// {
// if (SerializationUtilityMethods.isFixedLengthType(mySchema[i].colType))
// sz -= SerializationUtilityMethods.getFixedLengthTypeSize(mySchema[i].colType);
// }
// }
// }
// //now serialize the null map
// myNullMap.CopyTo(myNullMapArray, 0);
// int thisRowSize = myGetRowSizeInBytes(sz, r);
// bw.Write(thisRowSize);
// //tehcnically the size calculatinos are all off by 4 becuase tehy don't account for integer telling how big the row is
// long startPos = bw.BaseStream.Position;
// bw.Write(myNullMapArray);
// //write the ID first, for easy access
// bw.Write((int)r[myIdColumn]);
// //now write all the value stored in lookup table groups
// //we don't do this with the first lookup group becuase it's the default one that isn't stored in lookup tables
// for (int groupNo = 1; groupNo < mySchema.lookupGroups.Length; ++groupNo)
// {
// BlueRowLookupGroup thisGroup = mySchema.lookupGroups[groupNo];
// LookupTable thisTable = myTables[groupNo - 1];
// int thisLookupID = thisTable.getIndexForValues(thisGroup.extractGroupFromRow(r));
// //now, depending on the group size, write this value as either a byte, a short, or an int.
// switch (thisGroup.tableSize)
// {
// case 8:
// bw.Write((byte)thisLookupID);
// break;
// case 16:
// bw.Write((short)thisLookupID);
// break;
// case 32:
// bw.Write(thisLookupID);
// break;
// default:
// throw new BlueDataException("Error: invalid size " + thisGroup.tableSize.ToString() + " specified for lookup group.");
// }
// }
// //now write the rest of the individual columns
// foreach (ColumnInfo ci in mySchema.columnInfo)
// //we have already serialized the id column, as well as all the column groups, and we don't serialize any columns that are null
// if ( (!myNullMap[ci.colNo])
// && ci.colNo != myIdColumn
// && ci.lookupTableGroup == 0)
// {
// myPrimitiveSerializationHelper.serializeColumn(r, ci.colNo, bw);
// }
// long stopPos = bw.BaseStream.Position;
// if (stopPos- startPos != thisRowSize)
// throw new BlueDataException("Error: Bad Size Calculation in LookupTableRowSerializer. Expected size was "+thisRowSize.ToString()+", actual size was "+(stopPos-startPos).ToString());
// }
// public void serializeSchema(BinaryWriter bw)
// {
// mySchema.serialize(bw);
// }
// #endregion
// #region size calculations
// /// <summary>
// /// Sets up the fixed length (size in bytes occupied by each of the primitive type)
// /// and an index for the non primitive type
// /// </summary>
// private void myInitSizeCalculations()
// {
// ArrayList temp = new ArrayList();
// myFixedSize = myNullMapArray.Length;
// //go through all the columns that aren't in lookup groups, and add up their sizes
// foreach (ColumnInfo ci in mySchema.columnInfo)
// {
// //if you're in a lookup table group, your size is always fixed.
// if (ci.lookupTableGroup == 0)
// {
// if (SerializationUtilityMethods.isFixedLengthType(ci.colType))
// myFixedSize += SerializationUtilityMethods.getFixedLengthTypeSize(ci.colType);
// else
// temp.Add(ci.colNo);
// }
// }
// //now add on the size of the lookup groups
// foreach (BlueRowLookupGroup group in mySchema.lookupGroups)
// myFixedSize += (group.tableSize / 8);
// myNonFixedColIndexes = temp.ToArray(typeof(int)) as int[];
// }
// /// <summary>
// /// Find the size of the serialized row, assumes that nullMap has already
// /// been built
// /// </summary>
// /// <param name="fixedSize"></param>
// /// <param name="r"></param>
// /// <returns></returns>
// private int myGetRowSizeInBytes(int fixedSize, IBlueRow r)
// {
// foreach (int colNo in myNonFixedColIndexes)
// {
// if (!myNullMap[colNo])
// fixedSize += //myGetStringSizeInBytes (r[colNo] as string);
// SerializationUtilityMethods.getObjectSerializedSize(r[colNo]);
// }
// return fixedSize;
// }
// #endregion
// }
// }
@@ -1,279 +0,0 @@
// using System;
// using System.IO;
// using BCG.Utilities;
// using System.Collections.Generic;
// using System.IO.Compression;
// namespace BCG
// {
// /// <summary>
// /// Serializes data stored in all IBlueRowCollections that use a LookupTableSchema.
// /// </summary>
// public class LookupTableSerializer : IBlueDataSerializer
// {
// private const string myPubSubDeser = "BCG.LookupTablePubSubDeserializer",
// myClientDeser = "BCG.LookupTableClientDeserializer";
// private LookupTable[] myLookupTables;
// private LookupTableSchema mySchema;
// private LookupTableRowSerializer myRowSerializer;
// //the header writer writes into a regular stream
// private BinaryWriter myHeaderWriter;
// //the body writer writes into a compressed binary stream
// //that way, we shrink the size of the data that we send over the wire
// private BinaryWriter myBodyWriter;
// public LookupTableSerializer (IBlueTableSchema inSchema, object[] groups)
// {
// //get the column infos from the input schema
// ColumnInfo[] inputColumns = inSchema.columnInfo;
// //build the lookup table groups
// //there is one more group than input string because we have the 'default' group, which isn't stored in a table
// int numGroups = groups.Length + 1;
// BlueRowLookupGroup[] lookupGroups = new BlueRowLookupGroup[numGroups];
// //we start the groups with group no 1, becuase groups with 0 are not stored in lookup tables
// short groupNo = 1;
// Dictionary<string, short> colNameToGroupNo = new Dictionary<string, short>();
// foreach (string group in groups)
// {
// string[] sizeAndColumns = group.Split(';');
// if (sizeAndColumns.Length != 2)
// throw new ArgumentException("Error, bad Lookup Table Group found: " + group);
// short tableSize = short.Parse(sizeAndColumns[0]);
// string[] columns = sizeAndColumns[1].Split(',');
// BlueRowLookupGroup thisGroup = new BlueRowLookupGroup(groupNo, tableSize, columns);
// lookupGroups[groupNo] = thisGroup;
// foreach (string column in columns)
// colNameToGroupNo[column] = groupNo;
// ++groupNo;
// }
// //make the lookup group correpsonding to columns that aren't stored in the lookup table
// List<string> nonLookupTableColumns = new List<string>();
// foreach (ColumnInfo ci in inputColumns)
// {
// if (!colNameToGroupNo.ContainsKey(ci.colName))
// {
// nonLookupTableColumns.Add(ci.colName);
// }
// }
// lookupGroups[0] = new BlueRowLookupGroup(0, 0, nonLookupTableColumns.ToArray());
// //now build new column infos to replace the old ones
// int numCols = inputColumns.Length;
// ColumnInfo[] newColumns = new ColumnInfo[numCols];
// for (int col = 0; col < numCols; ++col)
// {
// ColumnInfo inCol = inputColumns[col];
// if (inCol.isConstant)
// newColumns[col] = new ColumnInfo(inCol.colName, col, inCol.colType, inCol.colValue);
// else
// {
// string colName = inCol.colName;
// if (!colNameToGroupNo.ContainsKey(inCol.colName))
// newColumns[col] = new ColumnInfo(colName, col, inCol.colType);
// else
// newColumns[col] = new ColumnInfo(colName, col, inCol.colType, colNameToGroupNo[colName]);
// }
// }
// //now build the schema
// mySchema = new LookupTableSchema(inSchema.tableName, newColumns, lookupGroups);
// //build the lookup tables. There is one less than the length of the lookupTableGroups because the first one
// //is always the default group which isn't stored in a lookup table.
// myLookupTables = new LookupTable[mySchema.lookupGroups.Length - 1];
// //start with 1 because we don't store the 0th lookup table
// for (int i = 0; i < myLookupTables.Length; ++i)
// {
// myLookupTables[i] = new LookupTable(mySchema, i+1);
// }
// //now create the row serializer
// myRowSerializer = new LookupTableRowSerializer(mySchema, myLookupTables);
// }
// #region IBlueDataSerializer Members
// /// <summary>
// /// Serializes the data related to a single op.
// /// Supports All Ops
// ///
// /// First, a LookupTableUpdate, which is contained in each op, looks like this:
// ///
// ///
// /// -LookupTableUpdate:
// /// group 1 values size \
// /// group 1 value |- all serialized by a single LookupTableObject
// /// group 1 value /
// /// ....
// /// group 2 values size
// /// group 2 value key
// /// ....
// ///
// ///
// /// Next, the stream formats of the different op codes
// ///
// ///
// /// -create :
// /// op
// /// clientDeser name
// /// pubsubsDeser name
// /// schema (which is a LookupTableSchema)
// /// LookupTableUpdate
// /// num rows
// /// row 1
// /// row 2
// /// ....
// ///
// ///
// /// - add, remove, change:
// /// op
// /// LookupTableUpdate
// /// the row
// ///
// /// -multipleRemoveAdd :
// /// op
// /// LookupTableUpdate
// /// num deleted rows
// /// deleted row 1
// /// deleted row 2
// /// ....
// /// num added rows
// /// added row 1
// /// added row 2
// /// ...
// ///
// /// -multipleChange
// /// op
// /// LookupTableUpdate
// /// num changed rows
// /// changd row 1
// /// changed row 2
// /// ...
// /// changd row n
// ///
// /// </summary>
// /// <param name="op"></param>
// /// <param name="rows"></param>
// /// <param name="serializedData"></param>
// public void serialize(byte op, BCG.IBlueRowCollection[] rows, out byte[] serializedData)
// {
// myHeaderWriter = new BinaryWriter(new MemoryStream());
// myBodyWriter = new BinaryWriter(new MemoryStream());
// //everything start with an op
// myHeaderWriter.Write(op);
// /*go through each of the rows we're about the serialize, and populate the lookup table
// * that way, when we serialize the lookup tables, well have something to report*/
// foreach (IBlueRowCollection collection in rows)
// foreach (IBlueRow r in collection)
// myRowSerializer.updateLookupTables(r);
// if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change)
// mySerializeSingleRow(op, BlueDataUtility.extractRow(rows[0], 0));
// else if (op == StreamOpCode.create)
// mySerializeCreate(rows[0]);
// else if (op == StreamOpCode.multipleChange)
// mySerializeMultipleChange(rows[0]);
// else if (op == StreamOpCode.multipleRemoveAdd)
// mySerializeMultipleRemoveAdd(rows[0], rows[1]);
// else
// {
// serializedData = null;
// // BlueEventLog.logAssert(false, String.Format("{0} is not a valid op", op));
// }
// //copy the data from the header and the body of the message
// myHeaderWriter.BaseStream.Position = 0;
// myBodyWriter.BaseStream.Position = 0;
// int headerLength = (int) myHeaderWriter.BaseStream.Length;
// MemoryStream zippedMemStream = new MemoryStream();
// SharpZipCompressor Kompressor = new SharpZipCompressor();
// Kompressor.compress(myBodyWriter.BaseStream,zippedMemStream);
// byte[] compressedBytes = zippedMemStream.ToArray();
// serializedData = new byte[headerLength + zippedMemStream.Length];
// Console.WriteLine(String.Format("Lookuptable Serializer: Header is {0}. Body compresesd from {1} to {2} bytes.", headerLength , myBodyWriter.BaseStream.Length, zippedMemStream.Length));
// myHeaderWriter.BaseStream.Read(serializedData, 0, headerLength);
// BinaryWriter sizeWriter = new BinaryWriter(new MemoryStream(serializedData));
// zippedMemStream.Position = 0;
// zippedMemStream.Read(serializedData, headerLength, (int) zippedMemStream.Length);
// }
// #endregion
// void mySerializeSingleRow(byte op, IBlueRow row)
// {
// mySerializeLookupTableUpdates(myBodyWriter);
// myRowSerializer.serializeRow(row, op, myBodyWriter);
// }
// private void mySerializeCreate (IBlueRowCollection rows)
// {
// myHeaderWriter.Write(myPubSubDeser);
// myHeaderWriter.Write(myClientDeser);
// mySchema.serialize(myBodyWriter);
// mySerializeLookupTableUpdates(myBodyWriter);
// myBodyWriter.Write(rows.Count);
// foreach (IBlueRow r in rows)
// myRowSerializer.serializeRow(r, StreamOpCode.create, myBodyWriter);
// }
// private void mySerializeMultipleChange(IBlueRowCollection rows)
// {
// mySerializeLookupTableUpdates(myBodyWriter);
// myBodyWriter.Write(rows.Count);
// foreach (IBlueRow r in rows)
// myRowSerializer.serializeRow(r,StreamOpCode.change,myBodyWriter);
// }
// private void mySerializeMultipleRemoveAdd(IBlueRowCollection deletedRows, IBlueRowCollection addedRows)
// {
// mySerializeLookupTableUpdates(myBodyWriter);
// myBodyWriter.Write(deletedRows.Count);
// foreach (IBlueRow r in deletedRows)
// myRowSerializer.serializeRow(r, StreamOpCode.change, myBodyWriter);
// myBodyWriter.Write(addedRows.Count);
// foreach (IBlueRow r in addedRows)
// myRowSerializer.serializeRow(r, StreamOpCode.add, myBodyWriter);
// }
// private void mySerializeLookupTableUpdates(BinaryWriter bw)
// {
// for (int i = 0; i < myLookupTables.Length; ++i)
// myLookupTables[i].serializeUpdates(bw);
// }
// }
// }
@@ -1,69 +0,0 @@
using System;
using System.IO;
namespace BCG
{
public class ClientTheoDeserializer : IClientDeserializer
{
private IBlueTableSchema mySchema = null;
public ClientTheoDeserializer() {}
#region IClientDeserializer Members
/// <summary>
/// Deserialized the streams of bytes, as one operation and the resulting rows. The op
/// is either a create or multipleChange.
/// </summary>
/// <param name="streams"></param>
/// <param name="theOps"></param>
/// <param name="theRows"></param>
public void deserialize (Stream[] streams, out byte[] theOps, out Array theRows)
{
bool isCreate = false;
BinaryReader br = new BinaryReader (UtilityMethods.getAggregatedStream (streams));
byte op = Byte.MaxValue;
IBlueRowCollection rows = null;
while (true)
{
try
{
op = br.ReadByte ();
}
catch (EndOfStreamException)
{
break;
}
if (op == StreamOpCode.create)
{
isCreate = true; //we crush all ops (that may be part of this call) but if theres a create we want that to be the
//only op
//ie, we dont want creates to be crushed, as this might lead to clients
//not detecting creates (not sure if its really worth it)
br.ReadString (); //pubsubDeserializer
br.ReadString (); //clientDeserializer
(mySchema = new BlueTableSchema ()) .deserialze (br);
}
rows = myGetRows (br, br.ReadInt32 ());
}
theOps = new byte[] {isCreate ? StreamOpCode.create : op};
theRows = new IBlueRowCollection[] {rows};
}
private IBlueRowCollection myGetRows (BinaryReader br, int n)
{
IBlueRow[] rows = new IBlueRow[n];
for (int i = 0; i < n; i++)
rows[i] = new BlueRow (mySchema, new object[]{
i,
br.ReadByte (),
br.ReadByte (),
new DateTime (br.ReadInt64 ()),
br.ReadDouble ()
});
return new SequentialIDBlueTable (mySchema, rows);
}
#endregion
}
}
@@ -1,69 +0,0 @@
using System;
using System.IO;
using System.Collections;
namespace BCG
{
public class PubsubTheoDeserializer : IPubsubDeserializer
{
private string myClientDeserializerName;
private int myNRows;
private byte[] myState;
private int mySizeOfRow ;
private IBlueTableSchema mySchema;
public PubsubTheoDeserializer()
{
mySizeOfRow = BlueDataValidTypes.getValueTypeSize (BlueDataValidTypes.LONG) +
BlueDataValidTypes.getValueTypeSize (BlueDataValidTypes.BYTE) +
BlueDataValidTypes.getValueTypeSize (BlueDataValidTypes.DOUBLE) +
BlueDataValidTypes.getValueTypeSize (BlueDataValidTypes.BYTE);
}
#region IPubsubDeserializer Members
public void deserialize (Stream[] streams, out byte[] theOps, out Array theRows)
{
byte op = Byte.MaxValue;
BinaryReader br = new BinaryReader (UtilityMethods.getAggregatedStream (streams));
while (true)
{
try
{
op = br.ReadByte ();
}
catch (EndOfStreamException)
{
break;
}
if (op == StreamOpCode.create)
{
br.ReadString (); //pubsubDeser name
myClientDeserializerName = br.ReadString ();
(mySchema = new BlueTableSchema ()).deserialze (br);
}
myState = br.ReadBytes ((myNRows = br.ReadInt32()) * mySizeOfRow);
}
theOps = new byte[] {op};
theRows = new object[] {myState};
}
public byte[] getState()
{
if (myState == null)
return new byte[] {};
else
{
BinaryWriter bw = new BinaryWriter (new MemoryStream ());
bw.Write (StreamOpCode.create);
bw.Write (GetType().FullName);
bw.Write (myClientDeserializerName);
mySchema.serialize (bw);
bw.Write (myNRows);
bw.Write (myState);
return (bw.BaseStream as MemoryStream).ToArray ();
}
}
#endregion
}
}
@@ -1,98 +0,0 @@
using System;
using System.IO;
using BCG.Utilities;
namespace BCG
{
/// <summary>
/// Serializes the theos, as one table. This is independent of the operation; we transmit
/// the whole state everytime theres a change of any kind. However the operation is also
/// serialized as part of the data; its eaither a create of batchChange
/// </summary>
public class TheoSerializer : IBlueDataSerializer
{
private const string myPubSubDeser = "BCG.PubsubTheoDeserializer",
myClientDeser = "BCG.ClientTheoDeserializer";
public TheoSerializer (IBlueTableSchema schema) {}
public TheoSerializer () {}
#region IBlueDataSerializer Members
/// <summary>
/// Serializes the data related to a single op. The only ops written are either a create
/// or multipleChange
/// -create
/// op
/// clientDeser name
/// pubsubsDeser name
/// schema
/// nRows
/// rows
/// -multipleChange
/// op
/// nRows
/// rows
/// This method extracts the rows accprding to the op, massages the op either to be a create
/// or multipleChange and passes it on to mySerialize
/// </summary>
/// <param name="op"></param>
/// <param name="rows"></param>
/// <param name="serializedData"></param>
public void serialize (byte op, BCG.IBlueRowCollection[] rows, out byte[] serializedData)
{
IBlueRowCollection totalRows = null;
if (op == StreamOpCode.create || op == StreamOpCode.multipleChange)
totalRows = rows[0];
else if (op == StreamOpCode.multipleRemoveAdd)
totalRows = rows[1];
else
// BlueEventLog.logAssert (false, String.Format ("{0} is not a valid op", op));
op = op == StreamOpCode.multipleRemoveAdd ? StreamOpCode.multipleChange : op;
serializedData = mySerialize (op, totalRows);
}
/// <summary>
/// expects a create or multipleChange.
/// </summary>
/// <param name="op"></param>
/// <param name="rows"></param>
/// <returns></returns>
private byte[] mySerialize (byte op, IBlueRowCollection rows)
{
BinaryWriter bw = new BinaryWriter (new MemoryStream ());
bw.Write (op);
if (op == StreamOpCode.create)
{
bw.Write (myPubSubDeser);
bw.Write (myClientDeser);
myReorderSchema (rows.schema).serialize (bw);
}
bw.Write (rows.Count);
foreach (IBlueRow r in rows)
{
bw.Write ((byte)r["arrayIndex"]);
bw.Write ((byte)r["cell"]);
bw.Write ((long)((DateTime)r["expiration"]).Ticks);
bw.Write ((double)r["value"]);
}
bw.BaseStream.Position = 0;
return (bw.BaseStream as MemoryStream).ToArray ();
}
#endregion
public static IBlueTableSchema getSchema()
{return myReorderSchema(new BlueTableSchema("serializedTheos"));}
private static IBlueTableSchema myReorderSchema (IBlueTableSchema schema)
{
return new BlueTableSchema (schema.tableName,
new ColumnInfo ("id", 0, typeof(int)),
new ColumnInfo ("arrayIndex", 1, typeof(byte)),
new ColumnInfo ("cell", 2, typeof(byte)),
new ColumnInfo ("expiration", 3, typeof(DateTime)),
new ColumnInfo ("value", 4, typeof(double)));
}
}
}
@@ -0,0 +1,285 @@
// using System;
// using System.Collections;
// using System.IO;
// using BCG.Utilities;
// using BCG.Utilities.General;
// //using BCG.Utilities.BlueConfiguration;
// using System.Collections.Generic;
// using System.Threading;
// using mbts;
// using System.Linq;
// using mbts.spark;
// namespace BCG {
// public class TableAccumulator : ITableAccumulator {
// private HighPerformanceTimer _timer = new HighPerformanceTimer();
// private IClientDeserializer myDeserializer = new ClientDeserializationController();
// private BlueTable myStateTable;
// private ISubscriberCore mySubscriptionCore;
// private TableUpdateDelegate myOnUpdateDelegate;
// private string myTag;
// private bool myUseIncrementsForMultipleOps = DataStreamGatewaySettings.instance.useIncrementsForMultipleOps;
// public TableAccumulator(string tag, ISubscriberCore subCore) {
// myTag = tag;
// mySubscriptionCore = subCore;
// }
// #region ITableAccumulator Members
// public bool is_active() { return myOnUpdateDelegate != null; }
// public event TableUpdateDelegate on_update {
// add {
// // BlueEventLog.logEvent ("TableSubscriber being Subscribed for tag :" + myTag, BlueEventLevel.Information);
// if (myOnUpdateDelegate == null) {
// //ie first subscription
// // BlueEventLog.logEvent ("Subscription made to PubSub for tag : " + myTag, BlueEventLevel.Information);
// mySubscriptionCore.subscribe(myTag);
// }
// myOnUpdateDelegate =
// Delegate.Combine(myOnUpdateDelegate, value) as TableUpdateDelegate;
// if (myStateTable != null)
// value(myStateTable, myStateTable, new BlueTable(myStateTable.schema));
// }
// remove {
// // BlueEventLog.logEvent ("TableSubscriber being Unsubscribed for tag :" + myTag, BlueEventLevel.Information);
// if (myOnUpdateDelegate != null) {
// myOnUpdateDelegate =
// Delegate.Remove(myOnUpdateDelegate, value) as TableUpdateDelegate;
// if (myOnUpdateDelegate == null) {
// //all subscribers gone
// // BlueEventLog.logEvent ("Unubscription made to PubSub for tag : " + myTag, BlueEventLevel.Information);
// mySubscriptionCore.unsubscribe(myTag);
// myStateTable = null;
// }
// }
// }
// }
// public void handle_updates(IEnumerable<byte[]> msgs) {
// update(msgs.Select(t => new MemoryStream(t)).ToArray());
// }
// public void update(Stream[] streams) {
// bts.Log.Gen.info($"TableAccumulator.update tag: {myTag} n_streams: {streams.Length} stream[0].len: {streams[0].Length - streams[0].Position}");
// //checking the condition as a subscriber might unsubscribe in a callback and there might be more than one
// //data element for that tag in the ViewStreamDatabase
// if (myOnUpdateDelegate != null) {
// byte[] ops;
// Array rows;
// _timer.start();
// myDeserializer.deserialize(streams, out ops, out rows);
// _timer.stop();
// // BlueEventLog.logEvent(String.Format("deserializing {0} took {1} msecs", myTag, _timer.lastIntervalTime), BlueEventLevel.Performance);
// _timer.start();
// myAccumulateAndFire(ops, rows);
// _timer.stop();
// // BlueEventLog.logEvent(String.Format("myAccumulateAndFire for {0} took {1} msecs", myTag, _timer.lastIntervalTime), BlueEventLevel.Performance);
// }
// }
// #endregion
// private void myAccumulateAndFire(byte[] ops, Array rows) {
// //if a non create comes before the create, the deserializer returns 0 sized arrays
// if (ops.Length == 0 || rows.Length == 0) {
// //ie if one is zero sized so should be the other one
// // BlueEventLog.logAssert (ops.Length + rows.Length == 0);
// } else {
// if (ops.Length > 1) {
// int index = 0;
// if (myStateTable == null) {
// for (index = 0; index < ops.Length; index++)
// if (ops[index] == StreamOpCode.create)
// break;
// if (index == ops.Length)
// return;
// }
// if (myUseIncrementsForMultipleOps)
// myHandleMultipleOpsIncrementally(ops, rows, index);
// else
// myHandleMultipleOps(ops, rows, index);
// } else {
// if (myStateTable != null || ops[0] == StreamOpCode.create)
// myHandleSingleOp(ops, rows);
// }
// }
// }
// private void myHandleMultipleOps(byte[] ops, Array rows, int offset) {
// HighPerformanceTimer timer = new HighPerformanceTimer();
// timer.start();
// IBlueRowCollection stateAtStart = myStateTable == null ? null : new BlueTable(myStateTable);
// for (int i = offset; i < ops.Length; i++) {
// byte op = ops[i];
// IBlueRowCollection[] tableArray = rows.GetValue(i) as IBlueRowCollection[];
// if (op == StreamOpCode.add)
// myStateTable.addRows(tableArray[0]);
// else if (op == StreamOpCode.remove)
// myStateTable.deleteRows(tableArray[0]);
// else if (op == StreamOpCode.change)
// myStateTable.changeRows(tableArray[0]);
// else if (op == StreamOpCode.create)
// myStateTable = new BlueTable(tableArray[0]);
// else if (op == StreamOpCode.multipleChange)
// myStateTable.changeRows(tableArray[0]);
// else if (op == StreamOpCode.multipleRemoveAdd) {
// myStateTable.deleteRows(tableArray[0]);
// myStateTable.addRows(tableArray[1]);
// }
// }
// IBlueRowCollection dRows, aRows;
// if (stateAtStart == null) {
// dRows = new BlueTable(myStateTable.schema);
// aRows = myStateTable;
// } else {
// BlueDataUtility.computeDifferences(myStateTable, stateAtStart, out dRows, out aRows);
// }
// timer.stop();
// //Console.WriteLine("multiple ops for table " + myTag + " took " + timer.lastIntervalTime + " msecs");
// myOnUpdateDelegate(myStateTable, aRows, dRows);
// }
// #region handling mutliple ops in single update call, incrementally
// private void myHandleMultipleOpsIncrementally(byte[] ops, Array rows, int offset) {
// BlueTable dels = null, adds = null;
// if (myStateTable != null) {
// dels = new BlueTable(myStateTable.schema);
// adds = new BlueTable(myStateTable.schema);
// }
// for (int i = offset; i < ops.Length; i++) {
// byte op = ops[i];
// IBlueRowCollection[] tableArray = rows.GetValue(i) as IBlueRowCollection[];
// if (op == StreamOpCode.add) {
// myHandleAdd(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds);
// } else if (op == StreamOpCode.remove) {
// myHandleRemove(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds);
// } else if (op == StreamOpCode.change) {
// myHandleChange(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds);
// } else if (op == StreamOpCode.create) {
// myHandleCreate(tableArray[0], ref dels, ref adds);
// } else if (op == StreamOpCode.multipleChange) {
// myHandleMultipleChanges(tableArray[0], dels, adds);
// } else if (op == StreamOpCode.multipleRemoveAdd) {
// myHandleMultipleRemovesAdds(tableArray[0], tableArray[1], dels, adds);
// }
// }
// myOnUpdateDelegate(myStateTable, adds, dels);
// }
// private void myHandleAdd(IBlueRow r, BlueTable dels, BlueTable adds) {
// if (dels != null) {
// myStateTable.addRow(r);
// adds.addRow(r);
// }
// }
// private void myHandleRemove(IBlueRow r, BlueTable dels, BlueTable adds) {
// if (dels != null) {
// myStateTable.deleteRow(r);
// if (adds.containsID((int)r["id"]))
// adds.deleteRow(r);
// else
// dels.addRow(r);
// }
// }
// private void myHandleChange(IBlueRow r, BlueTable dels, BlueTable adds) {
// if (dels != null) {
// IBlueRow er = myStateTable[(int)r["id"]];
// myHandleRemove(er, dels, adds);
// myHandleAdd(r, dels, adds);
// }
// }
// private void myHandleCreate(IBlueRowCollection newTable, ref BlueTable dels, ref BlueTable adds) {
// if (dels == null) {
// // BlueEventLog.logAssert(adds == null);
// dels = new BlueTable(newTable.schema);
// adds = new BlueTable(newTable);
// myStateTable = new BlueTable(newTable);
// } else {
// dels = new BlueTable(myStateTable);
// myStateTable = new BlueTable(newTable);
// adds = new BlueTable(newTable);
// }
// }
// private void myHandleMultipleRemovesAdds(IBlueRowCollection deletedRows, IBlueRowCollection addedRows,
// BlueTable dels, BlueTable adds) {
// if (dels != null) {
// foreach (IBlueRow r in deletedRows)
// myHandleRemove(r, dels, adds);
// foreach (IBlueRow r in addedRows)
// myHandleAdd(r, dels, adds);
// }
// }
// private void myHandleMultipleChanges(IBlueRowCollection changedRows, BlueTable dels, BlueTable adds) {
// if (dels != null)
// foreach (IBlueRow r in changedRows)
// myHandleChange(r, dels, adds);
// }
// #endregion
// private void myHandleSingleOp(byte[] ops, Array rows) {
// byte op = ops[0];
// IBlueRowCollection[] tableArray = rows.GetValue(0) as IBlueRowCollection[];
// IBlueRowCollection dRows, aRows;
// if (op == StreamOpCode.create) {
// if (myStateTable == null) {
// aRows = myStateTable = new BlueTable(tableArray[0]);
// dRows = new BlueTable(myStateTable.schema);
// myOnUpdateDelegate(myStateTable, aRows, dRows);
// } else {
// IBlueRowCollection newState = new BlueTable(tableArray[0]);
// BlueDataUtility.computeDifferences(newState, myStateTable, out dRows, out aRows);
// if (aRows.Count > 0 || dRows.Count > 0) {
// myStateTable = new BlueTable(newState);
// myOnUpdateDelegate(myStateTable, aRows, dRows);
// }
// }
// } else {
// if (op == StreamOpCode.add) {
// myStateTable.addRows(aRows = tableArray[0]);
// dRows = new BlueTable(myStateTable.schema);
// } else if (op == StreamOpCode.remove) {
// myStateTable.deleteRows(dRows = tableArray[0]);
// aRows = new BlueTable(myStateTable.schema);
// } else if (op == StreamOpCode.change) {
// IBlueRow cRow = BlueDataUtility.extractRow(tableArray[0], 0);
// aRows = new BlueTable(myStateTable.schema);
// dRows = new BlueTable(myStateTable.schema);
// (aRows as BlueTable).addRow(cRow);
// (dRows as BlueTable).addRow(myStateTable[(int)cRow["id"]]);
// myStateTable.changeRow(cRow);
// } else if (op == StreamOpCode.multipleChange) {
// if (myStateTable.Count == tableArray[0].Count) {
// //all rows are changed, so we can optimize this
// dRows = myStateTable;
// aRows = myStateTable = new BlueTable(tableArray[0]);
// } else {
// dRows = new BlueTable(myStateTable.schema);
// aRows = tableArray[0];
// foreach (IBlueRow r in tableArray[0]) {
// (dRows as BlueTable).addRow(myStateTable[(int)r["id"]]);
// myStateTable.changeRow(r);
// }
// }
// } else if (op == StreamOpCode.multipleRemoveAdd) {
// BlueDataUtility.absorbChanges(myStateTable, dRows = tableArray[0], aRows = tableArray[1]);
// } else {
// aRows = dRows = null;
// // BlueEventLog.logAssert(false, "unknown opcode");
// }
// myOnUpdateDelegate(myStateTable, aRows, dRows);
// }
// }
// }
// }
@@ -1,236 +0,0 @@
// using System;
// using System.IO;
// using NUnit.Framework;
// namespace BCG
// {
// /// <summary>
// /// Tests the deserialization happening within the pubsub.
// /// </summary>
// [TestGroup]
// public class CompressionSerializationTest
// {
// protected IBlueTableSchema mySchema;
// protected BlueTable myTable;
// protected IBlueDataSerializer mySer;
// protected IPubsubDeserializer myPubsubDeser;
// protected IClientDeserializer myClientDeser;
// [Test]
// public static void test_compression()
// {
// int n_len = 10000000;
// for (int t = 0; t < 10; t++)
// {
// Random rnd = new Random(DateTime.Now.Millisecond);
// byte[] data = new byte[n_len];
// rnd.NextBytes(data);
// BCG.Utilities.General.HighPerformanceTimer timer = new Utilities.General.HighPerformanceTimer();
// timer.start();
// byte[] compr = UtilityMethods.compress(new MemoryStream(data));
// timer.stop();
// Console.WriteLine("Time to compress {0} bytes {1} msecs ratio {2}", n_len, timer.lastIntervalTime, (1.0 * data.Length)/(1.0 * compr.Length));
// timer.start();
// Stream decompr_str = UtilityMethods.decompress(compr);
// timer.stop();
// Console.WriteLine("Time to de-compress {0} bytes {1} msecs", n_len, timer.lastIntervalTime);
// decompr_str.Position = 0;
// Assert.AreEqual(data.Length, (int)decompr_str.Length);
// for (int i = 0; i < data.Length; i++)
// Assert.AreEqual(data[i], decompr_str.ReadByte());
// }
// }
// public CompressionSerializationTest()
// {
// mySchema = new BlueTableSchema(myGetTableName(),
// new ColumnInfo("id", 0, typeof(int)),
// new ColumnInfo("i1", 1, typeof(int)),
// new ColumnInfo("d1", 2, typeof(double)),
// new ColumnInfo("dt", 3, typeof(DateTime)),
// new ColumnInfo("x", 4, typeof(byte))
// );
// mySer = new CompressionSerializer(mySchema);
// myClientDeser = new ClientDeserializationController();
// myPubsubDeser = new PubsubDeserializationController();
// }
// protected virtual string myGetTableName()
// {
// return "testUniversal";
// }
// /// <summary>
// /// Tests the entire cycle. This is done by 'publishing' data to the serializationController (ie a series of
// /// adds, deletions, changes etc) and getting the bytes and feeding it into a pubsubDeseriazlier and a clientDeserializer.
// /// At every step the data is tested for consistency. At the end the getState is used to get all the data, and its passed
// /// through the clientDeserializer to get all the rows.
// /// </summary>
// [Test]
// public void testEntireProcess()
// {
// myTable = new BlueTable(mySchema);
// byte[] data;
// int id;
// //create
// for (id = 0; id < 100; id++)
// myTable.addRow(myGetRow(id));
// byte[] ops;
// Array rows;
// IBlueRowCollection del, add;
// IBlueRowCollection[] receivedData;
// IBlueRow r;
// //create
// mySer.serialize(StreamOpCode.create, new IBlueRowCollection[] { myTable }, out data);
// myWrite(StreamOpCode.create, data);
// myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// Assert.AreEqual(1, ops.Length);
// Assert.AreEqual(StreamOpCode.create, ops[0]);
// Assert.AreEqual(1, rows.Length);
// receivedData = rows.GetValue(0) as IBlueRowCollection[];
// Assert.AreEqual(1, receivedData.Length);
// BlueDataUtility.computeDifferences(receivedData[0], myTable, out del, out add);
// Assert.AreEqual(0, del.Count);
// Assert.AreEqual(0, add.Count);
// //single add
// r = myGetRow(id);
// BlueTable temp = new BlueTable(mySchema);
// myTable.addRow(r);
// temp.addRow(r);
// mySer.serialize(StreamOpCode.add, new IBlueRowCollection[] { temp }, out data);
// myWrite(StreamOpCode.add, data);
// myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// Assert.AreEqual(1, ops.Length);
// Assert.AreEqual(StreamOpCode.add, ops[0]);
// Assert.AreEqual(1, rows.Length);
// receivedData = rows.GetValue(0) as IBlueRowCollection[];
// Assert.AreEqual(1, receivedData.Length);
// BlueDataUtility.computeDifferences(receivedData[0], temp, out del, out add);
// Assert.AreEqual(0, del.Count);
// Assert.AreEqual(0, add.Count);
// //single delete
// r = myTable[0]; //delete row with id 0
// temp = new BlueTable(mySchema);
// myTable.deleteRow(0);
// temp.addRow(r);
// mySer.serialize(StreamOpCode.remove, new IBlueRowCollection[] { temp }, out data);
// myWrite(StreamOpCode.remove, data);
// myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// Assert.AreEqual(1, ops.Length);
// Assert.AreEqual(StreamOpCode.remove, ops[0]);
// Assert.AreEqual(1, rows.Length);
// receivedData = rows.GetValue(0) as IBlueRowCollection[];
// Assert.AreEqual(1, receivedData.Length);
// BlueDataUtility.computeDifferences(receivedData[0], temp, out del, out add);
// Assert.AreEqual(0, del.Count);
// Assert.AreEqual(0, add.Count);
// //batch deletes and adds
// //0 was removed earlier, so lets delete from 1 to 24 first, and then add 101 to 124
// BlueTable delRows = new BlueTable(mySchema), addRows = new BlueTable(mySchema);
// for (int i = 1; i < 25; i++)
// {
// delRows.addRow(myTable[i]);
// myTable.deleteRow(i);
// }
// for (id = 101; id < 125; id++)
// {
// r = myGetRow(id);
// addRows.addRow(r);
// myTable.addRow(r);
// }
// for (int i = 25; i < 125; i++)
// Assert.AreEqual(true, myTable.containsID(i));
// Assert.AreEqual(100, myTable.Count);
// mySer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { delRows, addRows }, out data);
// myWrite(StreamOpCode.multipleRemoveAdd, data);
// myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// Assert.AreEqual(1, ops.Length);
// Assert.AreEqual(StreamOpCode.multipleRemoveAdd, ops[0]);
// Assert.AreEqual(1, rows.Length);
// receivedData = rows.GetValue(0) as IBlueRowCollection[];
// Assert.AreEqual(2, receivedData.Length);
// BlueDataUtility.computeDifferences(receivedData[0], delRows, out del, out add);
// Assert.AreEqual(0, del.Count);
// Assert.AreEqual(0, add.Count);
// BlueDataUtility.computeDifferences(receivedData[1], addRows, out del, out add);
// Assert.AreEqual(0, del.Count);
// Assert.AreEqual(0, add.Count);
// //pure changes only
// //change the first 25, 25-49
// BlueTable changedRows = new BlueTable(mySchema);
// for (int i = 25; i < 50; i++)
// {
// MutableBlueRow mr = new MutableBlueRow(myTable[i]);
// mr.setValue(4, Byte.MaxValue);
// myTable.changeRow(mr);
// changedRows.addRow(mr);
// }
// mySer.serialize(StreamOpCode.multipleChange, new IBlueRowCollection[] { changedRows }, out data);
// myWrite(StreamOpCode.multipleChange, data);
// myPubsubDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// myClientDeser.deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// Assert.AreEqual(1, ops.Length);
// Assert.AreEqual(StreamOpCode.multipleChange, ops[0]);
// Assert.AreEqual(1, rows.Length);
// receivedData = rows.GetValue(0) as IBlueRowCollection[];
// Assert.AreEqual(1, receivedData.Length);
// BlueDataUtility.computeDifferences(receivedData[0], changedRows, out del, out add);
// Assert.AreEqual(0, del.Count);
// Assert.AreEqual(0, add.Count);
// //final test
// data = myPubsubDeser.getState();
// new ClientDeserializationController().deserialize(new Stream[] { new MemoryStream(data) }, out ops, out rows);
// Assert.AreEqual(1, ops.Length);
// Assert.AreEqual(StreamOpCode.create, ops[0]);
// Assert.AreEqual(1, rows.Length);
// receivedData = rows.GetValue(0) as IBlueRowCollection[];
// Assert.AreEqual(1, receivedData.Length);
// BlueDataUtility.computeDifferences(receivedData[0], myTable, out del, out add);
// Assert.AreEqual(0, del.Count);
// Assert.AreEqual(0, add.Count);
// }
// private IBlueRow myGetRow(int id)
// {
// return new BlueRow(mySchema, // the day pp fixed this test.
// id, id + 10, id * 3.0, new DateTime(2008, 6, 4, 0, 0, 0, DateTimeKind.Unspecified), Convert.ToByte(id));
// }
// private void myWrite(byte op, byte[] data)
// {
// string s = "";
// if (op == StreamOpCode.add)
// s = "add one row";
// else if (op == StreamOpCode.change)
// s = "change one row";
// else if (op == StreamOpCode.create)
// s = "create";
// else if (op == StreamOpCode.multipleChange)
// s = "multiple change";
// else if (op == StreamOpCode.multipleRemoveAdd)
// s = "multiple add remove";
// else if (op == StreamOpCode.remove)
// s = "remove one row";
// Console.WriteLine("op = {0}, size = {1}", s, data.Length);
// }
// }
// }
@@ -1,100 +0,0 @@
// using System;
// using System.IO;
// using System.Collections;
// using NUnit.Framework;
// namespace BCG
// {
// [TestGroup]
// public class FixedChangingRowSerializationTest
// {
// private IBlueTableSchema mySchema = new BlueTableSchema ("Test",
// new ColumnInfo ("i", 0, typeof(int)),
// new ColumnInfo ("b", 1, typeof(bool)),
// new ColumnInfo ("d", 2, typeof(double)),
// new ColumnInfo ("id", 3, typeof(int)),
// new ColumnInfo ("dt", 4, typeof(DateTime)),
// new ColumnInfo ("bt", 5, typeof(byte)));
// private BlueTable myState;
// private Random myRnd = new Random (DateTime.Now.Millisecond);
// private IBlueRowSerializer mySerializer;
// private IClientRowDeserializer myRowDeserializer;
// public FixedChangingRowSerializationTest ()
// {
// myState = new BlueTable (mySchema);
// mySerializer = new FixedChangingRowSerializer (mySchema, "bt");
// myRowDeserializer = new ClientFixedChangingRowDeserializer (mySchema, new string[] {"bt"});
// }
// [Test]
// public void addTest ()
// {
// for (int i = 0; i < 1000; i++)
// {
// BinaryWriter bw = new BinaryWriter (new MemoryStream ());
// IBlueRow r = myCreateRow (i);
// mySerializer.serializeRow (r, StreamOpCode.add, bw);
// BinaryReader br = new BinaryReader (new MemoryStream ((bw.BaseStream as MemoryStream).ToArray ()));
// Assert.AreEqual (r["i"], br.ReadInt32());
// Assert.AreEqual (r["b"], br.ReadBoolean ());
// Assert.AreEqual (r["d"], br.ReadDouble ());
// Assert.AreEqual (r["id"], br.ReadInt32 ());
// Assert.AreEqual (((DateTime)r["dt"]).Ticks, br.ReadInt64());
// Assert.AreEqual (r["bt"], br.ReadByte ());
// }
// }
// [Test]
// public void removeTest ()
// {
// for (int i = 0; i < 1000; i++)
// {
// BinaryWriter bw = new BinaryWriter (new MemoryStream ());
// IBlueRow r = myCreateRow (i);
// mySerializer.serializeRow (r, StreamOpCode.remove, bw);
// BinaryReader br = new BinaryReader (new MemoryStream ((bw.BaseStream as MemoryStream).ToArray ()));
// Assert.AreEqual (r["id"], br.ReadInt32());
// }
// }
// [Test]
// public void changeTest ()
// {
// for (int i = 0; i < 1000; i++)
// {
// BinaryWriter bw = new BinaryWriter (new MemoryStream ());
// IBlueRow r = myCreateRow (i);
// mySerializer.serializeRow (r, StreamOpCode.change, bw);
// BinaryReader br = new BinaryReader (new MemoryStream ((bw.BaseStream as MemoryStream).ToArray ()));
// Assert.AreEqual (r["id"], br.ReadInt32());
// Assert.AreEqual (r["bt"], br.ReadByte());
// bool isEnd = false;
// try
// {
// br.ReadByte ();
// }
// catch (EndOfStreamException)
// {
// isEnd = true;
// }
// Assert.That (isEnd);
// }
// }
// private IBlueRow myCreateRow (int id)
// {
// return new BlueRow (mySchema,
// myRnd.Next (1000),
// myRnd.Next (1000) % 2 == 0,
// myRnd.NextDouble (),
// id,
// DateTime.Now,
// Convert.ToByte (10));
// }
// }
// }
@@ -1,40 +0,0 @@
// using System;
// namespace BCG
// {
// public class TestFixedChangingSerializer : FixedChangingSerializer
// {
// public TestFixedChangingSerializer (IBlueTableSchema schema)
// : base (schema,
// new string[] {"x"},
// "BCG.PubsubTestFixedChangingDeserializer",
// "BCG.ClientTestFixedChangingDeserializer"
// )
// {}
// }
// public class PubsubTestFixedChangingDeserializer : PubsubFixedChangingDeserializer
// {
// public PubsubTestFixedChangingDeserializer () : base (new string[] {"x"})
// {}
// }
// public class ClientTestFixedChangingDeserializer : ClientFixedChangingDeserializer
// {
// public ClientTestFixedChangingDeserializer () : base (new string[] {"x"})
// {}
// }
// public class FixedChangingSerializationTest : UniversalSerializationTest
// {
// public FixedChangingSerializationTest () {}
// protected override string myGetTableName()
// {
// return "testFixedChanging";
// }
// }
// }
@@ -1,73 +0,0 @@
// using System;
// using System.Collections;
// using System.IO;
// using NUnit.Framework;
// using BCG.Utilities.General;
// namespace BCG
// {
// [TestFixture]
// public class MarketOptionTheoSerializationTest
// {
// private IBlueTableSchema mySchema = new BlueTableSchema ("marketOptionTheo",
// new ColumnInfo ("arrayIndex", 0, typeof(byte)),
// new ColumnInfo ("cell", 1, typeof(byte)),
// new ColumnInfo ("expiration", 2, typeof(DateTime)),
// new ColumnInfo ("id", 3, typeof(int)),
// new ColumnInfo ("value", 4, typeof(double))
// );
// public MarketOptionTheoSerializationTest () {}
// [Test]
// public void serializationPerformanceTest ()
// {
// MarketOptionTheoSerializer ser = new MarketOptionTheoSerializer (mySchema);
// IClientDeserializer deser = new MarketOptionTheoClientDeserializer ();
// BlueTable bt = new BlueTable (mySchema);
// int count = 3000;
// byte[] ops;
// Array rows;
// HighPerformanceTimer timer = new HighPerformanceTimer ();
// byte[] serData = null;
// for (int i = 0; i < count; i++)
// bt.addRow (myGetRow (i));
// ///serialization
// //create
// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out serData);
// deser.deserialize (new Stream [] {new MemoryStream (serData)}, out ops, out rows);
// ArrayList times = new ArrayList ();
// for (int i = 0; i < 25; i++)
// {
// timer.start ();
// ser.serialize (StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] {bt, bt}, out serData);
// timer.stop ();
// times.Add (timer.lastIntervalTime);
// Console.WriteLine (String.Format ("serialization of {0} rows, time = {1}, size = {2}", count, timer.lastIntervalTime, serData.Length));
// }
// times.Sort ();
// Console.WriteLine ("Median time = {0}", times[times.Count/2]);
// //deserialization
// times.Clear ();
// for (int i = 0; i < 25; i++)
// {
// timer.start ();
// deser.deserialize (new Stream [] {new MemoryStream (serData)}, out ops, out rows);
// timer.stop();
// times.Add (timer.lastIntervalTime);
// Console.WriteLine (String.Format ("deserialization of {0} rows, time = {1}", count, timer.lastIntervalTime));
// }
// times.Sort ();
// Console.WriteLine ("Median time = {0}", times[times.Count/2]);
// }
// private IBlueRow myGetRow (int id)
// {
// return new BlueRow (mySchema,
// Convert.ToByte (2), Convert.ToByte (4), DateTime.Now, id, 10.324555);
// }
// }
// }
@@ -1,46 +0,0 @@
// using System;
// using NUnit.Framework;
// using BCG.Utilities.General;
// namespace BCG
// {
// [TestFixture]
// public class Performance
// {
// public Performance()
// {
// }
// [Test]
// public void testContructionTime ()
// {
// double totalTime;
// string deserName = "BCG.ClientUniversalDeserializer";
// HighPerformanceTimer timer = new HighPerformanceTimer ();
// IClientDeserializer cd;
// for (int i = 0; i < 100; i++)
// {
// timer.start ();
// cd = Activator.CreateInstance (Type.GetType (deserName)) as IClientDeserializer;
// timer.stop ();
// //Console.Write (timer.lastIntervalTime.ToString() + " ");
// }
// Console.WriteLine ("average time = " + timer.totalMilliseconds / 100);
// totalTime = timer.totalMilliseconds;
// timer = new HighPerformanceTimer ();
// for (int i = 0; i < 100; i++)
// {
// timer.start ();
// cd = new ClientUniversalDeserializer ();
// timer.stop ();
// //Console.Write (timer.lastIntervalTime.ToString () + " ");
// }
// Console.WriteLine ("average time = " + timer.totalMilliseconds / 100);
// Console.WriteLine ("Ratio = " + totalTime/timer.totalMilliseconds);
// }
// }
// }
@@ -1,275 +0,0 @@
// using System;
// using System.IO;
// using NUnit.Framework;
// namespace BCG
// {
// [TestFixture]
// public class TableAccumulatorTest
// {
// private IBlueTableSchema mySchema = new BlueTableSchema ("test",
// new ColumnInfo ("id", 0, typeof(int)),
// new ColumnInfo ("x", 1, typeof(int)));
// private SubscriberCore mySubCore = new SubscriberCore ();
// private IBlueRowCollection myNr, myAr, myDr;
// public TableAccumulatorTest() {}
// [SetUp]
// public void tearDown ()
// {
// myNr = myAr = myDr = null;
// }
// #region create
// [Test]
// public void testCreate ()
// {
// TableAccumulator ta = new TableAccumulator ("test", mySubCore);
// IBlueDataSerializer ser = new BlueDataSerializationController ();
// byte[] data;
// BlueTable bt = new BlueTable (mySchema);
// for (int i = 0; i < 100; i++)
// bt.addRow (new BlueRow (mySchema, i, 2*i));
// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
// ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
// ta.update (new Stream[] {new MemoryStream (data)});
// //testing the data
// IBlueRowCollection dr, ar;
// //new rows
// BlueDataUtility.computeDifferences (bt, myNr, out dr, out ar);
// Assert.AreEqual (0, dr.Count);
// Assert.AreEqual (0, ar.Count);
// //added rows
// BlueDataUtility.computeDifferences (bt, myAr, out dr, out ar);
// Assert.AreEqual (0, dr.Count);
// Assert.AreEqual (0, ar.Count);
// //deleted rows;
// Assert.AreEqual (0, myDr.Count);
// ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
// }
// #endregion
// #region single add, delete and change
// [Test]
// public void testAddRemoveChange ()
// {
// TableAccumulator ta = new TableAccumulator ("test", mySubCore);
// IBlueDataSerializer ser = new BlueDataSerializationController ();
// byte[] data;
// BlueTable bt = new BlueTable (mySchema);
// for (int i = 0; i < 100; i++)
// bt.addRow (new BlueRow (mySchema, i, 2*i));
// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
// ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
// ta.update (new Stream[] {new MemoryStream (data)});
// //add
// //add a row with id 100
// bt.clear ();
// bt.addRow (new BlueRow (mySchema, 100, 200));
// ser.serialize (StreamOpCode.add, new IBlueRowCollection[] {bt}, out data);
// ta.update (new Stream[] {new MemoryStream (data)});
// //testing the add data
// Assert.AreEqual (101, myNr.Count);
// for (int i = 0; i < 101; i++)
// myNr.containsID (i);
// Assert.AreEqual (0, myDr.Count);
// Assert.AreEqual (1, myAr.Count);
// Assert.That (myAr.containsID (100));
// //remove
// //remove the row with id 0
// bt.clear ();
// bt.addRow (new BlueRow (mySchema, 0, 0));
// ser.serialize (StreamOpCode.remove, new IBlueRowCollection[] {bt}, out data);
// ta.update (new Stream[] {new MemoryStream (data)});
// //testing remove data
// Assert.AreEqual (100, myNr.Count);
// for (int i = 0; i < 101; i++)
// {
// Assert.That ((i == 0 && !myNr.containsID (i)) || (i != 0 && myNr.containsID (i)));
// }
// Assert.AreEqual (0, myAr.Count);
// Assert.AreEqual (1, myDr.Count);
// Assert.That (myDr.containsID (0));
// //change
// bt.clear ();
// bt.addRow (new BlueRow (mySchema, 100, -100));
// ser.serialize (StreamOpCode.change, new IBlueRowCollection[] {bt}, out data);
// ta.update (new Stream[] {new MemoryStream (data)});
// //testing changed data
// Assert.AreEqual (100, myNr.Count);
// Assert.AreEqual (1, myAr.Count);
// Assert.AreEqual (100, BlueDataUtility.extractRow (myAr, 0)["id"]);
// Assert.AreEqual (-100, BlueDataUtility.extractRow (myAr, 0)["x"]);
// Assert.AreEqual (1, myDr.Count);
// Assert.AreEqual (100, BlueDataUtility.extractRow (myDr, 0)["id"]);
// Assert.AreEqual (200, BlueDataUtility.extractRow (myDr, 0)["x"]);
// ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
// }
// #endregion
// #region multiple change
// public void testMultipleChange ()
// {
// TableAccumulator ta = new TableAccumulator ("test", mySubCore);
// IBlueDataSerializer ser = new BlueDataSerializationController ();
// byte[] data;
// BlueTable bt = new BlueTable (mySchema);
// for (int i = 0; i < 100; i++)
// bt.addRow (new BlueRow (mySchema, i, 2*i));
// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
// ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
// ta.update (new Stream[] {new MemoryStream (data)});
// //partial rows changed
// bt.clear ();
// for (int i = 0; i < 50; i++)
// bt.addRow (new BlueRow (mySchema, i, -2*i));
// ser.serialize (StreamOpCode.multipleChange, new IBlueRowCollection[] {bt}, out data);
// ta.update (new Stream[] {new MemoryStream (data)});
// //checking received data
// Assert.AreEqual (100, myNr.Count);
// foreach (IBlueRow r in myNr)
// {
// if ((int)r["id"] < 50)
// Assert.AreEqual ((int)r["id"] * -2, r["x"]);
// else
// Assert.AreEqual ((int)r["id"] * 2, r["x"]);
// }
// Assert.AreEqual (50, myAr.Count);
// foreach (IBlueRow r in myAr)
// Assert.AreEqual ((int)r["id"] * -2, r["x"]);
// Assert.AreEqual (50, myDr.Count);
// foreach (IBlueRow r in myDr)
// Assert.AreEqual ((int)r["id"] * 2, r["x"]);
// //full rows changed
// bt.clear ();
// for (int i = 0; i < 100; i++)
// bt.addRow (new BlueRow (mySchema, i, -2*i));
// ser.serialize (StreamOpCode.multipleChange, new IBlueRowCollection[] {bt}, out data);
// ta.update (new Stream[] {new MemoryStream (data)});
// //checking received data
// Assert.AreEqual (100, myNr.Count);
// foreach (IBlueRow r in myNr)
// {
// Assert.AreEqual ((int)r["id"] * -2, r["x"]);
// }
// Assert.AreEqual (100, myAr.Count);
// foreach (IBlueRow r in myAr)
// Assert.AreEqual ((int)r["id"] * -2, r["x"]);
// Assert.AreEqual (100, myDr.Count);
// foreach (IBlueRow r in myDr)
// {
// if ((int)r["id"] < 50)
// Assert.AreEqual ((int)r["id"] * -2, r["x"]);
// else
// Assert.AreEqual ((int)r["id"] * 2, r["x"]);
// }
// ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
// }
// #endregion
// #region multiple deletes/adds
// public void testMultipleDeletesAdds ()
// {
// TableAccumulator ta = new TableAccumulator ("test", mySubCore);
// IBlueDataSerializer ser = new BlueDataSerializationController ();
// byte[] data;
// BlueTable bt = new BlueTable (mySchema);
// for (int i = 0; i < 100; i++)
// bt.addRow (new BlueRow (mySchema, i, 2*i));
// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
// ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
// ta.update (new Stream[] {new MemoryStream (data)});
// BlueTable at = new BlueTable (mySchema),
// dt = new BlueTable (mySchema);
// for (int i = 0; i < 50; i++)
// dt.addRow (new BlueRow (mySchema, i, 2*i));
// for (int i = 100; i < 150; i++)
// at.addRow (new BlueRow (mySchema, i, 2*i));
// ser.serialize (StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] {dt, at}, out data);
// ta.update (new Stream[] {new MemoryStream (data)});
// //checking state
// Assert.AreEqual (100, myNr.Count);
// for (int i = 50; i < 150; i++)
// Assert.That (myNr.containsID (i));
// Assert.AreEqual (50, myDr.Count);
// for (int i = 0; i < 50; i++)
// Assert.That (myDr.containsID (i));
// Assert.AreEqual (50, myAr.Count);
// for (int i = 100; i < 150; i++)
// Assert.That (myAr.containsID (i));
// ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
// }
// #endregion
// #region multiple ops in one stream
// [Test]
// public void testMultipleOps ()
// {
// TableAccumulator ta = new TableAccumulator ("test", mySubCore);
// ta.onUpdate += new TableUpdateDelegate (myHandleUpdate);
// IBlueDataSerializer ser = new BlueDataSerializationController ();
// byte[] data;
// MemoryStream ms = new MemoryStream ();
// BlueTable bt = new BlueTable (mySchema);
// for (int i = 0; i < 100; i++)
// bt.addRow (new BlueRow (mySchema, i, 2*i));
// ser.serialize (StreamOpCode.create, new IBlueRowCollection[] {bt}, out data);
// ta.update (new Stream[] {new MemoryStream (data)});
// //ms.Write (data, 0, data.Length);
// //changing the first 5 rows
// bt.clear ();
// for (int i = 0; i < 5; i++)
// bt.addRow (new BlueRow (mySchema, i, 1000));
// ser.serialize (StreamOpCode.multipleChange, new IBlueRowCollection[] {bt}, out data);
// ms.Write (data, 0, data.Length);
// //delete id 5
// bt.clear ();
// bt.addRow (new BlueRow (mySchema, 5, 10));
// ser.serialize (StreamOpCode.remove, new IBlueRowCollection[] {bt}, out data);
// ms.Write (data, 0, data.Length);
// //add id 100
// bt.clear ();
// bt.addRow (new BlueRow (mySchema, 100, 200));
// ser.serialize (StreamOpCode.add, new IBlueRowCollection[] {bt}, out data);
// ms.Write (data, 0, data.Length);
// //delete 4 and add 5
// bt.clear ();
// BlueTable dels = new BlueTable (bt.schema);
// for (int i = 90; i < 94; i++)
// dels.addRow (new BlueRow (mySchema, i, i*2));
// for (int i = 101; i < 106; i++)
// bt.addRow (new BlueRow (mySchema, i, 2*i));
// ser.serialize (StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] {dels, bt}, out data);
// ms.Write (data, 0, data.Length);
// ms.Position = 0;
// ta.update (new Stream[] {ms});
// Assert.AreEqual (101, myNr.Count);
// Assert.AreEqual (10, myDr.Count);
// Assert.AreEqual (11, myAr.Count);
// ta.onUpdate -= new TableUpdateDelegate (myHandleUpdate);
// }
// #endregion
// private void myHandleUpdate (IBlueRowCollection nr, IBlueRowCollection ar, IBlueRowCollection dr)
// {
// myNr = nr;
// myAr = ar;
// myDr = dr;
// }
// }
// }
@@ -1,58 +0,0 @@
// using System;
// using System.IO;
// using NUnit.Framework;
// namespace BCG
// {
// [TestFixture]
// public class UniversalRowSerializerTest
// {
// private IBlueRowSerializer myRowSerializer;
// private IClientRowDeserializer myRowDeserializer;
// private Random myRnd = new Random (DateTime.Now.Millisecond);
// private IBlueTableSchema mySchema = new BlueTableSchema ("Test",
// new ColumnInfo ("id", 0, typeof(int)),
// new ColumnInfo ("int1", 1, typeof(int)),
// new ColumnInfo ("int2", 2, typeof(int)),
// new ColumnInfo ("double1", 3, typeof(double)),
// new ColumnInfo ("double2", 4, typeof(double)),
// new ColumnInfo ("byte1", 5, typeof(byte)),
// new ColumnInfo ("string1", 6, typeof(string)),
// new ColumnInfo ("string2", 7, typeof(string)));
// public UniversalRowSerializerTest () {}
// [Test]
// public void testSerialiazationCycle ()
// {
// myRowSerializer = new UniversalRowSerializer (mySchema);
// myRowDeserializer = new ClientUniversalRowDeserializer (mySchema);
// for (int i = 0; i < 10000; i++)
// {
// BinaryWriter bw = new BinaryWriter (new MemoryStream ());
// IBlueRow ir = myGetRandomRow ();
// myRowSerializer.serializeRow (ir, StreamOpCode.add, bw);
// IBlueRow or;
// (bw.BaseStream as MemoryStream).Position = 0;
// BinaryReader br = new BinaryReader (bw.BaseStream);
// myRowDeserializer.deserializeRow (null, br, StreamOpCode.add, out or);
// Assert.That (BlueDataUtility.computeRowEquality (ir, or));
// }
// }
// private IBlueRow myGetRandomRow ()
// {
// return new BlueRow (mySchema,
// myRnd.Next (1000),
// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : myRnd.Next (1000) as object,
// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object: myRnd.Next (1000) as object,
// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : myRnd.NextDouble () as object,
// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : myRnd.NextDouble () as object,
// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : Convert.ToByte (76) as object,
// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : "abcdefgh" as object,
// myRnd.Next (5)% 3 == 0 ? DBNull.Value as object : "1234567890" as object
// );
// }
// }
// }
@@ -81,6 +81,13 @@ namespace bts {
serialize_column_info(ci, ref bw);
}
public static byte[] serialize_schema(IBlueTableSchema schema) {
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
serialize_schema(schema, ref bw);
return str.ToArray();
}
public static IBlueTableSchema deserialize_schema(ref BinaryReader br) {
var hex_txt = to_hex(br.BaseStream);
bts.Log.Gen.info($"StreamSerializationUtils.deserialize_schema bytes: {hex_txt}");
@@ -94,6 +101,11 @@ namespace bts {
return new BlueTableSchema(table_name, cis.ToArray());
}
public static IBlueTableSchema deserialize_schema(byte[] ser_bytes) {
BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes));
return deserialize_schema(ref br);
}
public static void serialize_column_info(ColumnInfo ci, ref BinaryWriter bw) {
// <col_no, name, type>
bw.Write(ci.colNo);
@@ -162,6 +174,18 @@ namespace bts {
return str.ToArray();
}
public static byte[] serialize_row(this IStreamRowSerializer ser, IBlueRow row) {
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
ser.serialize_row(row, StreamIOMessage.StreamOp.CREATE, ref bw);
return str.ToArray();
}
public static IBlueRow deserialize_row(this IStreamRowDeserializer row_deser, byte[] row_bytes) {
BinaryReader br = new BinaryReader(new MemoryStream(row_bytes));
return row_deser.deserialize_row(ref br);
}
public static StreamIOMessage deserialize(this bts.IStreamMsgDeserilizer deser, byte[] ser_msg) {
MemoryStream str = new MemoryStream(ser_msg);
BinaryReader br = new BinaryReader(str);