Merging changes

This commit is contained in:
Taha committed 2023-09-04 16:25:40 -04:00
1 parent c4216d2272
commit 797816be7e
437 files changed
+54228 -28

No files matched your search

@@ -0,0 +1,12 @@
using System;
namespace BCG
{
/// <summary>
/// Exception in case of problems with BlueData formulation e.x. the schema not having an id column
/// </summary>
public class BlueDataException : ApplicationException
{
public BlueDataException (string message) : base (message) {}
}
}
@@ -0,0 +1,36 @@
using System;
using System.ComponentModel;
namespace BCG
{
/// <summary>
/// Used by GridBindableBlueTable to
/// </summary>
public class BlueDataPropertyDescriptor : PropertyDescriptor
{
private ColumnInfo myColumnInfo;
public BlueDataPropertyDescriptor (ColumnInfo ci)
: base (ci.colName, new Attribute[]{})
{
myColumnInfo = ci;
}
public override bool IsReadOnly {get{return true;}}
public override Type PropertyType {get{return myColumnInfo.colType;}}
public override bool CanResetValue (object component) {return false;}
public override object GetValue (object component) {return (component as IBlueRow)[myColumnInfo.colName];}
public override void ResetValue(object component) {}
public override void SetValue(object component, object value) {}
public override bool ShouldSerializeValue(object component) {return false;}
public override Type ComponentType {get {return typeof(IBlueRow);}}
}
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,45 @@
using System;
namespace BCG
{
/// <summary>
/// Allowed types
/// </summary>
public class BlueDataValidTypes
{
public static readonly Type BOOL = typeof(bool);
public static readonly Type BYTE = typeof(byte);
public static readonly Type CHAR = typeof(char);
public static readonly Type DECIMAL = typeof(decimal);
public static readonly Type DOUBLE = typeof(double);
public static readonly Type SHORT = typeof(short);
public static readonly Type INT = typeof(int);
public static readonly Type LONG = typeof(long);
public static readonly Type SBYTE = typeof(sbyte);
//public static readonly Type FLOAT = typeof(float);
public static readonly Type STRING = typeof(string);
//public static readonly Type USHORT = typeof(ushort);
public static readonly Type UINT = typeof(uint);
public static readonly Type ULONG = typeof(ulong);
public static readonly Type DATETIME = typeof(DateTime);
public static readonly Type ISERIALIZEDTYPE = typeof(ISerializedType);
public static readonly Type SINGLE = typeof(Single);
public static readonly Type BYTEARRAY = typeof(byte[]);
public static int getValueTypeSize (Type t)
{
if (t == BOOL || t == BYTE || t == SBYTE)
return 1;
else if (t == SHORT || t == CHAR)
return 2;
else if (t == INT || t == UINT || t == SINGLE)
return 4;
else if (t == DOUBLE || t == LONG || t == ULONG || t == DATETIME)
return 8;
else if (t == DECIMAL)
return 16;
else
throw new ApplicationException("Unsupported Data Type");
}
}
}
@@ -0,0 +1,70 @@
using System;
using BCG;
//using BCG.Utilities;
namespace BCG
{
[Serializable]
public class BlueIntHash
{
[Serializable]
private struct Element
{
public int k;
public int v;
public int next;
}
public BlueIntHash(int capacity)
{
myElementPool = new Element[capacity + 1];
myNumberOfBuckets = capacity * 3 + 1;
myBuckets = new int[myNumberOfBuckets];
}
public void add(int k, int v)
{
int index = myGetHash(k);
myElementPool[myFirstFreeElement].k = k;
myElementPool[myFirstFreeElement].v = v;
myElementPool[myFirstFreeElement].next = myBuckets[index];
myBuckets[index] = myFirstFreeElement;
myFirstFreeElement++;
}
[Serializable]
public class ElementNotFound : Exception
{ }
public int find(int k)
{
int b = myFind(k);
if (b == 0)
throw new ElementNotFound();
else
return myElementPool[b].v;
}
public bool isPresent(int k)
{ return myFind(k) != 0; }
private int myFind(int k)
{
int b = myBuckets[myGetHash(k)];
while (b != 0 && myElementPool[b].k != k)
b = myElementPool[b].next;
return b;
}
private int myGetHash(int k)
{ return k % myNumberOfBuckets; }
private int myFirstFreeElement = 1;
private Element[] myElementPool;
private int[] myBuckets;
private int myNumberOfBuckets;
}
}
@@ -0,0 +1,318 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Data;
using System.Reflection;
using System.Linq;
//using BCG.Utilities;
using System.Text;
namespace BCG
{
/// <summary>
/// BlueRow is an analog of DataRow, an immutable one though. It constructs itself either
/// from dotNet DataRow or IBlueRow. One row can belong to any number of tables, 0 or 1 or many.
/// All what the BlueRow requires is a schema to manage its internal information
/// </summary>
[Serializable]
public class BlueRow : IBlueRow
{
protected Object[] myValues;
protected IBlueTableSchema mySchema;
#region Private helper methods
/// <summary>
/// Initializes the internal data structures
/// </summary>
/// <param name="schema"></param>
private void myInit(IBlueTableSchema schema)
{
mySchema = schema;
myValues = new Object[schema.columnNames.Length];
}
#endregion
#region Constructors
protected BlueRow() { }
public BlueRow(IBlueRow r)
{
myInit(r.schema);
foreach (ColumnInfo ci in mySchema.columnInfo)
myValues[ci.colNo] = r[ci.colNo];
}
/// <summary>
/// The BlueRow initializes itself from the row passed. The row is expected
/// to have the same schema (including the col ordering) as the row being created
/// </summary>
/// <param name="table"></param>
/// <param name="row"></param>
public BlueRow(IBlueTableSchema schema, IBlueRow row)
{
myInit(schema);
foreach (ColumnInfo ci in mySchema.columnInfo)
myValues[ci.colNo] = row[ci.colNo];
}
/// <summary>
/// Constructs itself from the schema and the array of values that correspond
/// to the rows
/// </summary>
/// <param name="schema">
/// Schema of the row
/// </param>
/// <param name="values">
/// Values represent the values in the row, the order should be the same as
/// represented in the schema
/// </param>
public BlueRow(IBlueTableSchema schema, params Object[] values)
{
myValues = values;
mySchema = schema;
}
/// <summary>
/// Initilizes itself from a dotNet DataRow. The row is expected to
/// have the same schema (including the column ordering) as the row being created
/// </summary>
/// <param name="table"></param>
/// <param name="row"></param>
public BlueRow(IBlueTableSchema schema, DataRow row)
{
myInit(schema);
foreach (DataColumn col in row.Table.Columns)
myValues[col.Ordinal] = row[col.Ordinal];
}
public BlueRow(IBlueTableSchema schema, IDictionary row)
{
myInit(schema);
foreach (DictionaryEntry de in row)
if (mySchema[de.Key as string] != null)
myValues[mySchema[de.Key as string].colNo] = de.Value;
}
/// <summary>
/// Constructs a new BlueRow by extracting the values fields enumerated
/// in schema from the POD (Plain Old Data) object passed. Obviously
/// the object's type must contain at least all the fields defined
/// by the schema and the types of these fields should match their counterparts.
/// See new BlueTableSchema(string tableName, Type).
/// This is a static function rather than a constructor because
/// the type signature of the constructor would not be unique (it would
/// conflict with several of the constructurs above)
/// </summary>
/// <param name="schema">The schema of the row</param>
/// <param name="id">The id of the row</param>
/// <param name="pod">The Plain Old Data object containing the data</param>
static public IBlueRow fromPOD(IBlueTableSchema schema, int id, Object pod)
{ return myBlueRowFromPOD(myGetAccessors(schema, pod), schema, id, pod); }
private static Dictionary<string, FieldInfo> myGetAccessors(IBlueTableSchema schema, object o)
{
var accessors = new Dictionary<string, FieldInfo>();
foreach (string fn in schema.columnNames)
if (fn.ToLower() != "id")
{
FieldInfo fi = o.GetType().GetField(fn);
if (fi != null)
accessors[fn] = fi;
}
return accessors;
}
private static IBlueRow myBlueRowFromPOD(Dictionary<string, FieldInfo> accessors,
IBlueTableSchema schema,
int id,
object pod)
{
MutableBlueRow r = new MutableBlueRow(schema);
foreach (var kvp in accessors)
{
object v = kvp.Value.GetValue(pod);
if (v == null)
v = DBNull.Value;
r.setValue(kvp.Key, v);
}
r.setValue("id", id);
return r;
}
public delegate int IdGenerator(object pod);
static public IBlueRowCollection fromPODCollection(IBlueTableSchema schema, IdGenerator g, IEnumerable pods)
{
Dictionary<string, FieldInfo> accessors = null;
BlueTable result = new BlueTable(schema);
foreach (Object o in pods)
{
if (accessors == null)
accessors = myGetAccessors(schema, o);
result.addRow(myBlueRowFromPOD(accessors, schema, g(o), o));
}
return result;
}
private static void myPodFromBlueRow(Dictionary<string, FieldInfo> accessors,
IBlueRow r, object pod)
{
foreach (var kvp in accessors)
if (!(r[kvp.Key] is DBNull))
kvp.Value.SetValue(pod, r[kvp.Key]);
}
/// <summary>
/// Copies a blue row to a POD (Plain Old Data) object.
/// </summary>
/// <param name="toObject"></param>
public static T copyToPOD<T>(IBlueRow r) where T : new()
{
T pod = new T();
myPodFromBlueRow(myGetAccessors(r.schema, pod), r, pod);
return pod;
}
public static List<T> toPODCollection<T>(IBlueRowCollection tb) where T : new()
{
List<T> result = new List<T>();
Dictionary<string, FieldInfo> accessors = null;
foreach (IBlueRow r in tb)
{
T pod = new T();
if (accessors == null)
accessors = myGetAccessors(tb.schema, pod);
myPodFromBlueRow(accessors, r, pod);
result.Add(pod);
}
return result;
}
// For viewing BlueRow in the debugger more easily
public override string ToString()
{
return this.ToString(true);
}
public string ToString(bool names)
{
StringBuilder sb = new StringBuilder();
bool first = true;
for (int x = 0; x < this.myValues.Length; x++)
{
if (!first) sb.Append(", ");
first = false;
if (names)
{
sb.Append(this.mySchema.columnNames[x]);
sb.Append("=");
}
sb.Append((this.myValues[x] ?? "(null)").ToString());
}
return sb.ToString();
}
/// <summary>
/// Constructs the schema and then populates the values, both from the IDictionary.
/// The table name (the only thing that can not be derived from the dictionary) is
/// also provided
/// </summary>
/// <param name="tableName"></param>
/// <param name="row"></param>
public BlueRow(string tableName, IDictionary row)
{
if (row is IBlueRow)
myInit(((IBlueRow)row).schema);
else
{
ColumnInfo[] cis = new ColumnInfo[row.Count];
int i = 0;
foreach (DictionaryEntry de in row)
cis[i] = new ColumnInfo(de.Key as string, i++, de.Value.GetType());
myInit(new BlueTableSchema(cis, tableName));
}
foreach (DictionaryEntry de in row)
myValues[mySchema[de.Key as string].colNo] = de.Value;
}
/// <summary>
/// Constructs the schema and then populates the values, both from the IDictionary.
/// </summary>
/// <param name="row"></param>
public BlueRow(IDictionary row) : this("", row) { }
#endregion
#region IBlueRow members
public virtual IBlueTableSchema schema { get { return mySchema; } }
public virtual object this[string colName]
{
get
{
ColumnInfo ci = mySchema[colName];
if (ci != null)
return myValues[ci.colNo];
else
throw new Exception(colName + " is not a valid column name for " + schema.tableName);
}
}
public virtual object this[int index] { get { return myValues[index]; } }
#endregion
#region ICollection members
public virtual int Count { get { return myValues.Length; } }
public virtual bool IsSynchronized { get { return false; } }
public virtual object SyncRoot { get { return this; } }
public virtual void CopyTo(Array array, int index) { myValues.CopyTo(array, index); }
#endregion
#region IDictionary members
public virtual bool IsFixedSize { get { return true; } }
virtual public bool IsReadOnly { get { return true; } }
/// <summary>
/// The key can either be the string corrsponding to the column name or
/// the order of the column
/// </summary>
public virtual object this[object key]
{
get
{
if (key is string)
return this[(string)key];
else if (key is int)
return this[(int)key];
throw new BlueDataException("Wrong type while indexing the row");
}
set { throw new NotSupportedException("The IBlueRow is readonly"); }
}
public virtual ICollection Keys { get { return mySchema.columnNames; } }
public virtual ICollection Values { get { return myValues; } }
public virtual void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
public virtual void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
public virtual bool Contains(object key) { return mySchema[(string)key] != null; }
/// <summary>
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
/// iterate through the values, typecast this into an IEnumerable
/// </summary>
/// <returns></returns>
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
public virtual void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
#endregion
}
}
@@ -0,0 +1,60 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Collections;
namespace BCG
{
/// <summary>
/// Generic IComparer for IBlueRow. Accepts a list of column names to define the sort order. All included columns must be IComparable.
/// </summary>
public class BlueRowComparer : IComparer<IBlueRow>, IComparer
{
private IEnumerable<string> myCompareColumns;
private bool myIsAscending;
public BlueRowComparer(bool ascending, params string[] columns)
{
this.myCompareColumns = columns;
this.myIsAscending = ascending;
}
public BlueRowComparer(params string[] columns)
: this(true, columns)
{
}
public int Compare(IBlueRow x, IBlueRow y)
{
// Guard against nulls
if (x == y) return 0;
if (x == null) return -1;
if (y == null) return 1;
int result = 0;
foreach (string compareColumn in this.myCompareColumns)
{
object left = x[compareColumn];
object right = y[compareColumn];
if (left == right) result = 0;
else if (left == null || left == DBNull.Value) result = -1;
else if (right == null || right == DBNull.Value) result = 1;
else result = ((IComparable)left).CompareTo(right);
if (result != 0)
break;
}
return result * (this.myIsAscending ? 1 : -1);
}
#region IComparer Members
public int Compare(object x, object y)
{
return this.Compare(x as IBlueRow, y as IBlueRow);
}
#endregion
}
}
@@ -0,0 +1,23 @@
using System;
using System.Collections;
namespace BCG
{
/// <summary>
/// This enumertaor is used to iterate through an IBlueRow when the IBlueRow is
/// used as an IDictionary
/// </summary>
public class BlueRowDictionaryEnumerator : BlueRowEnumerator, IDictionaryEnumerator
{
public BlueRowDictionaryEnumerator (IBlueRow row) : base (row) {}
public override object Current {get{return Entry;}}
#region IDictionaryEnumerator members
public object Key {get{return myRow.schema[myPos].colName;}}
public object Value {get{return myRow[myPos];}}
public DictionaryEntry Entry {get{return new DictionaryEntry (Key, Value);}}
#endregion
}
}
@@ -0,0 +1,37 @@
using System;
using System.Collections;
namespace BCG
{
/// <summary>
/// Used for enumerating an IEnumerable (ex ICollection). Furthermore, this is the
/// default Enueramtor used when enumerating asn IBlueRow
/// </summary>
public class BlueRowEnumerator : IEnumerator
{
protected int myPos = -1;
protected int myRowSize = -1;
protected IBlueRow myRow;
public BlueRowEnumerator (IBlueRow row)
{
myRow = row;
myRowSize = row.schema.columnNames.Length;
}
#region IEnumerator members
public void Reset() {myPos = 0;}
public bool MoveNext ()
{
if (myPos == myRowSize - 1)
return false;
myPos ++;
return true;
}
public virtual object Current {get{return myRow[myPos];}}
#endregion
}
}
@@ -0,0 +1,103 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BCG
{
public class BlueRowHasher
{
private int _n_keys;
private int _hash_counter = 0;
private bool _thread_safe = false;
private IDictionary<object, object> _root_map = new Dictionary<object, object>();
public BlueRowHasher(int n_keys, bool thread_safe = false)
{
_n_keys = n_keys;
_thread_safe = thread_safe;
}
public string get_string_hash(IBlueRow row, int[] key_cols)
{
return get_hash(row, key_cols).ToString();
}
/// <summary>
/// gets the hash for a row with given keys,
/// if key does not exist is adds it adds it
/// and returns the hash
/// </summary>
public int get_hash(IBlueRow row, int[] key_cols)
{
IDictionary<object, object> d = _root_map;
for (int i = 0; ; i++)
{
object v = row[key_cols[i]];
if (i < _n_keys - 1)
{
object inner_d = null;
if (!d.TryGetValue(v, out inner_d))
{
inner_d = _get_new_dictionary();
d.Add(v, inner_d);
}
d = (IDictionary<object, object>)inner_d;
}
else
{
object rv;
if (!d.TryGetValue (v, out rv))
{
rv = _get_next_hash();
d.Add(v, rv);
}
return (int)rv;
}
}
}
public int get_hash(params object[] key_vals)
{
IDictionary<object, object> d = _root_map;
for (int i = 0; ; i++)
{
object v = key_vals[i];
if (i < _n_keys - 1)
{
object inner_d = null;
if (!d.TryGetValue(v, out inner_d))
{
inner_d = _get_new_dictionary();
d.Add(v, inner_d);
}
d = (IDictionary<object, object>)inner_d;
}
else
{
object rv;
if (!d.TryGetValue(v, out rv))
{
rv = _get_next_hash();
d.Add(v, rv);
}
return (int)rv;
}
}
}
private int _get_next_hash()
{
return _thread_safe ? System.Threading.Interlocked.Increment(ref _hash_counter) : ++_hash_counter;
}
private IDictionary<object, object> _get_new_dictionary()
{
if (_thread_safe)
return new System.Collections.Concurrent.ConcurrentDictionary<object, object>();
else
return new Dictionary<object, object>();
}
}
}
@@ -0,0 +1,116 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
/*
* This class reprsents one lookup group that is stored together in a lookup table because it contains repeated data patterns.
* if groupNo is zero, then the group is not stord in a lookup table at all.
*
*
*
*/
namespace BCG
{
public class BlueRowLookupGroup
{
BlueTableSchema myEntrySchema;
public IBlueTableSchema entrySchema
{
get { return myEntrySchema; }
//cant' set this value, have to build it from the row's schema
}
//for faster access and such
private int[] myGroupColNos;
public int[] groupColNos
{
get { return myGroupColNos; }
}
private string[] myGroupColNames;
public string[] groupColNames
{
get { return myGroupColNames; }
}
private short myGroupNumber;
public short groupNumber
{
get { return myGroupNumber; }
}
/* how many bits are the ids into the table?
* In c# everything is stored as integers, but when it goes across the wire
* the data can be represented as a byte, a short, or an int
*/
private short myTableSize;
public short tableSize
{
get { return myTableSize; }
}
//if the group number is 0, the group is not stored in tables for serialization
public bool isStoredInTable
{
get { return myGroupNumber != 0; }
}
public BlueRowLookupGroup(short num, short sizeInBits, params string[] cols)
{
myGroupNumber = num;
myTableSize = sizeInBits;
myGroupColNames = cols ;
}
/*Takes a LookupTableSchema for the rows this group caches for, and builds the schema representing
each of these entries in the lookup table. This function should be called by the LookupTableSchema
during its constructor.
This function also caches the column numbers for this group, for quicker access*/
public void buildEntrySchema(LookupTableSchema theSchema)
{
ColumnInfo[] columns = new ColumnInfo[myGroupColNames.Length + 1];
myGroupColNos = new int[myGroupColNames.Length];
//the first column in our table is always the ID
int idColumn = 0;
columns[idColumn] = new ColumnInfo("id", idColumn, typeof(int));
//go through each of the columns
for (int col = 1; col < columns.Length; ++col)
{
ColumnInfo columnInInputTable = theSchema[groupColNames[col - 1]];
if (columnInInputTable.isConstant)
columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colValue);
else
columns[col] = new ColumnInfo(columnInInputTable.colName, col, columnInInputTable.colType);
myGroupColNos[col - 1] = columnInInputTable.colNo;
}
myEntrySchema = new BlueTableSchema(columns, "Lookup Group " + theSchema.tableName + ":" + myGroupNumber);
}
/*takes a row as input, and extracts the members of the group from the row */
public object[] extractGroupFromRow(IBlueRow input)
{
int numCols = myGroupColNames.Length;
object[] results = new object[numCols];
for (int col = 0; col < numCols; ++col)
results[col] = input[myGroupColNos[col]];
return results;
}
}
}
@@ -0,0 +1,239 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Data;
//using BCG.Utilities;
using System.Linq;
namespace BCG
{
/// <summary>
/// Anologue of the dotNet DataTable.
/// </summary>
[Serializable]
public class BlueTable : IBlueRowCollection
{
/// <summary>
/// Hashtable indexed on "id" to speed up removes
/// </summary>
protected Hashtable myIdIndex = new Hashtable();
/// <summary>
/// Schema the table adheres to
/// </summary>
protected IBlueTableSchema mySchema;
#region Constructors
protected BlueTable() { }
/// <summary>
/// The BlueTable constructs the schema from the table being passed and copies
/// all the data over
/// </summary>
/// <param name="table"></param>
public BlueTable(DataTable table)
{
ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
int i = 0;
foreach (DataColumn col in table.Columns)
cis[i] = new ColumnInfo(col.ColumnName, i++, col.DataType);
mySchema = new BlueTableSchema(cis, table.TableName);
foreach (DataRow row in table.Rows)
{
IBlueRow newRow = new BlueRow(mySchema, row.ItemArray);
myIdIndex[newRow["id"]] = newRow;
}
}
public BlueTable(IBlueRowCollection table)
{
mySchema = table.schema;
foreach (IBlueRow row in table)
addRow(row);
}
public BlueTable(string tableName,
IEnumerable objects,
params CMap[] columnMappers)
{
// extract schema from mappers
ColumnInfo[] cols = new ColumnInfo[columnMappers.Length];
for (int i = 0; i < columnMappers.Length; i++)
{
cols[i] = new ColumnInfo(
columnMappers[i].getColumnName(), i,
columnMappers[i].getColumnType());
}
mySchema = new BlueTableSchema(cols, tableName);
// load rows
int rowNumber = 0;
foreach (object rowObject in objects)
{
object[] values = new object[columnMappers.Length];
for (int cn = 0; cn < columnMappers.Length; cn++)
values[cn] = columnMappers[cn].getColumnValue(rowNumber, rowObject);
addRow(new BlueRow(mySchema, values));
rowNumber++;
}
}
/// <summary>
/// Gets the schema, no data is iniliazed at construction time
/// </summary>
/// <param name="schema"></param>
public BlueTable(IBlueTableSchema schema) { mySchema = schema; }
#endregion
#region IBlueRowCollection members
/// <summary>
/// Returns the row by the index, interface property
/// </summary>
public virtual IBlueRow this[int id]
{
get
{
object row = myIdIndex[id];
if (row == null)
throw new BlueDataException("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
return (IBlueRow)row;
}
}
/// <summary>
/// Gets the schema
/// </summary>
public virtual IBlueTableSchema schema { get { return mySchema; } }
public virtual bool containsID(int id) { return myIdIndex[id] != null; }
public virtual bool tryGetValue(int id, out IBlueRow row)
{
row = (IBlueRow)myIdIndex[id];
return (row != null);
}
#endregion
#region BlueTable members
public virtual void clear()
{
myIdIndex.Clear();
}
/// <summary>
/// If a row with the same id exists, it replaces it
/// </summary>
/// <param name="r"></param>
public virtual void forceAddRow(IBlueRow r)
{
myIdIndex[r["id"]] = r;
}
/// <summary>
/// Adds a new row to the table.
/// </summary>
/// <param name="row"></param>
public virtual void addRow(IBlueRow row)
{
object val = row["id"];
if (val == null || val == DBNull.Value)
throw new BlueDataException("The ID of the row being added is null/DNBull");
try
{
myIdIndex.Add(val, row);
}
catch (ArgumentException)
{
throw new BlueDataException("ID already exists in the table " + mySchema.tableName +
" , duplication of field ID not allowed");
}
}
public virtual void addRow(DataRow row)
{
object val = row["id"];
if (val == null || val == DBNull.Value)
throw new BlueDataException("The ID of the row being added is null/DNBull");
if (myIdIndex[val] != null)
throw new BlueDataException("ID already exists in the table, duplication of field ID not allowed");
IBlueRow newRow = new BlueRow(mySchema, row);
myIdIndex[val] = newRow;
}
public virtual void addRows(IBlueRowCollection rows)
{
foreach (IBlueRow nr in rows)
addRow(nr);
}
public virtual void addRows(IEnumerable rows)
{
foreach (IBlueRow nr in rows)
addRow(nr);
}
/// <summary>
/// Removes a row from the table. Complexity = O(1)
/// </summary>
/// <param name="row"></param>
public virtual bool deleteRow(IBlueRow row) { return deleteRow((int)row["id"]); }
public virtual bool deleteRow(DataRow row) { return deleteRow((int)row["id"]); }
/// <summary>
/// Removes a row specified by the id. The index helps in making this operation O(1)
/// </summary>
/// <param name="id"></param>
public virtual bool deleteRow(int id)
{
if (myIdIndex[id] == null)
return false;
myIdIndex.Remove(id);
return true;
}
public virtual bool deleteRows(IBlueRowCollection rows)
{
bool retVal = true;
foreach (IBlueRow r in rows)
retVal &= deleteRow(r);
return retVal;
}
public virtual bool deleteRows(IEnumerable rows)
{
bool retVal = true;
foreach (IBlueRow r in rows)
retVal &= deleteRow(r);
return retVal;
}
public virtual void changeRow(IBlueRow r)
{
myIdIndex[r["id"]] = r;
}
public virtual void changeRows(IBlueRowCollection rows)
{
foreach (IBlueRow r in rows)
changeRow(r);
}
#endregion
#region ICollection members
public virtual IEnumerator GetEnumerator() //{return myRows.GetEnumerator ();}
{ return myIdIndex.Values.GetEnumerator(); }
public virtual bool IsSynchronized { get { return false; } }
public virtual object SyncRoot { get { return this; } }
public virtual void CopyTo(Array array, int index) { myIdIndex.Values.CopyTo(array, index); }
public virtual int Count { get { return myIdIndex.Count; } }
#endregion
}
}
@@ -0,0 +1,286 @@
using System;
using System.Data;
using System.Collections;
using System.IO;
using System.Reflection;
using System.Collections.Generic;
using System.Linq;
using BCG.Utilities.General;
namespace BCG
{
/// <summary>
/// The BlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
/// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
/// are the indexers (both order and name based), and the property columnInfo and columnNames, that return
/// the respective data, sorted according to the ColumnOrder.
/// </summary>
[Serializable]
public class BlueTableSchema : IBlueTableSchema
{
public class BlueTableSchemaSerializationHelper
{
public static object parse(Type t, string data)
{
if (t == BlueDataValidTypes.BOOL)
return Boolean.Parse(data);
else if (t == BlueDataValidTypes.BYTE)
return Byte.Parse(data);
else if (t == BlueDataValidTypes.CHAR)
return char.Parse(data);
else if (t == BlueDataValidTypes.DATETIME)
return DateTime.Parse(data);
else if (t == BlueDataValidTypes.DOUBLE)
return Double.Parse(data);
else if (t == BlueDataValidTypes.DECIMAL)
return Decimal.Parse(data);
else if (t == BlueDataValidTypes.INT)
return int.Parse(data);
else if (t == BlueDataValidTypes.LONG)
return long.Parse(data);
else if (t == BlueDataValidTypes.SBYTE)
return SByte.Parse(data);
else if (t == BlueDataValidTypes.STRING)
return data;
else if (t == BlueDataValidTypes.UINT)
return uint.Parse(data);
else if (t == BlueDataValidTypes.ULONG)
return ulong.Parse(data);
else if (t == BlueDataValidTypes.SINGLE)
return Single.Parse(data);
else
throw new ApplicationException("Can not be parsed");
}
public static void serialize(BinaryWriter bw, Type t, object o)
{
if (t == BlueDataValidTypes.BOOL)
bw.Write((bool)o);
else if (t == BlueDataValidTypes.BYTE)
bw.Write((byte)o);
else if (t == BlueDataValidTypes.CHAR)
bw.Write((char)o);
else if (t == BlueDataValidTypes.DATETIME)
bw.Write(((DateTime)o).Ticks);
else if (t == BlueDataValidTypes.DOUBLE)
bw.Write((double)o);
else if (t == BlueDataValidTypes.DECIMAL)
bw.Write((decimal)o);
else if (t == BlueDataValidTypes.INT)
bw.Write((int)o);
else if (t == BlueDataValidTypes.LONG)
bw.Write((long)o);
else if (t == BlueDataValidTypes.SBYTE)
bw.Write((sbyte)o);
else if (t == BlueDataValidTypes.STRING)
bw.Write(o as string);
else if (t == BlueDataValidTypes.SINGLE)
bw.Write((Single)o);
else if (t == BlueDataValidTypes.BYTEARRAY)
{
byte[] bs = o as byte[];
bw.Write(bs.Length);
bw.Write(bs);
}
else if (t == BlueDataValidTypes.UINT)
bw.Write((uint)o);
else if (t == BlueDataValidTypes.ULONG)
bw.Write((ulong)o);
else
throw new ApplicationException("Unsupported type");
}
public static object deserialize(BinaryReader br, Type t)
{
if (t == BlueDataValidTypes.BOOL)
return br.ReadBoolean();
else if (t == BlueDataValidTypes.BYTE)
return br.ReadByte();
else if (t == BlueDataValidTypes.CHAR)
return br.ReadChar();
else if (t == BlueDataValidTypes.DATETIME)
return new DateTime(br.ReadInt64());
else if (t == BlueDataValidTypes.DOUBLE)
return br.ReadDouble();
else if (t == BlueDataValidTypes.DECIMAL)
return br.ReadDecimal();
else if (t == BlueDataValidTypes.INT)
return br.ReadInt32();
else if (t == BlueDataValidTypes.LONG)
return br.ReadInt64();
else if (t == BlueDataValidTypes.SBYTE)
return br.ReadSByte();
else if (t == BlueDataValidTypes.STRING)
return br.ReadString();
else if (t == BlueDataValidTypes.BYTEARRAY)
return br.ReadBytes(br.ReadInt32());
else if (t == BlueDataValidTypes.UINT)
return br.ReadUInt32();
else if (t == BlueDataValidTypes.ULONG)
return br.ReadUInt64();
else if (t == BlueDataValidTypes.SINGLE)
return br.ReadSingle();
else
throw new ApplicationException("Unsupported type");
}
}
protected Hashtable myIndexByColName = new Hashtable();
protected ColumnInfo[] myIndexByColOrder;
private string[] myTableColNames;
protected string myTableName;
public BlueTableSchema() { }
public BlueTableSchema(DataTable table)
{
myIndexByColOrder = new ColumnInfo[table.Columns.Count];
myTableName = table.TableName;
foreach (DataColumn col in table.Columns)
{
ColumnInfo ci = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
myIndexByColName[ci.colName] = ci;
myIndexByColOrder[ci.colNo] = ci;
}
}
public BlueTableSchema(ColumnInfo[] cis, string tableName)
{
myConstruct(cis, tableName);
}
private void myConstruct(ColumnInfo[] cis, string tableName)
{
myIndexByColOrder = new ColumnInfo[cis.Length];
myTableName = tableName;
foreach (ColumnInfo ci in cis)
{
myIndexByColOrder[ci.colNo] = ci;
myIndexByColName[ci.colName] = ci;
}
}
public BlueTableSchema(string tableName, params ColumnInfo[] cis) : this(cis, tableName) { }
/// <summary>
/// Creates a BlueTableSchema that exactly mirrors the public fields
/// present in fromType except that an id field will be added regardless
/// of it being present in fromType or not.
/// </summary>
/// <param name="tableName">the table name</param>
/// <param name="fromType">the plain old data type whose fields will be mirrored in the schema</param>
public BlueTableSchema(string tableName, Type fromType)
{
List<ColumnInfo> cis = new List<ColumnInfo>();
int i = 0;
// we sort the field name so that we have a deterministic order given
// the same type.
bool foundId = false;
foreach (FieldInfo fi in fromType.GetFields().OrderBy((f) => { return f.Name; }))
{
if (fi.Name.ToLower().Equals("id"))
foundId = true;
cis.Add(new ColumnInfo(fi.Name, i++, fi.FieldType));
}
if (!foundId)
cis.Add(new ColumnInfo("id", cis.Count, typeof(Int32)));
myConstruct(cis.ToArray(), tableName);
}
public ColumnInfo this[string colName]
{
get
{
object retVal = myIndexByColName[colName];
return retVal != null ? retVal as ColumnInfo : null;
}
}
public ColumnInfo this[int colNo]
{
get { return myIndexByColOrder[colNo]; }
}
/// <summary>
/// The returned Array is sorted according to the Column No
/// </summary>
public string[] columnNames
{
get
{
if (myTableColNames == null)
{
myTableColNames = new string[myIndexByColName.Count];
foreach (DictionaryEntry de in myIndexByColName)
{
ColumnInfo temp = (ColumnInfo)de.Value;
myTableColNames[temp.colNo] = temp.colName;
}
}
return myTableColNames;
}
}
/// <summary>
/// The returned Array is sorted according to the Column No
/// </summary>
public ColumnInfo[] columnInfo
{
get { return myIndexByColOrder; }
}
public string tableName { get { return myTableName; } }
#region ISerializedType methods
public void serialize(BinaryWriter bw)
{
bw.Write(columnNames.Length);
bw.Write(tableName);
foreach (ColumnInfo ci in columnInfo)
{
bw.Write(ci.colName);
bw.Write(ci.colType.ToString());
if (ci.isConstant)
{
bw.Write(true);
BlueTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
}
else
{
bw.Write(false);
}
}
}
public void deserialze(BinaryReader br)
{
ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()];
string name = br.ReadString();
for (int i = 0; i < cis.Length; i++)
{
string colName = br.ReadString();
Type t = TypeInstanceLoader.loadType(br.ReadString(), true);
bool isConstant = br.ReadBoolean();
if (isConstant)
{
cis[i] = new ColumnInfo(colName, i, t, BlueTableSchemaSerializationHelper.deserialize(br, t));
}
else
{
cis[i] = new ColumnInfo(colName, i, t);
}
}
myConstruct(cis, name);
}
public int getSerializedSize()
{
throw new NotImplementedException("Feature for provided constant values to" +
"non-primitive types in schema, not provided as of now");
}
#endregion
}
}
@@ -0,0 +1,39 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BCG
{
public abstract class CMap
{
public abstract string getColumnName();
public abstract Type getColumnType();
public abstract object getColumnValue(int rowIndex,object rowObject);
}
public class CMap<T> : CMap
{
public CMap(string columnName, MapF mapperFunction)
{
myColumnName = columnName;
myExtractColumn = mapperFunction;
}
private string myColumnName;
public override string getColumnName()
{
return myColumnName;
}
public override Type getColumnType()
{
return typeof(T);
}
public delegate T MapF(int rowIndex, object rowObject);
private MapF myExtractColumn;
public override object getColumnValue(int rowIndex, object rowObject)
{
return myExtractColumn(rowIndex, rowObject);
}
}
}
@@ -0,0 +1,92 @@
using System;
using System.Data;
using System.Collections.Generic;
using System.Linq;
namespace BCG
{
/// <summary>
/// Circular buffer for rows.
/// </summary>
public class CircularRowCollection : OrderedBlueTable
{
private int myBufferSize;
public CircularRowCollection (IBlueTableSchema schema, int bufferSize)
: base (schema)
{
myBufferSize = bufferSize ;
}
public CircularRowCollection (IBlueRowCollection rows, int bufferSize)
: this (rows.schema, bufferSize)
{
foreach (IBlueRow r in rows)
addRow (r);
}
public int bufferSize
{
get
{
return (myBufferSize);
}
}
/// <summary>
/// Adds a new row to the collection. If the size of the
/// collection becomes greater that
/// buffer size, the first row is returned;
/// </summary>
/// <param name="r"></param>
/// <returns></returns>
public IBlueRow addRowWithReturn (IBlueRow r)
{
base.addRow (r);
if (myOrderedRows.Count > myBufferSize)
{
IBlueRow retVal = myOrderedRows [0] as IBlueRow;
deleteRow (retVal);
return retVal;
}
return null;
}
public override void addRow (IBlueRow row)
{
base.addRow (row);
if (myOrderedRows.Count > myBufferSize)
deleteRow (myOrderedRows [0] as IBlueRow);
}
public override void addRow (DataRow row)
{
base.addRow (row);
if (myOrderedRows.Count > myBufferSize)
deleteRow (myOrderedRows [0] as IBlueRow);
}
public void add_rows(IEnumerable<IBlueRow> rows_to_add, out IBlueRowCollection added_rows, out IBlueRowCollection deleted_rows)
{
BlueTable added_rv = new BlueTable(mySchema), deleted_rv = new BlueTable(mySchema);
if (rows_to_add.Count() < myBufferSize)
{
foreach (IBlueRow r in rows_to_add)
{
IBlueRow dr = addRowWithReturn(r);
added_rv.addRow(r);
if (dr != null)
deleted_rv.addRow(dr);
}
}
else
{
deleted_rv = new BlueTable(this);
clear();
addRows(rows_to_add);
added_rv = new BlueTable(this);
}
added_rows = added_rv;
deleted_rows = deleted_rv;
}
}
}
@@ -0,0 +1,95 @@
using System;
namespace BCG
{
/// <summary>
/// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
/// </summary>
[Serializable]
public class ColumnInfo : IComparable
{
public static bool isAllowedType (Type t)
{
return
t.Equals (typeof (System.Int32)) ||
t.Equals (typeof (System.Int64)) ||
t.Equals (typeof (System.Double)) ||
t.Equals (typeof (System.String)) ||
t.Equals (typeof (System.DateTime)) ? true :
false;
}
//the default column info does not cach values in a lookup table, therefore it is in group 0
public ColumnInfo(string colName, int colNo, Type colType)
: this(colName, colNo, colType, 0)
{
}
public ColumnInfo(string colName, int colNo, Type colType, short colGroup)
{
myColName = colName;
myColNo = colNo;
myColType = colType;
myLookupTableGroup = colGroup;
myColValue = null;
}
public ColumnInfo (string colName, int colNo, object colValue)
: this (colName, colNo, colValue.GetType()) {myColValue = colValue == null ? DBNull.Value : colValue;}
public ColumnInfo (string colName, int colNo, Type colType, object colValue)
: this (colName, colNo, colType)
{
myColValue = colValue == null ? DBNull.Value : colValue;
}
public int CompareTo (object obj)
{
int otherColNo = ((ColumnInfo)(obj)).colNo;
if (myColNo < otherColNo)
return -1;
else if (myColNo > otherColNo)
return 1;
return 0;
}
//public bool isCalculated{get{return myExpression != null;}}
public bool isConstant {get{return myColValue != null;}}
private int myColNo;
private string myColName;
private Type myColType;
//private string myExpression;
private object myColValue;
//no table will ever have more than 65,536 columns
//if it does, we are in serious trouble
private short myLookupTableGroup;
public int colNo{get{return myColNo;}}
public string colName {get{return myColName;}}
public Type colType {get{return myColType;}}
//this value corresponds to how the data in the column is serialized.
//if the group number is 0, the values are not stored in a table
//if the vlaue is non-zero, all columns with the the same lookup table group are stored in the same lookup table together
//are stored together in a lookup table
public short lookupTableGroup
{
get { return myLookupTableGroup; }
set { myLookupTableGroup = value; }
}
//public string expression {get{return myExpression;}}
public object colValue {get{return myColValue;}}
}
}
@@ -0,0 +1,73 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BCG
{
public class ColumnKeyedData
{
private int[] _key_col_indices = null;
private string[] _key_col_names;
private BlueRowHasher _row_hasher;
private Dictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
public ColumnKeyedData(params string[] key_col_names)
{
_key_col_names = key_col_names;
_row_hasher = new BlueRowHasher(key_col_names.Length);
}
public void delete_rows(IBlueRowCollection rows)
{
if (rows != null)
{
_init_schema(rows.schema);
foreach (IBlueRow r in rows)
{
int row_hash = _row_hasher.get_hash(r, _key_col_indices);
if (_row_map.ContainsKey(row_hash))
_row_map.Remove(row_hash);
}
}
}
public void add_rows(IBlueRowCollection rows)
{
if (rows != null)
{
_init_schema(rows.schema);
foreach (IBlueRow r in rows)
{
int row_hash = _row_hasher.get_hash(r, _key_col_indices);
_row_map[row_hash] = r;
}
}
}
public void reset_state (IBlueRowCollection rows)
{
_row_map.Clear();
add_rows(rows);
}
public IEnumerable<IBlueRow> get_all_rows() { return _row_map.Values; }
private void _init_schema(IBlueTableSchema schema)
{
if (_key_col_indices == null)
_key_col_indices = BlueDataUtility.get_column_indices(schema, _key_col_names);
}
public IBlueRow get_row(params object[] key_vals)
{
int row_hash = _row_hasher.get_hash(key_vals);
IBlueRow rv = null;
_row_map.TryGetValue(row_hash, out rv);
return rv;
}
}
}
@@ -0,0 +1,54 @@
using System;
using System.IO;
using System.Collections;
namespace BCG
{
public class FileRowCollection : CircularRowCollection
{
public FileRowCollection(string dataFileName,
string schemaFileName)
: this(dataFileName, schemaFileName, Int32.MaxValue)
{ }
/// <summary>
/// </summary>
/// <param name="dataFileName">file with the rows</param>
/// <param name="schemFileName">file with schema (written as columns names in the first line and types in the 2nd</param>
/// <param name="maxSize">if max size is mentioned then a buffer is mantained for only
/// that number of rows, ie the last maxSize number of rows in the file
/// </param>
public FileRowCollection(string dataFileName,
string schemaFileName,
int maxSize)
: this(new StreamReader(dataFileName),
new StreamReader(schemaFileName),
maxSize)
{ }
public FileRowCollection(StreamReader dataReader,
StreamReader schemaReader)
: this(dataReader, schemaReader, Int32.MaxValue)
{ }
public FileRowCollection(StreamReader dataReader,
StreamReader schemaReader,
int maxSize)
: base(BlueDataUtility.createSchemaFromCSVSchemaFile(schemaReader), maxSize)
{
int id = 0;
while (dataReader.Peek() > 0)
{
try
{
addRow(BlueDataUtility.createRowFromCSVFileRow(dataReader.ReadLine(), mySchema, id++));
}
catch (Exception)
{
//BlueEventLog.logEvent (e.Message, BlueEventLevel.Warning);
}
}
}
}
}
@@ -0,0 +1,384 @@
// using System;
// using System.Reflection;
// using System.Reflection.Emit;
// using System.Collections;
// using System.ComponentModel;
// namespace BCG
// {
// public class GridBindableBlueTable : IBindingList, IBlueRowCollection, ITypedList
// {
// private class MyComparer : IComparer
// {
// private string myColName;
// public MyComparer (string colName)
// {
// myColName = colName;
// }
// #region IComparer Members
// public int Compare (object x, object y)
// {
// IBlueRow r1 = x as IBlueRow;
// IBlueRow r2 = y as IBlueRow;
// try
// {
// return
// r1[myColName] == DBNull.Value ?
// r2[myColName] == DBNull.Value ?
// 0
// :
// -1
// :
// r2[myColName] == DBNull.Value ?
// 1
// :
// (r1[myColName] as IComparable).CompareTo (r2[myColName]);
// }
// catch (Exception e)
// {
// Console.WriteLine (e);
// return 0;
// }
// }
// #endregion
// }
// private PropertyDescriptor mySortPropertyDescriptor;
// private IBlueTableSchema mySchema;
// private PropertyDescriptorCollection myPDColl;
// protected ArrayList myList = new ArrayList ();
// protected ListSortDirection mySortDirection;
// protected IComparer myComparer;
// public event System.ComponentModel.ListChangedEventHandler ListChanged;
// public GridBindableBlueTable (IBlueTableSchema schema)
// : this (schema, new ArrayList())
// {}
// public GridBindableBlueTable (IBlueTableSchema schema, ICollection data)
// {
// myComparer = new MyComparer ("id");
// mySchema = schema;
// myInitPropertyDescriptors ();
// foreach (object o in data)
// Add (o);
// }
// private void myInitPropertyDescriptors ()
// {
// ArrayList temp = new ArrayList ();
// foreach (ColumnInfo ci in mySchema.columnInfo)
// temp.Add (new BlueDataPropertyDescriptor (ci));
// myPDColl = new PropertyDescriptorCollection (
// temp.ToArray (typeof(PropertyDescriptor)) as PropertyDescriptor[]);
// }
// #region IList Members
// public bool IsReadOnly {get {return false;}}
// public virtual object this [int index]
// {
// get {return myList[index];}
// set {throw new NotSupportedException ("Illegal operation for a Sorted list");}
// }
// public virtual void RemoveAt(int index)
// {
// myList.RemoveAt (index);
// if (ListChanged != null)
// ListChanged (this, new ListChangedEventArgs (ListChangedType.ItemDeleted, index));
// }
// public virtual void Insert (int index, object value)
// {
// throw new NotSupportedException ("Illegal operation for a Sorted list");
// }
// public virtual void Remove (object value)
// {
// RemoveAt (myList.IndexOf (value));
// }
// public virtual bool Contains (object value)
// {
// return myList.Contains (value);
// }
// public virtual void Clear()
// {
// myList.Clear ();
// if (ListChanged != null)
// ListChanged (this, new ListChangedEventArgs (ListChangedType.Reset, 0));
// }
// public virtual int IndexOf (object value)
// {
// return myList.IndexOf (value);
// }
// /// <summary>
// /// Adds the value to the right index in the sorted list
// /// </summary>
// /// <param name="value"></param>
// /// <returns></returns>
// public virtual int Add (object value)
// {
// int insertionIndex = -1;
// if (Count > 0)
// {
// for (int i = 0; i < myList.Count; i++)
// {
// if ((insertionIndex = myGetInsertionIndex (value, i)) > -1)
// {
// myList.Insert (insertionIndex, value);
// break;
// }
// insertionIndex = -1;
// }
// }
// else
// {
// insertionIndex = 0;
// myList.Add (value);
// }
// if (ListChanged != null)
// ListChanged (this, new ListChangedEventArgs (ListChangedType.ItemAdded, insertionIndex));
// return insertionIndex;
// }
// private int myGetInsertionIndex (object obj, int listIndex)
// {
// if (mySortPropertyDescriptor == null)
// return myList.Count;
// if (mySortDirection.Equals (ListSortDirection.Ascending))
// {
// if (myComparer.Compare (obj, myList[listIndex]) <= 0)
// return listIndex;
// else if (listIndex == myList.Count-1)
// return myList.Count;
// }
// else
// {
// if (myComparer.Compare (obj, myList[listIndex]) >= 0)
// return listIndex;
// else if (listIndex == myList.Count-1)
// return myList.Count;
// }
// return -1;
// }
// public bool IsFixedSize {get {return false;}}
// #endregion
// #region ICollection Members
// public bool IsSynchronized {get {return false;}}
// public int Count {get {return myList.Count;}}
// public void CopyTo(Array array, int index) {myList.CopyTo (array, index);}
// public object SyncRoot {get {return null;}}
// #endregion
// #region IEnumerable Members
// public IEnumerator GetEnumerator() {return myList.GetEnumerator ();}
// #endregion
// #region IBindingList Members
// public void AddIndex(PropertyDescriptor property)
// {
// throw new NotSupportedException ("Adding index is not supported");
// }
// public bool AllowNew
// {
// get {return false;}
// }
// public void ApplySort (PropertyDescriptor property, System.ComponentModel.ListSortDirection direction)
// {
// if (mySortPropertyDescriptor != null
// && mySortPropertyDescriptor.Name == property.Name)
// {
// if (mySortDirection == direction)
// return;
// else
// {
// mySortDirection = direction;
// myList.Reverse ();
// return;
// }
// }
// foreach (PropertyDescriptor pd in myPDColl)
// if (pd.Name == property.Name)
// mySortPropertyDescriptor = pd;
// myComparer = new MyComparer (property.Name);
// mySortDirection = direction;
// myList.Sort (myComparer);
// }
// public PropertyDescriptor SortProperty
// {
// get {return mySortPropertyDescriptor;}
// }
// public int Find (PropertyDescriptor property, object key)
// {
// throw new NotSupportedException ("Find not supported");
// }
// public bool SupportsSorting
// {
// get {return true;}
// }
// public bool IsSorted {get {return mySortPropertyDescriptor != null;}}
// public bool AllowRemove {get {return false;}}
// public bool SupportsSearching {get {return false;}}
// public System.ComponentModel.ListSortDirection SortDirection {get {return mySortDirection;}}
// public bool SupportsChangeNotification {get {return true;}}
// public void RemoveSort()
// {
// mySortPropertyDescriptor = null;
// }
// public object AddNew()
// {
// throw new NotSupportedException ("Find not supported");
// }
// public bool AllowEdit
// {
// get {return false;}
// }
// public void RemoveIndex(PropertyDescriptor property)
// {
// throw new NotSupportedException ("Find not supported");
// }
// #endregion
// #region IBlueRowCollection Members
// IBlueRow BCG.IBlueRowCollection.this[int id]
// {
// get
// {
// foreach (IBlueRow r in (this as IList))
// {
// if (r["id"].Equals(id))
// return r;
// }
// return null;
// }
// }
// public bool containsID (int id)
// {
// return (this as IBlueRowCollection) [id] != null;
// }
// public IBlueTableSchema schema {get {return mySchema;}}
// #endregion
// #region Dynamic Type Creation
// private IBlueRow myCreateDynamicRow (IBlueRow r)
// {
// //return Activator.CreateInstance (myRowType, new object[] {r}) as IBlueRow;
// return r;
// }
// private Type myGenerateType ()
// {
// ILGenerator ilg;
// AssemblyName an = new AssemblyName ();
// an.Name = "DynamicRowAssembly";
// AssemblyBuilder aBuilder =
// AppDomain.CurrentDomain.DefineDynamicAssembly (an, AssemblyBuilderAccess.Run);
// ModuleBuilder mBuilder =
// aBuilder.DefineDynamicModule ("DefaultModule");
// TypeBuilder tBuilder =
// mBuilder.DefineType ("DynamicBlueRow", TypeAttributes.Public, typeof(BlueRow));
// ConstructorBuilder cb =
// tBuilder.DefineConstructor (
// MethodAttributes.Public,
// CallingConventions.Standard,
// new Type[] {typeof(IBlueRow)}
// );
// ilg = cb.GetILGenerator ();
// ilg.Emit (OpCodes.Ldarg_0);
// ilg.Emit (OpCodes.Ldarg_1);
// ilg.Emit (OpCodes.Call,
// typeof(BlueRow).GetConstructor (new Type[] {typeof(IBlueRow)}));
// ilg.Emit (OpCodes.Ret);
// foreach (string col in mySchema.columnNames)
// {
// myAddProperty (col, tBuilder);
// }
// Type dynType = tBuilder.CreateType ();
// return dynType;
// }
// private void myAddProperty (string colName, TypeBuilder tBuilder)
// {
// MethodBuilder methodBuilder =
// tBuilder.DefineMethod ("get_" + colName,
// MethodAttributes.Public,
// CallingConventions.Standard,
// typeof(object),
// new Type[]{}
// );
// ILGenerator ilg =
// methodBuilder.GetILGenerator ();
// ilg.Emit (OpCodes.Ldarg_0);
// ilg.Emit (OpCodes.Ldstr, colName);
// ilg.Emit (OpCodes.Call, typeof(BlueRow).GetMethod ("get_Item", new Type[]{typeof(string)}));
// ilg.Emit (OpCodes.Ret);
// PropertyBuilder propBuilder =
// tBuilder.DefineProperty (colName, System.Reflection.PropertyAttributes.None , typeof(object), new Type[]{});
// propBuilder.SetGetMethod (methodBuilder);
// }
// #endregion
// #region ITypedList Members
// public PropertyDescriptorCollection GetItemProperties(PropertyDescriptor[] listAccessors)
// {
// return myPDColl;
// }
// public string GetListName(PropertyDescriptor[] listAccessors)
// {
// return "GridBindableBlueTable";
// }
// #endregion
// }
// }
@@ -0,0 +1,113 @@
using System;
using System.Collections;
namespace BCG
{
/// <summary>
/// An N-domensional hashtable , providing add and retrieve features
/// for the time being. Does not deal with null keys right now
/// </summary>
///
[Serializable]
public class HashTableND
{
private int myDim, myCount = 0;
private IDictionary myRootDic = new Hashtable ();
public HashTableND (int dim) {myDim = dim;}
public object add (object[] keys, object val)
{
ArrayList t = new ArrayList (keys);
t.Add (val);
return add (t.ToArray());
}
/// <summary>
/// Adds a value to the hashtable. the size of the arg array should be dim+1, with the
/// the first dim entries corresponding to keys and the last to the value to be added
/// </summary>
/// <param name="os"></param>
public object add (object[] os)
{
if (os.Length != myDim+1)
throw new ApplicationException ("size of arg array not right");
else
{
myCount ++;
IDictionary d = myRootDic;
for (int i = 0 ; ; i++)
{
if (i < myDim - 1)
{
if (!d.Contains (os[i]))
d.Add (os[i], new Hashtable ());
d = d[os[i]] as IDictionary;
}
else
{
d[os[i]] = os[i+1];
return os [myDim];
}
}
}
}
/// <summary>
/// Retrieves values from the hastable, with the arg array specifying the keys.
/// </summary>
/// <param name="keys"></param>
/// <returns></returns>
public object getValue (params object[] keys)
{
IDictionary d = myRootDic;
for (int i = 0; ; i++)
if (i < keys.Length-1)
{
d = d[keys[i]] as IDictionary;
if (d == null)
return null;
}
else
return d[keys[i]];
}
public void setValue (object[] keys, object val)
{
ArrayList t = new ArrayList (keys);
t.Add (val);
setValue (t.ToArray());
}
public void setValue (object[] os)
{
if (os.Length != myDim+1)
throw new ApplicationException ("size of arg array not right");
IDictionary d = myRootDic;
for (int i = 0; i < myDim-1; i++)
d = d[os[i]] as IDictionary;
d[os[myDim-1]] = os[myDim];
}
public void remove (params object[] keys)
{
myCount --;
IDictionary d = myRootDic;
for (int i = 0; ; i++)
if (i < keys.Length-1)
d = d[keys[i]] as IDictionary;
else
{
d.Remove (keys[i]);
return;
}
}
public int count {get {return myCount;}}
public ICollection topLevelKeys { get {return myRootDic.Keys;}}
}
}
@@ -0,0 +1,41 @@
using System;
using System.Collections;
namespace BCG
{
/// <summary>
/// Represents a primitive immutable anologue of a dotNet DataRow. The user assumes no info about the row
/// but the number of columns, and the way to access data through the indexer by providing either the
/// ordinal or the name. Also note that the default iteration through the IBlueRow return DictionaryEntry
/// (ie the default iteration is the same as an IDictionary's iteration)
/// </summary>
public interface IBlueRow : IDictionary
{
/// <summary>
/// Gets the schema of the row, which is same as that of the IBlueRowCollections to which it belongs
/// </summary>
IBlueTableSchema schema{get;}
/// <summary>
/// Though the IDictionary supports an untyped indexer, having the typed indexer makes retreivals
/// faster, specially in the case of large tables, where one type checking per cell would amount to
/// a sizeable time, eg for a table with 20000 rows and a schema of 20 columns, we will save about
/// half a million type checkings
/// </summary>
object this[int index] {get;}
object this[string colName] {get;}
}
namespace Extensions
{
public static class IBlueRowExtensions
{
public static bool isDBNull(this object val)
{
return (val == null || val == DBNull.Value);
}
public static bool isNotDBNull(this object val)
{
return (val != null && val != DBNull.Value);
}
}
}
}
@@ -0,0 +1,19 @@
using System;
using System.Collections;
namespace BCG
{
/// <summary>
/// Represents a readonly, very primitive form of dotNet DataTable. All it has has an indexer for
/// getting the rows
/// </summary>
public interface IBlueRowCollection : ICollection
{
/// <summary>
/// Gets the row corresponding to the id
/// </summary>
IBlueRow this[int id]{get;}
IBlueTableSchema schema {get;}
bool containsID (int id);
}
}
@@ -0,0 +1,228 @@
using System;
using System.Diagnostics;
using System.Collections;
//using BCG.Utilities;
//using BCG.Utilities.General;
namespace BCG
{
public class IBlueRowCollectionBuilder
{
static public bool isBlueDataType(Type t)
{
bool a = t.Equals(BlueDataValidTypes.BOOL) || t.Equals(BlueDataValidTypes.BYTE)
|| t.Equals(BlueDataValidTypes.BYTEARRAY) || t.Equals(BlueDataValidTypes.CHAR)
|| t.Equals(BlueDataValidTypes.DATETIME) || t.Equals(BlueDataValidTypes.DECIMAL)
|| t.Equals(BlueDataValidTypes.DOUBLE) || t.Equals(BlueDataValidTypes.INT)
|| t.Equals(BlueDataValidTypes.ISERIALIZEDTYPE) || t.Equals(BlueDataValidTypes.LONG)
|| t.Equals(BlueDataValidTypes.SBYTE) || t.Equals(BlueDataValidTypes.SINGLE)
|| t.Equals(BlueDataValidTypes.STRING) || t.Equals(BlueDataValidTypes.UINT)
|| t.Equals(BlueDataValidTypes.ULONG);
return a;
}
static public double[] subtract(object o1, object o2)
{
Type t = o1.GetType();
if (t == typeof(DateTime))
return new double[] { ((DateTime)o1 - (DateTime)o2).TotalDays };
else if (t.IsPrimitive && t.IsValueType)
return new double[] { Convert.ToDouble(o1) - Convert.ToDouble(o2) };
else if (t.IsArray && t.GetElementType().IsValueType && t.GetElementType().IsPrimitive)
{
double[] a1 = (double[])Array.ConvertAll<object, double>((object[])o1,
delegate (object o) { return Convert.ToDouble(o); });
double[] a2;
if (o2.GetType().IsArray)
a2 = (double[])Array.ConvertAll<object, double>((object[])o2,
delegate (object o) { return Convert.ToDouble(o); });
else
{
a2 = new double[a1.Length];
for (int i = 0; i < a1.Length; i++)
a2[i] = (double)o2;
}
double[] op = new double[a1.Length];
for (int i = 0; i < a1.Length; i++)
op[i] = a1[i] - a2[i];
return op;
}
else
return new double[] { 0.0 };
}
static public double[] add(object o1, object o2)
{
Type t = o1.GetType();
if (t.IsPrimitive && t.IsValueType)
return new double[] { Convert.ToDouble(o1) + Convert.ToDouble(o2) };
else if (t.IsArray && t.GetElementType().IsValueType && t.GetElementType().IsPrimitive)
{
double[] a1 = (double[])Array.ConvertAll<object, double>((object[])o1,
delegate (object o) { return Convert.ToDouble(o); });
double[] a2;
if (o2.GetType().IsArray)
a2 = (double[])Array.ConvertAll<object, double>((object[])o2,
delegate (object o) { return Convert.ToDouble(o); });
else
{
a2 = new double[a1.Length];
for (int i = 0; i < a1.Length; i++)
a2[i] = (double)o2;
}
double[] op = new double[a1.Length];
for (int i = 0; i < a1.Length; i++)
op[i] = a1[i] + a2[i];
return op;
}
else
return new double[] { 0.0 };
}
static public double[] mult(object o1, object o2)
{
Type t = o1.GetType();
if (t.IsPrimitive && t.IsValueType)
return new double[] { Convert.ToDouble(o1) * Convert.ToDouble(o2) };
else if (t.IsArray && t.GetElementType().IsValueType && t.GetElementType().IsPrimitive)
{
double[] a1 = (double[])Array.ConvertAll<object, double>((object[])o1,
delegate (object o) { return Convert.ToDouble(o); });
double[] a2;
if (o2.GetType().IsArray)
a2 = (double[])Array.ConvertAll<object, double>((object[])o2,
delegate (object o) { return Convert.ToDouble(o); });
else
{
a2 = new double[a1.Length];
for (int i = 0; i < a1.Length; i++)
a2[i] = (double)o2;
}
double[] op = new double[a1.Length];
for (int i = 0; i < a1.Length; i++)
op[i] = a1[i] * a2[i];
return op;
}
else
return new double[] { 0.0 };
}
public delegate double[] BinaryDelegate(object o1, object o2);
static public IBlueRowCollection operate(IBlueRowCollection d1, IBlueRowCollection d2,
BinaryDelegate del)
{
int jj = 0;
ColumnInfo[] newinfo = new ColumnInfo[d1.schema.columnInfo.Length];
foreach (ColumnInfo info in d1.schema.columnInfo)
newinfo[jj++] = new ColumnInfo(info.colName, info.colNo, typeof(double));
IBlueTableSchema schem = new BlueTableSchema(newinfo, d1.schema.tableName + "_" +
d2.schema.tableName + "_processed");
BlueTable res = new BlueTable(schem);
for (int i = 0; i < d1.Count; i++)
{
IBlueRow r1 = BlueDataUtility.extractRow(d1, i);
IBlueRow r2 = BlueDataUtility.extractRow(d2, i);
object[] vals = new object[r1.Count];
int idx = 0;
foreach (ColumnInfo info in res.schema.columnInfo)
vals[idx++] = del(r1[info.colName], r2[info.colName]);
res.addRow(new BlueRow(schem, vals));
}
return res;
}
static public IBlueRowCollection merge(string mergedName, IBlueRowCollection[] data)
{
SortedList colinfos = new SortedList();
foreach (IBlueRowCollection col in data)
{
IBlueTableSchema s = col.schema;
foreach (ColumnInfo info in s.columnInfo)
if (info.colName != "id")
colinfos[info.colName] = info;
}
int colidx = 1;
ColumnInfo[] ciary = new ColumnInfo[colinfos.Count + 1];
ciary[0] = new ColumnInfo("id", 0, typeof(int));
foreach (ColumnInfo ci in colinfos.Values)
{
ciary[colidx] = new ColumnInfo(ci.colName, colidx, ci.colType);
colidx++;
}
BlueTableSchema newschem = new BlueTableSchema(mergedName, ciary);
BlueTable tab = new BlueTable(newschem);
int numRows = 0;
foreach (IBlueRowCollection col in data)
if (col.Count > numRows)
numRows = col.Count;
for (int i = 0; i < numRows; i++)
{
Hashtable ht = new Hashtable();
ht["id"] = i;
foreach (string name in newschem.columnNames)
{
if (name != "id")
{
ht[name] = DBNull.Value;
foreach (IBlueRowCollection col in data)
{
IBlueRow r = BlueDataUtility.extractRow(col, i);
if (r != null && r.Contains(name))
ht[name] = r[name];
}
}
}
BlueRow br = new BlueRow(newschem, ht);
tab.addRow(br);
}
return tab;
}
static public IBlueRowCollection build(string tblname, object data)
{
return build(tblname, "value", data);
}
static public IBlueRowCollection build(string tblname, string dataname, object data)
{
BlueTable result = null;
if (data == null || tblname == null || tblname == string.Empty)
return null;
if (data is IBlueRowCollection)
{
IBlueRowCollection rows = data as IBlueRowCollection;
result = new BlueTable(new BlueTableSchema(rows.schema.columnInfo, tblname));
result.addRows(rows);
}
else if (data is Array)
{
Array da = (Array)data;
Type et = da.GetType().GetElementType();
if (!isBlueDataType(et))
et = typeof(string);
ColumnInfo ci = new ColumnInfo(dataname, 1, et);
ColumnInfo cii = new ColumnInfo("id", 0, typeof(int));
BlueTableSchema schem = new BlueTableSchema(tblname, cii, ci);
result = new BlueTable(schem);
int i = 0;
foreach (object dat in da)
{
object obj = (isBlueDataType(dat.GetType()) ? dat : dat.ToString());
result.addRow(new BlueRow(schem, i++, obj));
}
}
else if (isBlueDataType(data.GetType()))
{
ColumnInfo cii = new ColumnInfo("id", 0, typeof(int));
Type dt = data.GetType();
ColumnInfo ci = new ColumnInfo(dataname, 1, dt);
BlueTableSchema schem = new BlueTableSchema(tblname, cii, ci);
result = new BlueTable(schem);
result.addRow(new BlueRow(schem, 0, data));
}
else
{
ColumnInfo cii = new ColumnInfo("id", 0, typeof(int));
ColumnInfo ci = new ColumnInfo("value", 1, typeof(string));
BlueTableSchema schem = new BlueTableSchema(tblname, cii, ci);
result = new BlueTable(schem);
result.addRow(new BlueRow(schem, 0, data.ToString()));
}
return result;
}
}
}
@@ -0,0 +1,252 @@
using System;
using System.Collections;
//using BCG.Utilities;
namespace BCG
{
/// <summary>
/// Provides a multiple indexing mechanism, required to increase the peformance of the streams
/// </summary>
public class BlueTableIndexException : Exception
{
public BlueTableIndexException(string msg) : base(msg) { }
}
public interface IBlueTableIndex
{
void addRow(IBlueRow row);
void deleteRowOnPrimaryKey(object pkValue);
void deleteRowOnPrimaryKey(IBlueRow row);
void deleteRowOnBothKeys(object pkValue, object skValue);
void deleteRowOnBothKeys(IBlueRow row);
ArrayList searchRowOnPrimaryKey(object pkValue);
ArrayList searchRowOnPrimaryKey(IBlueRow row);
ArrayList searchRowOnBothKeys(object pkValue, object skValue);
ArrayList searchRowOnBothKeys(IBlueRow row);
}
/*
public class BlueTableDoubleIndex : IBlueTableIndex
{
private IBlueRowCollection myTable;
private string myPrimaryIndex;
private string mySecondaryIndex;
private Hashtable myHt = new Hashtable ();
/// <summary>
/// Builds an index with 2 columns of the table acting as primary index and secondary index
/// respectively
/// </summary>
/// <param name="table">
/// table to index
/// </param>
/// <param name="primaryIndex">
/// primary key to index upon
/// </param>
/// <param name="secondaryIndex">
/// column serving as the secondary key
/// </param>
public BlueTableDoubleIndex (IBlueRowCollection table, string primaryIndex, string secondaryIndex)
{
myTable = table;
myPrimaryIndex = primaryIndex;
mySecondaryIndex = secondaryIndex;
myInit ();
}
private void myInit()
{
foreach (IBlueRow row in myTable)
addRow (row);
}
/// <summary>
/// To add a row to the index
/// </summary>
/// <param name="row">
/// row to be added
/// </param>
public void addRow (IBlueRow row)
{
if (myHt[row[myPrimaryIndex]] != null)
((Hashtable)(myHt[row[myPrimaryIndex]]))[row[mySecondaryIndex]] = row;
else
{
TimeManager.start ();
Hashtable ht = new Hashtable();
TimeManager.stop ();
ht[row[mySecondaryIndex]] = row;
myHt[row[myPrimaryIndex]] = ht;
}
}
/// <summary>
/// If index or the primary key has already been deleted appropriate exceptions are thrown
/// </summary>
/// <param name="row"></param>
public void deleteRow (IBlueRow row)
{
if (myHt[row[myPrimaryIndex]] == null)
{
throw new BlueTableIndexException ("Theres no index (primary hash table key) for this index value");
}
((Hashtable)(myHt[row[myPrimaryIndex]])).Remove(row[mySecondaryIndex]);
if (((Hashtable)(myHt[row[myPrimaryIndex]])).Keys.Count == 0)
myHt.Remove(row[myPrimaryIndex]);
}
/// <summary>
/// Returns an array of DataRow that match the primary key being passed as an argument
/// </summary>
/// <param name="o">
/// Primary Key of the table
/// </param>
/// <returns>
/// Array of rows matching the criteria
/// </returns>
public IBlueRow[] findValueFromPrimaryKey (object o)
{
object oht = myHt[o];
if (oht == null)
return new IBlueRow[0];
Hashtable ht = (Hashtable)oht;
IBlueRow[] rows = new IBlueRow[ht.Keys.Count];
ht.Values.CopyTo(rows, 0);
return rows;
}
/// <summary>
/// returns as array of the primary keys
/// </summary>
/// <returns>
/// object[] of primary keys
/// </returns>
public Array getPrimaryKeys()
{
object[] retval = new object [myHt.Keys.Count];
myHt.Keys.CopyTo (retval, 0);
return retval;
}
}
*/
/// <summary>
/// Provides a 1-Level indexing structure.
/// </summary>
public class BlueTableSingleIndex : IBlueTableIndex
{
string myPrimaryKey;
string mySecondaryKey;
Hashtable myHt = new Hashtable();
public BlueTableSingleIndex(string primaryKey, string secondaryKey)
{
myPrimaryKey = primaryKey;
mySecondaryKey = secondaryKey;
}
/// <summary>
/// The primary key is the indexing key, and the secondary key is used as the key for
/// searching and hence deletion
/// </summary>
/// <param name="table"></param>
/// <param name="primaryKey"></param>
/// <param name="secondaryKey"></param>
public BlueTableSingleIndex(IBlueRowCollection table, string primaryKey, string secondaryKey)
: this(primaryKey, secondaryKey)
{
foreach (IBlueRow row in table)
addRow(row);
}
/// <summary>
/// Adds row to the indexing new structure
/// </summary>
/// <param name="row"></param>
public void addRow(IBlueRow row)
{
if (myHt[row[myPrimaryKey]] == null)
myHt[row[myPrimaryKey]] = new ArrayList();
((ArrayList)(myHt[row[myPrimaryKey]])).Add(row);
}
/// <summary>
/// Getts rows based on the Primary Key. Matching of the primary key is the only criteria
/// </summary>
/// <param name="row"></param>
/// <returns></returns>
public ArrayList searchRowOnPrimaryKey(IBlueRow row)
{
return searchRowOnPrimaryKey(row[myPrimaryKey]);
}
public ArrayList searchRowOnPrimaryKey(object pkValue)
{
if (myHt[pkValue] == null)
return new ArrayList();
return (ArrayList)myHt[pkValue];
}
/// <summary>
/// Gets rows based on the primary AND the secondary key
/// </summary>
/// <param name="row"></param>
/// <returns></returns>
public ArrayList searchRowOnBothKeys(IBlueRow row)
{
return searchRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
}
public ArrayList searchRowOnBothKeys(object pkValue, object skValue)
{
if (myHt[pkValue] == null)
return new ArrayList();
ArrayList matchingList = (ArrayList)myHt[pkValue];
ArrayList retVal = new ArrayList();
foreach (IBlueRow r in matchingList)
if (r[mySecondaryKey].Equals(skValue))
retVal.Add(r);
return retVal;
}
/// <summary>
/// Deletes rows based on the primary key only
/// </summary>
/// <param name="row"></param>
public void deleteRowOnPrimaryKey(IBlueRow row)
{
deleteRowOnPrimaryKey(row[myPrimaryKey]);
}
public void deleteRowOnPrimaryKey(object pkValue)
{
if (myHt[pkValue] == null)
throw new BlueTableIndexException("Key to be deleted was not found in the index");
myHt.Remove(pkValue);
}
public void deleteRowOnBothKeys(IBlueRow row)
{
deleteRowOnBothKeys(row[myPrimaryKey], row[mySecondaryKey]);
}
public void deleteRowOnBothKeys(object pkValue, object skValue)
{
if (myHt[pkValue] == null)
throw new BlueTableIndexException("Key to be deleted was not found in the index");
ArrayList matchingList = (ArrayList)myHt[pkValue];
ArrayList aux = new ArrayList();
foreach (IBlueRow r in matchingList)
if (r[mySecondaryKey].Equals(skValue))
aux.Add(r);
foreach (IBlueRow r in aux)
matchingList.Remove(r);
}
}
}
@@ -0,0 +1,33 @@
using System;
using System.Collections;
namespace BCG
{
/// <summary>
/// Represents a schema for the table, which can be indexed either with the column name or
/// the index of the column
/// </summary>
public interface IBlueTableSchema : ISerializedType
{
/// <summary>
/// index w.r.t. the ordinal
/// </summary>
ColumnInfo this[int index]{get;}
/// <summary>
/// index w.r.t. column name
/// </summary>
ColumnInfo this[string colName]{get;}
/// <summary>
/// Utility method to get all the column names
/// </summary>
string[] columnNames {get;}
/// <summary>
/// Utility method to get all the info about all the columns
/// </summary>
ColumnInfo[] columnInfo{get;}
/// <summary>
/// Name of the table, whose schema it holds
/// </summary>
string tableName {get;}
}
}
@@ -0,0 +1,10 @@
using System;
namespace BCG
{
/// <summary>
/// Tables that dont need to be cloned by the ViewStream into an ImmutableBlueTable should
/// implement this interface
/// </summary>
public interface IImmutableRowCollection : IBlueRowCollection {}
}
@@ -0,0 +1,31 @@
using System;
using System.IO;
namespace BCG
{
/// <summary>
/// Each type that needs to be transported through the PubSub mechanism, should
/// implement this interface, so that PubSub knows how to serialize it and
/// how to contruct an object out of a byte array
/// </summary>
public interface ISerializedType
{
/// <summary>
/// serializes itself and writes into a BinaryWriter; note that the underlying stream's pointer
/// moves to the end of the written bytes
/// </summary>
/// <returns></returns>
void serialize (BinaryWriter bw);
/// <summary>
/// construct itself from the bytes read off the BinaryReader; note that the underlying stream's pointer
/// moves to the end of the bytes read
/// </summary>
/// <param name="br"></param>
void deserialze (BinaryReader br);
/// <summary>
/// Should return the size in bytes, when the instance is written to a stream of binary data
/// </summary>
/// <returns></returns>
int getSerializedSize ();
}
}
@@ -0,0 +1,30 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
//using BCG.Utilities.General;
/*
* An instance of this class represents one particular type of tag. The class encapsulates
* the grammer of the tag, and contains a method for extracting metadata from tags of this type,
* as well as a method for converting metatdata back into a tag
*
* */
namespace BCG
{
public interface ITagType
{
//parses a given tag to see if it matches this type. If the tag does not match this type, null is returned.
//If the tag DOES match this type, then the metadata about the tag is returned.
IDictionary<MetaDataField, string> parse(string tag);
//takes metadata about the tag and turns it back into the tag
string generateTag(IDictionary<MetaDataField, string> metaData);
}
}
@@ -0,0 +1,103 @@
using System;
using BCG;
//using BCG.Utilities;
namespace BCG
{
[Serializable]
public class ImmutableBlueTable : IImmutableRowCollection
{
public ImmutableBlueTable(IBlueRowCollection c)
{
mySchema = c.schema;
myHash = new BlueIntHash(c.Count);
myRows = new IBlueRow[c.Count];
int id_col = c.schema["id"].colNo;
int n = 0;
foreach (IBlueRow r in c)
{
myRows[n] = r;
var id = (int)r[id_col];
if (myHash.isPresent(id))
throw new Exception("Duplicate row");
myHash.add(id, n);
n++;
}
}
#region IBlueRowCollection Members}
public IBlueRow this[int id]
{
get
{
try
{
return myRows[myHash.find(id)];
}
catch (BlueIntHash.ElementNotFound)
{
throw new BlueDataException("Row with id " + id.ToString() +
" was not found in table " +
mySchema.tableName);
}
}
}
public IBlueTableSchema schema
{
get
{
return mySchema;
}
}
public bool containsID(int id)
{ return myHash.isPresent(id); }
#endregion
#region ICollection Members
public bool IsSynchronized
{
get
{
return false;
}
}
public int Count
{
get
{ return myRows.Length; }
}
public void CopyTo(Array array, int index)
{
myRows.CopyTo(array, index);
}
public object SyncRoot
{
get
{ return null; }
}
#endregion
#region IEnumerable Members
public System.Collections.IEnumerator GetEnumerator()
{ return myRows.GetEnumerator(); }
#endregion
private BlueIntHash myHash;
private IBlueRow[] myRows;
private IBlueTableSchema mySchema;
}
}
@@ -0,0 +1,15 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BCG
{
public class InvalidMetaDataException : Exception
{
public InvalidMetaDataException(MetaDataField field) : base("Error: Invalid metadata: field " + field.ToString() + " was expected but missing.") { }
}
}
@@ -0,0 +1,139 @@
using System;
namespace BCG
{
public class KeysBasedTable2 : IBlueRowCollection
{
private int myID = 0;
private HashTableND myHash;
private BlueTable myTable;
private string[] myKeys;
public KeysBasedTable2 (IBlueTableSchema schema, params string[] keys)
{
myTable = new BlueTable (schema);
myKeys = keys;
myHash = new HashTableND (myKeys.Length);
}
#region IBlueRowCollection Members
public IBlueRow this[int id] {get {return myTable[id];}}
public bool containsID (int id) {return myTable.containsID (id);}
public IBlueTableSchema schema {get {return myTable.schema;}}
#endregion
#region ICollection Members
public bool IsSynchronized {get {return myTable.IsSynchronized;}}
public int Count {get {return myTable.Count;}}
public void CopyTo (Array array, int index) {myTable.CopyTo (array, index);}
public object SyncRoot {get {return myTable.SyncRoot;}}
#endregion
#region IEnumerable Members
public System.Collections.IEnumerator GetEnumerator() {return myTable.GetEnumerator ();}
#endregion
public void update (IBlueRow row, out IBlueRow replacedRow)
{
object[] keyVals = myGetKeyVals (row);
replacedRow = myHash.getValue (keyVals) as IBlueRow;
if (replacedRow == null)
{
myHash.add (keyVals, row);
}
else
{
myTable.deleteRow ((int)replacedRow["id"]);
myHash.setValue (keyVals, row);
}
myTable.addRow (row);
}
public void updateWithIDGeneration(IBlueRow row, out IBlueRow addedRow, out IBlueRow deletedRow)
{
object[] keyVals = myGetKeyVals(row);
deletedRow = myHash.getValue(keyVals) as IBlueRow;
if (deletedRow == null)
{
addedRow = BlueDataUtility.getNewReplacedRow(row, myID++, "id");
myHash.add(keyVals, addedRow);
myTable.addRow(addedRow);
}
else
{
addedRow = BlueDataUtility.getNewReplacedRow(row, deletedRow["id"], "id");
myHash.setValue(keyVals, addedRow);
myTable.deleteRow(deletedRow);
myTable.addRow(addedRow);
}
}
public void updateMultipleRows(IBlueRowCollection updatesRows,
out IBlueRowCollection removedRows, out IBlueRowCollection addedRows)
{
BlueTable preUpdateState = new BlueTable(myTable);
removedRows = new BlueTable(schema); addedRows = new BlueTable(schema);
foreach (IBlueRow r in updatesRows)
{
IBlueRow rRow, aRow;
updateWithIDGeneration(r, out aRow, out rRow);
if (rRow != null && preUpdateState.containsID ((int)rRow["id"]))
(removedRows as BlueTable).forceAddRow(rRow);
(addedRows as BlueTable).forceAddRow(aRow);
}
}
public void deleteRow (IBlueRow row, out IBlueRow deletedRow)
{
object[] keys = myGetKeyVals(row);
deletedRow = myHash.getValue(keys) as IBlueRow;
if (deletedRow != null) {
myTable.deleteRow(deletedRow);
myHash.remove(keys);
}
}
public void deleteRow (IBlueRow row)
{
IBlueRow t1;
deleteRow (row, out t1);
}
public void deleteMultipleRows(IBlueRowCollection rows)
{
foreach (IBlueRow r in rows)
deleteRow(r);
}
public void update (IBlueRow row)
{
IBlueRow t1;
update (row, out t1);
}
public IBlueRow getRow (object[] keys)
{
return myHash.getValue (keys) as IBlueRow;
}
public IBlueRow getRow(IBlueRow r)
{
return getRow(myGetKeyVals(r));
}
public IBlueRow this[params object[] keyVals] {get {return myHash.getValue (keyVals) as IBlueRow;}}
private object[] myGetKeyVals (IBlueRow r) {return BlueDataUtility.getSelectedValues (r, myKeys);}
}
}
@@ -0,0 +1,324 @@
using System;
using System.Data;
using System.Collections;
using System.IO;
using System.Reflection;
using System.Collections.Generic;
using System.Linq;
using BCG.Utilities.General;
namespace BCG
{
/// <summary>
/// The LookupTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
/// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
/// are the indexers (both order and name based), and the property columnInfo and columnNames, that return
/// the respective data, sorted according to the ColumnOrder.
/// </summary>
[Serializable]
public class LookupTableSchema : IBlueTableSchema
{
public class LookupTableSchemaSerializationHelper
{
public static object parse(Type t, string data)
{
if (t == BlueDataValidTypes.BOOL)
return Boolean.Parse(data);
else if (t == BlueDataValidTypes.BYTE)
return Byte.Parse(data);
else if (t == BlueDataValidTypes.CHAR)
return char.Parse(data);
else if (t == BlueDataValidTypes.DATETIME)
return DateTime.Parse(data);
else if (t == BlueDataValidTypes.DOUBLE)
return Double.Parse(data);
else if (t == BlueDataValidTypes.DECIMAL)
return Decimal.Parse(data);
else if (t == BlueDataValidTypes.INT)
return int.Parse(data);
else if (t == BlueDataValidTypes.LONG)
return long.Parse(data);
else if (t == BlueDataValidTypes.SBYTE)
return SByte.Parse(data);
else if (t == BlueDataValidTypes.STRING)
return data;
else if (t == BlueDataValidTypes.UINT)
return uint.Parse(data);
else if (t == BlueDataValidTypes.ULONG)
return ulong.Parse(data);
else if (t == BlueDataValidTypes.SINGLE)
return Single.Parse(data);
else
throw new ApplicationException("Can not be parsed");
}
public static void serialize(BinaryWriter bw, Type t, object o)
{
if (t == BlueDataValidTypes.BOOL)
bw.Write((bool)o);
else if (t == BlueDataValidTypes.BYTE)
bw.Write((byte)o);
else if (t == BlueDataValidTypes.CHAR)
bw.Write((char)o);
else if (t == BlueDataValidTypes.DATETIME)
bw.Write(((DateTime)o).Ticks);
else if (t == BlueDataValidTypes.DOUBLE)
bw.Write((double)o);
else if (t == BlueDataValidTypes.DECIMAL)
bw.Write((decimal)o);
else if (t == BlueDataValidTypes.INT)
bw.Write((int)o);
else if (t == BlueDataValidTypes.LONG)
bw.Write((long)o);
else if (t == BlueDataValidTypes.SBYTE)
bw.Write((sbyte)o);
else if (t == BlueDataValidTypes.STRING)
bw.Write(o as string);
else if (t == BlueDataValidTypes.SINGLE)
bw.Write((Single)o);
else if (t == BlueDataValidTypes.BYTEARRAY)
{
byte[] bs = o as byte[];
bw.Write(bs.Length);
bw.Write(bs);
}
else if (t == BlueDataValidTypes.UINT)
bw.Write((uint)o);
else if (t == BlueDataValidTypes.ULONG)
bw.Write((ulong)o);
else
throw new ApplicationException("Unsupported type");
}
public static object deserialize(BinaryReader br, Type t)
{
if (t == BlueDataValidTypes.BOOL)
return br.ReadBoolean();
else if (t == BlueDataValidTypes.BYTE)
return br.ReadByte();
else if (t == BlueDataValidTypes.CHAR)
return br.ReadChar();
else if (t == BlueDataValidTypes.DATETIME)
return new DateTime(br.ReadInt64());
else if (t == BlueDataValidTypes.DOUBLE)
return br.ReadDouble();
else if (t == BlueDataValidTypes.DECIMAL)
return br.ReadDecimal();
else if (t == BlueDataValidTypes.INT)
return br.ReadInt32();
else if (t == BlueDataValidTypes.LONG)
return br.ReadInt64();
else if (t == BlueDataValidTypes.SBYTE)
return br.ReadSByte();
else if (t == BlueDataValidTypes.STRING)
return br.ReadString();
else if (t == BlueDataValidTypes.BYTEARRAY)
return br.ReadBytes(br.ReadInt32());
else if (t == BlueDataValidTypes.UINT)
return br.ReadUInt32();
else if (t == BlueDataValidTypes.ULONG)
return br.ReadUInt64();
else if (t == BlueDataValidTypes.SINGLE)
return br.ReadSingle();
else
throw new ApplicationException("Unsupported type");
}
}
protected Hashtable myIndexByColName = new Hashtable();
protected ColumnInfo[] myIndexByColOrder;
private string[] myTableColNames;
private BlueRowLookupGroup[] myLookupGroups;
protected string myTableName;
public LookupTableSchema() { }
public LookupTableSchema(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
{
myConstruct(tableName, cis, groups);
}
private void myConstruct(string tableName, ColumnInfo[] cis, BlueRowLookupGroup[] groups)
{
myIndexByColOrder = new ColumnInfo[cis.Length];
myTableName = tableName;
foreach (ColumnInfo ci in cis)
{
myIndexByColOrder[ci.colNo] = ci;
myIndexByColName[ci.colName] = ci;
}
myLookupGroups = groups;
foreach (BlueRowLookupGroup group in groups)
{
group.buildEntrySchema(this);
}
}
public ColumnInfo this[string colName]
{
get
{
object retVal = myIndexByColName[colName];
return retVal != null ? retVal as ColumnInfo : null;
}
}
public BlueRowLookupGroup[] lookupGroups
{
get { return myLookupGroups; }
}
public ColumnInfo this[int colNo]
{
get { return myIndexByColOrder[colNo]; }
}
/// <summary>
/// The returned Array is sorted according to the Column No
/// </summary>
public string[] columnNames
{
get
{
if (myTableColNames == null)
{
myTableColNames = new string[myIndexByColName.Count];
foreach (DictionaryEntry de in myIndexByColName)
{
ColumnInfo temp = (ColumnInfo)de.Value;
myTableColNames[temp.colNo] = temp.colName;
}
}
return myTableColNames;
}
}
/// <summary>
/// The returned Array is sorted according to the Column No
/// </summary>
public ColumnInfo[] columnInfo
{
get { return myIndexByColOrder; }
}
public string tableName { get { return myTableName; } }
#region ISerializedType methods
public void serialize(BinaryWriter bw)
{
bw.Write(columnNames.Length);
bw.Write(tableName);
//first, write the column names
foreach (ColumnInfo ci in columnInfo)
{
bw.Write(ci.colName);
bw.Write(ci.colType.ToString());
if (ci.isConstant)
{
bw.Write(true);
LookupTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
}
else
{
bw.Write(false);
bw.Write(ci.lookupTableGroup);
}
}
//this is here for error checking
bw.Write(myLookupGroups.Length);
//now, write the size of each lookup table group
foreach (BlueRowLookupGroup group in myLookupGroups)
{
bw.Write(group.groupNumber);
bw.Write(group.tableSize);
bw.Write(group.groupColNames.Length);
foreach (string colName in group.groupColNames)
{
//write the index of this column in the table
bw.Write(this[colName].colNo);
}
}
}
public void deserialze(BinaryReader br)
{
int numCols = br.ReadInt32();
//these are the final output columns that will be created
ColumnInfo[] columnInfos = new ColumnInfo[numCols];
Dictionary<int, List<string>> colLookupGroupMap = new Dictionary<int, List<string>>();
string tableName = br.ReadString();
for (int colNo = 0; colNo < numCols; ++colNo)
{
string colName = br.ReadString();
Type colType = TypeInstanceLoader.loadType(br.ReadString(), true);
bool colIsConst = br.ReadBoolean();
if (colIsConst)
{
columnInfos[colNo] = new ColumnInfo(colName, colNo, LookupTableSchemaSerializationHelper.deserialize(br, colType));
}
else
{
short thisColGroup = br.ReadInt16();
columnInfos[colNo] = new ColumnInfo(colName, colNo, colType, thisColGroup);
}
}
int numGroups = br.ReadInt32();
BlueRowLookupGroup[] theGroups = new BlueRowLookupGroup[numGroups];
for (int group = 0; group < numGroups; ++group)
{
short groupNo = br.ReadInt16();
short groupSizeInBits = br.ReadInt16();
int numColsInGroup = br.ReadInt32();
int[] groupCols = new int[numColsInGroup];
string[] groupColNames = new string[numColsInGroup];
for (int col = 0; col < numColsInGroup; ++col)
{
groupCols[col] = br.ReadInt32();
groupColNames[col] = columnInfos[groupCols[col]].colName;
}
//create the group
theGroups[groupNo] = new BlueRowLookupGroup(groupNo, groupSizeInBits, groupColNames);
}
myConstruct(tableName, columnInfos, theGroups);
}
public int getSerializedSize()
{
throw new NotImplementedException("Feature for provided constant values to" +
"non-primitive types in schema, not provided as of now");
}
#endregion
}
}
@@ -0,0 +1,22 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BCG
{
/*
These fields are first going to be used by BlueRay to help it determine what differnet tags mean, but eventually
* the goal is that they will be used for all services in this system.
*
* not all metadata fields are present in all metadata; it depends upon the class. For example, a bluepages service
* probably won't have a 'symbol' or 'exchange' field populated, but it'll probably have the 'type' of service populated
*/
public enum MetaDataField { Protocol, //is this a bluepages service, a pubsub tag, a blueray tag, a 29 west thing, or what
Type, //if it's a multicast, is it option, fireprice, whatever. If it's bluepages, is it persistence, pubsub, whatever
Symbol,
Exchange,
Platform,
MarketDataType, // BAM (bid, ask, mid), last, etc
RealName }; //does this thing have a better name we could call it
}
@@ -0,0 +1,141 @@
using System;
using System.Reflection;
using System.Collections;
using System.Text;
using System.Text.RegularExpressions;
namespace BCG.Utilities.General
{
public class CSVParser
{
enum state
{
waitingForToken,
inQuote,
waitingForSeparator,
possibleEndOfQuote,
inToken,
}
class CSVParseException : Exception
{
public CSVParseException (string msg) : base (msg)
{}
}
public static string [] parse (string csv)
{
ArrayList tokens = new ArrayList();
StringBuilder currentToken = null;
state s = state.waitingForToken;
int n = 0;
foreach (char c in csv)
{
n++;
switch (s)
{
case state.waitingForToken:
if (c == ',')
tokens.Add("");
else if (c == '"')
{
s = state.inQuote;
currentToken = new StringBuilder();
}
else if (Char.IsWhiteSpace(c))
continue;
else
{
s = state.inToken;
currentToken = new StringBuilder();
currentToken.Append(c);
}
break;
case state.inQuote:
if (c == '"')
{
s = state.possibleEndOfQuote;
}
else
{
currentToken.Append(c);
}
break;
case state.possibleEndOfQuote:
if (c == '"')
{
s = state.inQuote;
currentToken.Append(c);
}
else if (Char.IsWhiteSpace(c))
{
tokens.Add(currentToken.ToString());
s = state.waitingForSeparator;
}
else if (c == ',')
{
tokens.Add(currentToken.ToString());
s = state.waitingForToken;
}
else
throw new Exception("Expected ',' or end of line at column " + n.ToString());
break;
case state.waitingForSeparator :
if (c == ',')
{
s = state.waitingForToken;
}
else if (Char.IsWhiteSpace(c))
continue;
else
throw new Exception("Expected ',' or end of line at column " + n.ToString());
break;
case state.inToken :
if (c == ',')
{
s = state.waitingForToken;
tokens.Add(currentToken.ToString());
}
else
{
currentToken.Append(c);
}
break;
}
}
if (s == state.inToken || s == state.possibleEndOfQuote)
tokens.Add(currentToken.ToString());
else if (s == state.waitingForToken)
tokens.Add("");
else if ( s != state.waitingForSeparator)
throw new Exception("Missing column or unterminated string at end of line");
string [] result = new string[tokens.Count];
tokens.CopyTo(result,0);
return result;
}
static void test()
{
for (;;)
{
try
{
foreach (string s in parse(Console.ReadLine()))
{
Console.WriteLine(s);
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
}
}
}
@@ -0,0 +1,397 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
namespace BCG
{
[Serializable]
public class CollectionHelpers
{
public delegate bool EqualDelegate(object a, object b);
public static ArrayList difference (ICollection a, ICollection b,
EqualDelegate ed)
{
ArrayList result = new ArrayList();
foreach (Object aa in a)
{
bool found = false;
foreach (Object bb in b)
{
if (ed(aa, bb))
{
found = true;
break;
}
}
if (!found)
result.Add (aa);
}
return result;
}
public static ArrayList difference (ICollection a, ICollection b)
{
ArrayList result = new ArrayList();
foreach (Object aa in a)
{
bool found = false;
foreach (Object bb in b)
{
if (aa.Equals(bb))
{
found = true;
break;
}
}
if (!found)
result.Add (aa);
}
return result;
}
public static ArrayList sorted_difference (ICollection sa, ICollection sb) {
ArrayList result = new ArrayList ();
IEnumerator ia = sa.GetEnumerator ();
IEnumerator ib = sb.GetEnumerator ();
IEnumerator iremainder = ia;
bool moreA,moreB;
for (moreA = ia.MoveNext (), moreB = ib.MoveNext (); moreA && moreB;) {
int cmp = ((IComparable)ia.Current).CompareTo (ib.Current);
if (cmp < 0) {
result.Add (ia.Current);
moreA = ia.MoveNext ();
iremainder = ia;
}
else if (cmp > 0)
moreB = ib.MoveNext ();
else {
moreA = ia.MoveNext ();
moreB = ib.MoveNext ();
}
}
for (; moreA; moreA = iremainder.MoveNext ())
result.Add (iremainder.Current);
return result;
}
public static bool typedequal<T>(ICollection<T> lhs, ICollection<T> rhs)
{
if(lhs == null || rhs == null)
{
return false;
}
T[] l = new T[lhs.Count];
T[] r = new T[rhs.Count];
lhs.CopyTo(l, 0);
rhs.CopyTo(r, 0);
return equal((IList)l, (IList)r);
}
public static bool equal(ICollection lhs, ICollection rhs)
{
if ( lhs == null || rhs == null )
{
return false;
}
Array l = new object[lhs.Count];
Array r = new object[rhs.Count];
lhs.CopyTo(l, 0);
rhs.CopyTo(r,0);
return equal(l,r);
}
public static bool equal(IList lhs, IList rhs)
{
if ( lhs == null || rhs == null )
{
return false;
}
if(lhs.Count != rhs.Count)
return false;
else
{
for(int i = 0; i < lhs.Count; i++)
{
if(!lhs[i].Equals(rhs[i]))
return false;
}
return true;
}
}
public static Array fromArrayList(ArrayList al, Type t)
{
Array result = Array.CreateInstance(t, al.Count);
al.CopyTo(result,0);
return result;
}
public static Array append (Array a1, Array a2, Type t)
{
Array result = Array.CreateInstance(t, a1.Length + a2.Length);
a1.CopyTo(result,0);
a2.CopyTo(result,a1.Length);
return result;
}
public static Array subtract (Array from, params object [] elemens)
{
return fromArrayList(
difference(from, elemens),from.GetType().
GetElementType());
}
public static bool hasDuplicates (ICollection collection)
{
Hashtable h = new Hashtable();
foreach (object o in collection)
h[o] = true;
return h.Keys.Count != collection.Count;
}
public static ICollection subtract (IDictionary d1, IDictionary d2)
{
ArrayList result = new ArrayList();
foreach (object d1k in d1.Keys)
if (!d2.Contains(d1k))
result.Add(d1k);
return result;
}
/// <summary>
/// Is c1 a subset of c2?
/// </summary>
/// <param name="c1"></param>
/// <param name="c2"></param>
/// <returns></returns>
public static bool isSubset (ICollection c1, ICollection c2)
{return difference(c1,c2).Count == 0;}
public static void orderedUniqueInsert (ArrayList coll,object item) {
int ifnd = coll.BinarySearch (item);
if (ifnd < 0)
coll.Insert (~ifnd,item);
else
coll[ifnd] = item;
}
public static void orderedRemove (ArrayList coll,object item) {
int ifnd = coll.BinarySearch (item);
if (ifnd >= 0)
coll.RemoveAt (ifnd);
}
public static bool t_equals<T>(T t1, T t2) { return (t1 != null && t2 != null && t1.Equals(t2)) || (t1 == null && t2 == null); }
public static bool dictionaries_equal<K, V>(IDictionary<K, V> new_d, IDictionary<K, V> old_d, Func<V, V, bool> comp = null)
{
IDictionary<K, V> deleted_d, added_d, changed_d;
compare_dictionaries(new_d, old_d, out deleted_d, out added_d, out changed_d, comp);
return deleted_d.Count == 0 && added_d.Count == 0 && changed_d.Count == 0;
}
public static void compare_dictionaries<K, V>(IDictionary<K, V> new_d, IDictionary<K, V> old_d,
out IDictionary<K, V> deleted_d, out IDictionary<K, V> added_d, out IDictionary<K, V> changed_d,
Func<V, V, bool> comp = null)
{
new_d = new_d ?? new Dictionary<K, V>();
old_d = old_d ?? new Dictionary<K, V>();
deleted_d = new Dictionary<K, V>();
added_d = new Dictionary<K, V>();
changed_d = new Dictionary<K, V>();
comp = comp ?? t_equals<V>;
foreach (KeyValuePair<K, V> p in new_d)
{
V old_d_val = default(V);
if (!old_d.TryGetValue(p.Key, out old_d_val))
added_d.Add(p);
else if (!comp(p.Value, old_d_val))
changed_d.Add(p);
}
foreach (KeyValuePair<K, V> p in old_d)
if (!new_d.ContainsKey(p.Key))
deleted_d.Add(p);
}
/// Compares 2 dictionaries with the same key types
/// but different data types. Takes a function to
/// compare the 2 data types
public static void compare_dictionaries<K, N, O>(IReadOnlyDictionary<K, N> new_d, IReadOnlyDictionary<K, O> old_d,
out IDictionary<K, O> deleted_d, out IDictionary<K, N> added_d, out IDictionary<K, Tuple<N,O>> changed_d,
Func<N, O, bool> comp)
{
new_d = new_d ?? new Dictionary<K, N>();
old_d = old_d ?? new Dictionary<K, O>();
deleted_d = new Dictionary<K, O>();
added_d = new Dictionary<K, N>();
changed_d = new Dictionary<K, Tuple<N,O>>();
foreach (KeyValuePair<K, N> p in new_d)
{
O old_val = default(O);
if (!old_d.TryGetValue(p.Key, out old_val))
added_d.Add(p);
else if (!comp(p.Value, old_val))
changed_d.Add(p.Key, new Tuple<N, O>(p.Value, old_val));
}
foreach (KeyValuePair<K, O> p in old_d)
if (!new_d.ContainsKey(p.Key))
deleted_d.Add(p);
}
public static void compare_dictionary_keys<K, V>(IDictionary<K, V> new_d, IDictionary<K, V> old_d,
out HashSet<K> deleted_keys, out HashSet<K> added_keys)
{
new_d = new_d ?? new Dictionary<K, V>();
old_d = old_d ?? new Dictionary<K, V>();
deleted_keys = new HashSet<K>();
added_keys = new HashSet<K>();
foreach (KeyValuePair<K, V> p in new_d)
if (!old_d.ContainsKey(p.Key))
added_keys.Add(p.Key);
foreach (KeyValuePair<K, V> p in old_d)
if (!new_d.ContainsKey(p.Key))
deleted_keys.Add(p.Key);
}
public static void compare_hashsets<T>(HashSet<T> new_d, HashSet<T> old_d,
out HashSet<T> deleted_d, out HashSet<T> added_d)
{
added_d = new HashSet<T>();
deleted_d = new HashSet<T>();
foreach (T t in new_d)
if (!old_d.Contains(t))
added_d.Add(t);
foreach (T t in old_d)
if (!new_d.Contains(t))
deleted_d.Add(t);
}
public static IEnumerable<T> append_arrays<T>(params IEnumerable<T>[] ienum_list)
{
List<T> rv = new List<T>();
foreach (IEnumerable<T> ienum in ienum_list)
rv.AddRange(ienum);
return rv;
}
public static bool hashset_equals<T>(HashSet<T> new_d, HashSet<T> old_d)
{
HashSet<T> deleted_d, added_d;
compare_hashsets(new_d, old_d, out deleted_d, out added_d);
return deleted_d.Count == 0 && added_d.Count == 0;
}
public static void find_neighbors<T>(IEnumerable<T> items, Comparison<T> comp_func, T comp, out int low_index, out int high_index)
{
low_index = high_index = -1;
for (int i = 0; i < items.Count(); i++)
{
T p = items.ElementAt (i);
if (comp_func (comp, p) <= 0)
{
high_index = i;
low_index = i - 1;
if (low_index >= 0)
{
T low_index_val = items.ElementAt (low_index);
if (comp_func(comp, low_index_val) >= 0)
break;
}
else
break;
}
else
{
low_index = i;
high_index = i + 1;
if (high_index <= items.Count() - 1)
{
T high_index_val = items.ElementAt (high_index);
if (comp_func(comp, high_index_val) <= 0)
break;
}
else
{
high_index = -1;
break;
}
}
}
}
public static void find_neighbors<T, P>(IEnumerable<T> list, P nearest_to, Func<T, P, double> diff_measure,
out T low_neighbor, out T high_neighbor)
{
low_neighbor = default(T); high_neighbor = default(T);
list = list ?? new List<T>();
double low_min_diff = double.MaxValue, high_min_diff = double.MaxValue;
foreach (T t in list)
{
double this_diff = Math.Abs(diff_measure(t, nearest_to));
if (this_diff <= 0)
{
this_diff = Math.Abs(this_diff);
if (this_diff < low_min_diff)
{
low_min_diff = this_diff;
low_neighbor = t;
}
}
else
{
if (this_diff < high_min_diff)
{
high_min_diff = this_diff;
high_neighbor = t;
}
}
}
}
public static T find_nearest<T,P>(IEnumerable<T> list, P nearest_to, Func<T, double> diff_measure)
{
list = list ?? new List<T>();
double min_diff = double.MaxValue;
T rv = default(T);
foreach (T t in list)
{
double this_diff = Math.Abs(diff_measure(t));
if (this_diff < min_diff)
{
rv = t;
min_diff = this_diff;
}
}
return rv;
}
public static bool are_maps_equal<K, T>(IDictionary<K, T> m1, IDictionary<K, T> m2, Func<T, T, bool> equality_comparer = null)
{
m1 = m1 ?? new Dictionary<K, T>();
m2 = m2 ?? new Dictionary<K, T>();
if (equality_comparer == null)
equality_comparer = (t1, t2) => { return t1.Equals(t2); };
IDictionary<K, T> deleted_d, added_d;
IDictionary<K, Tuple<T, T>> changed_d;
compare_dictionaries<K, T, T>(new Dictionary<K, T>(m1), new Dictionary<K, T>(m2),
out deleted_d, out added_d, out changed_d, equality_comparer);
return deleted_d.Count + added_d.Count + changed_d.Count == 0;
}
}
}
@@ -0,0 +1,51 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BCG
{
public class TypeHelper
{
private static HashSet<TypeCode> _numeric_type_codes = new HashSet<TypeCode>
{
TypeCode.Byte, TypeCode.Decimal, TypeCode.Double, TypeCode.Int16, TypeCode.Int32,
TypeCode.Int64, TypeCode.SByte, TypeCode.Single, TypeCode.UInt16, TypeCode.UInt32,
TypeCode.UInt64
};
public static bool is_dbnull(object o) { return o == null || o == DBNull.Value; }
public static bool is_numeric_type(object o)
{
return is_dbnull(o) ? false : _numeric_type_codes.Contains(Type.GetTypeCode(o.GetType()));
}
public static T[] parse_string_to_typed_array<T>(string text, params char[] seperators)
{
if (text == null || text.Length == 0)
return new T[] { };
else
{
List<T> rv = new List<T>();
string[] toks = text.Split(seperators);
foreach (string t in toks)
{
string fin_t = t.Trim();
if (!String.IsNullOrEmpty(fin_t))
{
try
{
T ct = (T)Convert.ChangeType(fin_t, typeof(T));
if (ct != null)
rv.Add(ct);
}
catch (Exception) { }
}
}
return rv.ToArray();
}
}
}
}
@@ -0,0 +1,48 @@
using System;
using System.Reflection;
using System.Diagnostics;
namespace BCG.Utilities.General
{
public class TypeInstanceLoader
{
private TypeInstanceLoader ()
{
}
public static object loadInstance (string typename, params object[] parameters)
{
Type t = null;
try
{
t = loadType (typename);
}
catch (ApplicationException e)
{
Debug.Assert (false, e.Message);
}
return Activator.CreateInstance (t, parameters);
}
public static Type loadType (string name)
{
return loadType(name, false);
}
public static Type loadType (string name, bool throwOnError)
{
Type t = null;
Assembly[] ass = AppDomain.CurrentDomain.GetAssemblies();
foreach (Assembly a in ass)
{
t = a.GetType(name, false, true);
if (t != null)
return t;
}
if(throwOnError)
throw new ApplicationException ("Can't find type '" + name + "' in all assemblies.");
else
return null;
}
}
}
@@ -0,0 +1,145 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Diagnostics;
namespace BCG
{
/// <summary>
/// A generalization of TimeWindowTable. Derive a class from this and implement the IsInTable() method to create a table with a rolling window.
/// Note: When the window is rolled forward it will only call IsInTable() until it finds a row that it can keep. Make sure that the sort is ascending and that IsInTable works on a sorted collection.
/// </summary>
public abstract class MovingWindowTable : IBlueRowCollection
{
protected IComparer<IBlueRow> myRowComparer;
protected IBlueTableSchema mySchema;
protected List<IBlueRow> mySortedRows = new List<IBlueRow>();
protected Lazy<Dictionary<int, IBlueRow>> myIDToRows;
#region constructors
public MovingWindowTable(IBlueTableSchema schema, IComparer<IBlueRow> comparer)
{
this.myRowComparer = comparer;
this.mySchema = schema;
this.myIDToRows = new Lazy<Dictionary<int, IBlueRow>>(InitializeIDToRows);
}
#endregion
public bool addRow(IBlueRow r)
{
if (windowContains(r))
{
int insertIndex = this.mySortedRows.BinarySearch(r, this.myRowComparer);
if (insertIndex < 0) insertIndex = ~insertIndex;
this.mySortedRows.Insert(insertIndex, r);
if (this.myIDToRows.IsValueCreated)
{
this.myIDToRows.Value[(int)r["id"]] = r;
}
return true;
}
return false;
}
public IBlueRowCollection moveWindow()
{
BlueTable removed = new BlueTable(this.mySchema);
if (this.mySortedRows.Count == 0)
return removed;
// we don't necessarily know whether the table is sorted ascending or descending since we let the user pass in a comparer
// so we'll test both ends of the table and start pulling from the end that is outside the window
int index;
if (windowContains(this.mySortedRows[0]))
{
for (index = this.mySortedRows.Count - 1; index >= 0; index--)
{
if (windowContains(this.mySortedRows[index])) break;
removed.addRow(this.mySortedRows[index]);
}
index++;
this.mySortedRows.RemoveRange(index, this.mySortedRows.Count - index);
}
else
{
for (index = 0; index < this.mySortedRows.Count; index++)
{
if (windowContains(this.mySortedRows[index])) break;
removed.addRow(this.mySortedRows[index]);
}
this.mySortedRows.RemoveRange(0, index);
}
return removed;
}
protected abstract bool windowContains(IBlueRow row);
private Dictionary<int, IBlueRow> InitializeIDToRows()
{
Dictionary<int, IBlueRow> idMap = new Dictionary<int, IBlueRow>(this.mySortedRows.Count);
foreach (var row in this.mySortedRows)
{
idMap[(int)row["id"]] = row;
}
return idMap;
}
#region IBlueRowCollection Members
public IBlueRow this[int id]
{
get { return this.myIDToRows.Value[id]; }
}
public IBlueTableSchema schema
{
get { return this.mySchema; }
}
public bool containsID(int id)
{
return this.myIDToRows.Value.ContainsKey(id);
}
#endregion
#region ICollection Members
public void CopyTo(Array array, int index)
{
((System.Collections.ICollection)this.mySortedRows).CopyTo(array, index);
}
public int Count
{
get { return this.mySortedRows.Count; }
}
public bool IsSynchronized
{
get { return false; }
}
public object SyncRoot
{
// Throw notimplemented rather than return null which will probably do bad things
get { throw new NotImplementedException(); }
}
#endregion
#region IEnumerable Members
public System.Collections.IEnumerator GetEnumerator()
{
return this.mySortedRows.GetEnumerator();
}
#endregion
}
}
@@ -0,0 +1,56 @@
using System;
using System.Data;
using System.Collections;
namespace BCG
{
[Serializable]
public class MutableBlueRow : BlueRow
{
public MutableBlueRow (IBlueTableSchema schema)
: this (schema, MutableBlueRow.myCreateArray (schema.columnNames.Length, DBNull.Value))
{}
public MutableBlueRow (IBlueTableSchema schema, params object[] values)
: base (schema, MutableBlueRow.myCloneArray (schema,values))
{}
public MutableBlueRow (string tableName, IDictionary row)
: base (tableName, row)
{}
public MutableBlueRow (IBlueTableSchema schema, IDictionary row)
: base (schema, row) { }
public MutableBlueRow (IBlueRow r)
: base (r)
{}
public void setValue (string columnName, object newValue)
{
setValue (mySchema[columnName].colNo, newValue);
}
public void setValue (int colNo, object newValue)
{
myValues[colNo] = newValue;
}
private static object[] myCloneArray (IBlueTableSchema schema,object[] vals)
{
object[] retVal = new object[schema.columnNames.Length];
for (int i =0; i < vals.Length; i++)
retVal[i] = vals[i];
return retVal;
}
private static object[] myCreateArray (int sz, object o)
{
object[] rv = new object[sz];
for (int i = 0; i < sz; i++)
rv[i] = DBNull.Value;
return rv;
}
}
}
@@ -0,0 +1,70 @@
using System;
using System.Collections;
using System.Data;
//using BCG.Utilities;
namespace BCG
{
[Serializable]
public class OrderedBlueTable : BlueTable
{
protected ArrayList myOrderedRows = new ArrayList();
protected OrderedBlueTable() { }
public OrderedBlueTable(IBlueTableSchema schema) : base(schema) { }
public OrderedBlueTable(IBlueRowCollection table) : base(table) { }
public OrderedBlueTable(IBlueRowCollection table, int count) : this(table, 0, count) { }
public OrderedBlueTable(IBlueRowCollection table, int startIndex, int count)
{
//BlueEventLog.logAssert (count >= 0 && startIndex + count <= table.Count);
mySchema = table.schema;
int index = 0;
foreach (IBlueRow r in table)
{
if (index++ >= startIndex)
addRow(r);
if (index == startIndex + count)
break;
}
}
public override void clear()
{
base.clear();
myOrderedRows.Clear();
}
public override void addRow(IBlueRow row)
{
base.addRow(row);
myOrderedRows.Add(row);
}
public override void addRow(DataRow row)
{
base.addRow(row);
myOrderedRows.Add(myIdIndex[row["id"]]);
}
public override bool deleteRow(int id)
{
if (myIdIndex[id] == null)
return false;
myOrderedRows.Remove(myIdIndex[id]);
myIdIndex.Remove(id);
return true;
}
#region ICollection members
public override IEnumerator GetEnumerator() { return myOrderedRows.GetEnumerator(); }
public override void CopyTo(Array array, int index) { myOrderedRows.CopyTo(array, index); }
#endregion
}
}
@@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BCG
{
public class OrderedInplaceCircularBlueTable
{
}
}
@@ -0,0 +1,27 @@
using System;
namespace BCG.Utilities.General
{
/// <summary>
/// Properties or public fields of a type can be mapped by this attribute, governing
/// their visiblity in the schema that can be constructed out of the object
/// </summary>
public class SchemaVisibility : Attribute
{
private bool myVisible = true;
/// <summary>
/// Tagging visbible as true, results in the property/field to
/// map to itself in the schema. This has the same result as
/// not tagging a field/property
/// </summary>
/// <param name="visible"></param>
public SchemaVisibility (bool visible)
{
myVisible = visible;
}
public bool visible{get{return myVisible;}}
}
}
@@ -0,0 +1,65 @@
using System;
using System.Collections;
namespace BCG
{
/// <summary>
/// Optimized for the scneario when id's are sequential starting from 0 (no indexing is necessary etc)
/// </summary>
public class SequentialIDBlueTable : IBlueRowCollection
{
private IBlueRow[] myRows;
private IBlueTableSchema mySchema;
/// <summary>
/// The refernce to the rows is kept. So if user wants to reuse the variable, its best
/// to pass in a copied value
/// </summary>
/// <param name="schema"></param>
/// <param name="rows"></param>
public SequentialIDBlueTable (IBlueTableSchema schema, IBlueRow[] rows)
{
myRows = rows;
mySchema = schema;
}
public SequentialIDBlueTable(IBlueTableSchema schema, System.Collections.Generic.IReadOnlyList<object[]> rows)
{
mySchema = schema;
myRows = new IBlueRow[rows.Count];
for (int i = 0; i < myRows.Length; i++)
myRows[i] = new BlueRow(mySchema, rows[i]);
}
#region IBlueRowCollection Members
public IBlueRow this[int id] { get { return myRows[id]; } }
public IBlueTableSchema schema {get {return mySchema;}}
public bool containsID (int id)
{
return id > -1 && id < myRows.Length;
}
#endregion
#region ICollection Members
public bool IsSynchronized {get {return false;}}
public int Count {get {return myRows.Length;}}
public void CopyTo (Array array, int index) {myRows.CopyTo (array, index);}
public object SyncRoot {get {return null;}}
#endregion
#region IEnumerable Members
public System.Collections.IEnumerator GetEnumerator() {return myRows.GetEnumerator ();}
#endregion
}
}
@@ -0,0 +1,400 @@
using System;
using System.Collections;
using System.Data;
//using BCG.Utilities;
using System.IO;
namespace BCG
{
/// <summary>
/// SerializedBlueRow is an analog of DataRow, an immutable one though. It constructs itself either
/// from dotNet DataRow or IBlueRow. One row can belong to any number of tables, 0 or 1 or many.
/// All what the SerializedBlueRow requires is a schema to manage its internal information
///
///
/// Data in serialized rows is stored in two places: a byte array contains all nonstring data,
/// and a string array contains the string data. The strings in this array are all just references
/// to strings stored in the internin module (STring.intern)
/// Additdionally, the ids are cached locally in the row, so that they can be accessed faster.
/// </summary>
[Serializable]
public class SerializedBlueRow : IBlueRow
{
protected SerializedBlueTableSchema mySchema;
byte[] myRowData;
string[] myRowStrings;
#region Private helper methods
/// <summary>
/// Initializes the internal data structures
/// </summary>
/// <param name="schema"></param>
private void myInit(IBlueTableSchema schema)
{
if (schema is SerializedBlueTableSchema)
mySchema = schema as SerializedBlueTableSchema;
else
mySchema = new SerializedBlueTableSchema(schema);
myRowData = null;
}
#endregion
#region Constructors
protected SerializedBlueRow() { }
public SerializedBlueRow(IBlueRow r)
{
myInit(r.schema);
Object[] thisRowObjects = new Object[r.schema.columnInfo.Length];
foreach (ColumnInfo col in r.schema.columnInfo)
thisRowObjects[col.colNo] = r[col.colNo];
myBuildFromObjects(thisRowObjects);
}
/// <summary>
/// The SerializedBlueRow initializes itself from the row passed. The row is expected
/// to have the same schema (including the col ordering) as the row being created
/// </summary>
/// <param name="table"></param>
/// <param name="row"></param>
public SerializedBlueRow(IBlueTableSchema schema, IBlueRow row)
{
myInit(schema);
}
/// <summary>
/// Constructs itself from the schema and the array of values that correspond
/// to the rows
/// </summary>
/// <param name="schema">
/// Schema of the row
/// </param>
/// <param name="values">
/// Values represent the values in the row, the order should be the same as
/// represented in the schema
/// </param>
public SerializedBlueRow(IBlueTableSchema schema, params Object[] values)
{
myInit(schema);
myBuildFromObjects(values);
}
/// <summary>
/// Initilizes itself from a dotNet DataRow. The row is expected to
/// have the same schema (including the column ordering) as the row being created
/// </summary>
/// <param name="table"></param>
/// <param name="row"></param>
public SerializedBlueRow(IBlueTableSchema schema, DataRow row)
{
myInit(schema);
Object[] thisRowObjects = new Object[row.Table.Columns.Count];
foreach (DataColumn col in row.Table.Columns)
thisRowObjects[col.Ordinal] = row[col.Ordinal];
myBuildFromObjects(thisRowObjects);
}
public SerializedBlueRow(IBlueTableSchema schema, IDictionary row)
{
myInit(schema);
Object[] thisRowObjects = new Object[row.Count];
foreach (DictionaryEntry de in row)
if (mySchema[de.Key as string] != null)
thisRowObjects[mySchema[de.Key as string].colNo] = de.Value;
myBuildFromObjects(thisRowObjects);
}
/// <summary>
/// Constructs the schema and then populates the values, both from the IDictionary.
/// The table name (the only thing that can not be derived from the dictionary) is
/// also provided
/// </summary>
/// <param name="tableName"></param>
/// <param name="row"></param>
public SerializedBlueRow(string tableName, IDictionary row)
{
if (row is IBlueRow)
myInit(((IBlueRow)row).schema);
else
{
ColumnInfo[] cis = new ColumnInfo[row.Count];
int i = 0;
foreach (DictionaryEntry de in row)
cis[i] = new ColumnInfo(de.Key as string, i++, de.Value.GetType());
myInit(new SerializedBlueTableSchema(cis, tableName));
}
Object[] thisRowObjects = new Object[row.Count];
foreach (DictionaryEntry de in row)
thisRowObjects[mySchema[de.Key as string].colNo] = de.Value;
myBuildFromObjects(thisRowObjects);
}
/// <summary>
/// Constructs the schema and then populates the values, both from the IDictionary.
/// </summary>
/// <param name="row"></param>
public SerializedBlueRow(IDictionary row) : this("", row) { }
#endregion
/// <summary>
/// Creates the serialized blue row from the list of objects. Serializes them using the schema.
/// </summary>
/// <param name="theObjects"></param>
private void myBuildFromObjects(Object[] theObjects)
{
using(BinaryWriter theBinaryWriter = new BinaryWriter(new MemoryStream()))
{
//store the id
int idCol = mySchema["id"].colNo;
theBinaryWriter.Write((int)theObjects[idCol]);
//
//serialize all the data
//
int numCols = theObjects.Length;
//first, write the null map
BitArray NullMap = new BitArray(numCols);
for (int col = 0; col < numCols; ++col)
NullMap.Set(col,(theObjects[col] == null || theObjects[col] == DBNull.Value));
byte[] NullMapBytes = new Byte[mySchema.getNullMapSize()];
NullMap.CopyTo(NullMapBytes,0);
theBinaryWriter.Write(NullMapBytes);
//then, write the non null, non string objects into the byte array
//and copy the string references into the string array
string[] thisColStrings = new string[numCols];
int numStringCols = 0;
for (int col = 0; col < numCols; ++col)
{
//don't serialize null values into the array. They'll just take up space.
//also don't serialize the id
if(NullMap.Get(col) == false && col != idCol)
{
Type thisColType = mySchema[col].colType;
if (thisColType == typeof(string))
thisColStrings[numStringCols++] = String.Intern(((string)theObjects[col]));
else
{
long numBytesBefore = theBinaryWriter.BaseStream.Length;
mySchema.serializeObject(theObjects[col], theBinaryWriter, col);
long numBytesAfter = theBinaryWriter.BaseStream.Length;
}
}
}
//now that we knmow how many string columns are in this row, we should store those references
//we do it thsi way to avoid storing null string refernces.
//it complicates string access a tiny but, it the savings is probably worth it
//becuase a lot of strings are null in tables like trade
myRowStrings = new string[numStringCols];
for (int col = 0; col < numStringCols; ++col)
{
myRowStrings[col] = thisColStrings[col];
}
//now save the bytes of the memory writer
long numBytes = theBinaryWriter.BaseStream.Length;
myRowData = new Byte[numBytes];
theBinaryWriter.Seek(0, SeekOrigin.Begin);
for (int i = 0; i < numBytes; ++i)
myRowData[i] = Convert.ToByte(theBinaryWriter.BaseStream.ReadByte());
/*/This code is for verifying that the row construction actually worked.
//it is test code, and should be commented out when actually used
ICollection thisRowValues = Values;
BlueRow thisBlueRow = new BlueRow(mySchema, theObjects);
string[] diffcolumns = BlueDataUtility.compareRows(this, thisBlueRow);
if(diffcolumns.Length != 0)
throw new BlueDataException(string.Format("Row Serialization Failed on column: {0}", diffcolumns[0]));
//*/
}
}
public string Bytes
{
get { return new System.Text.ASCIIEncoding().GetString(myRowData); }
}
#region IBlueRow members
public virtual IBlueTableSchema schema { get { return mySchema; } }
public virtual object this[string colName]
{
get
{
//see if they're askign for the id. These should be much faster than querying the hashtable
//(this is slow right now, but soon we should swithc to unsafe code and just typecase the byte poitner);
if (colName == "id")
return this[mySchema.IdColumn];
ColumnInfo ci = mySchema[colName];
if (ci != null)
return this[ci.colNo];
else
throw new Exception(colName + " is not a valid column name for " + schema.tableName);
}
}
public unsafe virtual object this[int index]
{
get
{
if (index == mySchema.IdColumn)
{
//the id is stored at the start of the row, to speed up access to it
fixed (byte* id = &myRowData[0])
{
return *((int*)id);
}
}
//the id is the first thing stored in the row. WE want to eat this up first.
if (mySchema[index].colType == typeof(string))
{
BitArray nullMap = mySchema.getNullMap(myRowData);
int indexIntoStringTable = 0;
for (int colNo = 0; colNo < mySchema.columnInfo.Length; ++colNo )
{
if (nullMap.Get(colNo))
{
//this is the string they are asking for
if (colNo == index)
return DBNull.Value; //we want to break out of all the loops and such here
//otherwise, don't increment the index into the string table
}
else
{
if (colNo == index)
return myRowStrings[indexIntoStringTable];
else if (mySchema[colNo].colType == typeof(string))
++indexIntoStringTable;
}
}
}//end if this column is a string
else
return mySchema.getValueNonString(myRowData, index);
//you should never get to this point.
throw new BlueDataException("Error: Malformed Serialized Blue Row, couldn't pull data item");
}
}
#endregion
#region ICollection members
public virtual int Count { get { return mySchema.columnInfo.Length; } }
public virtual bool IsSynchronized { get { return false; } }
public virtual object SyncRoot { get { return this; } }
public virtual void CopyTo(Array array, int index) {
Values.CopyTo(array, index);
}
#endregion
#region IDictionary members
public virtual bool IsFixedSize { get { return true; } }
virtual public bool IsReadOnly { get { return true; } }
/// <summary>
/// The key can either be the string corrsponding to the column name or
/// the order of the column
/// </summary>
public virtual object this[object key]
{
get
{
if (key is string)
return this[(string)key];
else if (key is int)
return this[(int)key];
throw new BlueDataException("Wrong type while indexing the row");
}
set { throw new NotSupportedException("The IBlueRow is readonly"); }
}
public virtual ICollection Keys { get { return mySchema.columnNames; } }
public virtual ICollection Values
{
get
{
return mySchema.getAllValues(myRowData, myRowStrings);
}
}
public virtual void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
public virtual void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
public virtual bool Contains(object key) { return mySchema[(string)key] != null; }
/// <summary>
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
/// iterate through the values, typecast this into an IEnumerable
/// </summary>
/// <returns></returns>
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
public virtual void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
#endregion
#region IComparable Members
public int CompareTo(IBlueRow other)
{
int thisID = (int)this["id"];
int otherID = (int)other["id"];
if (thisID > otherID)
return 1;
else if (thisID == otherID)
return 0;
else
return -1;
}
#endregion
}
}
@@ -0,0 +1,476 @@
using System;
using System.Data;
using System.Collections;
using System.IO;
using BCG.Utilities.General;
//using BCG.Utilities;
namespace BCG
{
/// <summary>
/// The SerializedBlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
/// either from a DataTable or from an Array of ColumnInfo. The methods through which the user can access it
/// are the indexers (both order and name based), and the property columnInfo and columnNames, that return
/// the respective data, sorted according to the ColumnOrder.
///
/// It is designed to work in concert with serialized blue rows. The serialized blue rows hold most information inside
/// memory streams, and uses this schema to unpack the values when they are needed.
/// </summary>
[Serializable]
public class SerializedBlueTableSchema : IBlueTableSchema
{
public class SerializedBlueTableSchemaSerializationHelper
{
public static object parse(Type t, string data)
{
if (t == BlueDataValidTypes.BOOL)
return Boolean.Parse(data);
else if (t == BlueDataValidTypes.BYTE)
return Byte.Parse(data);
else if (t == BlueDataValidTypes.CHAR)
return char.Parse(data);
else if (t == BlueDataValidTypes.DATETIME)
return DateTime.Parse(data);
else if (t == BlueDataValidTypes.DOUBLE)
return Double.Parse(data);
else if (t == BlueDataValidTypes.DECIMAL)
return Decimal.Parse(data);
else if (t == BlueDataValidTypes.INT)
return int.Parse(data);
else if (t == BlueDataValidTypes.LONG)
return long.Parse(data);
else if (t == BlueDataValidTypes.SBYTE)
return SByte.Parse(data);
else if (t == BlueDataValidTypes.STRING)
return data;
else if (t == BlueDataValidTypes.UINT)
return uint.Parse(data);
else if (t == BlueDataValidTypes.ULONG)
return ulong.Parse(data);
else if (t == BlueDataValidTypes.SINGLE)
return Single.Parse(data);
else
throw new ApplicationException("Can not be parsed");
}
public static void serialize(BinaryWriter bw, Type t, object o)
{
if (t == BlueDataValidTypes.BOOL)
bw.Write((bool)o);
else if (t == BlueDataValidTypes.BYTE)
bw.Write((byte)o);
else if (t == BlueDataValidTypes.CHAR)
bw.Write((char)o);
else if (t == BlueDataValidTypes.DATETIME)
bw.Write(((DateTime)o).Ticks);
else if (t == BlueDataValidTypes.DOUBLE)
bw.Write((double)o);
else if (t == BlueDataValidTypes.DECIMAL)
bw.Write((decimal)o);
else if (t == BlueDataValidTypes.INT)
bw.Write((int)o);
else if (t == BlueDataValidTypes.LONG)
bw.Write((long)o);
else if (t == BlueDataValidTypes.SBYTE)
bw.Write((sbyte)o);
else if (t == BlueDataValidTypes.STRING)
bw.Write(o as string);
else if (t == BlueDataValidTypes.SINGLE)
bw.Write((Single)o);
else if (t == BlueDataValidTypes.BYTEARRAY)
{
byte[] bs = o as byte[];
bw.Write(bs.Length);
bw.Write(bs);
}
else if (t == BlueDataValidTypes.UINT)
bw.Write((uint)o);
else if (t == BlueDataValidTypes.ULONG)
bw.Write((ulong)o);
else
throw new ApplicationException("Unsupported type");
}
public static object deserialize(BinaryReader br, Type t)
{
if (t == BlueDataValidTypes.BOOL)
return br.ReadBoolean();
else if (t == BlueDataValidTypes.BYTE)
return br.ReadByte();
else if (t == BlueDataValidTypes.CHAR)
return br.ReadChar();
else if (t == BlueDataValidTypes.DATETIME)
return new DateTime(br.ReadInt64());
else if (t == BlueDataValidTypes.DOUBLE)
return br.ReadDouble();
else if (t == BlueDataValidTypes.DECIMAL)
return br.ReadDecimal();
else if (t == BlueDataValidTypes.INT)
return br.ReadInt32();
else if (t == BlueDataValidTypes.LONG)
return br.ReadInt64();
else if (t == BlueDataValidTypes.SBYTE)
return br.ReadSByte();
else if (t == BlueDataValidTypes.UINT)
return br.ReadUInt32();
else if (t == BlueDataValidTypes.ULONG)
return br.ReadUInt64();
else if (t == BlueDataValidTypes.SINGLE)
return br.ReadSingle();
else
throw new ApplicationException("Unsupported type");
}
}
protected Hashtable myIndexByColName = new Hashtable();
protected ColumnInfo[] myIndexByColOrder;
private string[] myTableColNames;
protected string myTableName;
private int[] myColumnSizesTable;
int myIdCol;
public SerializedBlueTableSchema() { }
//copies the schema from an existing schema, building the dispatch table
public SerializedBlueTableSchema(IBlueTableSchema toCopy)
{
int numCols = toCopy.columnInfo.Length;
ColumnInfo[] theColumns = new ColumnInfo[toCopy.columnInfo.Length];
foreach (ColumnInfo toCopyCI in toCopy.columnInfo)
{
theColumns[toCopyCI.colNo] = toCopyCI;
}
myConstruct(theColumns, toCopy.tableName);
}
public SerializedBlueTableSchema(DataTable table)
{
ColumnInfo[] theseCols = new ColumnInfo[table.Columns.Count];
myTableName = table.TableName;
foreach (DataColumn col in table.Columns)
{
theseCols[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
}
myConstruct(theseCols, myTableName);
}
public SerializedBlueTableSchema(ColumnInfo[] cis, string tableName)
{
myConstruct(cis, tableName);
}
public void serializeObject(object data, BinaryWriter bw, int colNo)
{
mySerializeDispatchTable[colNo](data, bw);
}
private void myConstruct(ColumnInfo[] cis, string tableName)
{
myIndexByColOrder = new ColumnInfo[cis.Length];
myTableName = tableName;
foreach (ColumnInfo ci in cis)
{
myIndexByColOrder[ci.colNo] = ci;
myIndexByColName[ci.colName] = ci;
}
myBuildDispatchTable();
}
public SerializedBlueTableSchema(string tableName, params ColumnInfo[] cis) : this(cis, tableName) { }
public ColumnInfo this[string colName]
{
get
{
object retVal = myIndexByColName[colName];
return retVal != null ? retVal as ColumnInfo : null;
}
}
public ColumnInfo this[int colNo]
{
get { return myIndexByColOrder[colNo]; }
}
/// <summary>
/// The returned Array is sorted according to the Column No
/// </summary>
public string[] columnNames
{
get
{
if (myTableColNames == null)
{
myTableColNames = new string[myIndexByColName.Count];
foreach (DictionaryEntry de in myIndexByColName)
{
ColumnInfo temp = (ColumnInfo)de.Value;
myTableColNames[temp.colNo] = temp.colName;
}
}
return myTableColNames;
}
}
/// <summary>
/// The returned Array is sorted according to the Column No
/// </summary>
public ColumnInfo[] columnInfo
{
get { return myIndexByColOrder; }
}
public string tableName { get { return myTableName; } }
public int IdColumn
{
get { return myIdCol; }
}
#region ISerializedType methods
public void serialize(BinaryWriter bw)
{
bw.Write(columnNames.Length);
bw.Write(tableName);
foreach (ColumnInfo ci in columnInfo)
{
bw.Write(ci.colName);
bw.Write(ci.colType.ToString());
if (ci.isConstant)
{
bw.Write(true);
SerializedBlueTableSchemaSerializationHelper.serialize(bw, ci.colType, ci.colValue);
}
else
{
bw.Write(false);
}
}
}
public void deserialze(BinaryReader br)
{
ColumnInfo[] cis = new ColumnInfo[br.ReadInt32()];
string name = br.ReadString();
for (int i = 0; i < cis.Length; i++)
{
string colName = br.ReadString();
Type t = TypeInstanceLoader.loadType(br.ReadString(), true);
bool isConstant = br.ReadBoolean();
if (isConstant)
{
cis[i] = new ColumnInfo(colName, i, t, SerializedBlueTableSchemaSerializationHelper.deserialize(br, t));
}
else
{
cis[i] = new ColumnInfo(colName, i, t);
}
}
myConstruct(cis, name);
}
public int getSerializedSize()
{
throw new NotImplementedException("Feature for provided constant values to" +
"non-primitive types in schema, not provided as of now");
}
#endregion
//builds a dispatch table used to pull values out of the memory streams in the rows.
private void myBuildDispatchTable()
{
int numCols = myIndexByColOrder.Length;
mySerializeDispatchTable = new SerializeDelegate[numCols];
myPullFromByteBufferDispatchTable = new PullFromByteBufferDelegate[numCols];
myColumnSizesTable = new int[numCols];
ColumnInfo idColumn = (ColumnInfo)myIndexByColName["id"];
myIdCol = idColumn.colNo;
for (int col = 0; col < numCols; ++col)
{
Type thisColType = myIndexByColOrder[col].colType;
//we don't store serializers for the id column, or string columns.
if (col == myIdCol || thisColType == typeof(string))
{
mySerializeDispatchTable[col] = null;
myColumnSizesTable[col] = 0;
myPullFromByteBufferDispatchTable[col] = null;
}
else
{
mySerializeDispatchTable[col] = myGetDispatchMethodSerialize(thisColType);
myPullFromByteBufferDispatchTable[col] = myGetDispatchMethodPullFromBuffer(thisColType);
myColumnSizesTable[col] = BlueDataValidTypes.getValueTypeSize(thisColType);
}
}
}
public int getNullMapSize()
{
int numCols = myIndexByColOrder.Length;
return Convert.ToInt32(Math.Ceiling((Convert.ToDouble(numCols) / 8.0)));
}
//this function reads the null array from inside the byte array.
//it has to copy the bytes over.
public BitArray getNullMap(byte[] theBytes)
{
int nullMapSize = getNullMapSize();
//the null map starts after the id, so that we have instant access to the id.
int byteCursor = BlueDataValidTypes.getValueTypeSize(typeof(Int32));
byte[] nullMapArray = new byte[nullMapSize];
for (int i = 0; i < nullMapSize; ++i)
nullMapArray[i] = theBytes[byteCursor++];
return new BitArray(nullMapArray);
}
//gets the value either from the bytes stream but only if the value is not a string.
//doesn't work if you're getting the id.
public unsafe object getValueNonString(byte[] theBytes, int colNo)
{
if (colNo == myIdCol)
throw new BlueDataException("Error: do not use getValueNonString to try to fetch the id of a row");
BitArray nullMap = getNullMap(theBytes);
if (nullMap.Get(colNo))
return DBNull.Value;
//the rest of the row starts after the id and the null map
int thisOffset = BlueDataValidTypes.getValueTypeSize(typeof(Int32)) + getNullMapSize();
for (int currentColumn = 0; currentColumn < colNo; ++currentColumn)
{
//pull the data from the binary reader, but only if this type is not null or a string
//if the type wer'e looking for is a string, it would have been returned already by the accesor
//in the bluerow.
//if it's the id, it also would have been returned already
if (!nullMap.Get(currentColumn)
&& myIndexByColOrder[currentColumn].colType != typeof(string)
&& currentColumn != myIdCol)
thisOffset += myColumnSizesTable[currentColumn];
}
fixed (byte* thisBytePtr = &theBytes[thisOffset])
{
return myPullFromByteBufferDispatchTable[colNo](thisBytePtr);
}
}
public unsafe object[] getAllValues(byte[] theBytes, string[] stringArray)
{
int numCols = myColumnSizesTable.Length;
object[] theObjs = new Object[numCols];
BitArray nullMap = getNullMap(theBytes);
int thisOffset = BlueDataValidTypes.getValueTypeSize(typeof(Int32)) + getNullMapSize();
int thisStringNo = 0;
for (int currentColumn = 0; currentColumn < numCols; ++currentColumn)
{
//pull the data from the binary reader, but only if this type is not null or a string
//if the type wer'e looking for is a string, it would have been returned already by the accesor
//in the bluerow.
//if it's the id, it also would have been returned already
if (nullMap.Get(currentColumn))
theObjs[currentColumn] = DBNull.Value;
else if (myIndexByColOrder[currentColumn].colType == typeof(string))
theObjs[currentColumn] = stringArray[thisStringNo++];
else if (currentColumn != myIdCol)
{
fixed (byte* thisGuy = &theBytes[thisOffset])
{
theObjs[currentColumn] = myPullFromByteBufferDispatchTable[currentColumn](thisGuy);
}
thisOffset += myColumnSizesTable[currentColumn];
}
}
return theObjs;
}
private delegate void SerializeDelegate(object data, BinaryWriter bw);
private SerializeDelegate[] mySerializeDispatchTable;
private void mySerializeInt32(object data, BinaryWriter bw) { bw.Write((Int32)data); }
private void mySerializeDouble(object data, BinaryWriter bw) { bw.Write((double)data); }
private void mySerializeChar(object data, BinaryWriter bw) { bw.Write((char)data); }
private void mySerializeLong(object data, BinaryWriter bw) { bw.Write((long)data); }
private void mySerializeSByte(object data, BinaryWriter bw) { bw.Write((sbyte)data); }
private void mySerializeBool(object data, BinaryWriter bw) { bw.Write((bool)data); }
private void mySerializeByte(object data, BinaryWriter bw) { bw.Write((byte)data); }
private void mySerializeDecimal(object data, BinaryWriter bw) { bw.Write((decimal)data); }
private void mySerializeUInt(object data, BinaryWriter bw) { bw.Write((uint)data); }
private void mySerializeString(object data, BinaryWriter bw) { bw.Write((string)data); }
private void mySerializeULong(object data, BinaryWriter bw) { bw.Write((ulong)data); }
private void mySerializeDateTime(object data, BinaryWriter bw) { bw.Write(((DateTime)data).ToOADate()); }
private void mySerializeSingle(object data, BinaryWriter bw) { bw.Write((Single)data); }
private SerializeDelegate myGetDispatchMethodSerialize(Type t)
{
if (t.Equals(BlueDataValidTypes.INT)) return new SerializeDelegate(mySerializeInt32);
else if (t.Equals(BlueDataValidTypes.DOUBLE)) return new SerializeDelegate(mySerializeDouble);
else if (t.Equals(BlueDataValidTypes.CHAR)) return new SerializeDelegate(mySerializeChar);
else if (t.Equals(BlueDataValidTypes.LONG)) return new SerializeDelegate(mySerializeLong);
else if (t.Equals(BlueDataValidTypes.SBYTE)) return new SerializeDelegate(mySerializeSByte);
else if (t.Equals(BlueDataValidTypes.BOOL)) return new SerializeDelegate(mySerializeBool);
else if (t.Equals(BlueDataValidTypes.BYTE)) return new SerializeDelegate(mySerializeByte);
else if (t.Equals(BlueDataValidTypes.DECIMAL)) return new SerializeDelegate(mySerializeDecimal);
else if (t.Equals(BlueDataValidTypes.UINT)) return new SerializeDelegate(mySerializeUInt);
else if (t.Equals(BlueDataValidTypes.ULONG)) return new SerializeDelegate(mySerializeULong);
else if (t.Equals(BlueDataValidTypes.DATETIME)) return new SerializeDelegate(mySerializeDateTime);
else if (t.Equals(BlueDataValidTypes.SINGLE)) return new SerializeDelegate(mySerializeSingle);
else if (t.Equals(BlueDataValidTypes.STRING)) return new SerializeDelegate(mySerializeString);
else throw new ApplicationException(t.FullName + " is not a supported type");
}
private unsafe delegate object PullFromByteBufferDelegate(byte* bytePtr);
private PullFromByteBufferDelegate[] myPullFromByteBufferDispatchTable;
private unsafe object myPullInt32(byte* bytePtr) { return *((int*)bytePtr); }
private unsafe object myPullDouble(byte* bytePtr) { return *((double*)bytePtr); }
private unsafe object myPullChar(byte* bytePtr) { return *((char*)bytePtr); }
private unsafe object myPullLong(byte* bytePtr) { return *((long*)bytePtr); }
private unsafe object myPullSByte(byte* bytePtr) { return *((SByte*)bytePtr); }
private unsafe object myPullBool(byte* bytePtr) { return *((bool*)bytePtr); }
private unsafe object myPullByte(byte* bytePtr) { return *(bytePtr); }
private unsafe object myPullDecimal(byte* bytePtr) { return *((decimal*)bytePtr); }
private unsafe object myPullUInt(byte* bytePtr) { return *((UInt32*)bytePtr); }
private unsafe object myPullULong(byte* bytePtr) { return *((UInt64*)bytePtr); }
private unsafe object myPullDateTime(byte* bytePtr)
{
double thisDateTime = *((double*)bytePtr);
return DateTime.FromOADate(thisDateTime);
}
private unsafe object myPullSingle(byte* bytePtr) { return *((Single*)bytePtr); ; }
private unsafe PullFromByteBufferDelegate myGetDispatchMethodPullFromBuffer(Type t)
{
if (t.Equals(BlueDataValidTypes.INT)) return new PullFromByteBufferDelegate(myPullInt32);
else if (t.Equals(BlueDataValidTypes.DOUBLE)) return new PullFromByteBufferDelegate(myPullDouble);
else if (t.Equals(BlueDataValidTypes.CHAR)) return new PullFromByteBufferDelegate(myPullChar);
else if (t.Equals(BlueDataValidTypes.LONG)) return new PullFromByteBufferDelegate(myPullLong);
else if (t.Equals(BlueDataValidTypes.SBYTE)) return new PullFromByteBufferDelegate(myPullSByte);
else if (t.Equals(BlueDataValidTypes.BOOL)) return new PullFromByteBufferDelegate(myPullBool);
else if (t.Equals(BlueDataValidTypes.BYTE)) return new PullFromByteBufferDelegate(myPullByte);
else if (t.Equals(BlueDataValidTypes.DECIMAL)) return new PullFromByteBufferDelegate(myPullDecimal);
else if (t.Equals(BlueDataValidTypes.UINT)) return new PullFromByteBufferDelegate(myPullUInt);
else if (t.Equals(BlueDataValidTypes.ULONG)) return new PullFromByteBufferDelegate(myPullULong);
else if (t.Equals(BlueDataValidTypes.DATETIME)) return new PullFromByteBufferDelegate(myPullDateTime);
else if (t.Equals(BlueDataValidTypes.SINGLE)) return new PullFromByteBufferDelegate(myPullSingle);
else throw new ApplicationException(t.FullName + " is not a supported type to pull from a buffer");
}
}
}
@@ -0,0 +1,121 @@
using System;
using System.ComponentModel;
using System.Collections;
namespace BCG
{
public class SimpleBindableCollection : IBlueRowCollection, ITypedList, IList
{
private ArrayList myRows;
private IBlueTableSchema mySchema;
private PropertyDescriptorCollection myPDColl;
public SimpleBindableCollection (IBlueTableSchema schema, ICollection rows)
{
mySchema = schema;
ArrayList temp = new ArrayList ();
foreach (ColumnInfo ci in mySchema.columnInfo)
temp.Add (new BlueDataPropertyDescriptor (ci));
myPDColl = new PropertyDescriptorCollection (temp.ToArray (typeof(PropertyDescriptor)) as PropertyDescriptor[]);
myRows = new ArrayList (rows);
}
public SimpleBindableCollection (IBlueTableSchema schema)
: this (schema, new ArrayList ())
{}
#region IBlueRowCollection Members
public IBlueRow this[int id]
{
get
{
foreach (IBlueRow r in this)
if (r["id"].Equals (id))
return r;
throw new ApplicationException ("ID + " + id.ToString () + " not found");
}
}
public bool containsID(int id)
{
try
{
return this[id] != null;
}
catch (ApplicationException)
{
return false;
}
}
public IBlueTableSchema schema {get {return mySchema;}}
#endregion
#region ICollection Members
public bool IsSynchronized {get {return false;}}
public int Count {get {return myRows.Count;}}
public void CopyTo(Array array, int index) {myRows.CopyTo (array, index);}
public object SyncRoot { get {return null;}}
#endregion
#region IEnumerable Members
public System.Collections.IEnumerator GetEnumerator()
{
return myRows.GetEnumerator ();
}
#endregion
#region ITypedList Members
public PropertyDescriptorCollection GetItemProperties(PropertyDescriptor[] listAccessors)
{
return myPDColl;
}
public string GetListName(PropertyDescriptor[] listAccessors)
{
return "SimpleBindableCollection";
}
#endregion
#region IList Members
public bool IsReadOnly {get {return true;}}
object System.Collections.IList.this[int index]
{
get {return myRows[index];}
set {throw new ApplicationException ("Collection is read only");}
}
public void RemoveAt(int index) {throw new ApplicationException ("Collection is read only");}
public void Insert(int index, object value) {throw new ApplicationException ("Collection is read only");}
public void Remove(object value) {throw new ApplicationException ("Collection is read only");}
public bool Contains(object value) {return myRows.Contains (value);}
public void Clear() {throw new ApplicationException ("Collection is read only");}
public int IndexOf(object value) {return myRows.IndexOf (value);}
public int Add(object value) {throw new ApplicationException ("Collection is read only");}
public bool IsFixedSize {get {return true;}}
#endregion
public void reset (ICollection rows) {myRows = new ArrayList(rows);}
}
}
@@ -0,0 +1,46 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BCG
{
public class SparseVectorBlueTable : IBlueRowCollection
{
private List<IBlueRow> _rows = new List<IBlueRow>(50);
public SparseVectorBlueTable(IBlueTableSchema schema) {
this.schema = schema;
}
public void reserve(int length) {
if (_rows.Capacity < length) {
_rows.Capacity += 1024;
}
while (_rows.Count <= length) {
_rows.Add(null);
}
}
public IBlueRow this[int id] {
get {
return _rows[id];
}
set {
this.reserve(id);
_rows[id] = value;
}
}
public IBlueTableSchema schema { get; private set; }
public bool containsID(int id) {
return (id >= 0 && id < _rows.Count && this[id] != null);
}
public void CopyTo(Array array, int index) { _rows.CopyTo((IBlueRow[])array, index);}
public int Count { get { return _rows.Count; } }
public bool IsSynchronized {get { return false; }}
public object SyncRoot { get { return this; } }
public System.Collections.IEnumerator GetEnumerator() { return _rows.Where(r => r != null).GetEnumerator(); }
}
}
@@ -0,0 +1,46 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BCG
{
public class SpreadIndexer
{
public enum IndexerDataType
{
SpreadSpec = 0,
Family,
LongDesc,
ShortDesc,
LegArray
}
private static Dictionary<string, Dictionary<IndexerDataType, object>> _cached_data = new Dictionary<string, Dictionary<IndexerDataType, object>>();
public static object get_value(string spread_def, IndexerDataType data_type)
{
Dictionary<IndexerDataType, object> type_to_data = null;
if (!_cached_data.TryGetValue(spread_def, out type_to_data))
{
type_to_data = new Dictionary<IndexerDataType,object> ();
_cached_data.Add(spread_def, type_to_data);
}
object rv = null;
type_to_data.TryGetValue(data_type, out rv);
return rv;
}
public static void set_value(string spread_def, IndexerDataType data_type, object val)
{
Dictionary<IndexerDataType, object> type_to_data = null;
if (!_cached_data.TryGetValue(spread_def, out type_to_data))
{
type_to_data = new Dictionary<IndexerDataType, object>();
_cached_data.Add(spread_def, type_to_data);
}
type_to_data[data_type] = val;
}
}
}
@@ -0,0 +1,41 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BCG
{
/// <summary>
/// keeps rows that are within n mseconds of now
/// </summary>
public class TimeWindowTable : MovingWindowTable
{
private int myWindowSizeMS;
private string myTimeCol;
public TimeWindowTable(IBlueTableSchema schema, int windowSizeMS, string timeCol)
:base(schema, new BlueRowComparer(timeCol, "id"))
{
this.myTimeCol = timeCol;
this.myWindowSizeMS = windowSizeMS;
}
public bool RemoveRow(IBlueRow row)
{
int index = this.mySortedRows.BinarySearch(row, this.myRowComparer);
if (index >= 0)
{
this.mySortedRows.RemoveAt(index);
return true;
}
return false;
}
protected override bool windowContains(IBlueRow row)
{
DateTime rowTime = (DateTime)row[this.myTimeCol];
return DateTime.Now.Subtract(rowTime).TotalMilliseconds <= this.myWindowSizeMS;
}
}
}
@@ -0,0 +1,73 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BCG
{
public class WrapperRow<T> : IBlueRow
{
private readonly IBlueTableSchema _schema;
private readonly T _data;
private readonly int? _id = null;
private Func<int, T, object> _getter;
private readonly int _id_col_index;
public WrapperRow(IBlueTableSchema schema, T data, Func<int, T, object> getter, int? id = 0)
{
_schema = schema;
_data = data;
_getter = getter;
_id = id;
_id_col_index = schema["id"].colNo;
}
public IBlueTableSchema schema { get { return _schema; } }
public object this[int i] { get { return (i == _id_col_index && _id != null) ? _id.Value : _getter(i, _data); } }
public object this[string c] { get { return this[_schema[c].colNo]; } }
public void Add(object key, object value) { throw new NotImplementedException(); }
public void Clear() { throw new NotImplementedException(); }
public bool Contains(object key) { return key != null && key is string && schema.columnNames.Contains(key as string); }
public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
public bool IsFixedSize { get { return true; } }
public bool IsReadOnly { get { return true; } }
public System.Collections.ICollection Keys { get { return schema.columnNames; } }
public void Remove(object key) { throw new NotImplementedException("IBlueRow is read only"); }
public System.Collections.ICollection Values { get { return _schema.columnInfo.Select(c => this[c.colNo]).ToArray(); } }
public virtual object this[object key]
{
get
{
if (key is string)
return this[(string)key];
else if (key is int)
return this[(int)key];
throw new BlueDataException("Wrong type while indexing the row");
}
set { throw new NotSupportedException("The IBlueRow is readonly"); }
}
public void CopyTo(Array array, int index) { Values.CopyTo(array, index); }
public int Count { get { return schema.columnNames.Length; } }
public virtual bool IsSynchronized { get { return false; } }
public virtual object SyncRoot { get { return this; } }
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
}
}
@@ -0,0 +1,316 @@
using System;
using System.Collections;
using System.Data;
using BCG.Utilities.General;
namespace BCG
{
[Serializable]
public class DataPacket : ICloneable, IDataPacket
{
public class BlueRowIdComparator
{
private string[] myIdColumns = new string[]{"id"};
public BlueRowIdComparator(string[] idCols)
{
if ( idCols.Length != 0 )
{
myIdColumns = idCols;
}
}
public bool blueRowIdEqualEvaluator(object a, object b)
{
if ( a == null || b == null )
{
return false;
}
if ( !(a is IBlueRow) || !(b is IBlueRow) )
{
return false;
}
IBlueRow lhs = (IBlueRow)a;
IBlueRow rhs = (IBlueRow)b;
bool result = true;
foreach(string idColumn in myIdColumns)
{
result = result && lhs[idColumn].Equals(rhs[idColumn]);
if ( !result )
break;
}
return result;
}
}
public IBlueRowCollection newRows, addedRows, deletedRows;
public DataPacket () {}
public DataPacket (IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows)
{
this.newRows = newRows;
this.addedRows = addedRows;
this.deletedRows = deletedRows;
}
public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows)
{
new_rows = newRows;
added_rows = addedRows;
deleted_rows = deletedRows;
}
public IBlueRowCollection getTrulyAddedRows(params string[] idColumns)
{
BlueTable truly = new BlueTable(addedRows.schema);
BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
CollectionHelpers.EqualDelegate ed =
new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
foreach(IBlueRow r in ta)
{
truly.addRow(r);
}
return truly;
}
public virtual IBlueRowCollection getTrulyAddedRows ()
{
BlueTable retVal = new BlueTable (schema);
foreach (IBlueRow r in addedRows)
if (!deletedRows.containsID ((int)r["id"]))
retVal.addRow (r);
return retVal;
}
public virtual IBlueRowCollection getTrulyDeletedRows ()
{
BlueTable retVal = new BlueTable (schema);
foreach (IBlueRow r in deletedRows)
if (!addedRows.containsID ((int)r["id"]))
retVal.addRow (r);
return retVal;
}
public virtual IBlueRowCollection getTrulyChangedRows ()
{
BlueTable retVal = new BlueTable (schema);
foreach (IBlueRow r in addedRows)
if (deletedRows.containsID ((int)r["id"]))
retVal.addRow (r);
return retVal;
}
public IBlueRowCollection getTrulyDeletedRows(params string[] idColumns)
{
BlueTable truly = new BlueTable(deletedRows.schema);
BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
CollectionHelpers.EqualDelegate ed =
new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
foreach(IBlueRow r in td)
{
truly.addRow(r);
}
return truly;
}
public IBlueRowCollection getTrulyChangedRows(params string[] idColumns)
{
BlueTable truly = new BlueTable(addedRows.schema);
BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
CollectionHelpers.EqualDelegate ed =
new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
ICollection ta = CollectionHelpers.difference(addedRows, deletedRows, ed);
ICollection tc = CollectionHelpers.difference(addedRows, ta, ed);
foreach(IBlueRow r in tc)
{
truly.addRow(r);
}
return truly;
}
public IBlueRowCollection getTrulyChangedOldRows(params string[] idColumns)
{
BlueTable truly = new BlueTable(deletedRows.schema);
BlueRowIdComparator c = new BlueRowIdComparator(idColumns);
CollectionHelpers.EqualDelegate ed =
new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
ICollection td = CollectionHelpers.difference(deletedRows, addedRows, ed);
ICollection tco = CollectionHelpers.difference(deletedRows, td, ed);
foreach(IBlueRow r in tco)
{
truly.addRow(r);
}
return truly;
}
public TablePacket toTablePacket ()
{
return new TablePacket (BlueDataUtility.toDataTable (newRows), BlueDataUtility.toDataTable (addedRows),
BlueDataUtility.toDataTable (deletedRows));
}
public IBlueTableSchema schema
{
get
{
IBlueTableSchema retVal = newRows.schema != null ? newRows.schema :
addedRows.schema != null ? addedRows.schema : deletedRows.schema;
return retVal;
}
}
/*
status D total A total D this A This D' total A' Total
x 0 0 0 0 0 0
0 0 0 1 1 this+
0 0 1 0 1this
0 0 1 1 1 this 1 this+
x 0 1 0 0
error 0 1 0 1
0 1 1 0 1 total-
0 1 1 1 1 total-, 1 this +
x 1 0 0 0
1 0 0 1 1 this+
error 1 0 1 0
error 1 0 1 1
x 1 1 0 0
error 1 1 0 1
1 1 1 0 1 total -1 keep the old delete and remove the old add
1 1 1 1 1 total-, 1 this +
d' = dthis - (dthis intersect a) + d // keep the old delete
a' = a - (dthis intersect a) + athis // keep the new add
* */
public static DataPacket combineWithIds(ICollection packets, string[] idColumns)
{
if ( packets.Count == 0 )
{
return new DataPacket(
new BlueTable(new DataTable()),
new BlueTable(new DataTable()),
new BlueTable(new DataTable()));
}
DataPacket.BlueRowIdComparator c = new DataPacket.BlueRowIdComparator(idColumns);
CollectionHelpers.EqualDelegate ed =
new CollectionHelpers.EqualDelegate(c.blueRowIdEqualEvaluator);
ArrayList added = new ArrayList();
ArrayList deleted = new ArrayList();
BlueTable dt = null;
BlueTable at = null;
IBlueRowCollection nt = null;
foreach(DataPacket dp in packets)
{
ArrayList oldDeleted = deleted;
ArrayList oldAdded = added;
added = new ArrayList();
deleted = new ArrayList();
if ( dt == null )
{
dt = new BlueTable(dp.schema);
at = new BlueTable(dp.schema);
}
deleted.AddRange(oldDeleted);
deleted.AddRange( CollectionHelpers.difference(
CollectionHelpers.difference(dp.deletedRows, oldAdded, ed),
oldDeleted, ed) );
added.AddRange(dp.addedRows);
added.AddRange( CollectionHelpers.difference(
CollectionHelpers.difference(oldAdded, dp.deletedRows, ed),
added, ed) );
nt = dp.newRows;
}
foreach ( IBlueRow dr in added )
{
at.addRow(dr);
}
foreach ( IBlueRow dr in deleted )
{
dt.addRow(dr);
}
return new DataPacket(nt,at,dt);
}
public DataPacket deepCopy ()
{
return new DataPacket (new OrderedBlueTable (newRows),
new OrderedBlueTable (newRows),
new BlueTable (deletedRows));
}
public virtual DataPacket createImmutable ()
{
return
new DataPacket(
newRows is IImmutableRowCollection ? newRows : new ImmutableBlueTable(newRows),
addedRows,
deletedRows);
}
#region ICloneable Members
public object Clone()
{
return MemberwiseClone();
}
#endregion
public string toIncrementalIDString()
{
System.Text.StringBuilder sb = new System.Text.StringBuilder(schema.tableName);
sb.Append ("\ndeleted : ");
foreach (IBlueRow r in deletedRows)
{
sb.Append(r["id"]);
sb.Append(',');
}
sb.Append("\nadded : ");
foreach (IBlueRow r in addedRows)
{
sb.Append(r["id"]);
sb.Append(',');
}
sb.Append("\n");
return sb.ToString();
}
public override string ToString()
{
return String.Format("DataPacket, schema {0} n {1} d {2} a {3}", schema.tableName, newRows.Count, deletedRows.Count, addedRows.Count);
}
}
}
@@ -1,8 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<ItemGroup>
<ProjectReference Include="..\pubsub\pubsub.csproj" />
</ItemGroup>
<PropertyGroup>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<TargetFramework>net7.0</TargetFramework>
<Nullable>enable</Nullable>
<Nullable>disable</Nullable>
</PropertyGroup>
</Project>
@@ -0,0 +1,15 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BCG
{
public interface IDataPacket
{
//this is an ICollection of IDictionary
void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows);
}
}
@@ -0,0 +1,18 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BCG
{
public interface IDataPump
{
string tag
{
get;
}
void start();
void stop();
}
}
@@ -0,0 +1,17 @@
using System;
namespace BCG
{
/// <summary>
/// A central place to get all ITableAccumulators. This is helpful, since specific implementations
/// of ITableAccumulators can be instantiated using config.
/// </summary>
public class AccumulatorFactory
{
public static ITableAccumulator getAccumulator (string tag, ISubscriberCore subCore)
{
return new TableAccumulator (tag, subCore);
}
}
}
@@ -0,0 +1,24 @@
using System;
using System.IO;
namespace BCG
{
public delegate void TableUpdateDelegate (IBlueRowCollection newRows,
IBlueRowCollection addedRows,
IBlueRowCollection deletedRows);
/// <summary>
/// This interface is supposed to hook upto pubsub on one end, receive data in bytes,
/// deserialize and call the client (that wil hook upto this interface) and give tables
/// </summary>
public interface ITableAccumulator
{
event TableUpdateDelegate onUpdate;
/// <summary>
/// return trues if this table acumulator has any subscribers.
/// </summary>
/// <returns></returns>
bool isActive();
void update (Stream[] streams);
}
}
@@ -0,0 +1,349 @@
using System;
using System.IO;
//using BCG.Utilities;
using mbts;
namespace BCG
{
public class TableAccumulator : ITableAccumulator
{
private HighPerformanceTimer _timer = new HighPerformanceTimer();
private IClientDeserializer myDeserializer = new ClientDeserializationController ();
private BlueTable myStateTable;
private ISubscriberCore mySubscriptionCore;
private TableUpdateDelegate myOnUpdateDelegate;
private string myTag;
private bool myUseIncrementsForMultipleOps = DataStreamGatewaySettings.instance.useIncrementsForMultipleOps;
public TableAccumulator (string tag, ISubscriberCore subCore)
{
myTag = tag;
mySubscriptionCore = subCore;
}
#region ITableAccumulator Members
public bool isActive()
{return myOnUpdateDelegate != null;}
public event TableUpdateDelegate onUpdate
{
add
{
// BlueEventLog.logEvent ("TableSubscriber being Subscribed for tag :" + myTag, BlueEventLevel.Information);
if (myOnUpdateDelegate == null)
{
//ie first subscription
// BlueEventLog.logEvent ("Subscription made to PubSub for tag : " + myTag, BlueEventLevel.Information);
mySubscriptionCore.subscribe (myTag);
}
myOnUpdateDelegate =
Delegate.Combine (myOnUpdateDelegate, value) as TableUpdateDelegate;
if (myStateTable != null)
value (myStateTable, myStateTable, new BlueTable (myStateTable.schema));
}
remove
{
// BlueEventLog.logEvent ("TableSubscriber being Unsubscribed for tag :" + myTag, BlueEventLevel.Information);
if (myOnUpdateDelegate != null)
{
myOnUpdateDelegate =
Delegate.Remove (myOnUpdateDelegate, value) as TableUpdateDelegate;
if (myOnUpdateDelegate == null)
{
//all subscribers gone
// BlueEventLog.logEvent ("Unubscription made to PubSub for tag : " + myTag, BlueEventLevel.Information);
mySubscriptionCore.unsubscribe (myTag);
myStateTable = null;
}
}
}
}
public void update(Stream[] streams)
{
//checking the condition as a subscriber might unsubscribe in a callback and there might be more than one
//data element for that tag in the ViewStreamDatabase
if (myOnUpdateDelegate != null)
{
byte[] ops;
Array rows;
_timer.start();
myDeserializer.deserialize (streams, out ops, out rows);
_timer.stop();
// BlueEventLog.logEvent(String.Format("deserializing {0} took {1} msecs", myTag, _timer.lastIntervalTime), BlueEventLevel.Performance);
_timer.start();
myAccumulateAndFire (ops, rows);
_timer.stop();
// BlueEventLog.logEvent(String.Format("myAccumulateAndFire for {0} took {1} msecs", myTag, _timer.lastIntervalTime), BlueEventLevel.Performance);
}
}
#endregion
private void myAccumulateAndFire (byte[] ops, Array rows)
{
//if a non create comes before the create, the deserializer returns 0 sized arrays
if (ops.Length == 0 || rows.Length == 0)
{
//ie if one is zero sized so should be the other one
// BlueEventLog.logAssert (ops.Length + rows.Length == 0);
}
else
{
if (ops.Length > 1)
{
int index = 0;
if (myStateTable == null)
{
for(index=0; index<ops.Length; index++)
if (ops[index] == StreamOpCode.create)
break;
if (index == ops.Length)
return;
}
if (myUseIncrementsForMultipleOps)
myHandleMultipleOpsIncrementally(ops, rows, index);
else
myHandleMultipleOps(ops, rows, index);
}
else
{
if (myStateTable != null || ops[0] == StreamOpCode.create)
myHandleSingleOp (ops, rows);
}
}
}
private void myHandleMultipleOps (byte[] ops, Array rows, int offset)
{
HighPerformanceTimer timer = new HighPerformanceTimer();
timer.start();
IBlueRowCollection stateAtStart = myStateTable == null ? null : new BlueTable (myStateTable);
for (int i = offset; i < ops.Length; i++)
{
byte op = ops[i];
IBlueRowCollection[] tableArray = rows.GetValue (i) as IBlueRowCollection[];
if (op == StreamOpCode.add)
myStateTable.addRows (tableArray[0]);
else if (op == StreamOpCode.remove)
myStateTable.deleteRows (tableArray[0]);
else if (op == StreamOpCode.change)
myStateTable.changeRows (tableArray[0]);
else if (op == StreamOpCode.create)
myStateTable = new BlueTable (tableArray[0]);
else if (op == StreamOpCode.multipleChange)
myStateTable.changeRows (tableArray[0]);
else if (op == StreamOpCode.multipleRemoveAdd)
{
myStateTable.deleteRows (tableArray[0]);
myStateTable.addRows (tableArray[1]);
}
}
IBlueRowCollection dRows, aRows;
if (stateAtStart == null)
{
dRows = new BlueTable (myStateTable.schema);
aRows = myStateTable;
}
else
{
BlueDataUtility.computeDifferences (myStateTable, stateAtStart, out dRows, out aRows);
}
timer.stop();
//Console.WriteLine("multiple ops for table " + myTag + " took " + timer.lastIntervalTime + " msecs");
myOnUpdateDelegate (myStateTable, aRows, dRows);
}
#region handling mutliple ops in single update call, incrementally
private void myHandleMultipleOpsIncrementally(byte[] ops, Array rows, int offset)
{
BlueTable dels = null, adds = null;
if (myStateTable != null)
{
dels = new BlueTable(myStateTable.schema);
adds = new BlueTable(myStateTable.schema);
}
for (int i = offset; i < ops.Length; i++)
{
byte op = ops[i];
IBlueRowCollection[] tableArray = rows.GetValue(i) as IBlueRowCollection[];
if (op == StreamOpCode.add)
{
myHandleAdd(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds);
}
else if (op == StreamOpCode.remove)
{
myHandleRemove(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds);
}
else if (op == StreamOpCode.change)
{
myHandleChange(BlueDataUtility.extractRow(tableArray[0], 0), dels, adds);
}
else if (op == StreamOpCode.create)
{
myHandleCreate(tableArray[0], ref dels, ref adds);
}
else if (op == StreamOpCode.multipleChange)
{
myHandleMultipleChanges(tableArray[0], dels, adds);
}
else if (op == StreamOpCode.multipleRemoveAdd)
{
myHandleMultipleRemovesAdds(tableArray[0], tableArray[1], dels, adds);
}
}
myOnUpdateDelegate(myStateTable, adds, dels);
}
private void myHandleAdd(IBlueRow r, BlueTable dels, BlueTable adds)
{
if (dels != null)
{
myStateTable.addRow(r);
adds.addRow(r);
}
}
private void myHandleRemove(IBlueRow r, BlueTable dels, BlueTable adds)
{
if (dels != null)
{
myStateTable.deleteRow(r);
if (adds.containsID((int)r["id"]))
adds.deleteRow(r);
else
dels.addRow(r);
}
}
private void myHandleChange(IBlueRow r, BlueTable dels, BlueTable adds)
{
if (dels != null)
{
IBlueRow er = myStateTable[(int)r["id"]];
myHandleRemove(er, dels, adds);
myHandleAdd(r, dels, adds);
}
}
private void myHandleCreate(IBlueRowCollection newTable, ref BlueTable dels, ref BlueTable adds)
{
if (dels == null)
{
// BlueEventLog.logAssert(adds == null);
dels = new BlueTable(newTable.schema);
adds = new BlueTable(newTable);
myStateTable = new BlueTable(newTable);
}
else
{
dels = new BlueTable(myStateTable);
myStateTable = new BlueTable(newTable);
adds = new BlueTable(newTable);
}
}
private void myHandleMultipleRemovesAdds(IBlueRowCollection deletedRows, IBlueRowCollection addedRows,
BlueTable dels, BlueTable adds)
{
if (dels != null)
{
foreach (IBlueRow r in deletedRows)
myHandleRemove(r, dels, adds);
foreach (IBlueRow r in addedRows)
myHandleAdd(r, dels, adds);
}
}
private void myHandleMultipleChanges(IBlueRowCollection changedRows, BlueTable dels, BlueTable adds)
{
if (dels != null)
foreach (IBlueRow r in changedRows)
myHandleChange(r, dels, adds);
}
#endregion
private void myHandleSingleOp(byte[] ops, Array rows)
{
byte op = ops[0];
IBlueRowCollection[] tableArray = rows.GetValue (0) as IBlueRowCollection[];
IBlueRowCollection dRows, aRows;
if (op == StreamOpCode.create)
{
if (myStateTable == null)
{
aRows = myStateTable = new BlueTable(tableArray[0]);
dRows = new BlueTable(myStateTable.schema);
myOnUpdateDelegate(myStateTable, aRows, dRows);
}
else
{
IBlueRowCollection newState = new BlueTable(tableArray[0]);
BlueDataUtility.computeDifferences(newState, myStateTable, out dRows, out aRows);
if (aRows.Count > 0 || dRows.Count > 0)
{
myStateTable = new BlueTable(newState);
myOnUpdateDelegate(myStateTable, aRows, dRows);
}
}
}
else
{
if (op == StreamOpCode.add)
{
myStateTable.addRows(aRows = tableArray[0]);
dRows = new BlueTable(myStateTable.schema);
}
else if (op == StreamOpCode.remove)
{
myStateTable.deleteRows(dRows = tableArray[0]);
aRows = new BlueTable(myStateTable.schema);
}
else if (op == StreamOpCode.change)
{
IBlueRow cRow = BlueDataUtility.extractRow(tableArray[0], 0);
aRows = new BlueTable(myStateTable.schema);
dRows = new BlueTable(myStateTable.schema);
(aRows as BlueTable).addRow(cRow);
(dRows as BlueTable).addRow(myStateTable[(int)cRow["id"]]);
myStateTable.changeRow(cRow);
}
else if (op == StreamOpCode.multipleChange)
{
if (myStateTable.Count == tableArray[0].Count)
{
//all rows are changed, so we can optimize this
dRows = myStateTable;
aRows = myStateTable = new BlueTable(tableArray[0]);
}
else
{
dRows = new BlueTable(myStateTable.schema);
aRows = tableArray[0];
foreach (IBlueRow r in tableArray[0])
{
(dRows as BlueTable).addRow(myStateTable[(int)r["id"]]);
myStateTable.changeRow(r);
}
}
}
else if (op == StreamOpCode.multipleRemoveAdd)
{
BlueDataUtility.absorbChanges(myStateTable, dRows = tableArray[0], aRows = tableArray[1]);
}
else
{
aRows = dRows = null;
// BlueEventLog.logAssert(false, "unknown opcode");
}
myOnUpdateDelegate(myStateTable, aRows, dRows);
}
}
}
}
@@ -0,0 +1,69 @@
using System;
using System.IO;
using System.Collections;
using BCG.Utilities;
namespace BCG
{
/// <summary>
/// Controls (ie creates dynamically) an IClientDeserializer and then passes data to it.
/// </summary>
public class ClientDeserializationController : IClientDeserializer
{
private IClientDeserializer myDeserializer;
public ClientDeserializationController () {}
#region IClientDeserializer Members
/// <summary>
/// Deserializes bytes into operation and data. If it gets a non create before a create, both
/// the out arrays get initialized to empty arrays
/// </summary>
/// <param name="streams"></param>
/// <param name="theOps"></param>
/// <param name="theRows"></param>
public void deserialize (Stream[] streams, out byte[] theOps, out Array theRows)
{
if (myDeserializer != null)
{
myDeserializer.deserialize (streams, out theOps, out theRows);
}
else
{
Stream totalStream = UtilityMethods.getAggregatedStream (streams);
long initialPos = totalStream.Position;
BinaryReader br = new BinaryReader (totalStream);
byte op;
try
{
op = br.ReadByte ();
}
catch (EndOfStreamException)
{
op = Byte.MaxValue;
}
if (op == StreamOpCode.create)
{
br.ReadString (); //pubsubclientdeserializer
string deserName = br.ReadString ();
myDeserializer = Activator.CreateInstance (Type.GetType (deserName))
as IClientDeserializer;
totalStream.Position = initialPos;
myDeserializer.deserialize (new Stream[] {totalStream}, out theOps, out theRows);
}
else
{
// BlueEventLog.logEvent ("Received a non create before a create", BlueEventLevel.Warning);
theOps = new byte[]{};
theRows = new object[]{};
}
}
}
#endregion
}
}
@@ -0,0 +1,124 @@
using System;
using System.Collections;
using System.IO;
using BCG.Utilities;
//using BCG.Utilities.BlueConfiguration;
namespace BCG
{
/// <summary>
/// Pairs up with the UniversalSerializer to deserialize the corresponding data. Everythign anologous
/// to UniversalSerializer.
/// </summary>
public abstract class ClientDeserializer : IClientDeserializer
{
protected IBlueTableSchema mySchema;
protected IBlueTableSchema myFirstSchema;
protected IClientRowDeserializer myRowDeserializer;
public ClientDeserializer()
{
myFirstSchema = null;
mySchema = null;
}
#region IClientDeserializer Members
void BCG.IClientDeserializer.deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
{
BinaryReader br = new BinaryReader(UtilityMethods.getAggregatedStream(streams));
// BlueTrace.WriteDeserializeStart(streams.Length, br.BaseStream.Length);
// emit # of streams and total length
ArrayList ops = new ArrayList(), rows = new ArrayList();
IBlueRowCollection[] result;
while (true)
{
byte op;
try
{
int value = br.BaseStream.ReadByte();
if (value == -1) break;
op = (byte)value;
// BlueTrace.WriteDeserializeOperation(op);
}
catch (EndOfStreamException)
{
break;
}
if (op == StreamOpCode.create)
{
br.ReadString(); //pubsubDeserializer
br.ReadString(); //clientDeserializer
mySchema = new BlueTableSchema();
mySchema.deserialze(br);
//keep track of the first schema you've ever seen, so you know how to reorder columns
if (myFirstSchema == null)
myFirstSchema = mySchema;
result = new IBlueRowCollection[] { myGetRows(br, op, br.ReadInt32()) };
myEncorporateRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else
{
if (mySchema != null)
{
if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
{
result = new IBlueRowCollection[] { myGetRows(br, op, 1) };
myEncorporateRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else if (op == StreamOpCode.multipleChange)
{
result = new IBlueRowCollection[] { myGetRows(br, op, br.ReadInt32()) };
myEncorporateRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else if (op == StreamOpCode.multipleRemoveAdd)
{
result =
new IBlueRowCollection[] {
myGetRows (br, StreamOpCode.remove, br.ReadInt32 ()),
myGetRows (br, StreamOpCode.add, br.ReadInt32 ())
};
myEncorporateRows(result[0], StreamOpCode.remove);
myEncorporateRows(result[1], StreamOpCode.add);
ops.Add(op);
rows.Add(result);
}
else
{
// BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error);
}
}
else
{
// BlueEventLog.logEvent("Received a non create operation even though the schema" +
// "has never been created", BlueEventLevel.Warning);
}
}
}
theOps = ops.ToArray(typeof(byte)) as byte[];
theRows = rows.ToArray();
// BlueTrace.WriteDeserializeStop(rows.Count);
}
#endregion
protected abstract IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows);
protected abstract void myEncorporateRows(IBlueRowCollection rows, byte op);
}
}
@@ -0,0 +1,34 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
namespace BCG
{
class ClientReorderingRowDeserializer : IClientRowDeserializer
{
private IBlueTableSchema myOriginalSchema;
private IBlueTableSchema myOutputSchema;
private ClientUniversalRowDeserializer myDeserializer;
public ClientReorderingRowDeserializer(IBlueTableSchema originalSchema, IBlueTableSchema newSchema)
{
myOriginalSchema = originalSchema;
myOutputSchema = newSchema;
myDeserializer = new ClientUniversalRowDeserializer(newSchema);
}
#region IClientRowDeserializer Members
public void deserializeRow(IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
{
IBlueRow temp = null;
myDeserializer.deserializeRow(currentState, br, op, out temp);
r = BlueDataUtility.changeSchema(temp, myOutputSchema);
}
#endregion
}
}
@@ -0,0 +1,35 @@
using System;
using System.Collections;
using System.IO;
namespace BCG
{
public class ClientUniversalDeserializer : ClientDeserializer
{
public ClientUniversalDeserializer () {}
protected override IBlueRowCollection myGetRows (BinaryReader br, byte op, int nRows)
{
BlueTable retVal = new BlueTable (mySchema);
if (myRowDeserializer == null)
myRowDeserializer = new ClientUniversalRowDeserializer(mySchema);
else if (op == StreamOpCode.create)
myRowDeserializer = new ClientReorderingRowDeserializer(myFirstSchema, mySchema);
for (int i = 0; i < nRows; i++)
{
IBlueRow row;
myRowDeserializer.deserializeRow (null, br, op, out row);
retVal.addRow (row);
}
return retVal;
}
protected override void myEncorporateRows (IBlueRowCollection rows, byte op)
{
//this does not keep state so ignore calls
}
}
}
@@ -0,0 +1,69 @@
using System;
using System.Collections;
using System.IO;
using BCG.Utilities;
//using BCG.Utilities.BlueConfiguration;
namespace BCG
{
/// <summary>
/// Helps the ClientUniversalDeserializer in deserilizing by dealing with the deserialization process
/// of one row. This leaves the ClientUniversalDeserializer to deserializer headers (indicating operations)
/// </summary>
internal class ClientUniversalRowDeserializer : IClientRowDeserializer
{
private IBlueTableSchema mySchema;
private int myNullMapLength = 0;
private PrimitiveDeserializationHelper myDeserializationHelper;
private BitArray myNullMap;
private int[] mySerIndexToSchemaIndexMap;
public ClientUniversalRowDeserializer (IBlueTableSchema schema)
{
myDeserializationHelper = new PrimitiveDeserializationHelper (mySchema = schema);
myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8)));
myInitReorderSchema ();
}
#region Schema Reordering initialization
private void myInitReorderSchema ()
{
int idColIndex = mySchema["id"].colNo;
int[] temp = new int [mySchema.columnInfo.Length];
foreach (ColumnInfo ci in mySchema.columnInfo)
{
temp[ci.colNo] =
ci.colNo == idColIndex ? 0 :
ci.colNo < idColIndex ? ci.colNo + 1 : ci.colNo;
}
mySerIndexToSchemaIndexMap = new int[temp.Length];
for (int i = 0; i < temp.Length; i ++)
mySerIndexToSchemaIndexMap [temp[i]] = i;
}
#endregion
#region IClientRowDeserializer Members
public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
{
br.ReadInt32 (); //length information for the row not needed
myNullMap = new BitArray(br.ReadBytes (myNullMapLength));
object[] values = new object [mySchema.columnNames.Length];
for (int i = 0; i < values.Length; i++)
{
int colNo = mySerIndexToSchemaIndexMap[i];
values[mySerIndexToSchemaIndexMap[i]] = myNullMap[colNo] ? DBNull.Value : myDeserializationHelper.deserializeColumn (colNo, br);
}
r = new BlueRow(mySchema, values);
}
#endregion
}
}
@@ -0,0 +1,22 @@
using System;
using System.Collections;
using System.IO;
namespace BCG
{
/// <summary>
/// Performs deserialization inside the client app. The client gets either the complete data set or the
/// increments from PubSub and this interface makes sense out of them from a tabular/datastream stand point.
/// </summary>
public interface IClientDeserializer
{
/// <summary>
/// deserializes the data received by the client from the pubsub
/// </summary>
/// <param name="serializedData">the data in byte format received from pubsub</param>
/// <param name="theOps">the operations that the data maps onto</param>
/// <param name="theRows">the data corresponding to the operations, is an array of IblueRowCollection[]</param>
void deserialize (Stream[] streams, out byte[] theOps, out Array theRows);
}
}
@@ -0,0 +1,14 @@
using System;
using System.IO;
using System.Collections;
namespace BCG
{
/// <summary>
/// Deserializes one row. the current state is also provided as some deserialization schemes might use it.
/// </summary>
public interface IClientRowDeserializer
{
void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r);
}
}
@@ -0,0 +1,79 @@
using System;
using System.IO;
namespace BCG
{
/// <summary>
/// Deserializes primitive types. Contructs a hash based scheme based on the index, to make everythign O(1)
/// rather than searching it up and making it O(n)
/// </summary>
public class PrimitiveDeserializationHelper
{
private delegate object DeserializeDelegate (int colNo, BinaryReader br);
private DeserializeDelegate[] myDispatchTable;
private ColumnInfo[] myCis;
public PrimitiveDeserializationHelper (IBlueTableSchema schema)
: this (schema.columnInfo)
{}
public PrimitiveDeserializationHelper (ColumnInfo[] cis)
{
myDispatchTable = new DeserializeDelegate [(myCis = cis).Length];
foreach (ColumnInfo ci in cis)
myDispatchTable[ci.colNo] = myGetDispatchMethod (ci.colType);
}
public object deserializeColumn (int colNo, BinaryReader br)
{
return myDispatchTable[colNo] (colNo, br);
}
private DeserializeDelegate myGetDispatchMethod (Type t)
{
if (t.Equals (BlueDataValidTypes.INT)) return new DeserializeDelegate (myDeserializeInt32);
else if (t.Equals (BlueDataValidTypes.DOUBLE)) return new DeserializeDelegate (myDeserializeDouble);
else if (t.Equals (BlueDataValidTypes.STRING)) return new DeserializeDelegate (myDeserializeString);
else if (t.Equals (BlueDataValidTypes.BYTEARRAY)) return new DeserializeDelegate (myDeserializeByteArray);
else if (t.Equals (BlueDataValidTypes.CHAR)) return new DeserializeDelegate (myDeserializeChar);
else if (t.Equals (BlueDataValidTypes.LONG)) return new DeserializeDelegate (myDeserializeLong);
else if (t.Equals (BlueDataValidTypes.SBYTE)) return new DeserializeDelegate (myDeserializeSByte);
else if (t.Equals (BlueDataValidTypes.BOOL)) return new DeserializeDelegate (myDeserializeBool);
else if (t.Equals (BlueDataValidTypes.BYTE)) return new DeserializeDelegate (myDeserializeByte);
else if (t.Equals (BlueDataValidTypes.DECIMAL)) return new DeserializeDelegate (myDeserializeDecimal);
else if (t.Equals (BlueDataValidTypes.UINT)) return new DeserializeDelegate (myDeserializeUInt);
else if (t.Equals (BlueDataValidTypes.ULONG)) return new DeserializeDelegate (myDeserializeULong);
else if (t.Equals (BlueDataValidTypes.DATETIME)) return new DeserializeDelegate (myDeserializeDateTime);
else if (t.Equals (BlueDataValidTypes.SINGLE)) return new DeserializeDelegate (myDeserializeSingle);
else if (SerializationUtilityMethods.isSerializedType (t)) return new DeserializeDelegate (myDeserializeISerializedType);
else throw new ApplicationException (t.FullName + " is not a supported type");
}
private object myDeserializeInt32 (int colNo, BinaryReader br) {return br.ReadInt32 ();}
private object myDeserializeISerializedType (int colNo, BinaryReader br)
{
ISerializedType o = Activator.CreateInstance (myCis[colNo].colType)
as ISerializedType;
o.deserialze (br);
return o;
}
private object myDeserializeDouble (int colNo, BinaryReader br) {return br.ReadDouble ();}
private object myDeserializeString (int colNo, BinaryReader br) {return br.ReadString ();}
private object myDeserializeByteArray (int colNo, BinaryReader br) {return br.ReadBytes (br.ReadInt32 ());}
private object myDeserializeChar (int colNo, BinaryReader br) {return br.ReadChar ();}
private object myDeserializeLong (int colNo, BinaryReader br) {return br.ReadInt64 ();}
private object myDeserializeSByte (int colNo, BinaryReader br) {return br.ReadSByte ();}
private object myDeserializeBool (int colNo, BinaryReader br) {return br.ReadBoolean ();}
private object myDeserializeByte (int colNo, BinaryReader br) {return br.ReadByte ();}
private object myDeserializeDecimal (int colNo, BinaryReader br) {return br.ReadDecimal ();}
private object myDeserializeUInt (int colNo, BinaryReader br) {return br.ReadUInt32 ();}
private object myDeserializeDateTime (int colNo, BinaryReader br)
{
return new DateTime(br.ReadInt64());
//return DateTime.FromOADate (br.ReadDouble ());
//return new DateTime(br.ReadInt64());
}
private object myDeserializeULong (int colNo, BinaryReader br) {return br.ReadUInt64 ();}
private object myDeserializeSingle (int colNo, BinaryReader br) {return br.ReadSingle ();}
}
}
@@ -0,0 +1,134 @@
using System;
using System.Collections;
//using BCG.Utilities.BlueConfiguration;
using System.Text.RegularExpressions;
namespace BCG
{
public class DataStreamGatewaySettings //: ConfigurationRequest
{
private static object myLockObj = new object();
private static DataStreamGatewaySettings mySelf;
public DataStreamGatewaySettings ()
{
SendBlueCommMonitor = false;
}
private string[] myNewSerializationSchemeTableList;
private SerializationScheme[] mySerializationScheme;
private bool myEnablePubsubStats;
private int mySingleTagStatsUpdateInterval;
private int myPubsubStatsUpdateInterval;
private bool myUseIncrementsForMultipleOps;
private bool myUseCppDataStreams = false;
private bool myUseSerializedBlueRows = false;
//public override bool isValid() {return true;}
public string[] newSerializationSchemeTableList
{
get {return myNewSerializationSchemeTableList;}
set {myNewSerializationSchemeTableList = value;}
}
public bool enablePubsubStats
{
get { return myEnablePubsubStats; }
set { myEnablePubsubStats = Convert.ToBoolean(value); }
}
public int singleTagStatsUpdateInterval
{
get { return mySingleTagStatsUpdateInterval; }
set { mySingleTagStatsUpdateInterval = value; }
}
public int pubsubStatsUpdateInterval
{
get { return myPubsubStatsUpdateInterval; }
set { myPubsubStatsUpdateInterval = value; }
}
public SerializationScheme[] serializationScheme
{
get {return mySerializationScheme;}
set {mySerializationScheme = value;}
}
public bool useCppDataStreams
{
get {return myUseCppDataStreams;}
set { myUseCppDataStreams = value; }
}
public bool isSerializableUnderNewScheme (string tableName)
{
ArrayList temp = new ArrayList ();
foreach (string s in newSerializationSchemeTableList)
if (new Regex (s).Match (tableName).Success)
return true;
return false;
}
public string getSubscriber (string tableName)
{
foreach (SerializationScheme ss in mySerializationScheme)
if (new Regex (ss.tableRegExp).Match (tableName).Success)
return ss.subscriber;
return null;
}
public bool useIncrementsForMultipleOps
{
get { return myUseIncrementsForMultipleOps; }
set { myUseIncrementsForMultipleOps = value; }
}
public string getSerializer (string tableName)
{
foreach (SerializationScheme ss in mySerializationScheme)
if (new Regex (ss.tableRegExp).Match (tableName).Success)
return ss.serializer;
return null;
}
public object[] getSerializerArgs(string tableName)
{
foreach (SerializationScheme ss in mySerializationScheme)
if (new Regex(ss.tableRegExp).Match(tableName).Success)
return ss.args;
return null;
}
public bool useSerializedBlueRows
{
get { return myUseSerializedBlueRows; }
set { myUseSerializedBlueRows = value; }
}
public bool SendBlueCommMonitor
{
get;
set;
}
public static DataStreamGatewaySettings instance
{
get
{
lock(myLockObj)
{
if (mySelf == null)
{
// ConfigurationRequest req = ConfigurationRequester.request(typeof(DataStreamGatewaySettings));
mySelf = new DataStreamGatewaySettings();
}
}
return mySelf;
}
}
}
}
@@ -0,0 +1,20 @@
using System;
namespace BCG
{
public class SerializationScheme
{
public string tableRegExp, serializer, subscriber;
public object[] args;
public SerializationScheme () {}
public SerializationScheme (string theTableRegExp, string theSerializer, string theSubscriber, object[] theArgs)
{
tableRegExp = theTableRegExp;
serializer = theSerializer;
subscriber = theSubscriber;
args = theArgs;
}
}
}
@@ -0,0 +1,155 @@
using System;
using System.Collections;
using System.Reflection;
//using BCG.Utilities.BlueConfiguration;
using BCG.Utilities;
namespace BCG
{
public class CorePrimtiveManager
{
private IDictionary myTypeLookupTable = new Hashtable ();
private static CorePrimtiveManager mySelf;
private ArrayList myAssemblyNames = new ArrayList ();
private CorePrimtiveManager()
{
myLoadAllTypes ();
}
private void myLoadAllTypes ()
{
//we only load one type here
loadAssembly ("X:\\bcg\\blue2\\Debug\\CppToCSharpDataStreamBridge.dll");
}
private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args)
{
foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies())
if (args.Name.Equals(a.FullName))
return a;
return null;
}
/// <summary>
/// Loads all the types associated with the assembly
/// </summary>
/// <param name="assemblyName"></param>
public void loadAssembly (string assemblyName)
{
if (myAssemblyNames.Contains (assemblyName))
return;
myAssemblyNames.Add (assemblyName);
Assembly loadedAssembly = GetType ().Assembly.FullName.Equals (assemblyName) ?
GetType ().Assembly :
Assembly.LoadFrom (assemblyName);
foreach (Type type in loadedAssembly.GetTypes())
if (!type.IsInterface
&& !type.IsAbstract
&& type.IsPublic)
{
bool isVisible = true;
string userRep = type.Name;
if (isVisible)
{
// BlueEventLog.logAssert(myTypeLookupTable [userRep] == null);
myTypeLookupTable[userRep] = type;
}
}
}
public static CorePrimtiveManager instance
{
get {return mySelf = mySelf == null ? new CorePrimtiveManager () : mySelf;}
}
/// <summary>
/// Returns the Type name corresponding to the primitive name
/// </summary>
/// <param name="friendlyName"></param>
/// <returns></returns>
public string find (string friendlyName)
{
if (myTypeLookupTable[friendlyName] == null)
throw new ApplicationException (friendlyName + " is not the friendly name of any of the loaded types");
return (myTypeLookupTable[friendlyName] as Type).Name;
}
/// <summary>
/// Constructs and returns an object of type corresponding to the friendly name
/// </summary>
/// <param name="friendlyName">friendly primitive name </param>
/// <param name="args">args can be null or an empty ICollection if the paramterless constructor
/// is the desired one</param>
/// <returns></returns>
public Object build (string friendlyName, ICollection args)
{
object[] constructorArgs = null;
if (args != null && args.Count > 0)
{
constructorArgs = new object [args.Count];
args.CopyTo (constructorArgs, 0);
}
if (myTypeLookupTable[friendlyName] == null)
throw new ApplicationException (friendlyName + " is not the friendly name of any of the loaded types");
return
constructorArgs == null ?
Activator.CreateInstance (myTypeLookupTable[friendlyName] as Type):
Activator.CreateInstance (myTypeLookupTable[friendlyName] as Type, constructorArgs);
}
/// <summary>
/// Returns the name of all the primitives
/// </summary>
/// <returns></returns>
public string[] getPrimitiveNames ()
{
string[] retVal = new string [myTypeLookupTable.Count];
myTypeLookupTable.Keys.CopyTo (retVal, 0);
return retVal;
}
/// <summary>
/// Return the types of all the streams
/// </summary>
/// <returns></returns>
public Type[] getStreamTypes ()
{
Type[] retVal = new Type [myTypeLookupTable.Count];
myTypeLookupTable.Values.CopyTo (retVal, 0);
return retVal;
}
public ICollection getConstructorSigns (string primitiveName)
{
ArrayList retVal = new ArrayList ();
// BlueEventLog.logAssert (myTypeLookupTable[primitiveName] != null);
Type t = myTypeLookupTable[primitiveName] as Type;
foreach (ConstructorInfo ci in t.GetConstructors())
{
string sign = "";
sign = t.Name + " (";
foreach (ParameterInfo pi in ci.GetParameters ())
sign += pi.ParameterType.Name +
" " +
pi.Name + ", ";
sign = sign.Length == (t.Name + " (").Length ? sign + ")": sign.Remove (sign.Length - 2, 2) + ")";
retVal.Add (sign);
}
return retVal;
}
public string getAssemblyName (string primitiveName)
{
// BlueEventLog.logAssert (myTypeLookupTable[primitiveName] != null);
return (myTypeLookupTable[primitiveName] as Type).Assembly.GetName().Name;
}
}
}
@@ -0,0 +1,20 @@
using System;
namespace BCG
{
public class PubStreamPrefixer
{
private static readonly string myPrefixString = "DS";
public static string prefix (string tableName)
{
return myPrefixString + "_" + tableName.ToLower();
}
public static string extractTableNameFromTag (string tag)
{
return tag.Substring ((myPrefixString + "_").Length);
}
}
}
@@ -0,0 +1,339 @@
using System;
using System.IO;
using System.Collections;
namespace BCG
{
public class SerializationUtilityMethods
{
private static IDictionary mySerializationDispatchTable,
myDeserializationDispatchTable;
private delegate void MySerialize (object data, BinaryWriter bw);
private delegate object MyDeserialize (Type t, BinaryReader br);
static SerializationUtilityMethods ()
{
mySerializationDispatchTable = new Hashtable ();
mySerializationDispatchTable [BlueDataValidTypes.BOOL.Name] =
new MySerialize (mySerializeBool);
mySerializationDispatchTable [BlueDataValidTypes.BYTE.Name] =
new MySerialize (mySerializeByte);
mySerializationDispatchTable [BlueDataValidTypes.CHAR.Name] =
new MySerialize (mySerializeChar);
mySerializationDispatchTable [BlueDataValidTypes.DATETIME.Name] =
new MySerialize (mySerializeDateTime);
mySerializationDispatchTable [BlueDataValidTypes.DECIMAL.Name] =
new MySerialize (mySerializeDecimal);
mySerializationDispatchTable [BlueDataValidTypes.DOUBLE.Name] =
new MySerialize (mySerializeDouble);
mySerializationDispatchTable [BlueDataValidTypes.INT.Name] =
new MySerialize (mySerializeInt32);
mySerializationDispatchTable [BlueDataValidTypes.ISERIALIZEDTYPE.Name] =
new MySerialize (mySerializeISerializedType);
mySerializationDispatchTable [BlueDataValidTypes.LONG.Name] =
new MySerialize (mySerializeLong);
mySerializationDispatchTable [BlueDataValidTypes.SBYTE.Name] =
new MySerialize (mySerializeSByte);
mySerializationDispatchTable [BlueDataValidTypes.STRING.Name] =
new MySerialize (mySerializeString);
mySerializationDispatchTable [BlueDataValidTypes.BYTEARRAY.Name] =
new MySerialize (mySerializeByteArray);
mySerializationDispatchTable [BlueDataValidTypes.UINT.Name] =
new MySerialize (mySerializeUInt);
mySerializationDispatchTable [BlueDataValidTypes.ULONG.Name] =
new MySerialize (mySerializeULong);
mySerializationDispatchTable [BlueDataValidTypes.SINGLE.Name] =
new MySerialize (mySerializeSingle);
myDeserializationDispatchTable = new Hashtable ();
myDeserializationDispatchTable [BlueDataValidTypes.BOOL.Name] =
new MyDeserialize (myDeserializeBool);
myDeserializationDispatchTable [BlueDataValidTypes.BYTE.Name] =
new MyDeserialize (myDeserializeByte);
myDeserializationDispatchTable [BlueDataValidTypes.CHAR.Name] =
new MyDeserialize (myDeserializeChar);
myDeserializationDispatchTable [BlueDataValidTypes.DATETIME.Name] =
new MyDeserialize (myDeserializeDateTime);
myDeserializationDispatchTable [BlueDataValidTypes.DECIMAL.Name] =
new MyDeserialize (myDeserializeDecimal);
myDeserializationDispatchTable [BlueDataValidTypes.DOUBLE.Name] =
new MyDeserialize (myDeserializeDouble);
myDeserializationDispatchTable [BlueDataValidTypes.INT.Name] =
new MyDeserialize (myDeserializeInt32);
myDeserializationDispatchTable [BlueDataValidTypes.ISERIALIZEDTYPE.Name] =
new MyDeserialize (myDeserializeISerializedType);
myDeserializationDispatchTable [BlueDataValidTypes.LONG.Name] =
new MyDeserialize (myDeserializeLong);
myDeserializationDispatchTable [BlueDataValidTypes.SBYTE.Name] =
new MyDeserialize (myDeserializeSByte);
myDeserializationDispatchTable [BlueDataValidTypes.STRING.Name] =
new MyDeserialize (myDeserializeString);
myDeserializationDispatchTable [BlueDataValidTypes.BYTEARRAY.Name] =
new MyDeserialize (myDeserializeByteArray);
myDeserializationDispatchTable [BlueDataValidTypes.UINT.Name] =
new MyDeserialize (myDeserializeUInt);
myDeserializationDispatchTable [BlueDataValidTypes.ULONG.Name] =
new MyDeserialize (myDeserializeULong);
myDeserializationDispatchTable [BlueDataValidTypes.SINGLE.Name] =
new MyDeserialize (myDeserializeSingle);
}
public static int getObjectSerializedSize (object o)
{
Type t = o.GetType ();
if (t == BlueDataValidTypes.STRING)
{
int l = (o as string).Length;
if (l < 128)
return l + 1;
if (l < 16384)
return l + 2;
if (l < 2097152)
return l + 3;
if (l < Math.Pow (2, 28))
return l + 4;
throw new ApplicationException ("String too big");
}
else if (t == BlueDataValidTypes.BYTEARRAY)
{
return getFixedLengthTypeSize (BlueDataValidTypes.INT) +
(o as byte[]).Length;
}
else if (isSerializedType (t))
{
return (o as ISerializedType).getSerializedSize ();
}
return getFixedLengthTypeSize (t);
}
public static int getFixedLengthTypeSize (Type t)
{
if (t == BlueDataValidTypes.BOOL ||
t == BlueDataValidTypes.BYTE ||
t == BlueDataValidTypes.CHAR ||
t == BlueDataValidTypes.SBYTE)
return 1;
else if (t == BlueDataValidTypes.INT ||
t == BlueDataValidTypes.UINT||
t == BlueDataValidTypes.SINGLE)
return 4;
else if (t == BlueDataValidTypes.DOUBLE ||
t == BlueDataValidTypes.LONG ||
t== BlueDataValidTypes.ULONG)
return 8;
else if (t == BlueDataValidTypes.DECIMAL)
return 16;
else if (t == BlueDataValidTypes.DATETIME)
//as objects of type DateTime are stored as doubles
return getFixedLengthTypeSize (typeof(long));
else
throw new ApplicationException (t.FullName + " is not a supported type");
}
public static bool isFixedLengthType (Type t)
{
return !(t.Equals (BlueDataValidTypes.STRING) || t.Equals (BlueDataValidTypes.BYTEARRAY) ||
isSerializedType (t));
}
public static bool isSerializedType (Type t)
{
return t.GetInterface ("BCG.ISerializedType") != null;
}
#region Serialization
public static void serializeValue (object data, BinaryWriter bw)
{
Type t = data.GetType ();
if (isSerializedType (t))
(mySerializationDispatchTable [typeof(ISerializedType).Name]
as MySerialize)(data, bw);
else
(mySerializationDispatchTable [t.Name] as MySerialize) (data, bw);
}
public static void serializeEnum(object data, BinaryWriter bw)
{
mySerializeInt32(data, bw);
}
private static void mySerializeInt32 (object data, BinaryWriter bw)
{
bw.Write ((Int32)data);
}
private static void mySerializeDouble (object data, BinaryWriter bw)
{
bw.Write ((double)data);
}
private static void mySerializeString (object data, BinaryWriter bw)
{
bw.Write (data as string);
}
private static void mySerializeByteArray (object data, BinaryWriter bw)
{
byte[] bs = data as byte[];
bw.Write (bs.Length);
bw.Write (bs);
}
private static void mySerializeSingle (object data, BinaryWriter bw)
{
bw.Write ((Single)data);
}
private static void mySerializeChar (object data, BinaryWriter bw)
{
bw.Write ((char)data);
}
private static void mySerializeLong (object data, BinaryWriter bw)
{
bw.Write ((long)data);
}
private static void mySerializeSByte (object data, BinaryWriter bw)
{
bw.Write ((sbyte)data);
}
private static void mySerializeBool (object data, BinaryWriter bw)
{
bw.Write ((bool)data);
}
private static void mySerializeByte (object data, BinaryWriter bw)
{
bw.Write ((byte)data);
}
private static void mySerializeDecimal (object data, BinaryWriter bw)
{
bw.Write ((decimal)data);
}
private static void mySerializeUInt (object data, BinaryWriter bw)
{
bw.Write ((uint)data);
}
private static void mySerializeULong (object data, BinaryWriter bw)
{
bw.Write ((ulong)data);
}
private static void mySerializeDateTime (object data, BinaryWriter bw)
{
//DateTime is written is serialiazed as OADATE
bw.Write (((DateTime)data).ToOADate());
}
private static void mySerializeISerializedType (object data, BinaryWriter bw)
{
(data as ISerializedType).serialize (bw);
}
#endregion
#region Deserialization
public static object deserializeValue(Type t, BinaryReader br)
{
return myDeserialize(t, br);
}
private static object myDeserialize (Type t, BinaryReader br)
{
return isSerializedType (t) ?
(myDeserializationDispatchTable [typeof(ISerializedType).Name]
as MyDeserialize)(t, br)
:
(myDeserializationDispatchTable [t.Name] as MyDeserialize) (t, br);
}
public static object deserializeEnum(Type t, BinaryReader br)
{
return myDeserializeInt32(t, br);
}
private static object myDeserializeInt32 (Type t, BinaryReader br)
{
return br.ReadInt32 ();
}
private static object myDeserializeISerializedType (Type t, BinaryReader br)
{
ISerializedType o = Activator.CreateInstance (t)
as ISerializedType;
o.deserialze (br);
return o;
}
private static object myDeserializeDouble (Type t, BinaryReader br)
{
return br.ReadDouble ();
}
private static object myDeserializeString (Type t, BinaryReader br)
{
return br.ReadString ();
}
private static object myDeserializeByteArray (Type t, BinaryReader br)
{
return br.ReadBytes (br.ReadInt32 ());
}
private static object myDeserializeSingle (Type t, BinaryReader br)
{
return br.ReadSingle ();
}
private static object myDeserializeChar (Type t, BinaryReader br)
{
return br.ReadChar ();
}
private static object myDeserializeLong (Type t, BinaryReader br)
{
return br.ReadInt64 ();
}
private static object myDeserializeSByte (Type t, BinaryReader br)
{
return br.ReadSByte ();
}
private static object myDeserializeBool (Type t, BinaryReader br)
{
return br.ReadBoolean ();
}
private static object myDeserializeByte (Type t, BinaryReader br)
{
return br.ReadByte ();
}
private static object myDeserializeDecimal (Type t, BinaryReader br)
{
return br.ReadDecimal ();
}
private static object myDeserializeUInt (Type t, BinaryReader br)
{
return br.ReadUInt32 ();
}
private static object myDeserializeDateTime (Type t, BinaryReader br)
{
return DateTime.FromOADate(br.ReadDouble());
//return new DateTime(br.ReadInt64());
}
private static object myDeserializeULong (Type t, BinaryReader br)
{
return br.ReadUInt64 ();
}
#endregion
}
}
@@ -0,0 +1,44 @@
using System;
namespace BCG
{
/// <summary>
/// The table publishing happens here
/// </summary>
public interface IPubTablePublisher
{
/// <summary>
/// Adds a row to the existing table
/// </summary>
/// <param name="row"></param>
void addRow (IBlueRow row);
/// <summary>
/// Deletes a row from an existing table
/// </summary>
/// <param name="row"></param>
void deleteRow (IBlueRow row);
/// <summary>
/// Changes an existing row
/// </summary>
/// <param name="row"></param>
void changeRow (IBlueRow row);
/// <summary>
/// Creates a table. if already exists overwrites it.
/// </summary>
/// <param name="rows"></param>
void createTable (IBlueRowCollection rows);
/// <summary>
/// Removes and adds more than one row; however the assumption here is that delete is
/// applied first and then add. If that violates the current state, exception is thrown
/// </summary>
/// <param name="delRows"></param>
/// <param name="addRows"></param>
void batchRemoveAdd (IBlueRowCollection delRows, IBlueRowCollection addRows);
/// <summary>
/// Changes more than one row at a time. This is specially helpful, since changes make a huge
/// percentage of the pubsub traffic.
/// </summary>
/// <param name="rows"></param>
void batchChange (IBlueRowCollection rows);
}
}
@@ -0,0 +1,175 @@
using System;
using System.Collections;
using System.IO;
namespace BCG
{
/// <summary>
/// The only instance of IPubTableBroker. It handles the publishing of one tag, and holds a specialized serializer
/// that it initializes according to the table, to optimize serialiazation, hence bw and performance
/// This class is held (as an interface) by the PubTableBroker, which guarantees the sanity of calls,
/// ie the first call will always be create.
/// </summary>
public class PubTablePublisher : IPubTablePublisher
{
protected IBlueDataSerializer mySerializer;
protected PublisherCore myPubCore;
public PubTablePublisher (PublisherCore publisherCore)
{
myPubCore = publisherCore;
mySerializer = new BlueDataSerializationController ();
}
#region IPubTablePublisher Members
public virtual void addRow (IBlueRow row)
{
byte[] data;
mySerializer.serialize (StreamOpCode.add,
myGetRowCollectionArray (row),
out data);
myPublish (data, row.schema.tableName);
}
public virtual void deleteRow (IBlueRow row)
{
byte[] data;
mySerializer.serialize (StreamOpCode.remove,
myGetRowCollectionArray (row),
out data);
myPublish (data, row.schema.tableName);
}
public virtual void changeRow(IBlueRow row)
{
byte[] data;
mySerializer.serialize (StreamOpCode.change,
myGetRowCollectionArray (row),
out data);
myPublish (data, row.schema.tableName);
}
public virtual void createTable (IBlueRowCollection rows)
{
byte[] data;
mySerializer.serialize (StreamOpCode.create,
new IBlueRowCollection [] {rows},
out data);
myPublish (data, rows.schema.tableName);
}
public virtual void batchRemoveAdd (IBlueRowCollection delRows, IBlueRowCollection addRows)
{
byte[] data;
if (myIsRealChange (delRows, addRows))
batchChange (addRows);
else
{
mySerializer.serialize (StreamOpCode.multipleRemoveAdd,
new IBlueRowCollection []{delRows, addRows},
out data);
myPublish (data, delRows.schema.tableName);
}
}
public virtual void batchChange (IBlueRowCollection rows)
{
byte[] data;
mySerializer.serialize (StreamOpCode.multipleChange,
new IBlueRowCollection [] {rows},
out data);
myPublish (data, rows.schema.tableName);
}
#endregion
#region private methods
private void myPublish (byte[] data, string tableName)
{
if (data.Length > 0)
myPubCore.publish ("DataStreamTag",
PubStreamPrefixer.prefix (tableName),
new MemoryStream (data, 0, data.Length, false, true));
}
private IBlueRowCollection[] myGetRowCollectionArray (IBlueRow r)
{
return new IBlueRowCollection []{
BlueDataUtility.toBlueRowCollection (new IBlueRow[] {r})
};
}
private bool myIsRealChange (IBlueRowCollection delRows, IBlueRowCollection addRows)
{
if (delRows.Count == addRows.Count)
{
foreach (IBlueRow dRow in delRows)
if (!addRows.containsID ((int)dRow["id"]))
return false;
return true;
}
else
return false;
}
#endregion
}
public class IndependentTablePublisher : PubTablePublisher
{
bool myIsCreated = false;
public IndependentTablePublisher (PublisherCore pubCore) : base (pubCore)
{}
public override void addRow (IBlueRow row)
{
myCheckCreate (row.schema);
base.addRow (row);
}
public override void batchChange (IBlueRowCollection rows)
{
myCheckCreate (rows.schema);
base.batchChange (rows);
}
public override void batchRemoveAdd (IBlueRowCollection delRows, IBlueRowCollection addRows)
{
myCheckCreate (delRows.schema);
base.batchRemoveAdd (delRows, addRows);
}
public override void changeRow (IBlueRow row)
{
myCheckCreate (row.schema);
base.changeRow (row);
}
public override void deleteRow (IBlueRow row)
{
myCheckCreate (row.schema);
base.deleteRow (row);
}
public override void createTable(IBlueRowCollection rows)
{
myIsCreated = true;
base.createTable (rows);
}
private void myCheckCreate (IBlueTableSchema schema)
{
if (!myIsCreated)
{
//so that the initiazliation work is done on the serializer
byte[] placeHolder;
mySerializer.serialize (StreamOpCode.create, new IBlueRowCollection [] {new BlueTable (schema)}, out placeHolder);
myIsCreated = true;
}
}
}
}
@@ -0,0 +1,65 @@
using System;
namespace BCG
{
/// <summary>
/// Deals with the publication of one tag. Gets the specific implementaion of IBlueDataSerializer according to
/// the table name, from the config, and wraps around it. The external code that will deal with user calls for
/// row operations (PubTablePublisher) will only deal with this class, hence will be agnostic of the specalized
/// serialization.
/// </summary>
public class BlueDataSerializationController : IBlueDataSerializer
{
private IBlueDataSerializer mySerializer;
public BlueDataSerializationController () {}
#region IBlueDataSerializer Members
/// <summary>
/// Serializes an op and data; however if a non create is called before a create,
/// the serializedData is set to an empty byte array.
/// </summary>
/// <param name="op"></param>
/// <param name="rows"></param>
/// <param name="serializedData"></param>
public void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData)
{
if (op == StreamOpCode.create)
{
if (mySerializer == null)
mySerializer = myGetSerializerForTable(rows[0].schema);
mySerializer.serialize (op, rows, out serializedData);
}
else
{
if (mySerializer != null)
{
mySerializer.serialize (op, rows, out serializedData);
}
else
{
serializedData = new byte[] {};
}
}
}
#endregion
private IBlueDataSerializer myGetSerializerForTable(IBlueTableSchema theSchema)
{
string serializerName = DataStreamGatewaySettings.instance.getSerializer(theSchema.tableName);
object[] serializerArgs = DataStreamGatewaySettings.instance.getSerializerArgs(theSchema.tableName);
if(serializerArgs != null)
return Activator.CreateInstance(Type.GetType(serializerName), theSchema, serializerArgs) as IBlueDataSerializer;
else
return Activator.CreateInstance(Type.GetType(serializerName), theSchema) as IBlueDataSerializer;
}
}
}
@@ -0,0 +1,111 @@
using System;
using System.Collections;
using System.IO;
namespace BCG
{
/// <summary>
/// Abstract base class that can potentially serve all implementations of IBlueDataSerializer. The IBlueRowSerializer
/// is used to serialize individual rows. The implementaions should, in the constructor, contruct the myRowSerializer
/// to do the specific row serializations. Every nonabtract implementation of the this, should have a contructor
/// taking in an IBlueTableSchema.
/// </summary>
public abstract class BlueDataSerializer : IBlueDataSerializer
{
protected IBlueRowSerializer myRowSerializer;
protected string myClientDeserializerName, myPubsubDeserializerName;
public BlueDataSerializer(string pubsubDeserializerName,
string clientDeserializerName)
{
myPubsubDeserializerName = pubsubDeserializerName;
myClientDeserializerName = clientDeserializerName;
}
#region IBlueDataSerializer Members
/// <summary>
/// Provides the following serialization
///
/// -add, remove, change:
/// op
/// serializedRow (depending on the implementation of myRowSerializer)
///
/// -create
/// op
/// pubsubDeserializer name
/// clientDeserializer name
/// schema
/// # of rows
/// serializedRows
///
/// -multipleAddRemoves
/// op
/// # of deleted rows
/// serializedRows
/// # of added rows
/// serializedRows
///
/// -multipleChanges
/// op
/// # of changed rows
/// serializedRows
/// </summary>
/// <param name="op"></param>
/// <param name="rows"></param>
/// <param name="serializedData"></param>
public void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData)
{
// BlueTrace.WriteSerializeStart(op, rows.Length);
BinaryWriter bw = new BinaryWriter(new MemoryStream());
if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change)
{
bw.Write(op);
myRowSerializer.serializeRow(BlueDataUtility.extractRow(rows[0], 0), op, bw);
}
else if (op == StreamOpCode.create)
{
IBlueRowCollection createdRows = rows[0];
bw.Write(op);
bw.Write(myPubsubDeserializerName);
bw.Write(myClientDeserializerName);
int dummyIdCol = 0;
int[] dummyIdList = new int[1];
//the schema doesn't sort anything right now, because the C++ pub sub client depends on things being in a certain order
IBlueTableSchema sortedSchema = BlueDataUtility.getSortedSchema(createdRows.schema, out dummyIdList, out dummyIdCol);
sortedSchema.serialize(bw);
bw.Write(createdRows.Count);
foreach (IBlueRow r in createdRows)
myRowSerializer.serializeRow(r, StreamOpCode.add, bw);
}
else if (op == StreamOpCode.multipleRemoveAdd)
{
bw.Write(op);
IBlueRowCollection delRows = rows[0],
addRows = rows[1];
bw.Write(delRows.Count);
foreach (IBlueRow dr in delRows)
myRowSerializer.serializeRow(dr, StreamOpCode.remove, bw);
bw.Write(addRows.Count);
foreach (IBlueRow ar in addRows)
myRowSerializer.serializeRow(ar, StreamOpCode.add, bw);
}
else if (op == StreamOpCode.multipleChange)
{
bw.Write(op);
IBlueRowCollection changedRows = rows[0];
bw.Write(changedRows.Count);
foreach (IBlueRow cr in changedRows)
myRowSerializer.serializeRow(cr, StreamOpCode.change, bw);
}
bw.BaseStream.Position = 0;
serializedData = (bw.BaseStream as MemoryStream).ToArray();
// BlueTrace.WriteSerializeStop(serializedData.Length);
}
#endregion
}
}
@@ -0,0 +1,21 @@
using System;
using System.Collections;
namespace BCG
{
/// <summary>
/// All the data serialized to be send over to PubSub happens here. All it does is f(op, rows) = byte[].
/// </summary>
public interface IBlueDataSerializer
{
/// <summary>
/// f(op, rows) = byte[]
/// </summary>
/// <param name="op">Operation that can be add, delete, change etc</param>
/// <param name="rows">The data to serialize. The only case when it will be an array of more than 1 element
/// is when there are multiple deletes and adds, ie StreamOpCode.multipleRemoveAdd.
/// </param>
/// <param name="serializedData">The data serialized as bytes to be send over the wire </param>
void serialize (byte op, IBlueRowCollection[] rows, out byte[] serializedData);
}
}
@@ -0,0 +1,23 @@
using System;
using System.IO;
namespace BCG
{
/// <summary>
/// Responsible for serilializing rows
/// The way the serialization scheme works is that the header is same
/// for all serialization algorithms, only the way the row is serialized is different. Abstracting it out and we
/// can have each specific serializer have a unique implementation of IBlueRowSerializer.
/// </summary>
public interface IBlueRowSerializer
{
/// <summary>
/// Serializes a row and writes it into the BinaryWriter. Since its only one row, the ops should be appropriate
/// for one row; the ones allowed are StreamopCode.add, remove and change only.
/// </summary>
/// <param name="r"></param>
/// <param name="bw"></param>
void serializeRow (IBlueRow r, byte op, BinaryWriter bw);
}
}
@@ -0,0 +1,96 @@
using System;
using System.IO;
namespace BCG
{
/// <summary>
/// Serializes primitive types. Contructs a hash based scheme based on the index, to make everythign O(1)
/// rather than searching it up and making it O(n), n = types supported.
/// </summary>
public class PrimitiveSerializationHelper
{
private delegate void SerializeDelegate (object data, BinaryWriter bw);
private SerializeDelegate[] myDispatchTable;
#region constructors
public PrimitiveSerializationHelper (IBlueTableSchema schema)
: this (schema.columnInfo) {}
public PrimitiveSerializationHelper (ColumnInfo[] cis)
{
myDispatchTable = new SerializeDelegate [cis.Length];
foreach (ColumnInfo ci in cis)
myDispatchTable[ci.colNo] = myGetDispatchMethod (ci.colType);
}
#endregion
#region public methods
/// <summary>
/// Serializes the column no in the row and writes it into the BinaryWriter
/// </summary>
/// <param name="r"></param>
/// <param name="colNo"></param>
/// <param name="bw"></param>
public void serializeColumn (IBlueRow r, int colNo, BinaryWriter bw)
{
myDispatchTable[colNo] (r[colNo], bw);
}
/// <summary>
/// Writes the data to the BinaryWriter. The ColNo is provided to get the appropriate
/// serializer.
/// </summary>
/// <param name="data"></param>
/// <param name="colNo"></param>
/// <param name="bw"></param>
public void serializeColumn (object data, int colNo, BinaryWriter bw)
{
myDispatchTable[colNo] (data, bw);
}
#endregion
#region private methods
private void mySerializeInt32 (object data, BinaryWriter bw) {bw.Write ((Int32)data);}
private void mySerializeDouble (object data, BinaryWriter bw) {bw.Write ((double)data);}
private void mySerializeString (object data, BinaryWriter bw){bw.Write (data as string);}
private void mySerializeByteArray (object data, BinaryWriter bw)
{
byte[] bs = data as byte[];
bw.Write (bs.Length);
bw.Write (bs);
}
private void mySerializeChar (object data, BinaryWriter bw) {bw.Write ((char)data);}
private void mySerializeLong (object data, BinaryWriter bw) {bw.Write ((long)data);}
private void mySerializeSByte (object data, BinaryWriter bw) {bw.Write ((sbyte)data);}
private void mySerializeBool (object data, BinaryWriter bw) {bw.Write ((bool)data);}
private void mySerializeByte (object data, BinaryWriter bw) {bw.Write ((byte)data);}
private void mySerializeDecimal (object data, BinaryWriter bw) {bw.Write ((decimal)data);}
private void mySerializeUInt (object data, BinaryWriter bw) {bw.Write ((uint)data);}
private void mySerializeULong (object data, BinaryWriter bw){bw.Write ((ulong)data);}
private void mySerializeDateTime (object data, BinaryWriter bw) {bw.Write (((DateTime)data).Ticks);}
private void mySerializeSingle (object data, BinaryWriter bw) {bw.Write ((Single)data);}
private void mySerializeISerializedType (object data, BinaryWriter bw) {(data as ISerializedType).serialize (bw);}
private SerializeDelegate myGetDispatchMethod (Type t)
{
if (t.Equals (BlueDataValidTypes.INT)) return new SerializeDelegate (mySerializeInt32);
else if (t.Equals (BlueDataValidTypes.DOUBLE)) return new SerializeDelegate (mySerializeDouble);
else if (t.Equals (BlueDataValidTypes.STRING)) return new SerializeDelegate (mySerializeString);
else if (t.Equals (BlueDataValidTypes.BYTEARRAY)) return new SerializeDelegate (mySerializeByteArray);
else if (t.Equals (BlueDataValidTypes.CHAR)) return new SerializeDelegate (mySerializeChar);
else if (t.Equals (BlueDataValidTypes.LONG)) return new SerializeDelegate (mySerializeLong);
else if (t.Equals (BlueDataValidTypes.SBYTE)) return new SerializeDelegate (mySerializeSByte);
else if (t.Equals (BlueDataValidTypes.BOOL)) return new SerializeDelegate (mySerializeBool);
else if (t.Equals (BlueDataValidTypes.BYTE)) return new SerializeDelegate (mySerializeByte);
else if (t.Equals (BlueDataValidTypes.DECIMAL)) return new SerializeDelegate (mySerializeDecimal);
else if (t.Equals (BlueDataValidTypes.UINT)) return new SerializeDelegate (mySerializeUInt);
else if (t.Equals (BlueDataValidTypes.ULONG)) return new SerializeDelegate (mySerializeULong);
else if (t.Equals (BlueDataValidTypes.DATETIME)) return new SerializeDelegate (mySerializeDateTime);
else if (t.Equals (BlueDataValidTypes.SINGLE)) return new SerializeDelegate (mySerializeSingle);
else if (SerializationUtilityMethods.isSerializedType (t)) return new SerializeDelegate (mySerializeISerializedType);
else throw new ApplicationException (t.FullName + " is not a supported type");
}
#endregion
}
}
@@ -0,0 +1,120 @@
using System;
using System.Collections;
using System.IO;
using BCG.Utilities;
using System.Collections.Generic;
namespace BCG
{
/// <summary>
/// Helps the UniversalSerializer by serializing one row. Hence this class is agnostic to the operation
/// which is figured out by UniversalSerializer.
///
/// it sorts the schema it gets, to make sure that the rows are always published the same way.
/// For now, the sorting operation doesn't do anything. The reason for this is that C++ clients
/// are depending upon the data arriving in a certain order.
///
/// It performs a mapping : (row)->byte[]
/// </summary
internal class UniversalRowSerializer : IBlueRowSerializer
{
private PrimitiveSerializationHelper myPrimitiveSerializationHelper;
private int myFixedSize;
private int[] myNonFixedColIndexes;
private IBlueTableSchema mySchema;
//this null map is the null map that will be written to the wire
//therefore, its values represent the nulls in row, ordered by the-rearranged schema.
private BitArray myNullMap;
private byte[] myNullMapArray; //just so that a new instance is not created everytime
//this thing stores a mapping between the columns in the original schema that were passed in
private int[] mySortedSchemaToOriginalSchemaMapping;
private int myIdColumnInSortedSchema; //this is used because the id column will be different in the original row and the schema
public UniversalRowSerializer(IBlueTableSchema schema)
{
mySchema = schema;
myPrimitiveSerializationHelper = new PrimitiveSerializationHelper(mySchema);
//therefore well use the original code
//sort the schema after we give it to the serialiaztion helper, because the helper constructs a dispatch table
//so it needs to know about the original schema.
mySchema = BlueDataUtility.getSortedSchema(mySchema, out mySortedSchemaToOriginalSchemaMapping, out myIdColumnInSortedSchema);
myNullMapArray =
new byte[Convert.ToInt32(Math.Ceiling(Convert.ToDouble(mySchema.columnInfo.Length) / 8))];
myNullMap =
new BitArray(myNullMapArray);
myInitSizeCalculations();
}
#region IBlueRowSerializer methods
/// <summary>
/// Row serialized in the following order
/// row size (nullBitMap size + actual row size)
/// null bit map
/// non null row members (with id as the first col)
/// </summary>
/// <param name="r"></param>
/// <param name="bw"></param>
public void serializeRow(IBlueRow r, byte op, BinaryWriter bw)
{
myNullMap.SetAll(false);
for (int i = 0; i < mySchema.columnNames.Length; i++)
if (r[mySortedSchemaToOriginalSchemaMapping[i]] == DBNull.Value || r[mySortedSchemaToOriginalSchemaMapping[i]] == null)
myNullMap[i] = true;
myNullMap.CopyTo(myNullMapArray, 0);
long initPos = bw.BaseStream.Position;
bw.Write((int)0); ///place holder for size
bw.Write(myNullMapArray);
bw.Write((int)r[mySortedSchemaToOriginalSchemaMapping[myIdColumnInSortedSchema]]);
foreach (ColumnInfo ci in mySchema.columnInfo)
if (!myNullMap[ci.colNo] && ci.colNo != myIdColumnInSortedSchema)
myPrimitiveSerializationHelper.serializeColumn(r, mySortedSchemaToOriginalSchemaMapping[ci.colNo], bw);
long endPos = bw.BaseStream.Position;
bw.BaseStream.Position = initPos;
bw.Write(Convert.ToInt32(endPos - (initPos+4))); /// size does not include the meta data hence substracting 4
bw.BaseStream.Position = endPos;
}
public void serializeSchema(BinaryWriter bw)
{
mySchema.serialize(bw);
}
#endregion
#region size calculations
/// <summary>
/// Sets up the fixed length (size in bytes occupied by each of the primitive type)
/// and an index for the non primitive type
/// </summary>
private void myInitSizeCalculations()
{
ArrayList temp = new ArrayList();
myFixedSize = myNullMapArray.Length;
foreach (ColumnInfo ci in mySchema.columnInfo)
{
if (!SerializationUtilityMethods.isFixedLengthType(ci.colType))
temp.Add(ci.colNo);
else
myFixedSize += SerializationUtilityMethods.getFixedLengthTypeSize(ci.colType);
}
myNonFixedColIndexes = temp.ToArray(typeof(int)) as int[];
}
/// <summary>
/// Find the size of the serialized row, assumes that nullMap has already
/// been built
/// </summary>
/// <param name="fixedSize"></param>
/// <param name="r"></param>
/// <returns></returns>
private int myGetRowSizeInBytes(int fixedSize, IBlueRow r)
{
foreach (int colNo in myNonFixedColIndexes)
{
if (!myNullMap[colNo])
fixedSize += //myGetStringSizeInBytes (r[colNo] as string);
SerializationUtilityMethods.getObjectSerializedSize(r[mySortedSchemaToOriginalSchemaMapping[colNo]]);
}
return fixedSize;
}
#endregion
}
}
@@ -0,0 +1,16 @@
using System;
namespace BCG
{
/// <summary>
/// Uses the UniversalRowSerializer as the IBlueRowSerializer implementation.
/// </summary>
public class UniversalSerializer : BlueDataSerializer
{
public UniversalSerializer (IBlueTableSchema schema)
: base ("BCG.PubsubUniversalDeserializer, DataStreamGateway", "BCG.ClientUniversalDeserializer")
{
myRowSerializer = new UniversalRowSerializer (schema);
}
}
}
@@ -0,0 +1,15 @@
using System;
namespace BCG
{
/// <summary>
/// Exception thrown if the publishing violates the state of table, ex deleted row
/// does not exist in the table
/// </summary>
public class TablePublishingException : ApplicationException
{
public TablePublishingException (string message)
: base (message)
{}
}
}
@@ -0,0 +1,25 @@
using System;
using System.IO;
using System.Collections;
namespace BCG
{
/// <summary>
/// Deserializes a row. Returns the id of the row and its full content
/// </summary>
public interface IPubsubRowDeserializer
{
/// <summary>
/// Reads the data from the BinaryReader and deserializes it in the form of a row
/// </summary>
/// <param name="currentState">Some deserializers might need the current state, ie a Dictionary of rows
/// keyed off the id</param>
/// <param name="br">BinaryReader starting at the row</param>
/// <param name="op">The operation, as this might drive the deserializations scheme in some cases. This op
/// however is one of the basics operations, add remove or change as all other operations can be resolved into this</param>
/// <param name="id">The id of the deserialized row</param>
/// <param name="row">The actual row contents</param>
void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row);
}
}
@@ -0,0 +1,166 @@
using System;
using System.Collections;
using System.IO;
using BCG.Utilities;
using System.Text;
namespace BCG
{
/// <summary>
/// Base class for all deserializers.
/// </summary>
public abstract class PubsubDeserializer : IPubsubDeserializer
{
protected IDictionary myState = new Hashtable ();
protected IBlueTableSchema mySchema;
protected string myClientDeserializerName; //for the getState method
protected IPubsubRowDeserializer myDeserializer;
public PubsubDeserializer () {}
#region IPubSubDeserializer Members
public virtual void deserialize (Stream[] streams, out byte[] theOps, out Array theRows)
{
BinaryReader br = new BinaryReader (UtilityMethods.getAggregatedStream (streams));
ArrayList ops = new ArrayList (), rows = new ArrayList ();
while (true)
{
IDictionary[] result;
byte op;
try
{
int value = br.BaseStream.ReadByte();
if (value == -1) break;
op = (byte)value;
}
catch (EndOfStreamException)
{
break;
}
if (op == StreamOpCode.create)
{
myState.Clear ();
br.ReadString (); //the pubsub-deserializer name, should be the same as this's name
myClientDeserializerName = br.ReadString (); //the client-deserializer name
(mySchema = new BlueTableSchema()).deserialze (br);
result = new IDictionary[] {myGetRows (br, op, br.ReadInt32 ())};
myEncorporatRows (result[0], op);
ops.Add (op);
rows.Add (result);
}
else
{
if (mySchema != null)
{
if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
{
result = new IDictionary[] {myGetRows (br, op, 1)};
myEncorporatRows (result[0], op);
ops.Add (op);
rows.Add (result);
}
else if (op == StreamOpCode.multipleChange)
{
result = new IDictionary[] {myGetRows (br, op, br.ReadInt32 ())};
myEncorporatRows (result[0], op);
ops.Add (op);
rows.Add (result);
}
else if (op == StreamOpCode.multipleRemoveAdd)
{
ops.Add (op);
result = new IDictionary [] {
myGetRows (br, StreamOpCode.remove, br.ReadInt32 ()),
myGetRows (br, StreamOpCode.add, br.ReadInt32 ())
};
myEncorporatRows (result[0], StreamOpCode.remove);
myEncorporatRows (result[1], StreamOpCode.add);
rows.Add (result);
}
else
{
// BlueEventLog.logEvent ("Illegal op-code received", BlueEventLevel.Error);
}
}
else
{
// BlueEventLog.logEvent ("Received a non create operation even though the schema" +
// "has never been created", BlueEventLevel.Warning);
}
}
}
theOps = ops.ToArray (typeof(byte)) as byte[];
theRows = rows.ToArray ();
}
/// <summary>
/// Sends in a create signal, so that new clients can get a create everytime and start
/// afresh from there
/// </summary>
/// <returns></returns>
public virtual byte[] getState()
{
if (mySchema == null)
return new byte[] {};
else
{
BinaryWriter bw = new BinaryWriter (new MemoryStream ());
bw.Write (StreamOpCode.create);
bw.Write (GetType().AssemblyQualifiedName);
bw.Write (myClientDeserializerName);
mySchema.serialize (bw);
bw.Write (myState.Count);
foreach (byte[] row in myState.Values)
bw.Write (row);
return (bw.BaseStream as MemoryStream).ToArray ();
}
}
#endregion
/// <summary>
/// Every row that has been deserialized passes through this method, so that it can update the state.
/// </summary>
/// <param name="rows"></param>
/// <param name="op"></param>
protected virtual void myEncorporatRows (IDictionary rows, byte op)
{
if (op == StreamOpCode.add || op == StreamOpCode.create)
foreach (DictionaryEntry de in rows)
{
if (myState.Contains(de.Key)) {
// BlueEventLog2.instance().logEvent(String.Format("Duplicate row creation. t: {0} id:{1}", mySchema.tableName, de.Key),BlueEventLog2Level.Fatal);
StringBuilder b = new StringBuilder();
foreach (string n in mySchema.columnNames)
b.AppendFormat(" {0}", n);
// BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()), BlueEventLog2Level.Fatal);
}
myState.Add(de.Key, de.Value);
}
else if (op == StreamOpCode.remove)
foreach (DictionaryEntry de in rows)
{
if (!myState.Contains(de.Key))
{
// BlueEventLog2.instance().logEvent(String.Format("Unknown row deletion. t: {0} id:{1}", mySchema.tableName, de.Key), BlueEventLog2Level.Fatal);
StringBuilder b = new StringBuilder();
foreach (string n in mySchema.columnNames)
b.AppendFormat(" {0}", n);
// BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()),BlueEventLog2Level.Fatal);
}
myState.Remove(de.Key);
}
else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange)
foreach (DictionaryEntry de in rows)
myState[de.Key] = de.Value;
}
/// <summary>
/// Mass deserialization happens here, also since there is no knowledge of the schema
/// </summary>
/// <param name="br"></param>
/// <param name="op"></param>
/// <param name="noOfRows"></param>
/// <returns></returns>
protected abstract IDictionary myGetRows (BinaryReader br, byte op, int noOfRows);
}
}
@@ -0,0 +1,25 @@
using System;
using System.Collections;
using System.IO;
namespace BCG
{
public class PubsubUniversalDeserializer : PubsubDeserializer
{
public PubsubUniversalDeserializer() {}
protected override IDictionary myGetRows (BinaryReader br, byte op, int noOfRows)
{
myDeserializer = myDeserializer == null ? new PubsubUniversalRowDeserializer (mySchema) : myDeserializer;
Hashtable retVal = new Hashtable ();
for (int i = 0; i < noOfRows; i++)
{
int id; object row;
myDeserializer.deserializeRow (myState, br, op, out id, out row);
retVal.Add (id, row);
}
return retVal;
}
}
}
@@ -0,0 +1,34 @@
using System;
using System.IO;
using System.Collections;
namespace BCG
{
public class PubsubUniversalRowDeserializer : IPubsubRowDeserializer
{
private int myNullMapLength;
private IBlueTableSchema mySchema;
private int myRowOverheadSize = SerializationUtilityMethods.getFixedLengthTypeSize (typeof(int));
public PubsubUniversalRowDeserializer (IBlueTableSchema schema)
{
mySchema = schema;
myNullMapLength = Convert.ToInt32((Math.Ceiling(Convert.ToDouble(mySchema.columnNames.Length)/8)));
}
#region IRowDeserializer Members
public void deserializeRow (IDictionary currentState, BinaryReader br, byte op, out int id, out object row)
{
//current state is ignored
long pos = br.BaseStream.Position;
int rowSize = br.ReadInt32 ();
br.BaseStream.Position = pos;
row = br.ReadBytes (rowSize + myRowOverheadSize);
id = BitConverter.ToInt32 (row as byte[], myNullMapLength + myRowOverheadSize);
}
#endregion
}
}
@@ -0,0 +1,57 @@
using System;
using System.IO;
using System.Collections;
using BCG.Utilities;
namespace BCG
{
/// <summary>
/// Keeps a collection of TagStateBanks to hold the state of each tag
/// </summary>
public class PubsubServerTagHandler : IPubSubServerTagHandler
{
private IDictionary myTagStates = new Hashtable ();
private bool myPublishStatistics = //DataStreamGatewaySettings.instance.enablePubsubStats;
false;
public PubsubServerTagHandler() {}
#region IPubSubServerTagHandler Members
public byte[] getContent (string tag)
{
TagStateBank stateBank = myTagStates[tag] as TagStateBank;
if (stateBank != null)
{
return stateBank.getConent ();
}
else
{
// BlueEventLog.logEvent (String.Format ("Information requested for tag {0} which does not exist now", tag), BlueEventLevel.Warning);
return new byte[] {};
}
}
public void update (string tag, Stream[] messageBodies)
{
if (myPublishStatistics)
{
StatisticsLogger.instance.log(tag, messageBodies);
}
if (!myTagStates.Contains (tag))
{
// BlueEventLog.logEvent (String.Format ("Tag {0} received for the first time and its state created", tag), BlueEventLevel.Information);
myTagStates.Add (tag, new TagStateBank ());
}
(myTagStates[tag] as TagStateBank).update (messageBodies);
}
public void doWork() {}
public string[] supportedTagTypes {get {return new string [] {"DataStreamTag"};}}
public void register(InternalPublishDelegate callback) { }
#endregion
}
}
@@ -0,0 +1,102 @@
using System;
using System.IO;
using System.Collections;
using BCG.Utilities;
using System.Threading;
namespace BCG
{
class TagData
{
public DateTime lastUpdateTime;
public long totalSize;
public int lastUpdateSize;
public TagData(DateTime ut, long ts, int lus)
{
lastUpdateTime = ut;
totalSize = ts;
lastUpdateSize = lus;
}
public override string ToString()
{
return lastUpdateTime.ToString() + "," + totalSize.ToString() + "," + lastUpdateSize.ToString();
}
}
class StatisticsLogger
{
private IDictionary myTagData = new Hashtable();
private static StatisticsLogger mySelf;
private BlockingQueue myQueue = new BlockingQueue();
private DateTime myLastFireTime = DateTime.MinValue;
private TimeSpan myPubsubStatsUpdateInterval = new TimeSpan(0, 0, 0, 0, DataStreamGatewaySettings.instance.pubsubStatsUpdateInterval);
private TimeSpan myTagStatsUpdateInterval = new TimeSpan(0, 0, 0, 0, DataStreamGatewaySettings.instance.singleTagStatsUpdateInterval);
private StatisticsLogger()
{
Thread readThread = new Thread(new ThreadStart(myReadAndPublish));
readThread.Priority = ThreadPriority.BelowNormal;
readThread.Start();
}
public static StatisticsLogger instance
{
get
{
if (mySelf == null)
mySelf = new StatisticsLogger();
return mySelf;
}
}
public void log(string tag, Stream[] bodies)
{
long nBytes = 0;
foreach (Stream s in bodies)
nBytes += s.Length;
myQueue.enqueue(new object[] { tag, Convert.ToInt32 (nBytes) });
}
private void myReadAndPublish()
{
while (true)
{
object qe = myQueue.dequeue(50);
if (qe != null)
{
object[] args = qe as object[];
string tag = args[0].ToString();
int dataSize = (int)args[1];
if (!myTagData.Contains(tag))
{
myTagData.Add(tag, new TagData(DateTime.Now, Convert.ToInt64(dataSize), dataSize));
}
else
{
TagData tdToSend = myTagData[tag] as TagData;
tdToSend.lastUpdateSize += dataSize;
tdToSend.totalSize += dataSize;
}
}
myFireMonitors();
}
}
private void myFireMonitors()
{
if (DateTime.Now.Subtract(myLastFireTime) >= myPubsubStatsUpdateInterval)
{
foreach (string tag in myTagData.Keys)
{
TagData td = myTagData[tag] as TagData;
if (td.lastUpdateSize > 0 && DateTime.Now.Subtract(td.lastUpdateTime) >= myTagStatsUpdateInterval)
{
td.lastUpdateTime = DateTime.Now;
td.lastUpdateSize = 0;
}
}
myLastFireTime = DateTime.Now;
}
}
}
}
@@ -0,0 +1,26 @@
using System;
using System.Collections;
using System.IO;
namespace BCG
{
public class TagStateBank
{
//private IDictionary myState = new Hashtable ();
private PubsubDeserializationController myDeserializationController = new PubsubDeserializationController ();
public TagStateBank () {}
public byte[] getConent ()
{
return myDeserializationController.getState ();
}
public void update (Stream[] streams)
{
byte[] ops;
Array rowCollection;
myDeserializationController.deserialize (streams, out ops, out rowCollection);
}
}
}
@@ -0,0 +1,146 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using BCG.Utilities;
namespace BCG
{
public class CompressionClientDeserializer : IClientDeserializer
{
protected IBlueTableSchema mySchema;
protected IBlueTableSchema myFirstSchema;
protected IClientRowDeserializer myRowDeserializer;
public CompressionClientDeserializer()
{
myFirstSchema = null;
mySchema = null;
}
#region IClientDeserializer Members
public void deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
{
BinaryReader br = new BinaryReader(UtilityMethods.getAggregatedStream(streams));
// BlueTrace.WriteDeserializeStart(streams.Length, br.BaseStream.Length);
// emit # of streams and total length
ArrayList ops = new ArrayList(), rows = new ArrayList();
IBlueRowCollection[] result;
while (true)
{
byte op;
try
{
int value = br.BaseStream.ReadByte();
if (value == -1) break;
op = (byte)value;
// BlueTrace.WriteDeserializeOperation(op);
}
catch (EndOfStreamException)
{
break;
}
if (op == StreamOpCode.create)
{
br.ReadString(); //pubsubDeserializer
br.ReadString(); //clientDeserializer
mySchema = new BlueTableSchema();
mySchema.deserialze(br);
//keep track of the first schema you've ever seen, so you know how to reorder columns
if (myFirstSchema == null)
myFirstSchema = mySchema;
/// decompress here
int compressedMessageSize = br.ReadInt32();
byte[] compressedData = br.ReadBytes(compressedMessageSize);
BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(compressedData));
result = new IBlueRowCollection[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) };
myEncorporateRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else
{
if (mySchema != null)
{
if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
{
result = new IBlueRowCollection[] { myGetRows(br, op, 1) };
myEncorporateRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else if (op == StreamOpCode.multipleChange)
{
int compressedMessageSize = br.ReadInt32();
byte[] compressedData = br.ReadBytes(compressedMessageSize);
BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(compressedData));
result = new IBlueRowCollection[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) };
myEncorporateRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else if (op == StreamOpCode.multipleRemoveAdd)
{
int compressedMessageSize = br.ReadInt32();
byte[] compressedData = br.ReadBytes(compressedMessageSize);
BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(compressedData));
result =
new IBlueRowCollection[] {
myGetRows (decompressedBr, StreamOpCode.remove, decompressedBr.ReadInt32 ()),
myGetRows (decompressedBr, StreamOpCode.add, decompressedBr.ReadInt32 ())
};
myEncorporateRows(result[0], StreamOpCode.remove);
myEncorporateRows(result[1], StreamOpCode.add);
ops.Add(op);
rows.Add(result);
}
else
{
// BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error);
}
}
else
{
// BlueEventLog.logEvent("Received a non create operation even though the schema" +
// "has never been created", BlueEventLevel.Warning);
}
}
}
theOps = ops.ToArray(typeof(byte)) as byte[];
theRows = rows.ToArray();
// BlueTrace.WriteDeserializeStop(rows.Count);
}
#endregion
private IBlueRowCollection myGetRows(BinaryReader br, byte op, int nRows)
{
BlueTable retVal = new BlueTable(mySchema);
if (myRowDeserializer == null)
myRowDeserializer = new ClientUniversalRowDeserializer(mySchema);
else if (op == StreamOpCode.create)
myRowDeserializer = new ClientReorderingRowDeserializer(myFirstSchema, mySchema);
for (int i = 0; i < nRows; i++)
{
IBlueRow row;
myRowDeserializer.deserializeRow(null, br, op, out row);
retVal.addRow(row);
}
return retVal;
}
private void myEncorporateRows(IBlueRowCollection rows, byte op) { }
}
}
@@ -0,0 +1,192 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Collections;
using BCG.Utilities;
namespace BCG
{
public class CompressionPubsubDeserializer : IPubsubDeserializer
{
protected IDictionary myState = new Hashtable();
protected IBlueTableSchema mySchema;
protected string myClientDeserializerName; //for the getState method
protected IPubsubRowDeserializer myRowDeserializer;
public CompressionPubsubDeserializer() { }
#region IPubSubDeserializer Members
public virtual void deserialize(Stream[] streams, out byte[] theOps, out Array theRows)
{
BinaryReader br = new BinaryReader(UtilityMethods.getAggregatedStream(streams));
ArrayList ops = new ArrayList(), rows = new ArrayList();
while (true)
{
IDictionary[] result;
byte op;
try
{
int value = br.BaseStream.ReadByte();
if (value == -1) break;
op = (byte)value;
}
catch (EndOfStreamException)
{
break;
}
if (op == StreamOpCode.create)
{
myState.Clear();
br.ReadString(); //the pubsub-deserializer name, should be the same as this's name
myClientDeserializerName = br.ReadString(); //the client-deserializer name
(mySchema = new BlueTableSchema()).deserialze(br);
/// deserialize this part
int compressedMessageSize = br.ReadInt32();
BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(br.ReadBytes(compressedMessageSize)));
result = new IDictionary[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) };
myEncorporatRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else
{
if (mySchema != null)
{
if (op == StreamOpCode.add || op == StreamOpCode.change || op == StreamOpCode.remove)
{
result = new IDictionary[] { myGetRows(br, op, 1) };
myEncorporatRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else if (op == StreamOpCode.multipleChange)
{
int compressedMessageSize = br.ReadInt32();
BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(br.ReadBytes(compressedMessageSize)));
result = new IDictionary[] { myGetRows(decompressedBr, op, decompressedBr.ReadInt32()) };
myEncorporatRows(result[0], op);
ops.Add(op);
rows.Add(result);
}
else if (op == StreamOpCode.multipleRemoveAdd)
{
ops.Add(op);
int compressedMessageSize = br.ReadInt32();
BinaryReader decompressedBr = new BinaryReader(UtilityMethods.decompress(br.ReadBytes(compressedMessageSize)));
result = new IDictionary[] {
myGetRows (decompressedBr, StreamOpCode.remove, decompressedBr.ReadInt32 ()),
myGetRows (decompressedBr, StreamOpCode.add, decompressedBr.ReadInt32 ())
};
myEncorporatRows(result[0], StreamOpCode.remove);
myEncorporatRows(result[1], StreamOpCode.add);
rows.Add(result);
}
else
{
// BlueEventLog.logEvent("Illegal op-code received", BlueEventLevel.Error);
}
}
else
{
// BlueEventLog.logEvent("Received a non create operation even though the schema" +
// "has never been created", BlueEventLevel.Warning);
}
}
}
theOps = ops.ToArray(typeof(byte)) as byte[];
theRows = rows.ToArray();
}
/// <summary>
/// Sends in a create signal, so that new clients can get a create everytime and start
/// afresh from there
/// </summary>
/// <returns></returns>
public virtual byte[] getState()
{
if (mySchema == null)
return new byte[] { };
else
{
BinaryWriter bw = new BinaryWriter(new MemoryStream());
bw.Write(StreamOpCode.create);
bw.Write(GetType().AssemblyQualifiedName);
bw.Write(myClientDeserializerName);
mySchema.serialize(bw);
/// compress it here
BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream());
decompressedBw.Write(myState.Count);
foreach (byte[] row in myState.Values)
decompressedBw.Write(row);
decompressedBw.BaseStream.Position = 0;
byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream);
bw.Write(compressedData.Length);
bw.Write(compressedData);
return (bw.BaseStream as MemoryStream).ToArray();
}
}
#endregion
/// <summary>
/// Every row that has been deserialized passes through this method, so that it can update the state.
/// </summary>
/// <param name="rows"></param>
/// <param name="op"></param>
protected virtual void myEncorporatRows(IDictionary rows, byte op)
{
if (op == StreamOpCode.add || op == StreamOpCode.create)
foreach (DictionaryEntry de in rows)
{
if (myState.Contains(de.Key))
{
// BlueEventLog2.instance().logEvent(String.Format("Duplicate row creation. t: {0} id:{1}", mySchema.tableName, de.Key), BlueEventLog2Level.Fatal);
StringBuilder b = new StringBuilder();
foreach (string n in mySchema.columnNames)
b.AppendFormat(" {0}", n);
// BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()), BlueEventLog2Level.Fatal);
}
myState.Add(de.Key, de.Value);
}
else if (op == StreamOpCode.remove)
foreach (DictionaryEntry de in rows)
{
if (!myState.Contains(de.Key))
{
// BlueEventLog2.instance().logEvent(String.Format("Unknown row deletion. t: {0} id:{1}", mySchema.tableName, de.Key), BlueEventLog2Level.Fatal);
StringBuilder b = new StringBuilder();
foreach (string n in mySchema.columnNames)
b.AppendFormat(" {0}", n);
// BlueEventLog2.instance().logEvent(String.Format("Schema:{0}", b.ToString()), BlueEventLog2Level.Fatal);
}
myState.Remove(de.Key);
}
else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange)
foreach (DictionaryEntry de in rows)
myState[de.Key] = de.Value;
}
/// <summary>
/// Mass deserialization happens here, also since there is no knowledge of the schema
/// </summary>
/// <param name="br"></param>
/// <param name="op"></param>
/// <param name="noOfRows"></param>
/// <returns></returns>
private IDictionary myGetRows (BinaryReader br, byte op, int noOfRows)
{
myRowDeserializer = myRowDeserializer == null ? new PubsubUniversalRowDeserializer (mySchema) : myRowDeserializer;
Hashtable retVal = new Hashtable ();
for (int i = 0; i < noOfRows; i++)
{
int id; object row;
myRowDeserializer.deserializeRow (myState, br, op, out id, out row);
retVal.Add (id, row);
}
return retVal;
}
}
}
@@ -0,0 +1,90 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
namespace BCG
{
public class CompressionSerializer : IBlueDataSerializer
{
private IBlueRowSerializer myRowSerializer;
private string myPubsubDeserializerName = "BCG.CompressionPubsubDeserializer, DataStreamGateway", myClientDeserializerName = "BCG.CompressionClientDeserializer";
public CompressionSerializer (IBlueTableSchema schema)
{
myRowSerializer = new UniversalRowSerializer (schema);
}
public void serialize(byte op, IBlueRowCollection[] rows, out byte[] serializedData)
{
// BlueTrace.WriteSerializeStart(op, rows.Length);
BinaryWriter bw = new BinaryWriter(new MemoryStream());
if (op == StreamOpCode.add || op == StreamOpCode.remove || op == StreamOpCode.change)
{
bw.Write(op);
/// no need to compress the only row
myRowSerializer.serializeRow(BlueDataUtility.extractRow(rows[0], 0), op, bw);
}
else if (op == StreamOpCode.create)
{
IBlueRowCollection createdRows = rows[0];
bw.Write(op);
bw.Write(myPubsubDeserializerName);
bw.Write(myClientDeserializerName);
int dummyIdCol = 0;
int[] dummyIdList = new int[1];
//the schema doesn't sort anything right now, because the C++ pub sub client depends on things being in a certain order
IBlueTableSchema sortedSchema = BlueDataUtility.getSortedSchema(createdRows.schema, out dummyIdList, out dummyIdCol);
sortedSchema.serialize(bw);
/// compress it here
BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream());
decompressedBw.Write(createdRows.Count);
foreach (IBlueRow r in createdRows)
myRowSerializer.serializeRow(r, StreamOpCode.add, decompressedBw);
decompressedBw.BaseStream.Position = 0;
byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream);
bw.Write(compressedData.Length);
bw.Write(compressedData);
}
else if (op == StreamOpCode.multipleRemoveAdd)
{
bw.Write(op);
IBlueRowCollection delRows = rows[0], addRows = rows[1];
/// compress it here
BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream());
decompressedBw.Write(delRows.Count);
foreach (IBlueRow dr in delRows)
myRowSerializer.serializeRow(dr, StreamOpCode.remove, decompressedBw);
decompressedBw.Write(addRows.Count);
foreach (IBlueRow ar in addRows)
myRowSerializer.serializeRow(ar, StreamOpCode.add, decompressedBw);
decompressedBw.BaseStream.Position = 0;
byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream);
bw.Write(compressedData.Length);
bw.Write(compressedData);
}
else if (op == StreamOpCode.multipleChange)
{
bw.Write(op);
IBlueRowCollection changedRows = rows[0];
/// compress it here
BinaryWriter decompressedBw = new BinaryWriter(new MemoryStream());
decompressedBw.Write(changedRows.Count);
foreach (IBlueRow cr in changedRows)
myRowSerializer.serializeRow(cr, StreamOpCode.change, decompressedBw);
decompressedBw.BaseStream.Position = 0;
byte[] compressedData = UtilityMethods.compress(decompressedBw.BaseStream);
bw.Write(compressedData.Length);
bw.Write(compressedData);
}
bw.BaseStream.Position = 0;
serializedData = (bw.BaseStream as MemoryStream).ToArray();
// BlueTrace.WriteSerializeStop(serializedData.Length);
}
}
}
@@ -0,0 +1,11 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BCG
{
public class BBOOptionDeserializer : ClientFixedDeserializer
{
public BBOOptionDeserializer() { }
}
}
@@ -0,0 +1,49 @@
using System;
using BCG.Utilities;
namespace BCG
{
public abstract class ClientFixedDeserializer : ClientDeserializer
{
private BlueTable myCurrentState;
public ClientFixedDeserializer () {}
protected override IBlueRowCollection myGetRows (System.IO.BinaryReader br, byte op, int nRows)
{
BlueTable retVal = new BlueTable (mySchema);
op = op == StreamOpCode.create ? StreamOpCode.add :
op == StreamOpCode.multipleChange ? StreamOpCode.change : op;
if (myRowDeserializer == null)
{
myRowDeserializer = new ClientFixedRowDeserializer (mySchema);
myCurrentState = new BlueTable (mySchema);
}
for (int i = 0; i < nRows; i++)
{
IBlueRow r;
myRowDeserializer.deserializeRow (myCurrentState, br, op, out r);
retVal.addRow (r);
}
return retVal;
}
protected override void myEncorporateRows (IBlueRowCollection rows, byte op)
{
if (op == StreamOpCode.create)
myCurrentState = new BlueTable (rows);
else if (op == StreamOpCode.add)
myCurrentState.addRows (rows);
else if (op == StreamOpCode.remove)
myCurrentState.deleteRows (rows);
else if (op == StreamOpCode.change || op == StreamOpCode.multipleChange)
myCurrentState.changeRows (rows);
else
{
// BlueEventLog.logAssert (false, "Illegal operation for this context");
}
}
}
}
@@ -0,0 +1,38 @@
using System;
using System.IO;
using BCG.Utilities;
namespace BCG
{
public class ClientFixedRowDeserializer : IClientRowDeserializer
{
private PrimitiveDeserializationHelper myHelper;
private IBlueTableSchema mySchema;
public ClientFixedRowDeserializer (IBlueTableSchema schema)
{
myHelper = new PrimitiveDeserializationHelper (mySchema = schema);
}
#region IClientRowDeserializer Members
public void deserializeRow (IBlueRowCollection currentState, BinaryReader br, byte op, out IBlueRow r)
{
if (op == StreamOpCode.add || op == StreamOpCode.change)
{
object[] retVal = new object[mySchema.columnInfo.Length];
foreach (ColumnInfo ci in mySchema.columnInfo)
retVal[ci.colNo] = myHelper.deserializeColumn (ci.colNo, br);
r = new BlueRow (mySchema, retVal);
}
else if (op == StreamOpCode.remove)
r = currentState[br.ReadInt32 ()];
else
{
// BlueEventLog.logAssert (false, op.ToString() + " not valid in this context");
r = null;
}
}
#endregion
}
}
@@ -0,0 +1,11 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BCG
{
public class MarketOptionTheoClientDeserializer : ClientFixedDeserializer
{
public MarketOptionTheoClientDeserializer() { }
}
}
@@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BCG
{
public class BBOOptionSerializer : FixedSerializer
{
public BBOOptionSerializer(IBlueTableSchema schema)
: base(schema, "BCG.BBOOptionPubsubDeserializer", "BCG.BBOOptionDeserializer")
{ }
}
}
@@ -0,0 +1,35 @@
using System;
using System.IO;
namespace BCG
{
/// <summary>
/// Optimized to serialize fixed length rows, ie schemas that have only primitive types and non-nulls
/// </summary>
public class FixedRowSerializer : IBlueRowSerializer
{
private PrimitiveSerializationHelper myHelper;
private int myIdColNo;
private IBlueTableSchema mySchema;
public FixedRowSerializer (IBlueTableSchema schema)
{
myIdColNo = schema["id"].colNo;
myHelper = new PrimitiveSerializationHelper (schema);
mySchema = schema;
}
#region IBlueRowSerializer Members
public void serializeRow (IBlueRow r, byte op, BinaryWriter bw)
{
if (op == StreamOpCode.remove)
myHelper.serializeColumn (r, myIdColNo, bw);
else
foreach (ColumnInfo ci in r.schema.columnInfo)
myHelper.serializeColumn (r, ci.colNo, bw);
}
#endregion
}
}
@@ -0,0 +1,20 @@
using System;
using System.IO;
namespace BCG
{
/// <summary>
/// Tailored to serialize tables that haved fixed schema (ie no dynamic length types like strings, arrays etc and no nulls)
/// </summary>
public abstract class FixedSerializer : BlueDataSerializer
{
public FixedSerializer (IBlueTableSchema schema,
string pubsubDeserializer,
string clientDeserializer)
: base (pubsubDeserializer, clientDeserializer)
{
myRowSerializer = new FixedRowSerializer (schema);
}
}
}
@@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BCG
{
public class MarketOptionTheoSerializer : FixedSerializer
{
public MarketOptionTheoSerializer(IBlueTableSchema schema)
: base(schema, "BCG.MarketOptionTheoPubsubDeserializer",
"BCG.MarketOptionTheoClientDeserializer")
{ }
}
}
@@ -0,0 +1,11 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BCG
{
public class BBOOptionPubsubDeserializer : PubsubFixedDeserializer
{
public BBOOptionPubsubDeserializer() { }
}
}
@@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace BCG
{
public class MarketOptionTheoPubsubDeserializer : PubsubFixedDeserializer
{
public MarketOptionTheoPubsubDeserializer()
{ }
}
}
Loaded 100 of 437 files, more files were not shown because too many files have changed in this diff. Show more