Files
spark/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs
T

1946 lines
79 KiB
C#

using System;
using System.Data;
using System.Collections;
using System.Collections.Generic;
using System.Reflection;
using System.IO;
using System.Diagnostics;
using System.Linq;
//using BCG.Utilities;
using BCG.Utilities.General;
using System.Linq.Expressions;
//using mbts.spark;
using mbts;
namespace BCG {
public static class BlueDataUtility {
#region Extensions
public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) {
return BlueDataUtility.get_row_datetime_value(r, col_name, err_value);
}
public static DateTime get_datetime(this IBlueRow r, int col_no, DateTime err_value = default(DateTime)) {
return BlueDataUtility.get_row_datetime_value(r, col_no, err_value);
}
public static T get_enum<T>(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct {
return BlueDataUtility.get_row_enum_value<T>(r, col_name, err_value);
}
public static T get_enum<T>(this IBlueRow r, int col_no, T err_value = default(T)) where T : struct {
return BlueDataUtility.get_row_enum_value<T>(r, col_no, err_value);
}
public static T get<T>(this IBlueRow r, string col_name, T err_value = default(T)) {
return get_row_value(r, col_name, err_value);
}
public static T get<T>(this IBlueRow r, int col_no, T err_value = default(T)) {
return get_row_value(r, col_no, err_value);
}
public static IBlueRowCollection where(this IBlueRowCollection rows, Func<IBlueRow, bool> where_func) {
if (rows == null)
return null;
else {
BlueTable rv = new BlueTable(rows.schema);
foreach (IBlueRow r in rows)
if (where_func(r))
rv.addRow(r);
return rv;
}
}
public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
List<IBlueRow> rv = new List<IBlueRow>();
foreach (IBlueRow r in rows)
rv.Add(r);
return rv;
}
public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) {
List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
foreach (Tuple<Type, string> c in cols)
cis.Add(new ColumnInfo(c.Item2, cis.Count, c.Item1));
return new BlueTableSchema(input_schema.tableName, cis.ToArray());
}
public static IBlueRowCollection to_collection(this IBlueRow r) {
if (r == null)
return null;
else {
BlueTable rv = new BlueTable(r.schema);
rv.addRow(r);
return rv;
}
}
public static DataPacket build_dp(IBlueRowCollection new_rows, DataPacket last_fired_dp) {
IBlueRowCollection added_rows = new_rows;
IBlueRowCollection deleted_rows = last_fired_dp == null ? new BlueTable(new_rows.schema) : last_fired_dp.newRows;
return new DataPacket(new_rows, added_rows, deleted_rows);
}
public static IBlueRowCollection adapt_for_sql_storage(this IBlueRowCollection rows,
bool handle_datetime_min = false,
bool handle_double_nan = false) {
if (rows != null) {
BlueTable rv = new BlueTable(rows.schema);
foreach (IBlueRow r in rows) {
MutableBlueRow mr = new MutableBlueRow(r);
rv.addRow(mr);
foreach (ColumnInfo ci in rows.schema.columnInfo) {
if (handle_datetime_min && ci.colType == typeof(DateTime) && DateTime.MinValue.Equals(r[ci.colName]))
mr.setValue(ci.colName, DBNull.Value);
if (handle_double_nan && ci.colType == typeof(double) && Double.IsNaN(r.get<double>(ci.colName)))
mr.setValue(ci.colName, DBNull.Value);
}
}
return rv;
} else
return null;
}
#endregion
public static void print_table(this IBlueRowCollection table, Func<string, int> get_col_width) {
var rv = get_printed_rows(table, get_col_width);
Console.WriteLine(rv);
}
public static string get_printed_rows(this IBlueRowCollection table, Func<string, int> get_col_width) {
// Print column headers
string rv = String.Empty;// = $"{table.schema.tableName}______________________________________";
foreach (var col_name in table.schema.columnNames) {
int col_width = get_col_width(col_name);
rv += $"{col_name.PadRight(col_width)} ";
}
rv += "\n";
//rv += "______________________________________";
// Print table data
foreach (IBlueRow row in table) {
foreach (string col_name in table.schema.columnNames) {
int col_width = get_col_width(col_name);
object row_val = row[col_name];
string row_val_str = row_val.ToString();
if (row_val != null) {
rv += $"{row_val_str.PadRight(col_width)} ";
}
}
rv += "\n";
}
return rv;
}
private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
{
{ TypeCode.Boolean, typeof(bool) },
{ TypeCode.Byte, typeof(byte) },
{ TypeCode.Char, typeof(char) },
{ TypeCode.DateTime, typeof(DateTime) },
{ TypeCode.DBNull, typeof(DBNull) },
{ TypeCode.Decimal, typeof(decimal) },
{ TypeCode.Double, typeof(double) },
{ TypeCode.Empty, null },
{ TypeCode.Int16, typeof(short) },
{ TypeCode.Int32, typeof(int) },
{ TypeCode.Int64, typeof(long) },
{ TypeCode.Object, typeof(object) },
{ TypeCode.SByte, typeof(sbyte) },
{ TypeCode.Single, typeof(Single) },
{ TypeCode.String, typeof(string) },
{ TypeCode.UInt16, typeof(UInt16) },
{ TypeCode.UInt32, typeof(UInt32) },
{ TypeCode.UInt64, typeof(UInt64) }
};
public static IBlueTableSchema form_schema(string table_name, params Tuple<string, TypeCode>[] cols) {
List<ColumnInfo> cis = new List<ColumnInfo>();
foreach (Tuple<string, TypeCode> c in cols)
cis.Add(new ColumnInfo(c.Item1, cis.Count, _typecode_to_type[c.Item2]));
if (cis.FirstOrDefault(ci => ci.colName == "id") == null)
cis.Add(new ColumnInfo("id", cis.Count, typeof(int)));
return new BlueTableSchema(table_name, cis.ToArray());
}
/// <summary>
/// Gets the Hashcode of the row with the boolean indicating whether the order of the rows should
/// be alphabetic or not
/// </summary>
/// <param name="row"></param>
/// <param name="isColAlphabetically"></param>
/// <returns></returns>
public static int getRowHashCode(IBlueRow row, bool isColAlphabetically) {
string retVal = "";
if (isColAlphabetically) {
string[] colNames = row.schema.columnNames;
Array.Sort(colNames);
foreach (string col in colNames)
retVal += row[col].ToString();
} else
foreach (object o in row)
retVal += o.ToString();
return retVal.GetHashCode();
}
/// <summary>
/// Finds the hashcode only inluding the columns (in the order) listed in the column names argument
/// </summary>
/// <param name="row"></param>
/// <param name="colNames"></param>
/// <returns></returns>
public static int getRowHashCode(IBlueRow row, params string[] colNames) {
string retVal = "";
foreach (string colName in colNames)
retVal += row[colName].ToString();
return retVal.GetHashCode();
}
/// <summary>
/// Converts a IBlueRowCollection to a .Net DataTable
/// </summary>
/// <param name="rows"></param>
/// <returns></returns>
public static DataTable toDataTable(IBlueRowCollection rows) {
DataTable retTable = getTableFromSchema(rows.schema);
retTable.BeginLoadData();
foreach (IBlueRow row in rows)
retTable.Rows.Add(toDataRow(row, retTable));
retTable.EndLoadData();
return retTable;
}
/// <summary>
/// Converts to a DataRow, the preconditioner being that order of the columns should be same
/// in the schema of the row and the table
/// </summary>
/// <param name="row"></param>
/// <param name="table"></param>
public static DataRow toDataRow(IBlueRow row, DataTable table) {
DataRow retRow = table.NewRow();
foreach (string colName in row.schema.columnNames)
retRow[colName] = row[colName];
return retRow;
}
/// <summary>
/// Returns the IBlueTableSchema corresponding to the row
/// </summary>
/// <param name="row">
/// DataRow whose schmea is desired
/// </param>
/// <returns></returns>
public static IBlueTableSchema getSchema(DataRow row) { return getSchema(row.Table); }
/// <summary>
/// Returns the IBlueTableSchema of the DataTable
/// </summary>
/// <param name="table">
/// table whos Schema is desired
/// </param>
/// <returns></returns>
public static IBlueTableSchema getSchema(DataTable table) {
ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
foreach (DataColumn col in table.Columns)
cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
return new BlueTableSchema(cis, table.TableName);
}
/// <summary>
/// Constructs the schema from an object, using the SchemaMapping attribute. Only those
/// properties are converted into schmea columns that have a get accessor
/// </summary>
/// <param name="o"></param>
/// <returns></returns>
public static IBlueTableSchema getSchema(object o) {
return getSchema(o.GetType());
}
/// <summary>
/// Builds a new schema based on an existing one + a set of new columns
/// </summary>
/// <param name="schema">Base schema</param>
/// <param name="cis">Columns to add</param>
public static IBlueTableSchema addColumnsToSchema(IBlueTableSchema schema, params ColumnInfo[] cis) {
ArrayList rv = new ArrayList(schema.columnInfo);
rv.AddRange(cis);
return new BlueTableSchema(schema.tableName, rv.ToArray(typeof(ColumnInfo)) as ColumnInfo[]);
}
/// <summary>
/// appends the cols of s2, to s1, subtracting the id col
/// </summary>
/// <param name="s1"></param>
/// <param name="appendedSchema"></param>
/// <returns></returns>
public static IBlueTableSchema appendSchemas(IBlueTableSchema s1, IBlueTableSchema appendedSchema) {
List<ColumnInfo> cis = new List<ColumnInfo>(s1.columnInfo);
foreach (ColumnInfo c in appendedSchema.columnInfo)
if (c.colName != "id")
cis.Add(new ColumnInfo(c.colName, cis.Count, c.colType));
return new BlueTableSchema(s1.tableName, cis.ToArray());
}
public static IBlueTableSchema getSchema(Type t) {
return getSchema(t, false);
}
public static IBlueTableSchema getSchema(Type t, bool createIdField) {
return getSchema(t, t.Name, createIdField);
}
public static IBlueTableSchema getSchema(Type t, string tableName, bool createIdField) {
ArrayList colInfos = new ArrayList();
int i = 0;
foreach (PropertyInfo pi in t.GetProperties())
if (myIsActiveProperty(pi))
colInfos.Add(new ColumnInfo(pi.Name, i++,
ColumnInfo.isAllowedType(pi.PropertyType) ?
pi.PropertyType : typeof(String)));
foreach (FieldInfo fi in t.GetFields())
if (myIsActiveField(fi))
colInfos.Add(new ColumnInfo(fi.Name, i++,
ColumnInfo.isAllowedType(fi.FieldType) ? fi.FieldType : typeof(String)));
if (createIdField)
colInfos.Add(new ColumnInfo("id", i++, typeof(int)));
return new BlueTableSchema(colInfos.ToArray(typeof(ColumnInfo)) as ColumnInfo[],
tableName);
}
public static IBlueTableSchema getSchema(Type t, string tableName) {
return new BlueTableSchema(getSchema(t).columnInfo, tableName);
}
public static IBlueRow createRowFromType(IBlueTableSchema schema, object o) {
return createRowFromType(schema, o, null);
}
//creates a row from the given type, and if idVal is not null, using the given id
public static IBlueRow createRowFromType(IBlueTableSchema schema, object o, object idVal) {
object[] rowArray = new object[schema.columnInfo.Length];
Type t = o.GetType();
foreach (PropertyInfo pi in t.GetProperties())
if (myIsActiveProperty(pi))
rowArray[schema[pi.Name].colNo] =
ColumnInfo.isAllowedType(pi.PropertyType) ?
pi.GetValue(o, null) : pi.GetValue(o, null).ToString();
foreach (FieldInfo fi in t.GetFields())
if (myIsActiveField(fi))
rowArray[schema[fi.Name].colNo] =
ColumnInfo.isAllowedType(fi.FieldType) ?
fi.GetValue(o) : fi.GetValue(o).ToString();
if (idVal != null)
rowArray[schema["id"].colNo] = idVal;
return new BlueRow(schema, rowArray);
}
public static MutableBlueRow create_mutable_row(IBlueTableSchema schema, object o, object idVal,
Func<Type, object, object> type_converter) {
object[] rowArray = new object[schema.columnInfo.Length];
Type t = o.GetType();
foreach (PropertyInfo pi in t.GetProperties())
if (myIsActiveProperty(pi))
rowArray[schema[pi.Name].colNo] =
ColumnInfo.isAllowedType(pi.PropertyType) ?
pi.GetValue(o, null) : type_converter(pi.PropertyType, pi.GetValue(o, null));
foreach (FieldInfo fi in t.GetFields())
if (myIsActiveField(fi))
rowArray[schema[fi.Name].colNo] =
ColumnInfo.isAllowedType(fi.FieldType) ?
fi.GetValue(o) : type_converter(fi.FieldType, fi.GetValue(o));
if (idVal != null)
rowArray[schema["id"].colNo] = idVal;
return new MutableBlueRow(schema, rowArray);
}
public static IBlueTableSchema get_schema(Type t, string tableName, bool createIdField, Func<Type, Type> type_converter) {
List<ColumnInfo> colInfos = new List<ColumnInfo>();
int i = 0;
if (createIdField)
colInfos.Add(new ColumnInfo("id", i++, typeof(int)));
foreach (PropertyInfo pi in t.GetProperties())
colInfos.Add(new ColumnInfo(pi.Name, i++, type_converter(pi.PropertyType)));
foreach (FieldInfo fi in t.GetFields())
colInfos.Add(new ColumnInfo(fi.Name, i++, type_converter(fi.FieldType)));
return new BlueTableSchema(colInfos.ToArray(), tableName);
}
public static MutableBlueRow create_mutable_row_not_slow(IBlueTableSchema schema, FieldInfo[] fields, PropertyInfo[] props,
object o, object idVal, Func<Type, object, object> data_converter) {
object[] rowArray = new object[schema.columnInfo.Length];
if (props != null) {
foreach (PropertyInfo pi in props)
rowArray[schema[pi.Name].colNo] = data_converter(pi.PropertyType, pi.GetValue(o, null));
}
if (fields != null) {
foreach (FieldInfo fi in fields)
rowArray[schema[fi.Name].colNo] = data_converter(fi.FieldType, fi.GetValue(o));
}
if (idVal != null)
rowArray[schema["id"].colNo] = idVal;
return new MutableBlueRow(schema, rowArray);
}
public static T createTypeFromRow<T>(IBlueRow row) {
return (T)createTypeFromRow(row, typeof(T));
}
public static object createTypeFromRow(IBlueRow row, Type t) {
object outValue = Activator.CreateInstance(t);
PropertyInfo[] properties = t.GetProperties();
foreach (PropertyInfo pi in properties) {
if (myIsActiveProperty(pi))
if (row.Contains(pi.Name))
if (row[pi.Name].Equals(DBNull.Value) && !pi.PropertyType.Equals(typeof(Object)))
pi.SetValue(outValue, null, null);
else
pi.SetValue(outValue, row[pi.Name], null);
}
FieldInfo[] fields = t.GetFields();
foreach (FieldInfo fi in fields) {
if (myIsActiveField(fi))
if (row.Contains(fi.Name))
if (row[fi.Name].Equals(DBNull.Value) && !fi.FieldType.Equals(typeof(Object)))
fi.SetValue(outValue, null);
else
fi.SetValue(outValue, row[fi.Name]);
}
return outValue;
}
//these functions should originally have been made public
//but changing that would change a bunch of code
public static bool isActiveProperty(PropertyInfo pi) {
return myIsActiveProperty(pi);
}
public static bool isActiveField(FieldInfo fi) {
return myIsActiveField(fi);
}
private static bool myIsActiveProperty(PropertyInfo pi) {
if (pi.GetGetMethod() == null)
return false;
foreach (Attribute att in pi.GetCustomAttributes(true))
if (att is SchemaVisibility && !(att as SchemaVisibility).visible)
return false;
return true;
}
private static bool myIsActiveField(FieldInfo fi) {
if (!fi.IsPublic)
return false;
foreach (Attribute att in fi.GetCustomAttributes(true))
if (att is SchemaVisibility && !(att as SchemaVisibility).visible)
return false;
return true;
}
private static bool myIsActiveMember(MemberInfo mi) {
if (!(mi is FieldInfo) ||
!(mi is PropertyInfo) ||
(mi is PropertyInfo && (mi as PropertyInfo).GetGetMethod() == null) ||
(mi is FieldInfo && !(mi as FieldInfo).IsPublic))
return false;
foreach (Attribute att in mi.GetCustomAttributes(true))
if (att is SchemaVisibility && !(att as SchemaVisibility).visible)
return false;
return true;
}
private static bool myDoesRowExistsInTable(IBlueRowCollection brc2, string[] columns, object[] oarray) {
bool rowExists = false;
foreach (IBlueRow br in brc2) {
bool allElementsSame = true;
for (int i = 0; i < columns.Length; i++) {
string col = columns[i];
object o1 = br[col];
object o2 = oarray[i];
allElementsSame = (o1.Equals(o2) == false) ? false : allElementsSame;
}
if (allElementsSame == true) {
rowExists = true;
break;
}
}
return rowExists;
}
/// <summary>
/// Constructs a DataTable from the IBlueTableSchema
/// </summary>
/// <param name="schema">
/// Schema whose table needs to be created
/// </param>
/// <returns></returns>
public static DataTable getTableFromSchema(IBlueTableSchema schema) {
DataTable retTable = new DataTable(schema.tableName);
foreach (ColumnInfo ci in schema.columnInfo)
retTable.Columns.Add(ci.colName, ci.colType);
return retTable;
}
/// <summary>
/// Constructs a BlueRow from a DataRow.
/// </summary>
/// <param name="row">source row</param>
/// <returns></returns>
public static IBlueRow toBlueRow(DataRow row) {
return new BlueRow(getSchema(row), row);
}
public static IBlueRowCollection toIBlueRowCollection(DataTable dt) {
BlueTable retVal = new BlueTable(getSchema(dt));
foreach (DataRow dr in dt.Rows)
retVal.addRow(dr);
return retVal;
}
public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) {
BlueTable retVal = null;
foreach (DataRow dr in rows) {
IBlueRow r = toBlueRow(dr);
if (retVal == null)
retVal = new BlueTable(r.schema);
retVal.addRow(r);
}
return retVal;
}
/// <summary>
/// Returns the nth row of the IBlueRowCollection. This method could be used
/// for any of the nonRandom-Access-IBlueRowCollection-Implementations, as its
/// O(n)
/// </summary>
/// <param name="rows"></param>
/// <param name="index"></param>
/// <returns></returns>
public static IBlueRow extractRow(IBlueRowCollection rows, int index) {
int i = 0;
foreach (IBlueRow row in rows)
if (i++ == index)
return row;
return null;
}
/// <summary>
/// Computes the equality fo the 2 rows. The rows might have different columns orderings
/// </summary>
/// <param name="r1"></param>
/// <param name="r2"></param>
/// <returns></returns>
public static bool computeRowEquality(IBlueRow r1, IBlueRow r2) {
return computeRowEquality(r1, r2, r1.schema.columnNames);
}
public static bool computeRowEquality(IBlueRow r1,
IBlueRow r2,
params string[] colNames) {
return compareRows(r1, r2, colNames) == 0 ? true : false;
}
/// <summary>
/// Computes a string equivelant for the row, using the columns provided
/// in the argument
/// </summary>
/// <param name="row"></param>
/// <param name="colNames"></param>
/// <returns></returns>
public static string rowToString(IBlueRow row, params string[] colNames) {
string retVal = "";
foreach (string colName in colNames)
retVal += (row[colName] != null ? row[colName].ToString().Trim() : "NULL") + "\n";
return retVal;
}
public static string rowToString(IBlueRow row) { return rowToString(row, row.schema.columnNames); }
public static string rowCollectionToString(IBlueRowCollection rowColl) {
if (rowColl == null)
return "Null IBlueRowCollection";
else {
string retVal = "";
foreach (IBlueRow r in rowColl)
retVal += retVal == "" ? rowToString(r) : "\n" + rowToString(r);
return retVal;
}
}
/// <summary>
/// Compares 2 rows, using the column names in the order provided, ie the column name with
/// the lower index in the array has higher priority. Also a null is always assumed
/// to be less than a non-null
/// </summary>
/// <param name="r1"></param>
/// <param name="r2"></param>
/// <param name="colNames"></param>
/// <returns></returns>
public static int compareRows(IBlueRow r1,
IBlueRow r2,
params string[] colNames) {
int retVal = 0;
foreach (string colName in colNames) {
object val1 = r1[colName];
object val2 = r2[colName];
retVal = _compare_row_cols(val1, val2);
if (retVal != 0)
break;
}
return retVal;
}
private static int _compare_row_cols(object v1, object v2) {
if (_is_null(v1))
return _is_null(v2) ? 0 : -1;
else {
if (_is_null(v2))
return 1;
else {
if (v1 is IComparable && v2 is IComparable)
return (v1 as IComparable).CompareTo(v2);
else if (v1 is byte[] && v2 is byte[])
return _compare_bytes(v1 as byte[], v2 as byte[]) ? 0 : 1;
else
return 1;
}
}
}
private static bool _compare_bytes(byte[] v1, byte[] v2) {
if (v1.Length != v2.Length)
return false;
else
for (int i = 0; i < v1.Length; i++)
if (v1[i] != v2[i])
return false;
return true;
}
private static bool _is_null(object o) { return o == null || o == DBNull.Value; }
/// <summary>
/// Creates a new IBlueRow with a schema matching the schema provided in 'new schema'
/// </summary>
/// <param name="inRow">a row with the values to be converted</param>
/// <param name="newSchema">the desired schema for the output row</param>
/// <returns></returns>
public static IBlueRow changeSchema(IBlueRow inRow, IBlueTableSchema newSchema) {
object[] outputValues = new object[newSchema.columnInfo.Length];
for (int i = 0; i < outputValues.Length; ++i)
outputValues[i] = inRow[newSchema.columnNames[i]];
return new BlueRow(newSchema, outputValues);
}
/// <summary>
/// Combines input tables into a single output table, with different ID's
/// this will result in new rows being created, which is kind of expensive
/// </summary>
/// <param name="tables"></param>
/// <returns></returns>
public static IBlueRowCollection combineTables(params IBlueRowCollection[] tables) {
//get some base cases out of the way
if (tables.Length == 0)
return null;
if (tables.Length == 1)
return tables[0];
BlueTable result = new BlueTable(tables[0].schema);
int idToUseNext = 0;
int numCols = tables[0].schema.columnInfo.Length;
int idCol = tables[0].schema["id"].colNo;
IBlueRowCollection previousTable = null;
foreach (IBlueRowCollection table in tables) {
if (previousTable != null)
if (!compareSchemas(previousTable.schema, table.schema))
throw new ArgumentException("Error, tables must have the same schema to be combined");
foreach (IBlueRow r in table) {
object[] values = new object[numCols];
r.CopyTo(values, 0);
values[idCol] = idToUseNext++;
result.addRow(new BlueRow(table.schema, values));
}
previousTable = table;
}
return result;
}
/// <summary>
/// Compares two rows (having the same schema) and returns the
/// columns whose values dont match
/// </summary>
/// <param name="r1"></param>
/// <param name="r2"></param>
/// <returns></returns>
public static string[] compareRows(IBlueRow r1, IBlueRow r2) {
ArrayList retVal = new ArrayList();
foreach (string col in r1.schema.columnNames)
if (!r1[col].Equals(r2[col]))
retVal.Add(col);
return retVal.ToArray(typeof(string)) as string[];
}
public static BlueTable cloneRowCollection(IBlueRowCollection rows) {
BlueTable retVal = new BlueTable(rows.schema);
foreach (IBlueRow r in rows)
retVal.addRow(new BlueRow(r));
return retVal;
}
/// <summary>
/// Creates a new row with the values of the row passed, except that the newValues are forced into the
/// returned row, in places specified by the indexes (makes any sense ?)
/// </summary>
/// <param name="r"></param>
/// <param name="newValues"></param>
/// <param name="indexes"></param>
/// <returns></returns>
public static IBlueRow getNewReplacedRow(IBlueRow r, object[] newValues, int[] indexes) {
if (!newValues.Length.Equals(indexes.Length))
throw new Exception("Debug Assert");
object[] rowArray = new object[r.Count];
r.CopyTo(rowArray, 0);
for (int i = 0; i < indexes.Length; i++)
rowArray[indexes[i]] = newValues[i];
return new BlueRow(r.schema, rowArray);
}
public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, int index) {
return getNewReplacedRow(r, new object[] { newValue }, new int[] { index });
}
public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, string colName) {
return getNewReplacedRow(r, newValue, r.schema[colName].colNo);
}
/// <summary>
/// Computes the max row, based on the cols provided. If the collection is empty, null is returned
/// </summary>
/// <param name="rows"></param>
/// <param name="cols"></param>
/// <returns></returns>
public static IBlueRow computeMaxRow(IBlueRowCollection rows, params string[] cols) {
IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null;
foreach (IBlueRow r in rows)
retVal = compareRows(r, retVal, cols) > 0 ? r : retVal;
return retVal;
}
/// <summary>
/// Computes the max row, based on the cols provided. If the collection is empty, null is returned
/// </summary>
/// <param name="rows"></param>
/// <param name="cols"></param>
/// <returns></returns>
public static IBlueRow computeMinRow(IBlueRowCollection rows, params string[] cols) {
IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null;
foreach (IBlueRow r in rows)
retVal = compareRows(r, retVal, cols) < 0 ? r : retVal;
return retVal;
}
/// <summary>
/// Computes the sum of rows, based on the cols provided. If the collection is empty, null is returned
/// </summary>
/// <param name="rows"></param>
/// <param name="cols"></param>
/// <returns></returns>
public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) {
if (rows.Count == 0) return null;
IBlueRow retVal = BlueDataUtility.extractRow(rows, 0);
object[] sums = new object[cols.Length];
int[] idxs = new int[cols.Length];
for (int i = 0; i < cols.Length; i++) {
sums[i] = 0.0;
idxs[i] = rows.schema[cols[i]].colNo;
}
foreach (IBlueRow r in rows) {
for (int i = 0; i < cols.Length; i++) {
double dval = BlueDataUtility.get_row_value<double>(r, cols[i], 0.0);
//r[cols[i]] == null || r[cols[i]] is DBNull ? 0.0 : (double)r[cols[i]];
sums[i] = (double)sums[i] + dval;
}
}
return BlueDataUtility.getNewReplacedRow(retVal, sums, idxs);
}
public static IBlueRow computeSum<T>(IBlueRowCollection rows, params string[] cols) {
if (rows.Count == 0) return null;
IBlueRow retVal = BlueDataUtility.extractRow(rows, 0);
if (rows.Count == 1) return retVal;
object[] sums = new object[cols.Length];
int[] idxs = new int[cols.Length];
for (int i = 0; i < cols.Length; i++) {
sums[i] = default(T);
idxs[i] = rows.schema[cols[i]].colNo;
}
foreach (IBlueRow r in rows) {
for (int i = 0; i < cols.Length; i++) {
T tval = r[cols[i]] == null || r[cols[i]] is DBNull ? default(T) : (T)r[cols[i]];
sums[i] = MagicAdder<T>.Add((T)sums[i], tval);
}
}
return BlueDataUtility.getNewReplacedRow(retVal, sums, idxs);
}
/// <summary>
/// Uses the ID to identify the presence of a row in a collection
/// </summary>
/// <param name="table"></param>
/// <param name="row"></param>
/// <returns></returns>
public static bool rowPresent(IBlueRowCollection table, IBlueRow row) {
return table.containsID((int)row["id"]);
}
public static void computeDifferences(IBlueRowCollection newRows,
IBlueRowCollection oldRows,
out IBlueRowCollection dRows,
out IBlueRowCollection aRows) {
computeDifferences(
newRows,
oldRows,
newRows.schema.columnNames,
out dRows,
out aRows);
}
public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
int[] col_indices = new int[new_rows.schema.columnNames.Length];
for (int i = 0; i < col_indices.Length; i++)
col_indices[i] = i;
compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows);
}
public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, int[] col_indices,
out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
Func<IBlueRow, IBlueRow, bool> rows_equal = (r1, r2) => {
foreach (int col_no in col_indices)
if (!(r1[col_no] ?? DBNull.Value).Equals(r2[col_no] ?? DBNull.Value))
return false;
return true;
};
BlueTable del_rows = new BlueTable(new_rows.schema), add_rows = new BlueTable(new_rows.schema);
// adds + changes
foreach (IBlueRow r in new_rows)
if (!BlueDataUtility.rowPresent(old_rows, r))
add_rows.addRow(r);
else if (!rows_equal(r, old_rows[(int)r["id"]])) {
add_rows.addRow(r);
del_rows.addRow(old_rows[(int)r["id"]]);
}
// deletes
foreach (IBlueRow r in old_rows)
if (!BlueDataUtility.rowPresent(new_rows, r))
del_rows.addRow(r);
out_d_rows = del_rows;
out_a_rows = add_rows;
}
public static void computeDifferences(IBlueRowCollection c1,
IBlueRowCollection c2,
string[] columns,
out IBlueRowCollection dRows,
out IBlueRowCollection aRows) {
dRows = new BlueTable(c1.schema);
aRows = new BlueTable(c1.schema);
// adds + changes
foreach (IBlueRow nr in c1)
if (!BlueDataUtility.rowPresent(c2, nr))
(aRows as BlueTable).addRow(nr);
else if (BlueDataUtility.compareRows(nr, c2[(int)
nr["id"]], columns) != 0) {
(aRows as BlueTable).addRow(nr);
(dRows as BlueTable).addRow(c2[(int)nr["id"]]);
}
// deletes
foreach (IBlueRow nr in c2)
if (!BlueDataUtility.rowPresent(c1, nr))
(dRows as BlueTable).addRow(nr);
}
//brcDiff = brc1 - brc2
public static void diff(IBlueRowCollection brc1, IBlueRowCollection brc2, string[] columns, out IBlueRowCollection brcDiff) {
brcDiff = new BlueTable(brc1.schema);
foreach (IBlueRow br in brc1) {
object[] oarray = new object[columns.Length];
for (int i = 0; i < columns.Length; i++) {
string col = columns[i];
oarray[i] = br[col];
}
if (myDoesRowExistsInTable(brc2, columns, oarray) == false)
(brcDiff as BlueTable).addRow(br);
}
}
/// <summary>
/// Abosrbs the deletes and adds into the BlueTable
/// </summary>
/// <param name="t"></param>
/// <param name="dels"></param>
/// <param name="adds"></param>
public static void absorbChanges(BlueTable t, IBlueRowCollection dels, IBlueRowCollection adds) {
foreach (IBlueRow r in dels)
if (!t.deleteRow(r))
throw new Exception(String.Format("{0} does not have row with id {1} to delete", t.schema.tableName, r["id"]));
foreach (IBlueRow r in adds)
t.addRow(r);
}
public static IBlueRowCollection toBlueRowCollection(IBlueRow r) {
BlueTable rv = new BlueTable(r.schema);
rv.addRow(r);
return rv;
}
/// <summary>
/// Converts an ICollection of IBlueRows to IBlueRowCollection. If there are no rows in the
/// collection, a null is returned
/// </summary>
/// <param name="col"></param>
/// <returns></returns>
public static IBlueRowCollection toBlueRowCollection(ICollection col) {
BlueTable retVal = null;
foreach (IBlueRow r in col) {
retVal = retVal ?? new BlueTable(r.schema);
retVal.addRow(r);
}
return retVal;
}
/// <summary>
/// Computes the rows that are truely deleted, added or changed (with respect to relevant columns)
/// </summary>
/// <param name="dels">all the deleted rows in a certain callback</param>
/// <param name="adds">all the added rows in a certain callback</param>
/// <param name="truelyDels">the rows that are truely deleted or changed with respect to the relevant columns</param>
/// <param name="truelyAdds">the rows that are truely added or changed with respect to the relevant columns</param>
/// <param name="cols"></param>
public static void getTrueDeletesAndAdds(IBlueRowCollection dels,
IBlueRowCollection adds,
out IBlueRowCollection truelyDels,
out IBlueRowCollection truelyAdds,
string[] cols) {
truelyDels = new BlueTable(dels);
truelyAdds = new BlueTable(adds);
foreach (IBlueRow r in dels) {
if (adds.containsID((int)r["id"]) &&
BlueDataUtility.computeRowEquality(adds[(int)r["id"]], r, cols)) {
(truelyDels as BlueTable).deleteRow((int)r["id"]);
(truelyAdds as BlueTable).deleteRow((int)r["id"]);
}
}
}
public static IBlueRowCollection getSerializableCollection(IBlueRowCollection coll) {
BlueTable retVal = new OrderedBlueTable(coll.schema);
foreach (IBlueRow r in coll)
retVal.addRow(new BlueRow(r));
return retVal;
}
/// <summary>
/// extracts out a column from a table
/// </summary>
/// <param name="rows"></param>
/// <param name="colName"></param>
/// <returns></returns>
public static ArrayList extractColumn(IBlueRowCollection rows, string colName) {
ArrayList rv = new ArrayList();
foreach (IBlueRow r in rows)
rv.Add(r[colName]);
return rv;
}
/// <summary>
/// applies the deletedRows and addedRows to the oldState and asserts if it does not
/// match the newRows. for an old state with 10 K rows, and all 10 K rows being deleted and
/// added it takes ~ 35 ms to execute this method.
/// </summary>
/// <param name="oldState"></param>
/// <param name="newRows"></param>
/// <param name="delRows"></param>
/// <param name="addedRows"></param>
public static void checkConsistency(IBlueRowCollection oldState,
IBlueRowCollection newRows,
IBlueRowCollection delRows,
IBlueRowCollection addedRows) {
BlueTable calc = new BlueTable(oldState);
Debug.Assert(oldState != null && newRows != null && delRows != null && addedRows != null);
foreach (IBlueRow r in delRows) {
Debug.Assert(calc.containsID((int)r["id"]), "deleted row's id = " + r["id"] + " does not exist");
calc.deleteRow(r);
}
foreach (IBlueRow r in addedRows) {
Debug.Assert(!calc.containsID((int)r["id"]), "added rows's id = " + r["id"].ToString() + " already exists");
calc.addRow(r);
}
Debug.Assert(calc.Count == newRows.Count, "row count should be the same. Counts: calc=" + calc.Count + "; newRows=" + newRows.Count);
foreach (IBlueRow r in calc)
Debug.Assert(newRows.containsID((int)r["id"]), r["id"] + " is not found");
}
/// <summary>
/// Extracts values from an IBlueRow specified by the columnNames arg
/// </summary>
/// <param name="r"></param>
/// <param name="colNames"></param>
/// <returns></returns>
public static object[] getSelectedValues(IBlueRow r, params string[] colNames) {
object[] retVal = new object[colNames.Length];
for (int i = 0; i < colNames.Length; i++)
retVal[i] = r[colNames[i]];
return retVal;
}
/// <summary>
/// From an IBlueRowCollection, for a set of columns, extracts a row with the same values as
/// provided in the third arg
/// </summary>
/// <param name="rows"></param>
/// <param name="keyColumns"></param>
/// <param name="colValues"></param>
/// <returns></returns>
public static IBlueRow getRowWithKey(IBlueRowCollection rows, string[] keyColumns, object[] colValues) {
foreach (IBlueRow r in rows) {
object[] vals = getSelectedValues(r, keyColumns);
if (CollectionHelpers.equal(vals, colValues))
return r;
}
return null;
}
public static IBlueRowCollection deepCopy(IBlueRowCollection rows) {
BlueTable retVal = new BlueTable(rows.schema);
foreach (IBlueRow r in rows)
retVal.addRow(new BlueRow(r));
return retVal;
}
/// <summary>
/// Returns the columns, sorted alphabetically. The second arg
/// is for the inclusion of id
/// </summary>
/// <param name="schema"></param>
/// <param name="includeId"></param>
/// <returns></returns>
public static int[] getColumnsInSortedOrder(IBlueTableSchema schema, bool includeId) {
ArrayList names = new ArrayList(schema.columnNames),
retVal = new ArrayList();
names.Sort();
foreach (string n in names)
if (n != "id" || includeId)
retVal.Add(schema[n].colNo);
return retVal.ToArray(typeof(int)) as int[];
}
//this class is used so that we can sort a schema by column name
//that way, prepublished tables will allways get published with schemas in the same order
//regardless of how the underlying streams end up working
private class ColumnInfoComparisonWrapper : IComparable<ColumnInfoComparisonWrapper> {
ColumnInfo myColumn;
public string ColumnName {
get { return myColumn.colName; }
//can't set it, these are only for reading columns
}
public ColumnInfo Column {
get { return myColumn; }
}
public ColumnInfoComparisonWrapper(ColumnInfo theColumn) {
myColumn = theColumn;
}
public int CompareTo(ColumnInfoComparisonWrapper otherGuy) {
return ColumnName.CompareTo(otherGuy.ColumnName);
}
}
public static IBlueTableSchema getSortedSchema(IBlueTableSchema inSchema) {
List<string> sortedColNames = new List<string>(inSchema.columnNames);
sortedColNames.Sort();
List<ColumnInfo> sortedColInfos = new List<ColumnInfo>();
int colNo = 0;
foreach (string c in sortedColNames) {
ColumnInfo ci = inSchema[c];
sortedColInfos.Add(new ColumnInfo(c, colNo++, ci.colType));
}
return new BlueTableSchema(inSchema.tableName, sortedColInfos.ToArray());
}
public static IBlueTableSchema getSortedSchema(IBlueTableSchema theSchema, out int[] sortedSchemaToOriginalSchemaMapping, out int idColumnInSortedSchema) {
//for now this function will not do anything
//that is because we don't want to break the C++ deserialization code
int numCols = theSchema.columnNames.Length;
sortedSchemaToOriginalSchemaMapping = new int[numCols];
for (int col = 0; col < theSchema.columnNames.Length; ++col) {
//the mapping is just the identity mapping
sortedSchemaToOriginalSchemaMapping[col] = col;
}
idColumnInSortedSchema = theSchema["id"].colNo;
return theSchema;
/*
//set this to -1 in case the method doesn't find an id column
idColumnInSortedSchema = 01;
List<ColumnInfoComparisonWrapper> schemaColumns = new List<ColumnInfoComparisonWrapper>();
foreach (ColumnInfo ci in theSchema.columnInfo)
schemaColumns.Add(new ColumnInfoComparisonWrapper(ci));
schemaColumns.Sort();
int numCols = theSchema.columnInfo.Length;
ColumnInfo[] outputColumns = new ColumnInfo[numCols];
sortedSchemaToOriginalSchemaMapping = new int[numCols];
for(int i = 0; i < numCols; ++i)
{
string thisColName = schemaColumns[i].ColumnName;
outputColumns[i] = new ColumnInfo(thisColName, i, schemaColumns[i].Column.colType);
sortedSchemaToOriginalSchemaMapping[i] = theSchema[thisColName].colNo;
//we cache the ID column because it'll be differentin the serialized row than in our schema
if (thisColName == "id")
idColumnInSortedSchema = i;
}
theSchema = new BlueTableSchema(theSchema.tableName,outputColumns);
return theSchema;
*/
}
/// <summary>
/// Compares 2 schemas for column-by-column match
/// </summary>
/// <param name="s1"></param>
/// <param name="s2"></param>
/// <returns>true if equal</returns>
public static bool compareSchemas(IBlueTableSchema s1, IBlueTableSchema s2) {
bool result = true;
try {
if (s1.columnInfo.Length != s2.columnInfo.Length)
result = false;
else
foreach (ColumnInfo ci1 in s1.columnInfo) {
ColumnInfo ci2 = s2[ci1.colNo];
if (ci1.colName != ci2.colName || ci1.colType != ci2.colType) {
result = false;
break;
}
}
} catch {
result = false; // the col# wasn't found in s2
}
return result;
}
public static IBlueRowCollection removeRows(BlueTable table, string[] cols, object[] vals) {
IBlueRowCollection dels = findMatchingRows(table, cols, vals);
table.deleteRows(dels);
return dels;
}
public static IBlueRowCollection findMatchingRows(IBlueRowCollection rows, string[] cols, object[] vals) {
BlueTable rv = new BlueTable(rows.schema);
foreach (IBlueRow r in rows) {
bool match = true;
for (int i = 0; i < cols.Length; i++)
match &= vals[i].Equals(r[cols[i]]);
if (match)
rv.addRow(r);
}
return rv;
}
public static void checkRowTypes(IBlueRowCollection rows) {
foreach (IBlueRow r in rows)
foreach (string col in r.schema.columnNames)
if (r[col] != null && r[col] != DBNull.Value)
Debug.Assert(r[col].GetType() == rows.schema[col].colType,
String.Format("col {0} in table {1} is of type {2} in schema but is a {3} in the row", col, rows.schema.tableName, r.schema[col].colType.ToString(), r[col].GetType().ToString()));
}
public static void addRows(ref BlueTable table, ICollection rows) {
foreach (IBlueRow r in rows)
table.addRow(r);
}
public static void removeRows(ref BlueTable table, ICollection rows) {
foreach (IBlueRow r in rows)
table.deleteRow(r);
}
public static void sortOutPacket(IBlueRowCollection dels, IBlueRowCollection adds, string[] keyCols,
out IBlueRowCollection addedRows,
out IBlueRowCollection deletedRows,
out IBlueRowCollection changedRows) {
addedRows = new BlueTable(dels.schema);
deletedRows = new BlueTable(dels.schema);
changedRows = new BlueTable(dels.schema);
KeysBasedTable2 keyedDels = new KeysBasedTable2(dels.schema, keyCols),
keyedAdds = new KeysBasedTable2(dels.schema, keyCols);
foreach (IBlueRow r in dels)
keyedDels.update(r);
foreach (IBlueRow r in adds)
keyedAdds.update(r);
foreach (IBlueRow r in keyedDels) {
object[] key = BlueDataUtility.getSelectedValues(r, keyCols);
if (keyedAdds.getRow(key) != null) {
///change
(changedRows as BlueTable).addRow(r);
} else {
///delete
(deletedRows as BlueTable).addRow(r);
}
}
foreach (IBlueRow r in keyedAdds) {
object[] key = BlueDataUtility.getSelectedValues(r, keyCols);
if (keyedDels.getRow(key) == null) {
(addedRows as BlueTable).addRow(r);
}
}
}
public static int HashExpStrike(DateTime exp, double strike) {
return ((DateTime)exp).ToShortDateString().GetHashCode() ^ strike.GetHashCode();
}
public static IEnumerable<IBlueRow> collection_to_ienum(IBlueRowCollection rows) {
List<IBlueRow> rv = new List<IBlueRow>();
foreach (IBlueRow r in rows)
rv.Add(r);
return rv;
}
public static IBlueRowCollection ienum_to_collection(IEnumerable<IBlueRow> rows) {
BlueTable rv = null;
if (rows.Count() > 0) {
rv = new BlueTable(rows.ElementAt(0).schema);
foreach (IBlueRow r in rows)
rv.addRow(r);
}
return rv;
}
public static IBlueTableSchema add_column(IBlueTableSchema input_schema, string new_col_name, Type new_col_type) {
List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
cis.Add(new ColumnInfo(new_col_name, cis.Count, new_col_type));
return new BlueTableSchema(input_schema.tableName, cis.ToArray());
}
public static IBlueTableSchema remove_columns(IBlueTableSchema input_schema, params string[] col_names) {
Dictionary<string, bool> removed_col_names = new Dictionary<string, bool>();
foreach (string c in col_names)
removed_col_names[c] = true;
List<ColumnInfo> cis = new List<ColumnInfo>();
foreach (ColumnInfo ci in input_schema.columnInfo)
if (!removed_col_names.ContainsKey(ci.colName))
cis.Add(new ColumnInfo(ci.colName, cis.Count, ci.colType));
return new BlueTableSchema(cis.ToArray(), input_schema.tableName);
}
public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) {
List<int> rv = new List<int>();
foreach (string c in col_names)
rv.Add(schema[c].colNo);
return rv.ToArray();
}
public static string get_appended_string_val(IBlueRow r, int[] col_indices) {
System.Text.StringBuilder sb = new System.Text.StringBuilder();
for (int i = 0; i < col_indices.Length; i++)
sb.Append(r[col_indices[i]]);
return sb.ToString();
}
public delegate string RowValueToString(IBlueRow r, int col_index);
public static string get_appended_string_val(IBlueRow r, int[] col_indices, RowValueToString to_string_conv) {
System.Text.StringBuilder sb = new System.Text.StringBuilder();
for (int i = 0; i < col_indices.Length; i++)
sb.Append(to_string_conv(r, i));
return sb.ToString();
}
public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, string[] group_by_col_names) {
BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length);
return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher);
}
public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, string[] group_by_col_names, ref BlueRowHasher hasher) {
return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher);
}
public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, int[] group_by_col_nos, ref BlueRowHasher hasher) {
Dictionary<int, List<IBlueRow>> rv = new Dictionary<int, List<IBlueRow>>();
hasher = hasher ?? new BlueRowHasher(group_by_col_nos.Length);
foreach (IBlueRow r in rows) {
int row_hash = hasher.get_hash(r, group_by_col_nos);
List<IBlueRow> this_group_rows = null;
if (!rv.TryGetValue(row_hash, out this_group_rows)) {
this_group_rows = new List<IBlueRow>();
rv.Add(row_hash, this_group_rows);
}
this_group_rows.Add(r);
}
return rv;
}
public static void copy_to_row(IBlueRow row, string[] data_col_names, MutableBlueRow output_row) {
foreach (string cn in data_col_names)
output_row.setValue(cn, row[cn]);
}
/// <summary>
/// Copies the column nos specified in data_cols from row, to output_row
/// starting from index start_col_index
/// </summary>
public static void copy_to_row(IBlueRow row, int[] data_col_nos, object[] output_row, int start_col_index) {
foreach (int col_no in data_col_nos)
output_row[start_col_index++] = row[col_no];
}
public static object compute_min_max(object v1, object v2, bool is_max) {
if (!TypeHelper.is_dbnull(v1) && !TypeHelper.is_dbnull(v2)) {
if (TypeHelper.is_numeric_type(v1) && TypeHelper.is_numeric_type(v2)) {
double v1_num = (double)Convert.ChangeType(v1, TypeCode.Double),
v2_num = (double)Convert.ChangeType(v2, TypeCode.Double);
v1_num = Double.IsNaN(v1_num) ? (is_max ? Double.MinValue : Double.MaxValue) : v1_num;
v2_num = Double.IsNaN(v2_num) ? (is_max ? Double.MinValue : Double.MaxValue) : v2_num;
double rv = is_max ? Math.Max(v1_num, v2_num) : Math.Min(v1_num, v2_num);
return Convert.ChangeType(rv, v1.GetType());
} else
return v1;
} else
return TypeHelper.is_dbnull(v1) ? v2 : v1;
}
#region Sorting
public static class ParallelSort {
public class TreeRowEffComparer : IComparer<IBlueRow> {
private int _depth_in_tree_col_no;
private int[] _aggregation_tree_col_nos;
private Pair<int, bool>[] _sort_map;
public TreeRowEffComparer(int depth_in_tree_col_no, int[] aggregation_tree_col_nos, Pair<int, bool>[] sort_map) {
_depth_in_tree_col_no = depth_in_tree_col_no;
_aggregation_tree_col_nos = aggregation_tree_col_nos;
_sort_map = sort_map;
}
public int Compare(IBlueRow x, IBlueRow y) {
return BlueDataUtility.sort_in_a_tree(x, y, _depth_in_tree_col_no, _aggregation_tree_col_nos, _sort_map);
}
}
public class TabularRowComparer : IComparer<IBlueRow> {
private IEnumerable<Pair<int, bool>> _sort_map;
public TabularRowComparer(IEnumerable<Pair<int, bool>> sort_map) { _sort_map = sort_map; }
public int Compare(IBlueRow x, IBlueRow y) { return BlueDataUtility.compare_rows(x, y, _sort_map); }
}
public class TabularRowHashPairComparer : IComparer<Pair<IBlueRow, int>> {
private Pair<int, bool>[] _sort_map;
public TabularRowHashPairComparer(Pair<int, bool>[] sort_map) { _sort_map = sort_map; }
public int Compare(Pair<IBlueRow, int> x, Pair<IBlueRow, int> y) { return BlueDataUtility.compare_rows(x.first, y.first, _sort_map); }
}
#region Public Static Methods
public static void QuicksortSequential<T>(IList<T> arr, IComparer<T> comparer) {
QuicksortSequential(arr, comparer, 0, arr.Count - 1);
}
public static void QuicksortParallel<T>(IList<T> arr, IComparer<T> comparer) {
QuicksortParallel(arr, comparer, 0, arr.Count - 1);
}
#endregion
#region Private Static Methods
private static void QuicksortSequential<T>(IList<T> arr, IComparer<T> comparer, int left, int right) {
if (right > left) {
int pivot = Partition(arr, comparer, left, right);
QuicksortSequential(arr, comparer, left, pivot - 1);
QuicksortSequential(arr, comparer, pivot + 1, right);
}
}
private static void QuicksortParallel<T>(IList<T> arr, IComparer<T> comparer, int left, int right) {
const int SEQUENTIAL_THRESHOLD = 64;
if (right > left) {
if (right - left < SEQUENTIAL_THRESHOLD) {
QuicksortSequential(arr, comparer, left, right);
} else {
int pivot = Partition(arr, comparer, left, right);
System.Threading.Tasks.Parallel.Invoke(new Action[] { delegate {QuicksortParallel(arr, comparer, left, pivot - 1);},
delegate {QuicksortParallel(arr, comparer, pivot + 1, right); }
});
}
}
}
private static void Swap<T>(IList<T> arr, int i, int j) {
T tmp = arr[i];
arr[i] = arr[j];
arr[j] = tmp;
}
private static int Partition<T>(IList<T> arr, IComparer<T> comparer, int low, int high) {
// Simple partitioning implementation
int pivotPos = (high + low) / 2;
T pivot = arr[pivotPos];
Swap(arr, low, pivotPos);
int left = low;
for (int i = low + 1; i <= high; i++) {
if (comparer.Compare(arr[i], pivot) < 0) {
left++;
Swap(arr, i, left);
}
}
Swap(arr, low, left);
return left;
}
#endregion
}
public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) {
foreach (KeyValuePair<string, bool> p in sort_map) {
int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value);
if (comp != 0)
return comp;
}
return 0;
}
public static int compare_rows(IBlueRow r1, IBlueRow r2, IEnumerable<Pair<int, bool>> sort_map) {
foreach (Pair<int, bool> p in sort_map) {
int comp = compare_send_dontcare_to_last(r1, p.first, r2, p.first, p.second);
if (comp != 0)
return comp;
}
return 0;
}
public static int compare_send_dontcare_to_last(IBlueRow r1, int r1_col_no,
IBlueRow r2, int r2_col_no,
bool low_to_high) {
object o1 = BlueDataUtility.get_row_value<object>(r1, r1_col_no, DBNull.Value),
o2 = BlueDataUtility.get_row_value<object>(r2, r2_col_no, DBNull.Value);
return compare_send_dontcare_to_last(o1, o2, low_to_high);
}
public static List<Pair<int, bool>> get_indexed_sort_map(List<Pair<string, bool>> sort_map, IBlueTableSchema schema) {
List<Pair<int, bool>> rv = new List<Pair<int, bool>>();
foreach (Pair<string, bool> p in sort_map)
rv.Add(new Pair<int, bool>(schema[p.first].colNo, p.second));
return rv;
}
/// Comapares 2 rows in a way where nulls, nans, infinities,
/// empty strings etc are sent to the last
public static int compare_send_dontcare_to_last(IBlueRow r1, string r1_col_name,
IBlueRow r2, string r2_col_name,
bool low_to_high) {
object o1 = BlueDataUtility.get_row_value<object>(r1, r1_col_name, DBNull.Value),
o2 = BlueDataUtility.get_row_value<object>(r2, r2_col_name, DBNull.Value);
return compare_send_dontcare_to_last(o1, o2, low_to_high);
}
public static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) {
if (is_dont_care_val(o1))
return is_dont_care_val(o2) ? 0 : 1;
else
return is_dont_care_val(o2) ? -1 : (low_to_high ? ((IComparable)o1).CompareTo(o2)
: ((IComparable)o2).CompareTo(o1));
}
public static bool is_dont_care_val(object o) {
return o == null || o == DBNull.Value ||
o is string && String.IsNullOrEmpty((string)o) ||
o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) ||
o is float && (Single.IsInfinity((float)o) || Single.IsNaN((float)o)) ||
o is Int32 && (Int32.MaxValue.Equals(o) || Int32.MinValue.Equals(o));
}
#endregion
#region Tree Aggregation
public static int[] convert_aggregation_tree(string[] aggregation_tree_col_names, IBlueTableSchema schema) {
List<int> rv = new List<int>();
foreach (string col_name in aggregation_tree_col_names)
rv.Add(schema[col_name].colNo);
return rv.ToArray();
}
public static bool is_leaf_node(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) {
return BlueDataUtility.get_row_value<int>(r, depth_in_tree_col_no, -1) == (aggregation_tree.Count() - 1);
}
public static bool is_leaf_node(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) {
return BlueDataUtility.get_row_value<int>(r, depth_in_tree_col_name, -1) == (aggregation_tree.Count() - 1);
}
public static string generate_aggregation_based_id(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) {
int aggregation_priority = BlueDataUtility.get_row_value<int>(r, depth_in_tree_col_no, -1);
string rv = String.Empty;
for (int p = 0; p <= aggregation_priority; p++)
rv += (rv == String.Empty ? String.Empty : ":") + String.Format("{0},{1}", p, BlueDataUtility.get_row_value<object>(r, aggregation_tree[p], DBNull.Value));
return rv;
}
/// Generates a string based id for a row in a tree
/// by appending the values of the nodes above it
public static string generate_aggregation_based_id(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) {
int aggregation_priority = BlueDataUtility.get_row_value<int>(r, depth_in_tree_col_name, -1);
string rv = String.Empty;
for (int p = 0; p <= aggregation_priority; p++)
rv += (rv == String.Empty ? String.Empty : ":") +
String.Format("{0},{1}", p, BlueDataUtility.get_row_value<object>(r, aggregation_tree.ElementAt(p), DBNull.Value));
return rv;
}
/// <summary>
/// Given an aggregation hierarchy tree, figures if the first
/// arg is a parent of the second arg.
/// </summary>
public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, string depth_in_tree_col_name, string[] aggregation_tree) {
System.Diagnostics.Debug.Assert(parent_row.Contains(depth_in_tree_col_name) && child_row.Contains(depth_in_tree_col_name));
int parent_depth = BlueDataUtility.get_row_value<int>(parent_row, depth_in_tree_col_name, Int32.MaxValue),
child_depth = BlueDataUtility.get_row_value<int>(child_row, depth_in_tree_col_name, Int32.MaxValue);
if (parent_depth < child_depth) {
for (int p = 0; p <= parent_depth; p++) {
string col_name = aggregation_tree.ElementAt(p);
object parent_val = BlueDataUtility.get_row_value<object>(parent_row, col_name, DBNull.Value),
child_val = BlueDataUtility.get_row_value<object>(child_row, col_name, DBNull.Value);
if (!parent_val.Equals(child_val))
return false;
}
return true;
} else
return false;
}
public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, int depth_in_tree_col_no, int[] aggregation_tree) {
int parent_depth = BlueDataUtility.get_row_value<int>(parent_row, depth_in_tree_col_no, Int32.MaxValue),
child_depth = BlueDataUtility.get_row_value<int>(child_row, depth_in_tree_col_no, Int32.MaxValue);
if (parent_depth < child_depth) {
for (int p = 0; p <= parent_depth; p++) {
int col_no = aggregation_tree[p];
object parent_val = BlueDataUtility.get_row_value<object>(parent_row, col_no, DBNull.Value),
child_val = BlueDataUtility.get_row_value<object>(child_row, col_no, DBNull.Value);
if (!parent_val.Equals(child_val))
return false;
}
return true;
} else
return false;
}
public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, string depth_in_tree_col_name, string[] aggregation_tree, Dictionary<string, bool> sort_map) {
if (BlueDataUtility.is_parent(r1, r2, depth_in_tree_col_name, aggregation_tree))
return -1;
else if (BlueDataUtility.is_parent(r2, r1, depth_in_tree_col_name, aggregation_tree))
return 1;
else if (get_row_value<int>(r1, "id", Int32.MaxValue).Equals(get_row_value<int>(r2, "id", Int32.MaxValue)))
return 0;
else {
for (int p = 0; p < aggregation_tree.Count(); p++) {
/// sort_map is used to leaf nodes
if (p == aggregation_tree.Count() - 1) {
/// sorting for leaf nodes is special, since they can be
/// sorted by data columns
System.Diagnostics.Debug.Assert(BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree));
int leaf_comp = compare_rows(r1, r2, sort_map);
if (leaf_comp != 0)
return leaf_comp;
}
/// sort_map is used to find sorting order
string aggregation_col_name = aggregation_tree.ElementAt(p);
bool low_to_high = sort_map.TryGetValue(aggregation_col_name, out low_to_high) ? low_to_high : true;
int comp = BlueDataUtility.compare_send_dontcare_to_last(r1, aggregation_col_name, r2, aggregation_col_name, low_to_high);
if (comp != 0)
return comp;
}
return BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree) ?
compare_rows(r1, r2, sort_map) : 0;
}
}
public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, int depth_in_tree_col_no, int[] aggregation_tree, Pair<int, bool>[] sort_map) {
if (BlueDataUtility.is_parent(r1, r2, depth_in_tree_col_no, aggregation_tree))
return -1;
else if (BlueDataUtility.is_parent(r2, r1, depth_in_tree_col_no, aggregation_tree))
return 1;
else if (get_row_value<int>(r1, "id", Int32.MaxValue).Equals(get_row_value<int>(r2, "id", Int32.MaxValue)))
return 0;
else {
for (int p = 0; p < aggregation_tree.Count(); p++) {
/// sort_map is used to leaf nodes
if (p == aggregation_tree.Count() - 1) {
/// sorting for leaf nodes is special, since they can be
/// sorted by data columns
//System.Diagnostics.Debug.Assert(BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree));
int leaf_comp = compare_rows(r1, r2, sort_map);
if (leaf_comp != 0)
return leaf_comp;
}
/// sort_map is used to find sorting order
int aggregation_col_no = aggregation_tree[p];
bool low_to_high = true;//sort_map.TryGetValue(aggregation_col_name, out low_to_high) ? low_to_high : true;
int comp = BlueDataUtility.compare_send_dontcare_to_last(r1, aggregation_col_no, r2, aggregation_col_no, low_to_high);
if (comp != 0)
return comp;
}
return BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_no, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_no, aggregation_tree) ?
compare_rows(r1, r2, sort_map) : 0;
}
}
#endregion
#region Column accessors
public static T get_array_value<T>(Array r, int col_no, T error_val) {
try {
object rv = (r != null && col_no > -1 && col_no < r.Length) ? r.GetValue(col_no) : DBNull.Value;
if (rv == null || rv == DBNull.Value)
return error_val;
else
return (T)Convert.ChangeType(rv, typeof(T));
} catch {
return error_val;
}
}
public static DateTime get_array_datetime_value(Array r, int col_no, DateTime error_val) {
try {
object rv = (r != null && col_no > -1 && col_no < r.Length) ? r.GetValue(col_no) : error_val;
return _get_datetime_value(rv, error_val);
} catch {
return error_val;
}
}
public static T get_row_value<T>(IBlueRow r, int col_no, T error_val) {
try {
object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : DBNull.Value;
if (rv == null || rv == DBNull.Value)
return error_val;
else
return (T)Convert.ChangeType(rv, typeof(T));
} catch {
return error_val;
}
}
public static T get_row_value<T>(IBlueRow r, string col_name, T error_val) {
try {
object rv = (r != null && r.Contains(col_name)) ? r[col_name] : DBNull.Value;
if (rv == null || rv == DBNull.Value)
return error_val;
else
return (T)Convert.ChangeType(rv, typeof(T));
} catch {
return error_val;
}
}
public static T get_row_enum_value<T>(IBlueRow r, string col_name, T error_val) where T : struct {
try {
object rv = r.Contains(col_name) ? r[col_name] : DBNull.Value;
return _get_row_enum_value<T>(rv, error_val);
} catch {
return error_val;
}
}
public static T get_row_enum_value<T>(IBlueRow r, int col_no, T error_val) where T : struct {
try {
object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : DBNull.Value;
return _get_row_enum_value<T>(rv, error_val);
} catch {
return error_val;
}
}
private static T _get_row_enum_value<T>(object o, T error_val) where T : struct {
try {
if (o == null || o == DBNull.Value)
return error_val;
else if (o is string) {
T result = error_val;
if (Enum.TryParse<T>((string)o, true, out result)) return result;
return error_val;
} else if (o is Int32 || o is Int16 || o is UInt16 || o is UInt32)
return (T)o;
else if (o is T)
return (T)o;
else
return error_val;
} catch {
return error_val;
}
}
public static DateTime get_row_datetime_value(IBlueRow r, string col_name, DateTime error_val) {
try {
object rv = (r != null && r.Contains(col_name)) ? r[col_name] : error_val;
return _get_datetime_value(rv, error_val);
} catch {
return error_val;
}
}
public static DateTime get_row_datetime_value(IBlueRow r, int col_no, DateTime error_val) {
try {
object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : error_val;
return _get_datetime_value(rv, error_val);
} catch {
return error_val;
}
}
private static DateTime _get_datetime_value(object v, DateTime error_val) {
try {
if (v == null || v == DBNull.Value)
return error_val;
else {
if (v is DateTime)
return (DateTime)v;
else if (v is string) {
DateTime rv_dt = error_val;
DateTime.TryParse((string)v, out rv_dt);
return rv_dt;
} else {
double oa_date = (double)Convert.ChangeType(v, typeof(Double));
return DateTime.FromOADate(oa_date);
}
}
} catch {
return error_val;
}
}
#endregion
#region CSV methods
/// <summary>
/// Reads a schema file name, and return the constructed schema
/// </summary>
public static IBlueTableSchema createSchemaFromCSVSchemaFile(string schemaFileName) {
StreamReader sr = new StreamReader(schemaFileName);
IBlueTableSchema retVal = createSchemaFromCSVSchemaFile(sr);
sr.Close();
return retVal;
}
/// <summary>
/// Builds a schema from a schema file. Note that the stream is not closed, so
/// the calling code should do so
/// </summary>
public static IBlueTableSchema createSchemaFromCSVSchemaFile(TextReader sr) {
string tableName = sr.ReadLine();
ArrayList t = new ArrayList();
string l = "";
while ((l = sr.ReadLine()) != null)
t.Add(l.Trim());
string[] cols = t.ToArray(typeof(string)) as String[];
ColumnInfo[] cis = new ColumnInfo[cols.Length];
for (int i = 0; i < cols.Length; i++) {
string[] colTags = cols[i].Split('/');
cis[i] = new ColumnInfo(colTags[0], i, TypeInstanceLoader.loadType(colTags[1]));
}
IBlueTableSchema retVal = new BlueTableSchema(cis, tableName);
if (retVal["id"] == null) {
ArrayList temp = new ArrayList(retVal.columnInfo);
temp.Add(new ColumnInfo("id", temp.Count, typeof(System.Int32)));
retVal = new BlueTableSchema(retVal.tableName,
temp.ToArray(typeof(ColumnInfo)) as ColumnInfo[]);
}
return retVal;
}
/// <summary>
/// Creates an IBlueRow from a CSV data file row; if the string already contains an id,
/// the id argument is ignored
/// </summary>
public static IBlueRow createRowFromCSVFileRow(string fileRow, IBlueTableSchema schema, int id) {
string[] toks = CSVParser.parse(fileRow);
ArrayList rowList = new ArrayList();
if (toks.Length == schema.columnNames.Length) {
for (int i = 0; i < toks.Length; i++)
rowList.Add(myConvertTo(schema[i].colType, toks[i]));
} else {
int idIndex = schema["id"].colNo;
for (int i = 0; i < toks.Length; i++)
rowList.Add(myConvertTo(schema[i < idIndex ? i : i + 1].colType, toks[i]));
rowList.Insert(idIndex, id);
}
return new BlueRow(schema, rowList.ToArray());
}
/// <summary>
/// Returns the CSV data string corresponding to the row
/// </summary>
public static string toCSVDataString(IBlueRow r) {
string retVal = "";
for (int i = 0; i < r.schema.columnNames.Length; i++)
retVal +=
r.schema[i].colType.Equals(typeof(string)) ?
"\"" + r[i].ToString().Replace("\"", "\"\"") + "\"" :
r[i].ToString();
return retVal.Remove(retVal.Length - 1, 1);
}
public static string[] toCSVSchemaFileStrings(IBlueTableSchema schema) {
string[] retVal = new string[2];
retVal[0] = schema.tableName;
foreach (ColumnInfo ci in schema.columnInfo)
retVal[1] += ci.colName + "/" + ci.colType.ToString() + " ";
retVal[1] = retVal[1].Remove(retVal[1].Length - 1, 1);
return retVal;
}
/// <summary>
/// Converts a CSV data value to its original type.
/// </summary>
private static object myConvertTo(Type t, string val) {
val = val.Trim();
if (t.Equals(typeof(System.String)))
return val;
else if (val != "") {
if (t.Equals(typeof(System.Char)))
return Convert.ToChar(val);
if (t.Equals(typeof(System.Int16)))
return Convert.ToInt16(val);
if (t.Equals(typeof(System.Int32)))
return Convert.ToInt32(val);
if (t.Equals(typeof(System.Int64)))
return Convert.ToInt64(val);
if (t.Equals(typeof(System.Double)))
return Convert.ToDouble(val);
if (t.Equals(typeof(System.Single)))
return Convert.ToSingle(val);
if (t.Equals(typeof(System.DateTime)))
return myToDateTime(val);
}
return DBNull.Value;
}
private static object myToDateTime(string val) {
try {
return Convert.ToDateTime(val);
} catch (FormatException) //datetime is not stored in one of the string formats
{
return val.IndexOf('.') > -1 ?
DateTime.FromOADate(Convert.ToDouble(val)) : new DateTime(Convert.ToInt64(val));
}
}
public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) {
StreamReader sr = new StreamReader(fileAddress);
string cols = sr.ReadLine();
string[] columns = cols.Split(',');
ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
ci[0] = new ColumnInfo("id", 0, typeof(int));
for (int i = 1; i < ci.Length; i++)
ci[i] = new ColumnInfo(columns[i - 1], i, typeof(string));
IBlueTableSchema bts = new BlueTableSchema(ci, "csvTable");
BlueTable result = new BlueTable(bts);
int id = 0;
while (!sr.EndOfStream)
result.addRow(createRowFromCSVFileRow(sr.ReadLine(), bts, id++));
sr.Close();
return result;
}
#endregion
#region Magic
/// <summary>
/// Uses a cached compiled expression to implement a truly generic add function. Will throw if given types that cannot be added.
/// </summary>
internal class MagicAdder<T> {
static Func<T, T, T> add;
static MagicAdder() {
ParameterExpression left = Expression.Parameter(typeof(T), "a");
ParameterExpression right = Expression.Parameter(typeof(T), "b");
add = (Func<T, T, T>)Expression.Lambda(Expression.Add(left, right), left, right).Compile();
}
public static T Add(T a, T b) {
return add(a, b);
}
}
#endregion
#region Misc
public static void swap<T>(ref T t1, ref T t2) {
T temp = t1;
t1 = t2;
t2 = temp;
}
#endregion
}
}