From b03fec7f792e7c3dcf9b4db722a5bdb01058b94e Mon Sep 17 00:00:00 2001 From: John Colvin Date: Wed, 5 Jun 2024 10:49:13 -0500 Subject: [PATCH] Change line endings to crlf in a few cs files --- .../BlueData/BlueDataUtility.cs | 4526 ++++++++--------- .../DataStreamPublisher/Program.cs | 312 +- src/DataStreamSuite/PubSubServer/Program.cs | 152 +- 3 files changed, 2495 insertions(+), 2495 deletions(-) diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs index a9b65f3a6..aa09e83bb 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs @@ -1,2263 +1,2263 @@ -using System; -using System.Data; -using System.Collections; -using System.Collections.Generic; -using System.Reflection; -using System.IO; -using System.Diagnostics; -using System.Linq; - -//using BCG.Utilities; -using BCG.Utilities.General; -using System.Linq.Expressions; -//using mbts.spark; -using mbts; -using BCG.Extensions; -using System.Text.Json; -using System.Text.Json.Nodes; - -namespace BCG { - public static class BlueDataUtility { - #region Extensions - public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) { - return BlueDataUtility.get_row_datetime_value(r, col_name, err_value); - } - public static DateTime get_datetime(this IBlueRow r, int col_no, DateTime err_value = default(DateTime)) { - return BlueDataUtility.get_row_datetime_value(r, col_no, err_value); - } - public static T get_enum(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct { - return BlueDataUtility.get_row_enum_value(r, col_name, err_value); - } - public static T get_enum(this IBlueRow r, int col_no, T err_value = default(T)) where T : struct { - return BlueDataUtility.get_row_enum_value(r, col_no, err_value); - } - public static T get(this IBlueRow r, string col_name, T err_value = default(T)) { - return get_row_value(r, col_name, err_value); - } - public static T get(this IBlueRow r, int col_no, T err_value = default(T)) { - return get_row_value(r, col_no, err_value); - } - public static IEnumerable get_column_values(this IBlueRowCollection rows, string col) { - List rv = new List(); - foreach (IBlueRow r in rows) - rv.Add(r.get(col)); - return rv; - } - public static IEnumerable get_colvalues(IEnumerable rows, string col) { - List rv = new List(); - foreach (IBlueRow r in rows) - rv.Add(r.get(col)); - return rv; - } - - public static IDictionary> group_by(IBlueRowCollection rows, string gp_col) { - Dictionary> rv = new Dictionary>(); - foreach (IBlueRow r in rows) { - var gp_val = r.get(gp_col); - List gp_val_list = null; - if (!rv.TryGetValue(gp_val, out gp_val_list)) { - gp_val_list = new List(); - rv.Add(gp_val, gp_val_list); - } - gp_val_list.Add(r); - } - return rv.ToDictionary(kv => kv.Key, kv => kv.Value as IEnumerable); - } - - // this function takes in a key value table and converts it into a relational table, converting the key names into columns. - // The values are returned as string values - // doc_col is the column specifyng the doc name (following mongoDB's nomenclature, one set of key value rows ar a 'doc') - public static IBlueRowCollection keyvalue_to_relational(IBlueRowCollection kv_rows, string doc_col, string key_col, string val_col) { - HashSet keys = new HashSet(); - foreach (string col in get_column_values(kv_rows, key_col)) { - keys.Add(col); - } - if (keys.Contains("id")) { - keys.Remove("id"); - } - //unique_cols.Add(doc_col) - // all the keys become cols - var schema_cols = keys.Select(c => Tuple.Create(c, TypeCode.String)).ToList(); - //schema of the final table needs doc and id - schema_cols.Insert(0, Tuple.Create(doc_col, BlueDataUtility.get_typecode(typeof(T)))); - schema_cols.Insert(0, Tuple.Create("id", TypeCode.Int32)); - var schema = BlueDataUtility.form_schema("relational", schema_cols.ToArray()); - BlueTable rv = new BlueTable(schema); - var gp_map = group_by(kv_rows, doc_col); - foreach (var kv in gp_map) { - MutableBlueRow nr = new MutableBlueRow(schema); - nr.setValue("id", rv.Count); - nr.setValue(doc_col, kv.Key); - foreach (var r in kv.Value) { - var col = r.get(key_col); - if (!keys.Contains(col)) - continue; - var val = r.get(val_col); - nr.setValue(col, val); - } - rv.addRow(nr); - } - return rv; - } - - public static IBlueRowCollection where(this IBlueRowCollection rows, Func where_func) { - if (rows == null) - return null; - else { - BlueTable rv = new BlueTable(rows.schema); - foreach (IBlueRow r in rows) - if (where_func(r)) - rv.addRow(r); - return rv; - } - } - - public static IEnumerable to_ienum(this IBlueRowCollection rows) { - List rv = new List(); - foreach (IBlueRow r in rows) - rv.Add(r); - return rv; - } - - public static IDictionary to_map(this IBlueRowCollection rows) { - Dictionary rv = new Dictionary(); - foreach (IBlueRow r in rows) { - var id = r.get("id"); - rv.Add(id, r); - } - return rv; - } - - public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple[] cols) { - List cis = new List(input_schema.columnInfo); - foreach (Tuple c in cols) - cis.Add(new ColumnInfo(c.Item2, cis.Count, c.Item1)); - return new BlueTableSchema(input_schema.tableName, cis.ToArray()); - } - - public static IBlueRowCollection to_collection(this IBlueRow r) { - if (r == null) - return null; - else { - BlueTable rv = new BlueTable(r.schema); - rv.addRow(r); - return rv; - } - } - - public static DataPacket build_dp(IBlueRowCollection new_rows, DataPacket last_fired_dp) { - IBlueRowCollection added_rows = new_rows; - IBlueRowCollection deleted_rows = last_fired_dp == null ? new BlueTable(new_rows.schema) : last_fired_dp.newRows; - return new DataPacket(new_rows, added_rows, deleted_rows); - } - - public static DataPacket build_dp_inc(IBlueRowCollection new_rows, DataPacket last_fired_dp, bool deep_row_comapre = false) { - if (last_fired_dp == null) { - IBlueRowCollection added_rows = new_rows; - IBlueRowCollection deleted_rows = new BlueTable(new_rows.schema); - return new DataPacket(new_rows, added_rows, deleted_rows); - } else { - IBlueRowCollection added_rows = null; - IBlueRowCollection deleted_rows = null; - if (deep_row_comapre) - compute_differences_fast(new_rows, last_fired_dp.newRows, out deleted_rows, out added_rows); - else - compute_differences_id_based(new_rows, last_fired_dp.newRows, out deleted_rows, out added_rows); - return new DataPacket(new_rows, added_rows, deleted_rows); - } - } - - public static IBlueRowCollection adapt_for_sql_storage(this IBlueRowCollection rows, - bool handle_datetime_min = false, - bool handle_double_nan = false) { - if (rows != null) { - BlueTable rv = new BlueTable(rows.schema); - foreach (IBlueRow r in rows) { - MutableBlueRow mr = new MutableBlueRow(r); - rv.addRow(mr); - foreach (ColumnInfo ci in rows.schema.columnInfo) { - if (handle_datetime_min && ci.colType == typeof(DateTime) && DateTime.MinValue.Equals(r[ci.colName])) - mr.setValue(ci.colName, DBNull.Value); - if (handle_double_nan && ci.colType == typeof(double) && Double.IsNaN(r.get(ci.colName))) - mr.setValue(ci.colName, DBNull.Value); - } - } - return rv; - } else - return null; - } - - #endregion - - #region json - - public static List json_to_json_node_list(string jsonString) { - var jsonNodeList = new List(); - bool noe = String.IsNullOrEmpty(jsonString); - if (noe) - return jsonNodeList; - try { - // Parse the JSON string - JsonArray jsonArray = JsonNode.Parse(jsonString) as JsonArray; - - if (jsonArray == null) { - Console.WriteLine("The JSON string must represent a list."); - return jsonNodeList; - } - - // Iterate through each JsonNode in the JsonArray - foreach (JsonNode jsonNode in jsonArray) { - jsonNodeList.Add(jsonNode); - } - } catch (JsonException ex) { - Console.WriteLine($"Invalid JSON: {ex.Message}"); - } - - return jsonNodeList; - } - - #endregion - - public static void print_table(this IBlueRowCollection table, Func get_col_width) { - var rv = get_printed_rows(table, get_col_width); - Console.WriteLine(rv); - } - - public static string get_printed_rows(this IBlueRowCollection table, Func get_col_width) { - // Print column headers - string rv = String.Empty;// = $"{table.schema.tableName}______________________________________"; - foreach (var col_name in table.schema.columnNames) { - int col_width = get_col_width(col_name); - rv += $"{col_name.PadRight(col_width)} "; - } - rv += "\n"; - //rv += "______________________________________"; - - // Print table data - foreach (IBlueRow row in table) { - foreach (string col_name in table.schema.columnNames) { - int col_width = get_col_width(col_name); - object row_val = row[col_name]; - string row_val_str = row_val.ToString(); - if (row_val != null) { - rv += $"{row_val_str.PadRight(col_width)} "; - } - } - rv += "\n"; - } - return rv; - } - #region Schema types - // these are the only types allowed in schemas - private static IReadOnlyDictionary _typecode_to_type = new Dictionary - { - { TypeCode.Boolean, typeof(bool) }, //3 - { TypeCode.Byte, typeof(byte) }, //6 - { TypeCode.Char, typeof(char) }, //4 - // TODO - { TypeCode.DateTime, typeof(DateTime) },//16 - { TypeCode.Decimal, typeof(decimal) }, //15 - { TypeCode.Double, typeof(double) }, //14 - //{ TypeCode.Empty, null }, - { TypeCode.Int16, typeof(short) }, //7 - { TypeCode.Int32, typeof(int) }, //9 - { TypeCode.Int64, typeof(long) }, //11 - { TypeCode.SByte, typeof(sbyte) }, //5 - { TypeCode.Single, typeof(Single) }, //13 - { TypeCode.String, typeof(string) }, //18 - { TypeCode.UInt16, typeof(UInt16) }, //8 - { TypeCode.UInt32, typeof(UInt32) }, //10 - { TypeCode.UInt64, typeof(UInt64) } //12 - }; - - public static IEnumerable valid_column_typecodes() { - return _typecode_to_type.Keys; - } - - public static IEnumerable valid_column_types() { - return _typecode_to_type.Values; - } - - public static bool is_valid_col_type(Type t) { - TypeCode type_code = get_typecode(t); - return type_code != TypeCode.Empty; - } - - public static TypeCode get_typecode(Type t) { - foreach (var kv in _typecode_to_type) { - if (kv.Value == t) - return kv.Key; - } - //bts.Log.assert_d(false, $"type: {t} not found"); - return TypeCode.Empty; - } - - public static Type get_type(TypeCode type_code) { - Type rv = null; - if (_typecode_to_type.TryGetValue(type_code, out rv)) { - return rv; - } - return null; - } - - public static TypeCode type_code(this ColumnInfo ci) { - return get_typecode(ci.colType); - } - - #endregion - - public static Tuple guess_type(string value) { - if (string.IsNullOrEmpty(value)) { - return Tuple.Create(typeof(string), (object)value); - } - - int val = 0; - // Try to parse as integer - if (int.TryParse(value, out val)) { - return Tuple.Create(typeof(int), (object)val); - } - - double dval = 0.0; - // Try to parse as double - if (double.TryParse(value, out dval)) { - return Tuple.Create(typeof(double), (object)dval); - } - - DateTime dt; - // Try to parse as DateTime - if (DateTime.TryParse(value, out dt)) { - return Tuple.Create(typeof(DateTime), (object)dt); - } - - bool bval; - // Try to parse as boolean - if (bool.TryParse(value, out bval)) { - return Tuple.Create(typeof(bool), (object)bval); - } - - // If none of the above, return string type - return Tuple.Create(typeof(string), (object)value); - } - - public static IBlueTableSchema form_schema(string table_name, params Tuple[] cols) { - List cis = new List(); - foreach (Tuple c in cols) - cis.Add(new ColumnInfo(c.Item1, cis.Count, _typecode_to_type[c.Item2])); - if (cis.FirstOrDefault(ci => ci.colName == "id") == null) - cis.Add(new ColumnInfo("id", cis.Count, typeof(int))); - return new BlueTableSchema(table_name, cis.ToArray()); - } - - public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) { - // just use one row to guess - List cis = new List(); - foreach (IBlueRow r in rows) { - foreach (var ci in r.schema.columnInfo) { - if (ci.colType != typeof(System.String)) { - cis.Add(ci); - continue; - } - var (type, _) = guess_type(r[ci.colName] as string); - cis.Add(new ColumnInfo(ci.colName, ci.colNo, type)); - } - return new BlueTableSchema(cis.ToArray(), rows.schema.tableName); - } - return null; - } - - // converts a row where all columns are string types to a schema where the best types are guesed - public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) { - IBlueTableSchema typed_schema = get_typed_schema(string_rows); - var rv = new BlueTable(typed_schema); - foreach (IBlueRow r in string_rows) { - var typed_row = get_typed_row(r, typed_schema); - rv.addRow(typed_row); - } - return rv; - - static IBlueRow get_typed_row(IBlueRow string_row, IBlueTableSchema typed_schema) { - var rv = new MutableBlueRow(typed_schema); - foreach (var ci in typed_schema.columnInfo) { - var typed_val = convert(ci.colType, string_row[ci.colName].ToString()); - rv.setValue(ci.colName, typed_val); - } - return rv; - } - - static object convert(Type t, string value) { - // Attempt to convert the string to the specified type - try { - return Convert.ChangeType(value, t); - } catch (Exception) { - if (t == typeof(double)) { - // some doubles come with currency and ',' we clean that up here (there might be better ways to do this) - string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray()); - cleaned_val = cleaned_val.Replace('−', '-'); - double rv_d; - if (Double.TryParse(cleaned_val, out rv_d)) - return rv_d; - else - return null; - } - return null; - // Handle conversion failure gracefully - // throw new ArgumentException("Failed to convert the string to the specified type."); - } - } - } - - /// - /// Gets the Hashcode of the row with the boolean indicating whether the order of the rows should - /// be alphabetic or not - /// - /// - /// - /// - public static int getRowHashCode(IBlueRow row, bool isColAlphabetically) { - string retVal = ""; - if (isColAlphabetically) { - string[] colNames = row.schema.columnNames; - Array.Sort(colNames); - foreach (string col in colNames) - retVal += row[col].ToString(); - } else - foreach (object o in row) - retVal += o.ToString(); - return retVal.GetHashCode(); - } - - /// - /// Finds the hashcode only inluding the columns (in the order) listed in the column names argument - /// - /// - /// - /// - public static int getRowHashCode(IBlueRow row, params string[] colNames) { - string retVal = ""; - foreach (string colName in colNames) - retVal += row[colName].ToString(); - return retVal.GetHashCode(); - } - - /// - /// Converts a IBlueRowCollection to a .Net DataTable - /// - /// - /// - public static DataTable toDataTable(IBlueRowCollection rows) { - DataTable retTable = getTableFromSchema(rows.schema); - retTable.BeginLoadData(); - foreach (IBlueRow row in rows) - retTable.Rows.Add(toDataRow(row, retTable)); - retTable.EndLoadData(); - return retTable; - } - - /// - /// Converts to a DataRow, the preconditioner being that order of the columns should be same - /// in the schema of the row and the table - /// - /// - /// - public static DataRow toDataRow(IBlueRow row, DataTable table) { - DataRow retRow = table.NewRow(); - foreach (string colName in row.schema.columnNames) - retRow[colName] = row[colName]; - return retRow; - } - - /// - /// Returns the IBlueTableSchema corresponding to the row - /// - /// - /// DataRow whose schmea is desired - /// - /// - public static IBlueTableSchema getSchema(DataRow row) { return getSchema(row.Table); } - - /// - /// Returns the IBlueTableSchema of the DataTable - /// - /// - /// table whos Schema is desired - /// - /// - public static IBlueTableSchema getSchema(DataTable table) { - ColumnInfo[] cis = new ColumnInfo[table.Columns.Count]; - foreach (DataColumn col in table.Columns) - cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType); - return new BlueTableSchema(cis, table.TableName); - } - - /// - /// Constructs the schema from an object, using the SchemaMapping attribute. Only those - /// properties are converted into schmea columns that have a get accessor - /// - /// - /// - public static IBlueTableSchema getSchema(object o) { - return getSchema(o.GetType()); - } - - - /// - /// Builds a new schema based on an existing one + a set of new columns - /// - /// Base schema - /// Columns to add - public static IBlueTableSchema addColumnsToSchema(IBlueTableSchema schema, params ColumnInfo[] cis) { - ArrayList rv = new ArrayList(schema.columnInfo); - rv.AddRange(cis); - return new BlueTableSchema(schema.tableName, rv.ToArray(typeof(ColumnInfo)) as ColumnInfo[]); - } - - /// - /// appends the cols of s2, to s1, subtracting the id col - /// - /// - /// - /// - public static IBlueTableSchema appendSchemas(IBlueTableSchema s1, IBlueTableSchema appendedSchema) { - List cis = new List(s1.columnInfo); - foreach (ColumnInfo c in appendedSchema.columnInfo) - if (c.colName != "id") - cis.Add(new ColumnInfo(c.colName, cis.Count, c.colType)); - return new BlueTableSchema(s1.tableName, cis.ToArray()); - } - - - public static IBlueTableSchema getSchema(Type t) { - return getSchema(t, false); - } - - - public static IBlueTableSchema getSchema(Type t, bool createIdField) { - return getSchema(t, t.Name, createIdField); - } - public static IBlueTableSchema getSchema(Type t, string tableName, bool createIdField) { - ArrayList colInfos = new ArrayList(); - int i = 0; - foreach (PropertyInfo pi in t.GetProperties()) - if (myIsActiveProperty(pi)) - colInfos.Add(new ColumnInfo(pi.Name, i++, - is_valid_col_type(pi.PropertyType) ? - pi.PropertyType : typeof(String))); - foreach (FieldInfo fi in t.GetFields()) - if (myIsActiveField(fi)) - colInfos.Add(new ColumnInfo(fi.Name, i++, - is_valid_col_type(fi.FieldType) ? fi.FieldType : typeof(String))); - - if (createIdField) - colInfos.Add(new ColumnInfo("id", i++, typeof(int))); - - return new BlueTableSchema(colInfos.ToArray(typeof(ColumnInfo)) as ColumnInfo[], - tableName); - } - - public static IBlueTableSchema getSchema(Type t, string tableName) { - return new BlueTableSchema(getSchema(t).columnInfo, tableName); - } - - - public static IBlueRow createRowFromType(IBlueTableSchema schema, object o) { - return createRowFromType(schema, o, null); - } - - //creates a row from the given type, and if idVal is not null, using the given id - public static IBlueRow createRowFromType(IBlueTableSchema schema, object o, object idVal) { - object[] rowArray = new object[schema.columnInfo.Length]; - Type t = o.GetType(); - foreach (PropertyInfo pi in t.GetProperties()) - if (myIsActiveProperty(pi)) - rowArray[schema[pi.Name].colNo] = - is_valid_col_type(pi.PropertyType) ? - pi.GetValue(o, null) : pi.GetValue(o, null).ToString(); - foreach (FieldInfo fi in t.GetFields()) - if (myIsActiveField(fi)) - rowArray[schema[fi.Name].colNo] = - is_valid_col_type(fi.FieldType) ? - fi.GetValue(o) : fi.GetValue(o).ToString(); - if (idVal != null) - rowArray[schema["id"].colNo] = idVal; - return new BlueRow(schema, rowArray); - } - - public static MutableBlueRow create_mutable_row(IBlueTableSchema schema, object o, object idVal, - Func type_converter) { - object[] rowArray = new object[schema.columnInfo.Length]; - Type t = o.GetType(); - foreach (PropertyInfo pi in t.GetProperties()) - if (myIsActiveProperty(pi)) - rowArray[schema[pi.Name].colNo] = - is_valid_col_type(pi.PropertyType) ? - pi.GetValue(o, null) : type_converter(pi.PropertyType, pi.GetValue(o, null)); - foreach (FieldInfo fi in t.GetFields()) - if (myIsActiveField(fi)) - rowArray[schema[fi.Name].colNo] = - is_valid_col_type(fi.FieldType) ? - fi.GetValue(o) : type_converter(fi.FieldType, fi.GetValue(o)); - if (idVal != null) - rowArray[schema["id"].colNo] = idVal; - return new MutableBlueRow(schema, rowArray); - } - - - public static IBlueTableSchema get_schema(Type t, string tableName, bool createIdField, Func type_converter) { - List colInfos = new List(); - int i = 0; - if (createIdField) - colInfos.Add(new ColumnInfo("id", i++, typeof(int))); - - foreach (PropertyInfo pi in t.GetProperties()) - colInfos.Add(new ColumnInfo(pi.Name, i++, type_converter(pi.PropertyType))); - foreach (FieldInfo fi in t.GetFields()) - colInfos.Add(new ColumnInfo(fi.Name, i++, type_converter(fi.FieldType))); - return new BlueTableSchema(colInfos.ToArray(), tableName); - } - - public static MutableBlueRow create_mutable_row_not_slow(IBlueTableSchema schema, FieldInfo[] fields, PropertyInfo[] props, - object o, object idVal, Func data_converter) { - object[] rowArray = new object[schema.columnInfo.Length]; - if (props != null) { - foreach (PropertyInfo pi in props) - rowArray[schema[pi.Name].colNo] = data_converter(pi.PropertyType, pi.GetValue(o, null)); - } - if (fields != null) { - foreach (FieldInfo fi in fields) - rowArray[schema[fi.Name].colNo] = data_converter(fi.FieldType, fi.GetValue(o)); - } - if (idVal != null) - rowArray[schema["id"].colNo] = idVal; - return new MutableBlueRow(schema, rowArray); - } - - public static T createTypeFromRow(IBlueRow row) { - return (T)createTypeFromRow(row, typeof(T)); - } - - public static object createTypeFromRow(IBlueRow row, Type t) { - object outValue = Activator.CreateInstance(t); - - PropertyInfo[] properties = t.GetProperties(); - - foreach (PropertyInfo pi in properties) { - if (myIsActiveProperty(pi)) - if (row.Contains(pi.Name)) - if (row[pi.Name].Equals(DBNull.Value) && !pi.PropertyType.Equals(typeof(Object))) - pi.SetValue(outValue, null, null); - else - pi.SetValue(outValue, row[pi.Name], null); - } - FieldInfo[] fields = t.GetFields(); - foreach (FieldInfo fi in fields) { - if (myIsActiveField(fi)) - if (row.Contains(fi.Name)) - if (row[fi.Name].Equals(DBNull.Value) && !fi.FieldType.Equals(typeof(Object))) - fi.SetValue(outValue, null); - else - fi.SetValue(outValue, row[fi.Name]); - } - - return outValue; - - } - - //these functions should originally have been made public - //but changing that would change a bunch of code - public static bool isActiveProperty(PropertyInfo pi) { - return myIsActiveProperty(pi); - } - - public static bool isActiveField(FieldInfo fi) { - return myIsActiveField(fi); - } - - private static bool myIsActiveProperty(PropertyInfo pi) { - if (pi.GetGetMethod() == null) - return false; - foreach (Attribute att in pi.GetCustomAttributes(true)) - if (att is SchemaVisibility && !(att as SchemaVisibility).visible) - return false; - return true; - } - - private static bool myIsActiveField(FieldInfo fi) { - if (!fi.IsPublic) - return false; - foreach (Attribute att in fi.GetCustomAttributes(true)) - if (att is SchemaVisibility && !(att as SchemaVisibility).visible) - return false; - return true; - } - - private static bool myIsActiveMember(MemberInfo mi) { - if (!(mi is FieldInfo) || - !(mi is PropertyInfo) || - (mi is PropertyInfo && (mi as PropertyInfo).GetGetMethod() == null) || - (mi is FieldInfo && !(mi as FieldInfo).IsPublic)) - return false; - foreach (Attribute att in mi.GetCustomAttributes(true)) - if (att is SchemaVisibility && !(att as SchemaVisibility).visible) - return false; - return true; - } - - private static bool myDoesRowExistsInTable(IBlueRowCollection brc2, string[] columns, object[] oarray) { - bool rowExists = false; - foreach (IBlueRow br in brc2) { - bool allElementsSame = true; - for (int i = 0; i < columns.Length; i++) { - string col = columns[i]; - object o1 = br[col]; - object o2 = oarray[i]; - allElementsSame = (o1.Equals(o2) == false) ? false : allElementsSame; - } - if (allElementsSame == true) { - rowExists = true; - break; - } - } - return rowExists; - } - - /// - /// Constructs a DataTable from the IBlueTableSchema - /// - /// - /// Schema whose table needs to be created - /// - /// - public static DataTable getTableFromSchema(IBlueTableSchema schema) { - DataTable retTable = new DataTable(schema.tableName); - foreach (ColumnInfo ci in schema.columnInfo) - retTable.Columns.Add(ci.colName, ci.colType); - return retTable; - } - - /// - /// Constructs a BlueRow from a DataRow. - /// - /// source row - /// - public static IBlueRow toBlueRow(DataRow row) { - return new BlueRow(getSchema(row), row); - } - - public static IBlueRowCollection toIBlueRowCollection(DataTable dt) { - BlueTable retVal = new BlueTable(getSchema(dt)); - foreach (DataRow dr in dt.Rows) - retVal.addRow(dr); - return retVal; - } - - public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) { - BlueTable retVal = null; - foreach (DataRow dr in rows) { - IBlueRow r = toBlueRow(dr); - if (retVal == null) - retVal = new BlueTable(r.schema); - retVal.addRow(r); - } - return retVal; - } - - /// - /// Returns the nth row of the IBlueRowCollection. This method could be used - /// for any of the nonRandom-Access-IBlueRowCollection-Implementations, as its - /// O(n) - /// - /// - /// - /// - public static IBlueRow extractRow(IBlueRowCollection rows, int index) { - int i = 0; - foreach (IBlueRow row in rows) - if (i++ == index) - return row; - return null; - } - - /// - /// Computes the equality fo the 2 rows. The rows might have different columns orderings - /// - /// - /// - /// - - - /// - /// Computes a string equivelant for the row, using the columns provided - /// in the argument - /// - /// - /// - /// - public static string rowToString(IBlueRow row, params string[] colNames) { - string retVal = ""; - foreach (string colName in colNames) - retVal += (row[colName] != null ? row[colName].ToString().Trim() : "NULL") + "\n"; - return retVal; - } - - public static string rowToString(IBlueRow row) { return rowToString(row, row.schema.columnNames); } - - public static string rowCollectionToString(IBlueRowCollection rowColl) { - if (rowColl == null) - return "Null IBlueRowCollection"; - else { - string retVal = ""; - foreach (IBlueRow r in rowColl) - retVal += retVal == "" ? rowToString(r) : "\n" + rowToString(r); - return retVal; - } - } - - /// - /// Compares 2 rows, using the column names in the order provided, ie the column name with - /// the lower index in the array has higher priority. Also a null is always assumed - /// to be less than a non-null - /// - /// - /// - /// - /// - public static int compare_rows(IBlueRow r1, - IBlueRow r2, - params string[] colNames) { - int retVal = 0; - foreach (string colName in colNames) { - object val1 = r1[colName]; - object val2 = r2[colName]; - retVal = _compare_row_cols(val1, val2); - if (retVal != 0) - break; - } - return retVal; - } - - private static int _compare_row_cols(object v1, object v2) { - if (_is_null(v1)) - return _is_null(v2) ? 0 : -1; - else { - if (_is_null(v2)) - return 1; - else { - if (v1 is IComparable && v2 is IComparable) - return (v1 as IComparable).CompareTo(v2); - else if (v1 is byte[] && v2 is byte[]) - return _compare_bytes(v1 as byte[], v2 as byte[]) ? 0 : 1; - else - return 1; - } - } - } - - private static bool _compare_bytes(byte[] v1, byte[] v2) { - if (v1.Length != v2.Length) - return false; - else - for (int i = 0; i < v1.Length; i++) - if (v1[i] != v2[i]) - return false; - return true; - } - - private static bool _is_null(object o) { return o == null || o == DBNull.Value; } - /// - /// Creates a new IBlueRow with a schema matching the schema provided in 'new schema' - /// - /// a row with the values to be converted - /// the desired schema for the output row - /// - public static IBlueRow changeSchema(IBlueRow inRow, IBlueTableSchema newSchema) { - object[] outputValues = new object[newSchema.columnInfo.Length]; - - for (int i = 0; i < outputValues.Length; ++i) - outputValues[i] = inRow[newSchema.columnNames[i]]; - - return new BlueRow(newSchema, outputValues); - } - - /// - /// Combines input tables into a single output table, with different ID's - /// this will result in new rows being created, which is kind of expensive - /// - /// - /// - public static IBlueRowCollection combineTables(params IBlueRowCollection[] tables) { - //get some base cases out of the way - if (tables.Length == 0) - return null; - if (tables.Length == 1) - return tables[0]; - - BlueTable result = new BlueTable(tables[0].schema); - int idToUseNext = 0; - int numCols = tables[0].schema.columnInfo.Length; - int idCol = tables[0].schema["id"].colNo; - IBlueRowCollection previousTable = null; - foreach (IBlueRowCollection table in tables) { - if (previousTable != null) - if (!compareSchemas(previousTable.schema, table.schema)) - throw new ArgumentException("Error, tables must have the same schema to be combined"); - - foreach (IBlueRow r in table) { - object[] values = new object[numCols]; - r.CopyTo(values, 0); - values[idCol] = idToUseNext++; - result.addRow(new BlueRow(table.schema, values)); - } - - previousTable = table; - - } - - return result; - } - /// - /// Compares two rows (having the same schema) and returns the - /// columns whose values dont match - /// - /// - /// - /// - public static string[] compareRows(IBlueRow r1, IBlueRow r2) { - ArrayList retVal = new ArrayList(); - foreach (string col in r1.schema.columnNames) - if (!r1[col].Equals(r2[col])) - retVal.Add(col); - return retVal.ToArray(typeof(string)) as string[]; - } - - public static BlueTable cloneRowCollection(IBlueRowCollection rows) { - BlueTable retVal = new BlueTable(rows.schema); - foreach (IBlueRow r in rows) - retVal.addRow(new BlueRow(r)); - return retVal; - } - - /// - /// Creates a new row with the values of the row passed, except that the newValues are forced into the - /// returned row, in places specified by the indexes (makes any sense ?) - /// - /// - /// - /// - /// - public static IBlueRow getNewReplacedRow(IBlueRow r, object[] newValues, int[] indexes) { - if (!newValues.Length.Equals(indexes.Length)) - throw new Exception("Debug Assert"); - - object[] rowArray = new object[r.Count]; - r.CopyTo(rowArray, 0); - for (int i = 0; i < indexes.Length; i++) - rowArray[indexes[i]] = newValues[i]; - return new BlueRow(r.schema, rowArray); - } - - public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, int index) { - return getNewReplacedRow(r, new object[] { newValue }, new int[] { index }); - } - - public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, string colName) { - return getNewReplacedRow(r, newValue, r.schema[colName].colNo); - } - - /// - /// Computes the max row, based on the cols provided. If the collection is empty, null is returned - /// - /// - /// - /// - public static IBlueRow computeMaxRow(IBlueRowCollection rows, params string[] cols) { - IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null; - foreach (IBlueRow r in rows) - retVal = compare_rows(r, retVal, cols) > 0 ? r : retVal; - return retVal; - } - - /// - /// Computes the max row, based on the cols provided. If the collection is empty, null is returned - /// - /// - /// - /// - public static IBlueRow computeMinRow(IBlueRowCollection rows, params string[] cols) { - IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null; - foreach (IBlueRow r in rows) - retVal = compare_rows(r, retVal, cols) < 0 ? r : retVal; - return retVal; - } - - /// - /// Computes the sum of rows, based on the cols provided. If the collection is empty, null is returned - /// - /// - /// - /// - public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) { - if (rows.Count == 0) return null; - - IBlueRow retVal = BlueDataUtility.extractRow(rows, 0); - - object[] sums = new object[cols.Length]; - int[] idxs = new int[cols.Length]; - for (int i = 0; i < cols.Length; i++) { - sums[i] = 0.0; - idxs[i] = rows.schema[cols[i]].colNo; - } - - foreach (IBlueRow r in rows) { - for (int i = 0; i < cols.Length; i++) { - double dval = BlueDataUtility.get_row_value(r, cols[i], 0.0); - //r[cols[i]] == null || r[cols[i]] is DBNull ? 0.0 : (double)r[cols[i]]; - sums[i] = (double)sums[i] + dval; - } - } - - return BlueDataUtility.getNewReplacedRow(retVal, sums, idxs); - } - - - public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) { - if (rows.Count == 0) return null; - IBlueRow retVal = BlueDataUtility.extractRow(rows, 0); - if (rows.Count == 1) return retVal; - - object[] sums = new object[cols.Length]; - - int[] idxs = new int[cols.Length]; - for (int i = 0; i < cols.Length; i++) { - sums[i] = default(T); - idxs[i] = rows.schema[cols[i]].colNo; - } - - foreach (IBlueRow r in rows) { - for (int i = 0; i < cols.Length; i++) { - T tval = r[cols[i]] == null || r[cols[i]] is DBNull ? default(T) : (T)r[cols[i]]; - sums[i] = MagicAdder.Add((T)sums[i], tval); - } - } - - return BlueDataUtility.getNewReplacedRow(retVal, sums, idxs); - } - - /// - /// Uses the ID to identify the presence of a row in a collection - /// - /// - /// - /// - public static bool rowPresent(IBlueRowCollection table, IBlueRow row) { - return table.containsID((int)row["id"]); - } - - public static void computeDifferences(IBlueRowCollection newRows, - IBlueRowCollection oldRows, - out IBlueRowCollection dRows, - out IBlueRowCollection aRows) { - computeDifferences( - newRows, - oldRows, - newRows.schema.columnNames, - out dRows, - out aRows); - } - - public static bool compute_equality(IBlueRowCollection rows1, IBlueRowCollection rows2) { - IBlueRowCollection d_rows, a_rows; - compute_differences_fast(rows2, rows1, out d_rows, out a_rows); - return d_rows.Count == 0 && a_rows.Count == 0; - } - - public static bool compute_equality(ColumnInfo ci1, ColumnInfo ci2) { - return (ci1.colNo == ci2.colNo) && (ci1.colName == ci2.colName) && (ci1.colType == ci2.colType); - } - - public static bool compute_equality(IBlueTableSchema s1, IBlueTableSchema s2) { - if (s1.tableName != s1.tableName) - return false; - if (s1.columnInfo.Length != s2.columnInfo.Length) - return false; - for (int i = 0; i < s1.columnInfo.Length; i++) { - ColumnInfo ci1 = s2.columnInfo[i], ci2 = s2.columnInfo[i]; - if (!compute_equality(ci1, ci2)) - return false; - } - return true; - } - - public static bool compute_equality(IBlueRow r1, IBlueRow r2) { - return compute_row_equality(r1, r2, r1.schema.columnNames); - } - - public static bool compute_row_equality(IBlueRow r1, - IBlueRow r2, - params string[] col_names) { - return compare_rows(r1, r2, col_names) == 0 ? true : false; - } - - public static string to_string(ColumnInfo ci) { - return ci.colNo + ";" + ci.colName + ";" + ci.colType; - } - - public static string to_string(this IBlueRow r) { - string rv = String.Empty; - foreach (ColumnInfo c in r.schema.columnInfo) { - rv += $"{c.colName}: {r[c.colName]}\n"; - } - return rv; - } - - public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, - out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) { - if (old_rows == null || old_rows.Count == 0) { - out_d_rows = new BlueTable(new_rows.schema); - out_a_rows = new BlueTable(new_rows); - return; - } - int[] col_indices = new int[new_rows.schema.columnNames.Length]; - for (int i = 0; i < col_indices.Length; i++) - col_indices[i] = i; - compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows); - } - - public static void compute_differences_id_based(IBlueRowCollection new_rows, IBlueRowCollection old_rows, - out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) { - if (old_rows == null || old_rows.Count == 0) { - out_d_rows = new BlueTable(new_rows.schema); - out_a_rows = new BlueTable(new_rows); - return; - } - int[] col_indices = new int[] { new_rows.schema["id"].colNo }; - compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows); - } - - public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, int[] col_indices, - out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) { - Func rows_equal = (r1, r2) => { - foreach (int col_no in col_indices) - if (!(r1[col_no] ?? DBNull.Value).Equals(r2[col_no] ?? DBNull.Value)) - return false; - return true; - }; - - BlueTable del_rows = new BlueTable(new_rows.schema), add_rows = new BlueTable(new_rows.schema); - // adds + changes - foreach (IBlueRow r in new_rows) - if (!BlueDataUtility.rowPresent(old_rows, r)) - add_rows.addRow(r); - else if (!rows_equal(r, old_rows[(int)r["id"]])) { - add_rows.addRow(r); - del_rows.addRow(old_rows[(int)r["id"]]); - } - // deletes - foreach (IBlueRow r in old_rows) - if (!BlueDataUtility.rowPresent(new_rows, r)) - del_rows.addRow(r); - out_d_rows = del_rows; - out_a_rows = add_rows; - } - - - public static void computeDifferences(IBlueRowCollection c1, - IBlueRowCollection c2, - string[] columns, - out IBlueRowCollection dRows, - out IBlueRowCollection aRows) { - dRows = new BlueTable(c1.schema); - aRows = new BlueTable(c1.schema); - - // adds + changes - foreach (IBlueRow nr in c1) - if (!BlueDataUtility.rowPresent(c2, nr)) - (aRows as BlueTable).addRow(nr); - else if (BlueDataUtility.compare_rows(nr, c2[(int) - nr["id"]], columns) != 0) { - (aRows as BlueTable).addRow(nr); - (dRows as BlueTable).addRow(c2[(int)nr["id"]]); - - } - - // deletes - foreach (IBlueRow nr in c2) - if (!BlueDataUtility.rowPresent(c1, nr)) - (dRows as BlueTable).addRow(nr); - } - - //brcDiff = brc1 - brc2 - public static void diff(IBlueRowCollection brc1, IBlueRowCollection brc2, string[] columns, out IBlueRowCollection brcDiff) { - brcDiff = new BlueTable(brc1.schema); - foreach (IBlueRow br in brc1) { - object[] oarray = new object[columns.Length]; - for (int i = 0; i < columns.Length; i++) { - string col = columns[i]; - oarray[i] = br[col]; - } - if (myDoesRowExistsInTable(brc2, columns, oarray) == false) - (brcDiff as BlueTable).addRow(br); - } - } - - /// - /// Abosrbs the deletes and adds into the BlueTable - /// - /// - /// - /// - public static void absorbChanges(BlueTable t, IBlueRowCollection dels, IBlueRowCollection adds) { - foreach (IBlueRow r in dels) - if (!t.deleteRow(r)) - throw new Exception(String.Format("{0} does not have row with id {1} to delete", t.schema.tableName, r["id"])); - - foreach (IBlueRow r in adds) - t.addRow(r); - } - - public static IBlueRowCollection toBlueRowCollection(IBlueRow r) { - BlueTable rv = new BlueTable(r.schema); - rv.addRow(r); - return rv; - } - - /// - /// Converts an ICollection of IBlueRows to IBlueRowCollection. If there are no rows in the - /// collection, a null is returned - /// - /// - /// - public static IBlueRowCollection toBlueRowCollection(ICollection col) { - BlueTable retVal = null; - foreach (IBlueRow r in col) { - retVal = retVal ?? new BlueTable(r.schema); - retVal.addRow(r); - } - return retVal; - } - - /// - /// Computes the rows that are truely deleted, added or changed (with respect to relevant columns) - /// - /// all the deleted rows in a certain callback - /// all the added rows in a certain callback - /// the rows that are truely deleted or changed with respect to the relevant columns - /// the rows that are truely added or changed with respect to the relevant columns - /// - public static void getTrueDeletesAndAdds(IBlueRowCollection dels, - IBlueRowCollection adds, - out IBlueRowCollection truelyDels, - out IBlueRowCollection truelyAdds, - string[] cols) { - truelyDels = new BlueTable(dels); - truelyAdds = new BlueTable(adds); - foreach (IBlueRow r in dels) { - if (adds.containsID((int)r["id"]) && - BlueDataUtility.compute_row_equality(adds[(int)r["id"]], r, cols)) { - (truelyDels as BlueTable).deleteRow((int)r["id"]); - (truelyAdds as BlueTable).deleteRow((int)r["id"]); - } - } - } - - public static IBlueRowCollection getSerializableCollection(IBlueRowCollection coll) { - BlueTable retVal = new OrderedBlueTable(coll.schema); - foreach (IBlueRow r in coll) - retVal.addRow(new BlueRow(r)); - return retVal; - } - - /// - /// extracts out a column from a table - /// - /// - /// - /// - public static ArrayList extractColumn(IBlueRowCollection rows, string colName) { - ArrayList rv = new ArrayList(); - foreach (IBlueRow r in rows) - rv.Add(r[colName]); - return rv; - } - - /// - /// applies the deletedRows and addedRows to the oldState and asserts if it does not - /// match the newRows. for an old state with 10 K rows, and all 10 K rows being deleted and - /// added it takes ~ 35 ms to execute this method. - /// - /// - /// - /// - /// - public static void checkConsistency(IBlueRowCollection oldState, - IBlueRowCollection newRows, - IBlueRowCollection delRows, - IBlueRowCollection addedRows) { - BlueTable calc = new BlueTable(oldState); - Debug.Assert(oldState != null && newRows != null && delRows != null && addedRows != null); - - foreach (IBlueRow r in delRows) { - Debug.Assert(calc.containsID((int)r["id"]), "deleted row's id = " + r["id"] + " does not exist"); - calc.deleteRow(r); - } - - foreach (IBlueRow r in addedRows) { - Debug.Assert(!calc.containsID((int)r["id"]), "added rows's id = " + r["id"].ToString() + " already exists"); - calc.addRow(r); - } - - Debug.Assert(calc.Count == newRows.Count, "row count should be the same. Counts: calc=" + calc.Count + "; newRows=" + newRows.Count); - foreach (IBlueRow r in calc) - Debug.Assert(newRows.containsID((int)r["id"]), r["id"] + " is not found"); - - } - - /// - /// Extracts values from an IBlueRow specified by the columnNames arg - /// - /// - /// - /// - public static object[] getSelectedValues(IBlueRow r, params string[] colNames) { - object[] retVal = new object[colNames.Length]; - for (int i = 0; i < colNames.Length; i++) - retVal[i] = r[colNames[i]]; - return retVal; - } - - - /// - /// From an IBlueRowCollection, for a set of columns, extracts a row with the same values as - /// provided in the third arg - /// - /// - /// - /// - /// - public static IBlueRow getRowWithKey(IBlueRowCollection rows, string[] keyColumns, object[] colValues) { - foreach (IBlueRow r in rows) { - object[] vals = getSelectedValues(r, keyColumns); - if (CollectionHelpers.equal(vals, colValues)) - return r; - } - return null; - } - - public static IBlueRowCollection deepCopy(IBlueRowCollection rows) { - BlueTable retVal = new BlueTable(rows.schema); - foreach (IBlueRow r in rows) - retVal.addRow(new BlueRow(r)); - return retVal; - } - - /// - /// Returns the columns, sorted alphabetically. The second arg - /// is for the inclusion of id - /// - /// - /// - /// - public static int[] getColumnsInSortedOrder(IBlueTableSchema schema, bool includeId) { - ArrayList names = new ArrayList(schema.columnNames), - retVal = new ArrayList(); - names.Sort(); - foreach (string n in names) - if (n != "id" || includeId) - retVal.Add(schema[n].colNo); - return retVal.ToArray(typeof(int)) as int[]; - } - - //this class is used so that we can sort a schema by column name - //that way, prepublished tables will allways get published with schemas in the same order - //regardless of how the underlying streams end up working - private class ColumnInfoComparisonWrapper : IComparable { - - ColumnInfo myColumn; - - public string ColumnName { - get { return myColumn.colName; } - //can't set it, these are only for reading columns - } - - public ColumnInfo Column { - get { return myColumn; } - } - public ColumnInfoComparisonWrapper(ColumnInfo theColumn) { - myColumn = theColumn; - } - - public int CompareTo(ColumnInfoComparisonWrapper otherGuy) { - return ColumnName.CompareTo(otherGuy.ColumnName); - } - } - - public static IBlueTableSchema getSortedSchema(IBlueTableSchema inSchema) { - List sortedColNames = new List(inSchema.columnNames); - sortedColNames.Sort(); - List sortedColInfos = new List(); - int colNo = 0; - foreach (string c in sortedColNames) { - ColumnInfo ci = inSchema[c]; - sortedColInfos.Add(new ColumnInfo(c, colNo++, ci.colType)); - - } - return new BlueTableSchema(inSchema.tableName, sortedColInfos.ToArray()); - } - - public static IBlueTableSchema getSortedSchema(IBlueTableSchema theSchema, out int[] sortedSchemaToOriginalSchemaMapping, out int idColumnInSortedSchema) { - - //for now this function will not do anything - //that is because we don't want to break the C++ deserialization code - int numCols = theSchema.columnNames.Length; - sortedSchemaToOriginalSchemaMapping = new int[numCols]; - for (int col = 0; col < theSchema.columnNames.Length; ++col) { - //the mapping is just the identity mapping - sortedSchemaToOriginalSchemaMapping[col] = col; - - - } - - idColumnInSortedSchema = theSchema["id"].colNo; - - return theSchema; - - /* - //set this to -1 in case the method doesn't find an id column - idColumnInSortedSchema = 01; - List schemaColumns = new List(); - - foreach (ColumnInfo ci in theSchema.columnInfo) - schemaColumns.Add(new ColumnInfoComparisonWrapper(ci)); - - schemaColumns.Sort(); - - int numCols = theSchema.columnInfo.Length; - ColumnInfo[] outputColumns = new ColumnInfo[numCols]; - sortedSchemaToOriginalSchemaMapping = new int[numCols]; - - for(int i = 0; i < numCols; ++i) - { - string thisColName = schemaColumns[i].ColumnName; - outputColumns[i] = new ColumnInfo(thisColName, i, schemaColumns[i].Column.colType); - sortedSchemaToOriginalSchemaMapping[i] = theSchema[thisColName].colNo; - - //we cache the ID column because it'll be differentin the serialized row than in our schema - if (thisColName == "id") - idColumnInSortedSchema = i; - - } - - - theSchema = new BlueTableSchema(theSchema.tableName,outputColumns); - return theSchema; - */ - - - } - - - - - /// - /// Compares 2 schemas for column-by-column match - /// - /// - /// - /// true if equal - public static bool compareSchemas(IBlueTableSchema s1, IBlueTableSchema s2) { - bool result = true; - try { - if (s1.columnInfo.Length != s2.columnInfo.Length) - result = false; - else - foreach (ColumnInfo ci1 in s1.columnInfo) { - ColumnInfo ci2 = s2[ci1.colNo]; - if (ci1.colName != ci2.colName || ci1.colType != ci2.colType || ci1.colNo != ci2.colNo) { - result = false; - break; - } - } - } catch { - result = false; // the col# wasn't found in s2 - } - return result; - } - - public static void validate_row_schema(IBlueRow r) { - foreach (var ci in r.schema.columnInfo) { - object rval = r[ci.colName]; - if (rval == null) - continue; - bts.Log.assert(rval.isDBNull() || rval.GetType().Equals(ci.colType)); - } - } - - public static void validate_table_schema(IBlueRowCollection rows) { - foreach (IBlueRow r in rows) { - validate_row_schema(r); - } - } - - public static IBlueRowCollection removeRows(BlueTable table, string[] cols, object[] vals) { - IBlueRowCollection dels = findMatchingRows(table, cols, vals); - table.deleteRows(dels); - return dels; - } - - public static IBlueRowCollection findMatchingRows(IBlueRowCollection rows, string[] cols, object[] vals) { - BlueTable rv = new BlueTable(rows.schema); - foreach (IBlueRow r in rows) { - bool match = true; - for (int i = 0; i < cols.Length; i++) - match &= vals[i].Equals(r[cols[i]]); - if (match) - rv.addRow(r); - } - return rv; - } - - public static void checkRowTypes(IBlueRowCollection rows) { - foreach (IBlueRow r in rows) - foreach (string col in r.schema.columnNames) - if (r[col] != null && r[col] != DBNull.Value) - Debug.Assert(r[col].GetType() == rows.schema[col].colType, - String.Format("col {0} in table {1} is of type {2} in schema but is a {3} in the row", col, rows.schema.tableName, r.schema[col].colType.ToString(), r[col].GetType().ToString())); - } - - public static void addRows(ref BlueTable table, ICollection rows) { - foreach (IBlueRow r in rows) - table.addRow(r); - } - - public static void removeRows(ref BlueTable table, ICollection rows) { - foreach (IBlueRow r in rows) - table.deleteRow(r); - } - - public static void sortOutPacket(IBlueRowCollection dels, IBlueRowCollection adds, string[] keyCols, - out IBlueRowCollection addedRows, - out IBlueRowCollection deletedRows, - out IBlueRowCollection changedRows) { - addedRows = new BlueTable(dels.schema); - deletedRows = new BlueTable(dels.schema); - changedRows = new BlueTable(dels.schema); - - KeysBasedTable2 keyedDels = new KeysBasedTable2(dels.schema, keyCols), - keyedAdds = new KeysBasedTable2(dels.schema, keyCols); - - foreach (IBlueRow r in dels) - keyedDels.update(r); - foreach (IBlueRow r in adds) - keyedAdds.update(r); - - foreach (IBlueRow r in keyedDels) { - object[] key = BlueDataUtility.getSelectedValues(r, keyCols); - if (keyedAdds.getRow(key) != null) { - ///change - (changedRows as BlueTable).addRow(r); - } else { - ///delete - (deletedRows as BlueTable).addRow(r); - } - } - - foreach (IBlueRow r in keyedAdds) { - object[] key = BlueDataUtility.getSelectedValues(r, keyCols); - if (keyedDels.getRow(key) == null) { - (addedRows as BlueTable).addRow(r); - } - } - } - - public static int HashExpStrike(DateTime exp, double strike) { - return ((DateTime)exp).ToShortDateString().GetHashCode() ^ strike.GetHashCode(); - } - - public static IEnumerable collection_to_ienum(IBlueRowCollection rows) { - List rv = new List(); - foreach (IBlueRow r in rows) - rv.Add(r); - return rv; - } - - public static IBlueRowCollection ienum_to_collection(IEnumerable rows) { - BlueTable rv = null; - if (rows.Count() > 0) { - rv = new BlueTable(rows.ElementAt(0).schema); - foreach (IBlueRow r in rows) - rv.addRow(r); - } - return rv; - } - - public static IBlueTableSchema add_column(IBlueTableSchema input_schema, string new_col_name, Type new_col_type) { - List cis = new List(input_schema.columnInfo); - cis.Add(new ColumnInfo(new_col_name, cis.Count, new_col_type)); - return new BlueTableSchema(input_schema.tableName, cis.ToArray()); - } - - public static IBlueTableSchema remove_columns(IBlueTableSchema input_schema, params string[] col_names) { - Dictionary removed_col_names = new Dictionary(); - foreach (string c in col_names) - removed_col_names[c] = true; - List cis = new List(); - foreach (ColumnInfo ci in input_schema.columnInfo) - if (!removed_col_names.ContainsKey(ci.colName)) - cis.Add(new ColumnInfo(ci.colName, cis.Count, ci.colType)); - return new BlueTableSchema(cis.ToArray(), input_schema.tableName); - } - - public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) { - List rv = new List(); - foreach (string c in col_names) - rv.Add(schema[c].colNo); - return rv.ToArray(); - } - - public static string get_appended_string_val(IBlueRow r, int[] col_indices) { - System.Text.StringBuilder sb = new System.Text.StringBuilder(); - for (int i = 0; i < col_indices.Length; i++) - sb.Append(r[col_indices[i]]); - return sb.ToString(); - } - - public delegate string RowValueToString(IBlueRow r, int col_index); - public static string get_appended_string_val(IBlueRow r, int[] col_indices, RowValueToString to_string_conv) { - System.Text.StringBuilder sb = new System.Text.StringBuilder(); - for (int i = 0; i < col_indices.Length; i++) - sb.Append(to_string_conv(r, i)); - return sb.ToString(); - } - - public static IDictionary> group_together(IBlueRowCollection rows, string[] group_by_col_names) { - BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length); - return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher); - } - - public static IDictionary> group_together(IBlueRowCollection rows, string[] group_by_col_names, ref BlueRowHasher hasher) { - return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher); - } - - public static IDictionary> group_together(IBlueRowCollection rows, int[] group_by_col_nos, ref BlueRowHasher hasher) { - Dictionary> rv = new Dictionary>(); - hasher = hasher ?? new BlueRowHasher(group_by_col_nos.Length); - foreach (IBlueRow r in rows) { - int row_hash = hasher.get_hash(r, group_by_col_nos); - List this_group_rows = null; - if (!rv.TryGetValue(row_hash, out this_group_rows)) { - this_group_rows = new List(); - rv.Add(row_hash, this_group_rows); - } - this_group_rows.Add(r); - } - return rv; - } - - public static void copy_to_row(IBlueRow row, string[] data_col_names, MutableBlueRow output_row) { - foreach (string cn in data_col_names) - output_row.setValue(cn, row[cn]); - } - - /// - /// Copies the column nos specified in data_cols from row, to output_row - /// starting from index start_col_index - /// - public static void copy_to_row(IBlueRow row, int[] data_col_nos, object[] output_row, int start_col_index) { - foreach (int col_no in data_col_nos) - output_row[start_col_index++] = row[col_no]; - } - - public static object compute_min_max(object v1, object v2, bool is_max) { - if (!TypeHelper.is_dbnull(v1) && !TypeHelper.is_dbnull(v2)) { - if (TypeHelper.is_numeric_type(v1) && TypeHelper.is_numeric_type(v2)) { - double v1_num = (double)Convert.ChangeType(v1, TypeCode.Double), - v2_num = (double)Convert.ChangeType(v2, TypeCode.Double); - v1_num = Double.IsNaN(v1_num) ? (is_max ? Double.MinValue : Double.MaxValue) : v1_num; - v2_num = Double.IsNaN(v2_num) ? (is_max ? Double.MinValue : Double.MaxValue) : v2_num; - double rv = is_max ? Math.Max(v1_num, v2_num) : Math.Min(v1_num, v2_num); - return Convert.ChangeType(rv, v1.GetType()); - } else - return v1; - } else - return TypeHelper.is_dbnull(v1) ? v2 : v1; - } - - #region Sorting - - public static class ParallelSort { - public class TreeRowEffComparer : IComparer { - private int _depth_in_tree_col_no; - private int[] _aggregation_tree_col_nos; - private Pair[] _sort_map; - - public TreeRowEffComparer(int depth_in_tree_col_no, int[] aggregation_tree_col_nos, Pair[] sort_map) { - _depth_in_tree_col_no = depth_in_tree_col_no; - _aggregation_tree_col_nos = aggregation_tree_col_nos; - _sort_map = sort_map; - } - - public int Compare(IBlueRow x, IBlueRow y) { - return BlueDataUtility.sort_in_a_tree(x, y, _depth_in_tree_col_no, _aggregation_tree_col_nos, _sort_map); - } - } - - public class TabularRowComparer : IComparer { - private IEnumerable> _sort_map; - public TabularRowComparer(IEnumerable> sort_map) { _sort_map = sort_map; } - public int Compare(IBlueRow x, IBlueRow y) { return BlueDataUtility.compare_rows(x, y, _sort_map); } - } - - public class TabularRowHashPairComparer : IComparer> { - private Pair[] _sort_map; - - public TabularRowHashPairComparer(Pair[] sort_map) { _sort_map = sort_map; } - public int Compare(Pair x, Pair y) { return BlueDataUtility.compare_rows(x.first, y.first, _sort_map); } - } - - #region Public Static Methods - - public static void QuicksortSequential(IList arr, IComparer comparer) { - QuicksortSequential(arr, comparer, 0, arr.Count - 1); - } - - public static void QuicksortParallel(IList arr, IComparer comparer) { - QuicksortParallel(arr, comparer, 0, arr.Count - 1); - } - - #endregion - - #region Private Static Methods - - private static void QuicksortSequential(IList arr, IComparer comparer, int left, int right) { - if (right > left) { - int pivot = Partition(arr, comparer, left, right); - QuicksortSequential(arr, comparer, left, pivot - 1); - QuicksortSequential(arr, comparer, pivot + 1, right); - } - } - - private static void QuicksortParallel(IList arr, IComparer comparer, int left, int right) { - const int SEQUENTIAL_THRESHOLD = 64; - if (right > left) { - if (right - left < SEQUENTIAL_THRESHOLD) { - QuicksortSequential(arr, comparer, left, right); - } else { - int pivot = Partition(arr, comparer, left, right); - System.Threading.Tasks.Parallel.Invoke(new Action[] { delegate {QuicksortParallel(arr, comparer, left, pivot - 1);}, - delegate {QuicksortParallel(arr, comparer, pivot + 1, right); } - }); - } - } - } - - private static void Swap(IList arr, int i, int j) { - T tmp = arr[i]; - arr[i] = arr[j]; - arr[j] = tmp; - } - - private static int Partition(IList arr, IComparer comparer, int low, int high) { - // Simple partitioning implementation - int pivotPos = (high + low) / 2; - T pivot = arr[pivotPos]; - Swap(arr, low, pivotPos); - - int left = low; - for (int i = low + 1; i <= high; i++) { - if (comparer.Compare(arr[i], pivot) < 0) { - left++; - Swap(arr, i, left); - } - } - - Swap(arr, low, left); - return left; - } - - #endregion - } - - public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary sort_map) { - foreach (KeyValuePair p in sort_map) { - int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value); - if (comp != 0) - return comp; - } - return 0; - } - - public static int compare_rows(IBlueRow r1, IBlueRow r2, IEnumerable> sort_map) { - foreach (Pair p in sort_map) { - int comp = compare_send_dontcare_to_last(r1, p.first, r2, p.first, p.second); - if (comp != 0) - return comp; - } - return 0; - } - - public static int compare_send_dontcare_to_last(IBlueRow r1, int r1_col_no, - IBlueRow r2, int r2_col_no, - bool low_to_high) { - object o1 = BlueDataUtility.get_row_value(r1, r1_col_no, DBNull.Value), - o2 = BlueDataUtility.get_row_value(r2, r2_col_no, DBNull.Value); - return compare_send_dontcare_to_last(o1, o2, low_to_high); - } - - public static List> get_indexed_sort_map(List> sort_map, IBlueTableSchema schema) { - List> rv = new List>(); - foreach (Pair p in sort_map) - rv.Add(new Pair(schema[p.first].colNo, p.second)); - return rv; - } - - /// Comapares 2 rows in a way where nulls, nans, infinities, - /// empty strings etc are sent to the last - public static int compare_send_dontcare_to_last(IBlueRow r1, string r1_col_name, - IBlueRow r2, string r2_col_name, - bool low_to_high) { - object o1 = BlueDataUtility.get_row_value(r1, r1_col_name, DBNull.Value), - o2 = BlueDataUtility.get_row_value(r2, r2_col_name, DBNull.Value); - return compare_send_dontcare_to_last(o1, o2, low_to_high); - } - - public static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) { - if (is_dont_care_val(o1)) - return is_dont_care_val(o2) ? 0 : 1; - else - return is_dont_care_val(o2) ? -1 : (low_to_high ? ((IComparable)o1).CompareTo(o2) - : ((IComparable)o2).CompareTo(o1)); - } - - public static bool is_dont_care_val(object o) { - return o == null || o == DBNull.Value || - o is string && String.IsNullOrEmpty((string)o) || - o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) || - o is float && (Single.IsInfinity((float)o) || Single.IsNaN((float)o)) || - o is Int32 && (Int32.MaxValue.Equals(o) || Int32.MinValue.Equals(o)); - } - - #endregion - - #region Tree Aggregation - - public static int[] convert_aggregation_tree(string[] aggregation_tree_col_names, IBlueTableSchema schema) { - List rv = new List(); - foreach (string col_name in aggregation_tree_col_names) - rv.Add(schema[col_name].colNo); - return rv.ToArray(); - } - - public static bool is_leaf_node(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) { - return BlueDataUtility.get_row_value(r, depth_in_tree_col_no, -1) == (aggregation_tree.Count() - 1); - } - - public static bool is_leaf_node(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) { - return BlueDataUtility.get_row_value(r, depth_in_tree_col_name, -1) == (aggregation_tree.Count() - 1); - } - - public static string generate_aggregation_based_id(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) { - int aggregation_priority = BlueDataUtility.get_row_value(r, depth_in_tree_col_no, -1); - string rv = String.Empty; - for (int p = 0; p <= aggregation_priority; p++) - rv += (rv == String.Empty ? String.Empty : ":") + String.Format("{0},{1}", p, BlueDataUtility.get_row_value(r, aggregation_tree[p], DBNull.Value)); - return rv; - } - - /// Generates a string based id for a row in a tree - /// by appending the values of the nodes above it - public static string generate_aggregation_based_id(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) { - int aggregation_priority = BlueDataUtility.get_row_value(r, depth_in_tree_col_name, -1); - string rv = String.Empty; - for (int p = 0; p <= aggregation_priority; p++) - rv += (rv == String.Empty ? String.Empty : ":") + - String.Format("{0},{1}", p, BlueDataUtility.get_row_value(r, aggregation_tree.ElementAt(p), DBNull.Value)); - return rv; - } - /// - /// Given an aggregation hierarchy tree, figures if the first - /// arg is a parent of the second arg. - /// - public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, string depth_in_tree_col_name, string[] aggregation_tree) { - System.Diagnostics.Debug.Assert(parent_row.Contains(depth_in_tree_col_name) && child_row.Contains(depth_in_tree_col_name)); - int parent_depth = BlueDataUtility.get_row_value(parent_row, depth_in_tree_col_name, Int32.MaxValue), - child_depth = BlueDataUtility.get_row_value(child_row, depth_in_tree_col_name, Int32.MaxValue); - if (parent_depth < child_depth) { - for (int p = 0; p <= parent_depth; p++) { - string col_name = aggregation_tree.ElementAt(p); - object parent_val = BlueDataUtility.get_row_value(parent_row, col_name, DBNull.Value), - child_val = BlueDataUtility.get_row_value(child_row, col_name, DBNull.Value); - if (!parent_val.Equals(child_val)) - return false; - } - return true; - } else - return false; - } - - public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, int depth_in_tree_col_no, int[] aggregation_tree) { - int parent_depth = BlueDataUtility.get_row_value(parent_row, depth_in_tree_col_no, Int32.MaxValue), - child_depth = BlueDataUtility.get_row_value(child_row, depth_in_tree_col_no, Int32.MaxValue); - if (parent_depth < child_depth) { - for (int p = 0; p <= parent_depth; p++) { - int col_no = aggregation_tree[p]; - object parent_val = BlueDataUtility.get_row_value(parent_row, col_no, DBNull.Value), - child_val = BlueDataUtility.get_row_value(child_row, col_no, DBNull.Value); - if (!parent_val.Equals(child_val)) - return false; - } - return true; - } else - return false; - } - - public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, string depth_in_tree_col_name, string[] aggregation_tree, Dictionary sort_map) { - if (BlueDataUtility.is_parent(r1, r2, depth_in_tree_col_name, aggregation_tree)) - return -1; - else if (BlueDataUtility.is_parent(r2, r1, depth_in_tree_col_name, aggregation_tree)) - return 1; - else if (get_row_value(r1, "id", Int32.MaxValue).Equals(get_row_value(r2, "id", Int32.MaxValue))) - return 0; - else { - for (int p = 0; p < aggregation_tree.Count(); p++) { - /// sort_map is used to leaf nodes - if (p == aggregation_tree.Count() - 1) { - /// sorting for leaf nodes is special, since they can be - /// sorted by data columns - System.Diagnostics.Debug.Assert(BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree)); - int leaf_comp = compare_rows(r1, r2, sort_map); - if (leaf_comp != 0) - return leaf_comp; - } - /// sort_map is used to find sorting order - string aggregation_col_name = aggregation_tree.ElementAt(p); - bool low_to_high = sort_map.TryGetValue(aggregation_col_name, out low_to_high) ? low_to_high : true; - int comp = BlueDataUtility.compare_send_dontcare_to_last(r1, aggregation_col_name, r2, aggregation_col_name, low_to_high); - if (comp != 0) - return comp; - } - return BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree) ? - compare_rows(r1, r2, sort_map) : 0; - } - } - - public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, int depth_in_tree_col_no, int[] aggregation_tree, Pair[] sort_map) { - if (BlueDataUtility.is_parent(r1, r2, depth_in_tree_col_no, aggregation_tree)) - return -1; - else if (BlueDataUtility.is_parent(r2, r1, depth_in_tree_col_no, aggregation_tree)) - return 1; - else if (get_row_value(r1, "id", Int32.MaxValue).Equals(get_row_value(r2, "id", Int32.MaxValue))) - return 0; - else { - for (int p = 0; p < aggregation_tree.Count(); p++) { - /// sort_map is used to leaf nodes - if (p == aggregation_tree.Count() - 1) { - /// sorting for leaf nodes is special, since they can be - /// sorted by data columns - //System.Diagnostics.Debug.Assert(BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree)); - int leaf_comp = compare_rows(r1, r2, sort_map); - if (leaf_comp != 0) - return leaf_comp; - } - /// sort_map is used to find sorting order - int aggregation_col_no = aggregation_tree[p]; - bool low_to_high = true;//sort_map.TryGetValue(aggregation_col_name, out low_to_high) ? low_to_high : true; - int comp = BlueDataUtility.compare_send_dontcare_to_last(r1, aggregation_col_no, r2, aggregation_col_no, low_to_high); - if (comp != 0) - return comp; - } - return BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_no, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_no, aggregation_tree) ? - compare_rows(r1, r2, sort_map) : 0; - } - } - - #endregion - - #region Column accessors - - public static T get_array_value(Array r, int col_no, T error_val) { - try { - object rv = (r != null && col_no > -1 && col_no < r.Length) ? r.GetValue(col_no) : DBNull.Value; - if (rv == null || rv == DBNull.Value) - return error_val; - else - return (T)Convert.ChangeType(rv, typeof(T)); - } catch { - return error_val; - } - } - - public static DateTime get_array_datetime_value(Array r, int col_no, DateTime error_val) { - try { - object rv = (r != null && col_no > -1 && col_no < r.Length) ? r.GetValue(col_no) : error_val; - return _get_datetime_value(rv, error_val); - } catch { - return error_val; - } - } - - public static T get_row_value(IBlueRow r, int col_no, T error_val) { - try { - object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : DBNull.Value; - if (rv == null || rv == DBNull.Value) - return error_val; - else - return (T)Convert.ChangeType(rv, typeof(T)); - } catch { - return error_val; - } - } - - public static T get_row_value(IBlueRow r, string col_name, T error_val) { - try { - object rv = (r != null && r.Contains(col_name)) ? r[col_name] : DBNull.Value; - if (rv == null || rv == DBNull.Value) - return error_val; - else - return (T)Convert.ChangeType(rv, typeof(T)); - } catch { - return error_val; - } - } - - public static T get_row_enum_value(IBlueRow r, string col_name, T error_val) where T : struct { - try { - object rv = r.Contains(col_name) ? r[col_name] : DBNull.Value; - return _get_row_enum_value(rv, error_val); - } catch { - return error_val; - } - } - - public static T get_row_enum_value(IBlueRow r, int col_no, T error_val) where T : struct { - try { - object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : DBNull.Value; - return _get_row_enum_value(rv, error_val); - } catch { - return error_val; - } - } - - private static T _get_row_enum_value(object o, T error_val) where T : struct { - try { - if (o == null || o == DBNull.Value) - return error_val; - else if (o is string) { - T result = error_val; - if (Enum.TryParse((string)o, true, out result)) return result; - return error_val; - } else if (o is Int32 || o is Int16 || o is UInt16 || o is UInt32) - return (T)o; - else if (o is T) - return (T)o; - else - return error_val; - } catch { - return error_val; - } - } - - public static DateTime get_row_datetime_value(IBlueRow r, string col_name, DateTime error_val) { - try { - object rv = (r != null && r.Contains(col_name)) ? r[col_name] : error_val; - return _get_datetime_value(rv, error_val); - } catch { - return error_val; - } - } - - public static DateTime get_row_datetime_value(IBlueRow r, int col_no, DateTime error_val) { - try { - object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : error_val; - return _get_datetime_value(rv, error_val); - } catch { - return error_val; - } - } - - private static DateTime _get_datetime_value(object v, DateTime error_val) { - try { - if (v == null || v == DBNull.Value) - return error_val; - else { - if (v is DateTime) - return (DateTime)v; - else if (v is string) { - DateTime rv_dt = error_val; - DateTime.TryParse((string)v, out rv_dt); - return rv_dt; - } else { - double oa_date = (double)Convert.ChangeType(v, typeof(Double)); - return DateTime.FromOADate(oa_date); - } - } - } catch { - return error_val; - } - } - - #endregion - - #region CSV methods - /// - /// Reads a schema file name, and return the constructed schema - /// - public static IBlueTableSchema createSchemaFromCSVSchemaFile(string schemaFileName) { - StreamReader sr = new StreamReader(schemaFileName); - IBlueTableSchema retVal = createSchemaFromCSVSchemaFile(sr); - sr.Close(); - return retVal; - } - - /// - /// Builds a schema from a schema file. Note that the stream is not closed, so - /// the calling code should do so - /// - public static IBlueTableSchema createSchemaFromCSVSchemaFile(TextReader sr) { - string tableName = sr.ReadLine(); - ArrayList t = new ArrayList(); - string l = ""; - while ((l = sr.ReadLine()) != null) - t.Add(l.Trim()); - string[] cols = t.ToArray(typeof(string)) as String[]; - ColumnInfo[] cis = new ColumnInfo[cols.Length]; - for (int i = 0; i < cols.Length; i++) { - string[] colTags = cols[i].Split('/'); - cis[i] = new ColumnInfo(colTags[0], i, TypeInstanceLoader.loadType(colTags[1])); - } - IBlueTableSchema retVal = new BlueTableSchema(cis, tableName); - if (retVal["id"] == null) { - ArrayList temp = new ArrayList(retVal.columnInfo); - temp.Add(new ColumnInfo("id", temp.Count, typeof(System.Int32))); - retVal = new BlueTableSchema(retVal.tableName, - temp.ToArray(typeof(ColumnInfo)) as ColumnInfo[]); - } - return retVal; - } - - /// - /// Creates an IBlueRow from a CSV data file row; if the string already contains an id, - /// the id argument is ignored - /// - public static IBlueRow createRowFromCSVFileRow(string fileRow, IBlueTableSchema schema, int id) { - string[] toks = CSVParser.parse(fileRow); - ArrayList rowList = new ArrayList(); - if (toks.Length == schema.columnNames.Length) { - for (int i = 0; i < toks.Length; i++) - rowList.Add(myConvertTo(schema[i].colType, toks[i])); - } else { - int idIndex = schema["id"].colNo; - for (int i = 0; i < toks.Length; i++) - rowList.Add(myConvertTo(schema[i < idIndex ? i : i + 1].colType, toks[i])); - rowList.Insert(idIndex, id); - } - return new BlueRow(schema, rowList.ToArray()); - } - - /// - /// Returns the CSV data string corresponding to the row - /// - public static string toCSVDataString(IBlueRow r) { - string retVal = ""; - for (int i = 0; i < r.schema.columnNames.Length; i++) - retVal += - r.schema[i].colType.Equals(typeof(string)) ? - "\"" + r[i].ToString().Replace("\"", "\"\"") + "\"" : - r[i].ToString(); - return retVal.Remove(retVal.Length - 1, 1); - } - - public static string[] toCSVSchemaFileStrings(IBlueTableSchema schema) { - string[] retVal = new string[2]; - retVal[0] = schema.tableName; - foreach (ColumnInfo ci in schema.columnInfo) - retVal[1] += ci.colName + "/" + ci.colType.ToString() + " "; - retVal[1] = retVal[1].Remove(retVal[1].Length - 1, 1); - return retVal; - } - - /// - /// Converts a CSV data value to its original type. - /// - private static object myConvertTo(Type t, string val) { - val = val.Trim(); - if (t.Equals(typeof(System.String))) - return val; - else if (val != "") { - if (t.Equals(typeof(System.Char))) - return Convert.ToChar(val); - if (t.Equals(typeof(System.Int16))) - return Convert.ToInt16(val); - if (t.Equals(typeof(System.Int32))) - return Convert.ToInt32(val); - if (t.Equals(typeof(System.Int64))) - return Convert.ToInt64(val); - if (t.Equals(typeof(System.Double))) - return Convert.ToDouble(val); - if (t.Equals(typeof(System.Single))) - return Convert.ToSingle(val); - if (t.Equals(typeof(System.DateTime))) - return myToDateTime(val); - } - return DBNull.Value; - } - - private static object myToDateTime(string val) { - try { - return Convert.ToDateTime(val); - } catch (FormatException) //datetime is not stored in one of the string formats - { - return val.IndexOf('.') > -1 ? - DateTime.FromOADate(Convert.ToDouble(val)) : new DateTime(Convert.ToInt64(val)); - } - } - - public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) => - createRowsFromCSVFileWithoutTypes(fileAddress, out var _); - - public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress, out IBlueTableSchema schema) { - StreamReader sr = new StreamReader(fileAddress); - string cols = sr.ReadLine(); - string[] columns = cols.Split(','); - ColumnInfo[] ci = new ColumnInfo[columns.Length + 1]; - ci[0] = new ColumnInfo("id", 0, typeof(int)); - for (int i = 1; i < ci.Length; i++) - ci[i] = new ColumnInfo(columns[i - 1], i, typeof(string)); - schema = new BlueTableSchema(ci, "csvTable"); - BlueTable result = new BlueTable(schema); - int id = 0; - while (!sr.EndOfStream) - result.addRow(createRowFromCSVFileRow(sr.ReadLine(), schema, id++)); - sr.Close(); - return result; - } - - #endregion - - #region Magic - /// - /// Uses a cached compiled expression to implement a truly generic add function. Will throw if given types that cannot be added. - /// - internal class MagicAdder { - static Func add; - static MagicAdder() { - ParameterExpression left = Expression.Parameter(typeof(T), "a"); - ParameterExpression right = Expression.Parameter(typeof(T), "b"); - add = (Func)Expression.Lambda(Expression.Add(left, right), left, right).Compile(); - } - - public static T Add(T a, T b) { - return add(a, b); - } - } - #endregion - - #region Misc - - public static void swap(ref T t1, ref T t2) { - T temp = t1; - t1 = t2; - t2 = temp; - } - - #endregion - } -} +using System; +using System.Data; +using System.Collections; +using System.Collections.Generic; +using System.Reflection; +using System.IO; +using System.Diagnostics; +using System.Linq; + +//using BCG.Utilities; +using BCG.Utilities.General; +using System.Linq.Expressions; +//using mbts.spark; +using mbts; +using BCG.Extensions; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace BCG { + public static class BlueDataUtility { + #region Extensions + public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) { + return BlueDataUtility.get_row_datetime_value(r, col_name, err_value); + } + public static DateTime get_datetime(this IBlueRow r, int col_no, DateTime err_value = default(DateTime)) { + return BlueDataUtility.get_row_datetime_value(r, col_no, err_value); + } + public static T get_enum(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct { + return BlueDataUtility.get_row_enum_value(r, col_name, err_value); + } + public static T get_enum(this IBlueRow r, int col_no, T err_value = default(T)) where T : struct { + return BlueDataUtility.get_row_enum_value(r, col_no, err_value); + } + public static T get(this IBlueRow r, string col_name, T err_value = default(T)) { + return get_row_value(r, col_name, err_value); + } + public static T get(this IBlueRow r, int col_no, T err_value = default(T)) { + return get_row_value(r, col_no, err_value); + } + public static IEnumerable get_column_values(this IBlueRowCollection rows, string col) { + List rv = new List(); + foreach (IBlueRow r in rows) + rv.Add(r.get(col)); + return rv; + } + public static IEnumerable get_colvalues(IEnumerable rows, string col) { + List rv = new List(); + foreach (IBlueRow r in rows) + rv.Add(r.get(col)); + return rv; + } + + public static IDictionary> group_by(IBlueRowCollection rows, string gp_col) { + Dictionary> rv = new Dictionary>(); + foreach (IBlueRow r in rows) { + var gp_val = r.get(gp_col); + List gp_val_list = null; + if (!rv.TryGetValue(gp_val, out gp_val_list)) { + gp_val_list = new List(); + rv.Add(gp_val, gp_val_list); + } + gp_val_list.Add(r); + } + return rv.ToDictionary(kv => kv.Key, kv => kv.Value as IEnumerable); + } + + // this function takes in a key value table and converts it into a relational table, converting the key names into columns. + // The values are returned as string values + // doc_col is the column specifyng the doc name (following mongoDB's nomenclature, one set of key value rows ar a 'doc') + public static IBlueRowCollection keyvalue_to_relational(IBlueRowCollection kv_rows, string doc_col, string key_col, string val_col) { + HashSet keys = new HashSet(); + foreach (string col in get_column_values(kv_rows, key_col)) { + keys.Add(col); + } + if (keys.Contains("id")) { + keys.Remove("id"); + } + //unique_cols.Add(doc_col) + // all the keys become cols + var schema_cols = keys.Select(c => Tuple.Create(c, TypeCode.String)).ToList(); + //schema of the final table needs doc and id + schema_cols.Insert(0, Tuple.Create(doc_col, BlueDataUtility.get_typecode(typeof(T)))); + schema_cols.Insert(0, Tuple.Create("id", TypeCode.Int32)); + var schema = BlueDataUtility.form_schema("relational", schema_cols.ToArray()); + BlueTable rv = new BlueTable(schema); + var gp_map = group_by(kv_rows, doc_col); + foreach (var kv in gp_map) { + MutableBlueRow nr = new MutableBlueRow(schema); + nr.setValue("id", rv.Count); + nr.setValue(doc_col, kv.Key); + foreach (var r in kv.Value) { + var col = r.get(key_col); + if (!keys.Contains(col)) + continue; + var val = r.get(val_col); + nr.setValue(col, val); + } + rv.addRow(nr); + } + return rv; + } + + public static IBlueRowCollection where(this IBlueRowCollection rows, Func where_func) { + if (rows == null) + return null; + else { + BlueTable rv = new BlueTable(rows.schema); + foreach (IBlueRow r in rows) + if (where_func(r)) + rv.addRow(r); + return rv; + } + } + + public static IEnumerable to_ienum(this IBlueRowCollection rows) { + List rv = new List(); + foreach (IBlueRow r in rows) + rv.Add(r); + return rv; + } + + public static IDictionary to_map(this IBlueRowCollection rows) { + Dictionary rv = new Dictionary(); + foreach (IBlueRow r in rows) { + var id = r.get("id"); + rv.Add(id, r); + } + return rv; + } + + public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple[] cols) { + List cis = new List(input_schema.columnInfo); + foreach (Tuple c in cols) + cis.Add(new ColumnInfo(c.Item2, cis.Count, c.Item1)); + return new BlueTableSchema(input_schema.tableName, cis.ToArray()); + } + + public static IBlueRowCollection to_collection(this IBlueRow r) { + if (r == null) + return null; + else { + BlueTable rv = new BlueTable(r.schema); + rv.addRow(r); + return rv; + } + } + + public static DataPacket build_dp(IBlueRowCollection new_rows, DataPacket last_fired_dp) { + IBlueRowCollection added_rows = new_rows; + IBlueRowCollection deleted_rows = last_fired_dp == null ? new BlueTable(new_rows.schema) : last_fired_dp.newRows; + return new DataPacket(new_rows, added_rows, deleted_rows); + } + + public static DataPacket build_dp_inc(IBlueRowCollection new_rows, DataPacket last_fired_dp, bool deep_row_comapre = false) { + if (last_fired_dp == null) { + IBlueRowCollection added_rows = new_rows; + IBlueRowCollection deleted_rows = new BlueTable(new_rows.schema); + return new DataPacket(new_rows, added_rows, deleted_rows); + } else { + IBlueRowCollection added_rows = null; + IBlueRowCollection deleted_rows = null; + if (deep_row_comapre) + compute_differences_fast(new_rows, last_fired_dp.newRows, out deleted_rows, out added_rows); + else + compute_differences_id_based(new_rows, last_fired_dp.newRows, out deleted_rows, out added_rows); + return new DataPacket(new_rows, added_rows, deleted_rows); + } + } + + public static IBlueRowCollection adapt_for_sql_storage(this IBlueRowCollection rows, + bool handle_datetime_min = false, + bool handle_double_nan = false) { + if (rows != null) { + BlueTable rv = new BlueTable(rows.schema); + foreach (IBlueRow r in rows) { + MutableBlueRow mr = new MutableBlueRow(r); + rv.addRow(mr); + foreach (ColumnInfo ci in rows.schema.columnInfo) { + if (handle_datetime_min && ci.colType == typeof(DateTime) && DateTime.MinValue.Equals(r[ci.colName])) + mr.setValue(ci.colName, DBNull.Value); + if (handle_double_nan && ci.colType == typeof(double) && Double.IsNaN(r.get(ci.colName))) + mr.setValue(ci.colName, DBNull.Value); + } + } + return rv; + } else + return null; + } + + #endregion + + #region json + + public static List json_to_json_node_list(string jsonString) { + var jsonNodeList = new List(); + bool noe = String.IsNullOrEmpty(jsonString); + if (noe) + return jsonNodeList; + try { + // Parse the JSON string + JsonArray jsonArray = JsonNode.Parse(jsonString) as JsonArray; + + if (jsonArray == null) { + Console.WriteLine("The JSON string must represent a list."); + return jsonNodeList; + } + + // Iterate through each JsonNode in the JsonArray + foreach (JsonNode jsonNode in jsonArray) { + jsonNodeList.Add(jsonNode); + } + } catch (JsonException ex) { + Console.WriteLine($"Invalid JSON: {ex.Message}"); + } + + return jsonNodeList; + } + + #endregion + + public static void print_table(this IBlueRowCollection table, Func get_col_width) { + var rv = get_printed_rows(table, get_col_width); + Console.WriteLine(rv); + } + + public static string get_printed_rows(this IBlueRowCollection table, Func get_col_width) { + // Print column headers + string rv = String.Empty;// = $"{table.schema.tableName}______________________________________"; + foreach (var col_name in table.schema.columnNames) { + int col_width = get_col_width(col_name); + rv += $"{col_name.PadRight(col_width)} "; + } + rv += "\n"; + //rv += "______________________________________"; + + // Print table data + foreach (IBlueRow row in table) { + foreach (string col_name in table.schema.columnNames) { + int col_width = get_col_width(col_name); + object row_val = row[col_name]; + string row_val_str = row_val.ToString(); + if (row_val != null) { + rv += $"{row_val_str.PadRight(col_width)} "; + } + } + rv += "\n"; + } + return rv; + } + #region Schema types + // these are the only types allowed in schemas + private static IReadOnlyDictionary _typecode_to_type = new Dictionary + { + { TypeCode.Boolean, typeof(bool) }, //3 + { TypeCode.Byte, typeof(byte) }, //6 + { TypeCode.Char, typeof(char) }, //4 + // TODO + { TypeCode.DateTime, typeof(DateTime) },//16 + { TypeCode.Decimal, typeof(decimal) }, //15 + { TypeCode.Double, typeof(double) }, //14 + //{ TypeCode.Empty, null }, + { TypeCode.Int16, typeof(short) }, //7 + { TypeCode.Int32, typeof(int) }, //9 + { TypeCode.Int64, typeof(long) }, //11 + { TypeCode.SByte, typeof(sbyte) }, //5 + { TypeCode.Single, typeof(Single) }, //13 + { TypeCode.String, typeof(string) }, //18 + { TypeCode.UInt16, typeof(UInt16) }, //8 + { TypeCode.UInt32, typeof(UInt32) }, //10 + { TypeCode.UInt64, typeof(UInt64) } //12 + }; + + public static IEnumerable valid_column_typecodes() { + return _typecode_to_type.Keys; + } + + public static IEnumerable valid_column_types() { + return _typecode_to_type.Values; + } + + public static bool is_valid_col_type(Type t) { + TypeCode type_code = get_typecode(t); + return type_code != TypeCode.Empty; + } + + public static TypeCode get_typecode(Type t) { + foreach (var kv in _typecode_to_type) { + if (kv.Value == t) + return kv.Key; + } + //bts.Log.assert_d(false, $"type: {t} not found"); + return TypeCode.Empty; + } + + public static Type get_type(TypeCode type_code) { + Type rv = null; + if (_typecode_to_type.TryGetValue(type_code, out rv)) { + return rv; + } + return null; + } + + public static TypeCode type_code(this ColumnInfo ci) { + return get_typecode(ci.colType); + } + + #endregion + + public static Tuple guess_type(string value) { + if (string.IsNullOrEmpty(value)) { + return Tuple.Create(typeof(string), (object)value); + } + + int val = 0; + // Try to parse as integer + if (int.TryParse(value, out val)) { + return Tuple.Create(typeof(int), (object)val); + } + + double dval = 0.0; + // Try to parse as double + if (double.TryParse(value, out dval)) { + return Tuple.Create(typeof(double), (object)dval); + } + + DateTime dt; + // Try to parse as DateTime + if (DateTime.TryParse(value, out dt)) { + return Tuple.Create(typeof(DateTime), (object)dt); + } + + bool bval; + // Try to parse as boolean + if (bool.TryParse(value, out bval)) { + return Tuple.Create(typeof(bool), (object)bval); + } + + // If none of the above, return string type + return Tuple.Create(typeof(string), (object)value); + } + + public static IBlueTableSchema form_schema(string table_name, params Tuple[] cols) { + List cis = new List(); + foreach (Tuple c in cols) + cis.Add(new ColumnInfo(c.Item1, cis.Count, _typecode_to_type[c.Item2])); + if (cis.FirstOrDefault(ci => ci.colName == "id") == null) + cis.Add(new ColumnInfo("id", cis.Count, typeof(int))); + return new BlueTableSchema(table_name, cis.ToArray()); + } + + public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) { + // just use one row to guess + List cis = new List(); + foreach (IBlueRow r in rows) { + foreach (var ci in r.schema.columnInfo) { + if (ci.colType != typeof(System.String)) { + cis.Add(ci); + continue; + } + var (type, _) = guess_type(r[ci.colName] as string); + cis.Add(new ColumnInfo(ci.colName, ci.colNo, type)); + } + return new BlueTableSchema(cis.ToArray(), rows.schema.tableName); + } + return null; + } + + // converts a row where all columns are string types to a schema where the best types are guesed + public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) { + IBlueTableSchema typed_schema = get_typed_schema(string_rows); + var rv = new BlueTable(typed_schema); + foreach (IBlueRow r in string_rows) { + var typed_row = get_typed_row(r, typed_schema); + rv.addRow(typed_row); + } + return rv; + + static IBlueRow get_typed_row(IBlueRow string_row, IBlueTableSchema typed_schema) { + var rv = new MutableBlueRow(typed_schema); + foreach (var ci in typed_schema.columnInfo) { + var typed_val = convert(ci.colType, string_row[ci.colName].ToString()); + rv.setValue(ci.colName, typed_val); + } + return rv; + } + + static object convert(Type t, string value) { + // Attempt to convert the string to the specified type + try { + return Convert.ChangeType(value, t); + } catch (Exception) { + if (t == typeof(double)) { + // some doubles come with currency and ',' we clean that up here (there might be better ways to do this) + string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray()); + cleaned_val = cleaned_val.Replace('−', '-'); + double rv_d; + if (Double.TryParse(cleaned_val, out rv_d)) + return rv_d; + else + return null; + } + return null; + // Handle conversion failure gracefully + // throw new ArgumentException("Failed to convert the string to the specified type."); + } + } + } + + /// + /// Gets the Hashcode of the row with the boolean indicating whether the order of the rows should + /// be alphabetic or not + /// + /// + /// + /// + public static int getRowHashCode(IBlueRow row, bool isColAlphabetically) { + string retVal = ""; + if (isColAlphabetically) { + string[] colNames = row.schema.columnNames; + Array.Sort(colNames); + foreach (string col in colNames) + retVal += row[col].ToString(); + } else + foreach (object o in row) + retVal += o.ToString(); + return retVal.GetHashCode(); + } + + /// + /// Finds the hashcode only inluding the columns (in the order) listed in the column names argument + /// + /// + /// + /// + public static int getRowHashCode(IBlueRow row, params string[] colNames) { + string retVal = ""; + foreach (string colName in colNames) + retVal += row[colName].ToString(); + return retVal.GetHashCode(); + } + + /// + /// Converts a IBlueRowCollection to a .Net DataTable + /// + /// + /// + public static DataTable toDataTable(IBlueRowCollection rows) { + DataTable retTable = getTableFromSchema(rows.schema); + retTable.BeginLoadData(); + foreach (IBlueRow row in rows) + retTable.Rows.Add(toDataRow(row, retTable)); + retTable.EndLoadData(); + return retTable; + } + + /// + /// Converts to a DataRow, the preconditioner being that order of the columns should be same + /// in the schema of the row and the table + /// + /// + /// + public static DataRow toDataRow(IBlueRow row, DataTable table) { + DataRow retRow = table.NewRow(); + foreach (string colName in row.schema.columnNames) + retRow[colName] = row[colName]; + return retRow; + } + + /// + /// Returns the IBlueTableSchema corresponding to the row + /// + /// + /// DataRow whose schmea is desired + /// + /// + public static IBlueTableSchema getSchema(DataRow row) { return getSchema(row.Table); } + + /// + /// Returns the IBlueTableSchema of the DataTable + /// + /// + /// table whos Schema is desired + /// + /// + public static IBlueTableSchema getSchema(DataTable table) { + ColumnInfo[] cis = new ColumnInfo[table.Columns.Count]; + foreach (DataColumn col in table.Columns) + cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType); + return new BlueTableSchema(cis, table.TableName); + } + + /// + /// Constructs the schema from an object, using the SchemaMapping attribute. Only those + /// properties are converted into schmea columns that have a get accessor + /// + /// + /// + public static IBlueTableSchema getSchema(object o) { + return getSchema(o.GetType()); + } + + + /// + /// Builds a new schema based on an existing one + a set of new columns + /// + /// Base schema + /// Columns to add + public static IBlueTableSchema addColumnsToSchema(IBlueTableSchema schema, params ColumnInfo[] cis) { + ArrayList rv = new ArrayList(schema.columnInfo); + rv.AddRange(cis); + return new BlueTableSchema(schema.tableName, rv.ToArray(typeof(ColumnInfo)) as ColumnInfo[]); + } + + /// + /// appends the cols of s2, to s1, subtracting the id col + /// + /// + /// + /// + public static IBlueTableSchema appendSchemas(IBlueTableSchema s1, IBlueTableSchema appendedSchema) { + List cis = new List(s1.columnInfo); + foreach (ColumnInfo c in appendedSchema.columnInfo) + if (c.colName != "id") + cis.Add(new ColumnInfo(c.colName, cis.Count, c.colType)); + return new BlueTableSchema(s1.tableName, cis.ToArray()); + } + + + public static IBlueTableSchema getSchema(Type t) { + return getSchema(t, false); + } + + + public static IBlueTableSchema getSchema(Type t, bool createIdField) { + return getSchema(t, t.Name, createIdField); + } + public static IBlueTableSchema getSchema(Type t, string tableName, bool createIdField) { + ArrayList colInfos = new ArrayList(); + int i = 0; + foreach (PropertyInfo pi in t.GetProperties()) + if (myIsActiveProperty(pi)) + colInfos.Add(new ColumnInfo(pi.Name, i++, + is_valid_col_type(pi.PropertyType) ? + pi.PropertyType : typeof(String))); + foreach (FieldInfo fi in t.GetFields()) + if (myIsActiveField(fi)) + colInfos.Add(new ColumnInfo(fi.Name, i++, + is_valid_col_type(fi.FieldType) ? fi.FieldType : typeof(String))); + + if (createIdField) + colInfos.Add(new ColumnInfo("id", i++, typeof(int))); + + return new BlueTableSchema(colInfos.ToArray(typeof(ColumnInfo)) as ColumnInfo[], + tableName); + } + + public static IBlueTableSchema getSchema(Type t, string tableName) { + return new BlueTableSchema(getSchema(t).columnInfo, tableName); + } + + + public static IBlueRow createRowFromType(IBlueTableSchema schema, object o) { + return createRowFromType(schema, o, null); + } + + //creates a row from the given type, and if idVal is not null, using the given id + public static IBlueRow createRowFromType(IBlueTableSchema schema, object o, object idVal) { + object[] rowArray = new object[schema.columnInfo.Length]; + Type t = o.GetType(); + foreach (PropertyInfo pi in t.GetProperties()) + if (myIsActiveProperty(pi)) + rowArray[schema[pi.Name].colNo] = + is_valid_col_type(pi.PropertyType) ? + pi.GetValue(o, null) : pi.GetValue(o, null).ToString(); + foreach (FieldInfo fi in t.GetFields()) + if (myIsActiveField(fi)) + rowArray[schema[fi.Name].colNo] = + is_valid_col_type(fi.FieldType) ? + fi.GetValue(o) : fi.GetValue(o).ToString(); + if (idVal != null) + rowArray[schema["id"].colNo] = idVal; + return new BlueRow(schema, rowArray); + } + + public static MutableBlueRow create_mutable_row(IBlueTableSchema schema, object o, object idVal, + Func type_converter) { + object[] rowArray = new object[schema.columnInfo.Length]; + Type t = o.GetType(); + foreach (PropertyInfo pi in t.GetProperties()) + if (myIsActiveProperty(pi)) + rowArray[schema[pi.Name].colNo] = + is_valid_col_type(pi.PropertyType) ? + pi.GetValue(o, null) : type_converter(pi.PropertyType, pi.GetValue(o, null)); + foreach (FieldInfo fi in t.GetFields()) + if (myIsActiveField(fi)) + rowArray[schema[fi.Name].colNo] = + is_valid_col_type(fi.FieldType) ? + fi.GetValue(o) : type_converter(fi.FieldType, fi.GetValue(o)); + if (idVal != null) + rowArray[schema["id"].colNo] = idVal; + return new MutableBlueRow(schema, rowArray); + } + + + public static IBlueTableSchema get_schema(Type t, string tableName, bool createIdField, Func type_converter) { + List colInfos = new List(); + int i = 0; + if (createIdField) + colInfos.Add(new ColumnInfo("id", i++, typeof(int))); + + foreach (PropertyInfo pi in t.GetProperties()) + colInfos.Add(new ColumnInfo(pi.Name, i++, type_converter(pi.PropertyType))); + foreach (FieldInfo fi in t.GetFields()) + colInfos.Add(new ColumnInfo(fi.Name, i++, type_converter(fi.FieldType))); + return new BlueTableSchema(colInfos.ToArray(), tableName); + } + + public static MutableBlueRow create_mutable_row_not_slow(IBlueTableSchema schema, FieldInfo[] fields, PropertyInfo[] props, + object o, object idVal, Func data_converter) { + object[] rowArray = new object[schema.columnInfo.Length]; + if (props != null) { + foreach (PropertyInfo pi in props) + rowArray[schema[pi.Name].colNo] = data_converter(pi.PropertyType, pi.GetValue(o, null)); + } + if (fields != null) { + foreach (FieldInfo fi in fields) + rowArray[schema[fi.Name].colNo] = data_converter(fi.FieldType, fi.GetValue(o)); + } + if (idVal != null) + rowArray[schema["id"].colNo] = idVal; + return new MutableBlueRow(schema, rowArray); + } + + public static T createTypeFromRow(IBlueRow row) { + return (T)createTypeFromRow(row, typeof(T)); + } + + public static object createTypeFromRow(IBlueRow row, Type t) { + object outValue = Activator.CreateInstance(t); + + PropertyInfo[] properties = t.GetProperties(); + + foreach (PropertyInfo pi in properties) { + if (myIsActiveProperty(pi)) + if (row.Contains(pi.Name)) + if (row[pi.Name].Equals(DBNull.Value) && !pi.PropertyType.Equals(typeof(Object))) + pi.SetValue(outValue, null, null); + else + pi.SetValue(outValue, row[pi.Name], null); + } + FieldInfo[] fields = t.GetFields(); + foreach (FieldInfo fi in fields) { + if (myIsActiveField(fi)) + if (row.Contains(fi.Name)) + if (row[fi.Name].Equals(DBNull.Value) && !fi.FieldType.Equals(typeof(Object))) + fi.SetValue(outValue, null); + else + fi.SetValue(outValue, row[fi.Name]); + } + + return outValue; + + } + + //these functions should originally have been made public + //but changing that would change a bunch of code + public static bool isActiveProperty(PropertyInfo pi) { + return myIsActiveProperty(pi); + } + + public static bool isActiveField(FieldInfo fi) { + return myIsActiveField(fi); + } + + private static bool myIsActiveProperty(PropertyInfo pi) { + if (pi.GetGetMethod() == null) + return false; + foreach (Attribute att in pi.GetCustomAttributes(true)) + if (att is SchemaVisibility && !(att as SchemaVisibility).visible) + return false; + return true; + } + + private static bool myIsActiveField(FieldInfo fi) { + if (!fi.IsPublic) + return false; + foreach (Attribute att in fi.GetCustomAttributes(true)) + if (att is SchemaVisibility && !(att as SchemaVisibility).visible) + return false; + return true; + } + + private static bool myIsActiveMember(MemberInfo mi) { + if (!(mi is FieldInfo) || + !(mi is PropertyInfo) || + (mi is PropertyInfo && (mi as PropertyInfo).GetGetMethod() == null) || + (mi is FieldInfo && !(mi as FieldInfo).IsPublic)) + return false; + foreach (Attribute att in mi.GetCustomAttributes(true)) + if (att is SchemaVisibility && !(att as SchemaVisibility).visible) + return false; + return true; + } + + private static bool myDoesRowExistsInTable(IBlueRowCollection brc2, string[] columns, object[] oarray) { + bool rowExists = false; + foreach (IBlueRow br in brc2) { + bool allElementsSame = true; + for (int i = 0; i < columns.Length; i++) { + string col = columns[i]; + object o1 = br[col]; + object o2 = oarray[i]; + allElementsSame = (o1.Equals(o2) == false) ? false : allElementsSame; + } + if (allElementsSame == true) { + rowExists = true; + break; + } + } + return rowExists; + } + + /// + /// Constructs a DataTable from the IBlueTableSchema + /// + /// + /// Schema whose table needs to be created + /// + /// + public static DataTable getTableFromSchema(IBlueTableSchema schema) { + DataTable retTable = new DataTable(schema.tableName); + foreach (ColumnInfo ci in schema.columnInfo) + retTable.Columns.Add(ci.colName, ci.colType); + return retTable; + } + + /// + /// Constructs a BlueRow from a DataRow. + /// + /// source row + /// + public static IBlueRow toBlueRow(DataRow row) { + return new BlueRow(getSchema(row), row); + } + + public static IBlueRowCollection toIBlueRowCollection(DataTable dt) { + BlueTable retVal = new BlueTable(getSchema(dt)); + foreach (DataRow dr in dt.Rows) + retVal.addRow(dr); + return retVal; + } + + public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) { + BlueTable retVal = null; + foreach (DataRow dr in rows) { + IBlueRow r = toBlueRow(dr); + if (retVal == null) + retVal = new BlueTable(r.schema); + retVal.addRow(r); + } + return retVal; + } + + /// + /// Returns the nth row of the IBlueRowCollection. This method could be used + /// for any of the nonRandom-Access-IBlueRowCollection-Implementations, as its + /// O(n) + /// + /// + /// + /// + public static IBlueRow extractRow(IBlueRowCollection rows, int index) { + int i = 0; + foreach (IBlueRow row in rows) + if (i++ == index) + return row; + return null; + } + + /// + /// Computes the equality fo the 2 rows. The rows might have different columns orderings + /// + /// + /// + /// + + + /// + /// Computes a string equivelant for the row, using the columns provided + /// in the argument + /// + /// + /// + /// + public static string rowToString(IBlueRow row, params string[] colNames) { + string retVal = ""; + foreach (string colName in colNames) + retVal += (row[colName] != null ? row[colName].ToString().Trim() : "NULL") + "\n"; + return retVal; + } + + public static string rowToString(IBlueRow row) { return rowToString(row, row.schema.columnNames); } + + public static string rowCollectionToString(IBlueRowCollection rowColl) { + if (rowColl == null) + return "Null IBlueRowCollection"; + else { + string retVal = ""; + foreach (IBlueRow r in rowColl) + retVal += retVal == "" ? rowToString(r) : "\n" + rowToString(r); + return retVal; + } + } + + /// + /// Compares 2 rows, using the column names in the order provided, ie the column name with + /// the lower index in the array has higher priority. Also a null is always assumed + /// to be less than a non-null + /// + /// + /// + /// + /// + public static int compare_rows(IBlueRow r1, + IBlueRow r2, + params string[] colNames) { + int retVal = 0; + foreach (string colName in colNames) { + object val1 = r1[colName]; + object val2 = r2[colName]; + retVal = _compare_row_cols(val1, val2); + if (retVal != 0) + break; + } + return retVal; + } + + private static int _compare_row_cols(object v1, object v2) { + if (_is_null(v1)) + return _is_null(v2) ? 0 : -1; + else { + if (_is_null(v2)) + return 1; + else { + if (v1 is IComparable && v2 is IComparable) + return (v1 as IComparable).CompareTo(v2); + else if (v1 is byte[] && v2 is byte[]) + return _compare_bytes(v1 as byte[], v2 as byte[]) ? 0 : 1; + else + return 1; + } + } + } + + private static bool _compare_bytes(byte[] v1, byte[] v2) { + if (v1.Length != v2.Length) + return false; + else + for (int i = 0; i < v1.Length; i++) + if (v1[i] != v2[i]) + return false; + return true; + } + + private static bool _is_null(object o) { return o == null || o == DBNull.Value; } + /// + /// Creates a new IBlueRow with a schema matching the schema provided in 'new schema' + /// + /// a row with the values to be converted + /// the desired schema for the output row + /// + public static IBlueRow changeSchema(IBlueRow inRow, IBlueTableSchema newSchema) { + object[] outputValues = new object[newSchema.columnInfo.Length]; + + for (int i = 0; i < outputValues.Length; ++i) + outputValues[i] = inRow[newSchema.columnNames[i]]; + + return new BlueRow(newSchema, outputValues); + } + + /// + /// Combines input tables into a single output table, with different ID's + /// this will result in new rows being created, which is kind of expensive + /// + /// + /// + public static IBlueRowCollection combineTables(params IBlueRowCollection[] tables) { + //get some base cases out of the way + if (tables.Length == 0) + return null; + if (tables.Length == 1) + return tables[0]; + + BlueTable result = new BlueTable(tables[0].schema); + int idToUseNext = 0; + int numCols = tables[0].schema.columnInfo.Length; + int idCol = tables[0].schema["id"].colNo; + IBlueRowCollection previousTable = null; + foreach (IBlueRowCollection table in tables) { + if (previousTable != null) + if (!compareSchemas(previousTable.schema, table.schema)) + throw new ArgumentException("Error, tables must have the same schema to be combined"); + + foreach (IBlueRow r in table) { + object[] values = new object[numCols]; + r.CopyTo(values, 0); + values[idCol] = idToUseNext++; + result.addRow(new BlueRow(table.schema, values)); + } + + previousTable = table; + + } + + return result; + } + /// + /// Compares two rows (having the same schema) and returns the + /// columns whose values dont match + /// + /// + /// + /// + public static string[] compareRows(IBlueRow r1, IBlueRow r2) { + ArrayList retVal = new ArrayList(); + foreach (string col in r1.schema.columnNames) + if (!r1[col].Equals(r2[col])) + retVal.Add(col); + return retVal.ToArray(typeof(string)) as string[]; + } + + public static BlueTable cloneRowCollection(IBlueRowCollection rows) { + BlueTable retVal = new BlueTable(rows.schema); + foreach (IBlueRow r in rows) + retVal.addRow(new BlueRow(r)); + return retVal; + } + + /// + /// Creates a new row with the values of the row passed, except that the newValues are forced into the + /// returned row, in places specified by the indexes (makes any sense ?) + /// + /// + /// + /// + /// + public static IBlueRow getNewReplacedRow(IBlueRow r, object[] newValues, int[] indexes) { + if (!newValues.Length.Equals(indexes.Length)) + throw new Exception("Debug Assert"); + + object[] rowArray = new object[r.Count]; + r.CopyTo(rowArray, 0); + for (int i = 0; i < indexes.Length; i++) + rowArray[indexes[i]] = newValues[i]; + return new BlueRow(r.schema, rowArray); + } + + public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, int index) { + return getNewReplacedRow(r, new object[] { newValue }, new int[] { index }); + } + + public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, string colName) { + return getNewReplacedRow(r, newValue, r.schema[colName].colNo); + } + + /// + /// Computes the max row, based on the cols provided. If the collection is empty, null is returned + /// + /// + /// + /// + public static IBlueRow computeMaxRow(IBlueRowCollection rows, params string[] cols) { + IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null; + foreach (IBlueRow r in rows) + retVal = compare_rows(r, retVal, cols) > 0 ? r : retVal; + return retVal; + } + + /// + /// Computes the max row, based on the cols provided. If the collection is empty, null is returned + /// + /// + /// + /// + public static IBlueRow computeMinRow(IBlueRowCollection rows, params string[] cols) { + IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null; + foreach (IBlueRow r in rows) + retVal = compare_rows(r, retVal, cols) < 0 ? r : retVal; + return retVal; + } + + /// + /// Computes the sum of rows, based on the cols provided. If the collection is empty, null is returned + /// + /// + /// + /// + public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) { + if (rows.Count == 0) return null; + + IBlueRow retVal = BlueDataUtility.extractRow(rows, 0); + + object[] sums = new object[cols.Length]; + int[] idxs = new int[cols.Length]; + for (int i = 0; i < cols.Length; i++) { + sums[i] = 0.0; + idxs[i] = rows.schema[cols[i]].colNo; + } + + foreach (IBlueRow r in rows) { + for (int i = 0; i < cols.Length; i++) { + double dval = BlueDataUtility.get_row_value(r, cols[i], 0.0); + //r[cols[i]] == null || r[cols[i]] is DBNull ? 0.0 : (double)r[cols[i]]; + sums[i] = (double)sums[i] + dval; + } + } + + return BlueDataUtility.getNewReplacedRow(retVal, sums, idxs); + } + + + public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) { + if (rows.Count == 0) return null; + IBlueRow retVal = BlueDataUtility.extractRow(rows, 0); + if (rows.Count == 1) return retVal; + + object[] sums = new object[cols.Length]; + + int[] idxs = new int[cols.Length]; + for (int i = 0; i < cols.Length; i++) { + sums[i] = default(T); + idxs[i] = rows.schema[cols[i]].colNo; + } + + foreach (IBlueRow r in rows) { + for (int i = 0; i < cols.Length; i++) { + T tval = r[cols[i]] == null || r[cols[i]] is DBNull ? default(T) : (T)r[cols[i]]; + sums[i] = MagicAdder.Add((T)sums[i], tval); + } + } + + return BlueDataUtility.getNewReplacedRow(retVal, sums, idxs); + } + + /// + /// Uses the ID to identify the presence of a row in a collection + /// + /// + /// + /// + public static bool rowPresent(IBlueRowCollection table, IBlueRow row) { + return table.containsID((int)row["id"]); + } + + public static void computeDifferences(IBlueRowCollection newRows, + IBlueRowCollection oldRows, + out IBlueRowCollection dRows, + out IBlueRowCollection aRows) { + computeDifferences( + newRows, + oldRows, + newRows.schema.columnNames, + out dRows, + out aRows); + } + + public static bool compute_equality(IBlueRowCollection rows1, IBlueRowCollection rows2) { + IBlueRowCollection d_rows, a_rows; + compute_differences_fast(rows2, rows1, out d_rows, out a_rows); + return d_rows.Count == 0 && a_rows.Count == 0; + } + + public static bool compute_equality(ColumnInfo ci1, ColumnInfo ci2) { + return (ci1.colNo == ci2.colNo) && (ci1.colName == ci2.colName) && (ci1.colType == ci2.colType); + } + + public static bool compute_equality(IBlueTableSchema s1, IBlueTableSchema s2) { + if (s1.tableName != s1.tableName) + return false; + if (s1.columnInfo.Length != s2.columnInfo.Length) + return false; + for (int i = 0; i < s1.columnInfo.Length; i++) { + ColumnInfo ci1 = s2.columnInfo[i], ci2 = s2.columnInfo[i]; + if (!compute_equality(ci1, ci2)) + return false; + } + return true; + } + + public static bool compute_equality(IBlueRow r1, IBlueRow r2) { + return compute_row_equality(r1, r2, r1.schema.columnNames); + } + + public static bool compute_row_equality(IBlueRow r1, + IBlueRow r2, + params string[] col_names) { + return compare_rows(r1, r2, col_names) == 0 ? true : false; + } + + public static string to_string(ColumnInfo ci) { + return ci.colNo + ";" + ci.colName + ";" + ci.colType; + } + + public static string to_string(this IBlueRow r) { + string rv = String.Empty; + foreach (ColumnInfo c in r.schema.columnInfo) { + rv += $"{c.colName}: {r[c.colName]}\n"; + } + return rv; + } + + public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, + out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) { + if (old_rows == null || old_rows.Count == 0) { + out_d_rows = new BlueTable(new_rows.schema); + out_a_rows = new BlueTable(new_rows); + return; + } + int[] col_indices = new int[new_rows.schema.columnNames.Length]; + for (int i = 0; i < col_indices.Length; i++) + col_indices[i] = i; + compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows); + } + + public static void compute_differences_id_based(IBlueRowCollection new_rows, IBlueRowCollection old_rows, + out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) { + if (old_rows == null || old_rows.Count == 0) { + out_d_rows = new BlueTable(new_rows.schema); + out_a_rows = new BlueTable(new_rows); + return; + } + int[] col_indices = new int[] { new_rows.schema["id"].colNo }; + compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows); + } + + public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, int[] col_indices, + out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) { + Func rows_equal = (r1, r2) => { + foreach (int col_no in col_indices) + if (!(r1[col_no] ?? DBNull.Value).Equals(r2[col_no] ?? DBNull.Value)) + return false; + return true; + }; + + BlueTable del_rows = new BlueTable(new_rows.schema), add_rows = new BlueTable(new_rows.schema); + // adds + changes + foreach (IBlueRow r in new_rows) + if (!BlueDataUtility.rowPresent(old_rows, r)) + add_rows.addRow(r); + else if (!rows_equal(r, old_rows[(int)r["id"]])) { + add_rows.addRow(r); + del_rows.addRow(old_rows[(int)r["id"]]); + } + // deletes + foreach (IBlueRow r in old_rows) + if (!BlueDataUtility.rowPresent(new_rows, r)) + del_rows.addRow(r); + out_d_rows = del_rows; + out_a_rows = add_rows; + } + + + public static void computeDifferences(IBlueRowCollection c1, + IBlueRowCollection c2, + string[] columns, + out IBlueRowCollection dRows, + out IBlueRowCollection aRows) { + dRows = new BlueTable(c1.schema); + aRows = new BlueTable(c1.schema); + + // adds + changes + foreach (IBlueRow nr in c1) + if (!BlueDataUtility.rowPresent(c2, nr)) + (aRows as BlueTable).addRow(nr); + else if (BlueDataUtility.compare_rows(nr, c2[(int) + nr["id"]], columns) != 0) { + (aRows as BlueTable).addRow(nr); + (dRows as BlueTable).addRow(c2[(int)nr["id"]]); + + } + + // deletes + foreach (IBlueRow nr in c2) + if (!BlueDataUtility.rowPresent(c1, nr)) + (dRows as BlueTable).addRow(nr); + } + + //brcDiff = brc1 - brc2 + public static void diff(IBlueRowCollection brc1, IBlueRowCollection brc2, string[] columns, out IBlueRowCollection brcDiff) { + brcDiff = new BlueTable(brc1.schema); + foreach (IBlueRow br in brc1) { + object[] oarray = new object[columns.Length]; + for (int i = 0; i < columns.Length; i++) { + string col = columns[i]; + oarray[i] = br[col]; + } + if (myDoesRowExistsInTable(brc2, columns, oarray) == false) + (brcDiff as BlueTable).addRow(br); + } + } + + /// + /// Abosrbs the deletes and adds into the BlueTable + /// + /// + /// + /// + public static void absorbChanges(BlueTable t, IBlueRowCollection dels, IBlueRowCollection adds) { + foreach (IBlueRow r in dels) + if (!t.deleteRow(r)) + throw new Exception(String.Format("{0} does not have row with id {1} to delete", t.schema.tableName, r["id"])); + + foreach (IBlueRow r in adds) + t.addRow(r); + } + + public static IBlueRowCollection toBlueRowCollection(IBlueRow r) { + BlueTable rv = new BlueTable(r.schema); + rv.addRow(r); + return rv; + } + + /// + /// Converts an ICollection of IBlueRows to IBlueRowCollection. If there are no rows in the + /// collection, a null is returned + /// + /// + /// + public static IBlueRowCollection toBlueRowCollection(ICollection col) { + BlueTable retVal = null; + foreach (IBlueRow r in col) { + retVal = retVal ?? new BlueTable(r.schema); + retVal.addRow(r); + } + return retVal; + } + + /// + /// Computes the rows that are truely deleted, added or changed (with respect to relevant columns) + /// + /// all the deleted rows in a certain callback + /// all the added rows in a certain callback + /// the rows that are truely deleted or changed with respect to the relevant columns + /// the rows that are truely added or changed with respect to the relevant columns + /// + public static void getTrueDeletesAndAdds(IBlueRowCollection dels, + IBlueRowCollection adds, + out IBlueRowCollection truelyDels, + out IBlueRowCollection truelyAdds, + string[] cols) { + truelyDels = new BlueTable(dels); + truelyAdds = new BlueTable(adds); + foreach (IBlueRow r in dels) { + if (adds.containsID((int)r["id"]) && + BlueDataUtility.compute_row_equality(adds[(int)r["id"]], r, cols)) { + (truelyDels as BlueTable).deleteRow((int)r["id"]); + (truelyAdds as BlueTable).deleteRow((int)r["id"]); + } + } + } + + public static IBlueRowCollection getSerializableCollection(IBlueRowCollection coll) { + BlueTable retVal = new OrderedBlueTable(coll.schema); + foreach (IBlueRow r in coll) + retVal.addRow(new BlueRow(r)); + return retVal; + } + + /// + /// extracts out a column from a table + /// + /// + /// + /// + public static ArrayList extractColumn(IBlueRowCollection rows, string colName) { + ArrayList rv = new ArrayList(); + foreach (IBlueRow r in rows) + rv.Add(r[colName]); + return rv; + } + + /// + /// applies the deletedRows and addedRows to the oldState and asserts if it does not + /// match the newRows. for an old state with 10 K rows, and all 10 K rows being deleted and + /// added it takes ~ 35 ms to execute this method. + /// + /// + /// + /// + /// + public static void checkConsistency(IBlueRowCollection oldState, + IBlueRowCollection newRows, + IBlueRowCollection delRows, + IBlueRowCollection addedRows) { + BlueTable calc = new BlueTable(oldState); + Debug.Assert(oldState != null && newRows != null && delRows != null && addedRows != null); + + foreach (IBlueRow r in delRows) { + Debug.Assert(calc.containsID((int)r["id"]), "deleted row's id = " + r["id"] + " does not exist"); + calc.deleteRow(r); + } + + foreach (IBlueRow r in addedRows) { + Debug.Assert(!calc.containsID((int)r["id"]), "added rows's id = " + r["id"].ToString() + " already exists"); + calc.addRow(r); + } + + Debug.Assert(calc.Count == newRows.Count, "row count should be the same. Counts: calc=" + calc.Count + "; newRows=" + newRows.Count); + foreach (IBlueRow r in calc) + Debug.Assert(newRows.containsID((int)r["id"]), r["id"] + " is not found"); + + } + + /// + /// Extracts values from an IBlueRow specified by the columnNames arg + /// + /// + /// + /// + public static object[] getSelectedValues(IBlueRow r, params string[] colNames) { + object[] retVal = new object[colNames.Length]; + for (int i = 0; i < colNames.Length; i++) + retVal[i] = r[colNames[i]]; + return retVal; + } + + + /// + /// From an IBlueRowCollection, for a set of columns, extracts a row with the same values as + /// provided in the third arg + /// + /// + /// + /// + /// + public static IBlueRow getRowWithKey(IBlueRowCollection rows, string[] keyColumns, object[] colValues) { + foreach (IBlueRow r in rows) { + object[] vals = getSelectedValues(r, keyColumns); + if (CollectionHelpers.equal(vals, colValues)) + return r; + } + return null; + } + + public static IBlueRowCollection deepCopy(IBlueRowCollection rows) { + BlueTable retVal = new BlueTable(rows.schema); + foreach (IBlueRow r in rows) + retVal.addRow(new BlueRow(r)); + return retVal; + } + + /// + /// Returns the columns, sorted alphabetically. The second arg + /// is for the inclusion of id + /// + /// + /// + /// + public static int[] getColumnsInSortedOrder(IBlueTableSchema schema, bool includeId) { + ArrayList names = new ArrayList(schema.columnNames), + retVal = new ArrayList(); + names.Sort(); + foreach (string n in names) + if (n != "id" || includeId) + retVal.Add(schema[n].colNo); + return retVal.ToArray(typeof(int)) as int[]; + } + + //this class is used so that we can sort a schema by column name + //that way, prepublished tables will allways get published with schemas in the same order + //regardless of how the underlying streams end up working + private class ColumnInfoComparisonWrapper : IComparable { + + ColumnInfo myColumn; + + public string ColumnName { + get { return myColumn.colName; } + //can't set it, these are only for reading columns + } + + public ColumnInfo Column { + get { return myColumn; } + } + public ColumnInfoComparisonWrapper(ColumnInfo theColumn) { + myColumn = theColumn; + } + + public int CompareTo(ColumnInfoComparisonWrapper otherGuy) { + return ColumnName.CompareTo(otherGuy.ColumnName); + } + } + + public static IBlueTableSchema getSortedSchema(IBlueTableSchema inSchema) { + List sortedColNames = new List(inSchema.columnNames); + sortedColNames.Sort(); + List sortedColInfos = new List(); + int colNo = 0; + foreach (string c in sortedColNames) { + ColumnInfo ci = inSchema[c]; + sortedColInfos.Add(new ColumnInfo(c, colNo++, ci.colType)); + + } + return new BlueTableSchema(inSchema.tableName, sortedColInfos.ToArray()); + } + + public static IBlueTableSchema getSortedSchema(IBlueTableSchema theSchema, out int[] sortedSchemaToOriginalSchemaMapping, out int idColumnInSortedSchema) { + + //for now this function will not do anything + //that is because we don't want to break the C++ deserialization code + int numCols = theSchema.columnNames.Length; + sortedSchemaToOriginalSchemaMapping = new int[numCols]; + for (int col = 0; col < theSchema.columnNames.Length; ++col) { + //the mapping is just the identity mapping + sortedSchemaToOriginalSchemaMapping[col] = col; + + + } + + idColumnInSortedSchema = theSchema["id"].colNo; + + return theSchema; + + /* + //set this to -1 in case the method doesn't find an id column + idColumnInSortedSchema = 01; + List schemaColumns = new List(); + + foreach (ColumnInfo ci in theSchema.columnInfo) + schemaColumns.Add(new ColumnInfoComparisonWrapper(ci)); + + schemaColumns.Sort(); + + int numCols = theSchema.columnInfo.Length; + ColumnInfo[] outputColumns = new ColumnInfo[numCols]; + sortedSchemaToOriginalSchemaMapping = new int[numCols]; + + for(int i = 0; i < numCols; ++i) + { + string thisColName = schemaColumns[i].ColumnName; + outputColumns[i] = new ColumnInfo(thisColName, i, schemaColumns[i].Column.colType); + sortedSchemaToOriginalSchemaMapping[i] = theSchema[thisColName].colNo; + + //we cache the ID column because it'll be differentin the serialized row than in our schema + if (thisColName == "id") + idColumnInSortedSchema = i; + + } + + + theSchema = new BlueTableSchema(theSchema.tableName,outputColumns); + return theSchema; + */ + + + } + + + + + /// + /// Compares 2 schemas for column-by-column match + /// + /// + /// + /// true if equal + public static bool compareSchemas(IBlueTableSchema s1, IBlueTableSchema s2) { + bool result = true; + try { + if (s1.columnInfo.Length != s2.columnInfo.Length) + result = false; + else + foreach (ColumnInfo ci1 in s1.columnInfo) { + ColumnInfo ci2 = s2[ci1.colNo]; + if (ci1.colName != ci2.colName || ci1.colType != ci2.colType || ci1.colNo != ci2.colNo) { + result = false; + break; + } + } + } catch { + result = false; // the col# wasn't found in s2 + } + return result; + } + + public static void validate_row_schema(IBlueRow r) { + foreach (var ci in r.schema.columnInfo) { + object rval = r[ci.colName]; + if (rval == null) + continue; + bts.Log.assert(rval.isDBNull() || rval.GetType().Equals(ci.colType)); + } + } + + public static void validate_table_schema(IBlueRowCollection rows) { + foreach (IBlueRow r in rows) { + validate_row_schema(r); + } + } + + public static IBlueRowCollection removeRows(BlueTable table, string[] cols, object[] vals) { + IBlueRowCollection dels = findMatchingRows(table, cols, vals); + table.deleteRows(dels); + return dels; + } + + public static IBlueRowCollection findMatchingRows(IBlueRowCollection rows, string[] cols, object[] vals) { + BlueTable rv = new BlueTable(rows.schema); + foreach (IBlueRow r in rows) { + bool match = true; + for (int i = 0; i < cols.Length; i++) + match &= vals[i].Equals(r[cols[i]]); + if (match) + rv.addRow(r); + } + return rv; + } + + public static void checkRowTypes(IBlueRowCollection rows) { + foreach (IBlueRow r in rows) + foreach (string col in r.schema.columnNames) + if (r[col] != null && r[col] != DBNull.Value) + Debug.Assert(r[col].GetType() == rows.schema[col].colType, + String.Format("col {0} in table {1} is of type {2} in schema but is a {3} in the row", col, rows.schema.tableName, r.schema[col].colType.ToString(), r[col].GetType().ToString())); + } + + public static void addRows(ref BlueTable table, ICollection rows) { + foreach (IBlueRow r in rows) + table.addRow(r); + } + + public static void removeRows(ref BlueTable table, ICollection rows) { + foreach (IBlueRow r in rows) + table.deleteRow(r); + } + + public static void sortOutPacket(IBlueRowCollection dels, IBlueRowCollection adds, string[] keyCols, + out IBlueRowCollection addedRows, + out IBlueRowCollection deletedRows, + out IBlueRowCollection changedRows) { + addedRows = new BlueTable(dels.schema); + deletedRows = new BlueTable(dels.schema); + changedRows = new BlueTable(dels.schema); + + KeysBasedTable2 keyedDels = new KeysBasedTable2(dels.schema, keyCols), + keyedAdds = new KeysBasedTable2(dels.schema, keyCols); + + foreach (IBlueRow r in dels) + keyedDels.update(r); + foreach (IBlueRow r in adds) + keyedAdds.update(r); + + foreach (IBlueRow r in keyedDels) { + object[] key = BlueDataUtility.getSelectedValues(r, keyCols); + if (keyedAdds.getRow(key) != null) { + ///change + (changedRows as BlueTable).addRow(r); + } else { + ///delete + (deletedRows as BlueTable).addRow(r); + } + } + + foreach (IBlueRow r in keyedAdds) { + object[] key = BlueDataUtility.getSelectedValues(r, keyCols); + if (keyedDels.getRow(key) == null) { + (addedRows as BlueTable).addRow(r); + } + } + } + + public static int HashExpStrike(DateTime exp, double strike) { + return ((DateTime)exp).ToShortDateString().GetHashCode() ^ strike.GetHashCode(); + } + + public static IEnumerable collection_to_ienum(IBlueRowCollection rows) { + List rv = new List(); + foreach (IBlueRow r in rows) + rv.Add(r); + return rv; + } + + public static IBlueRowCollection ienum_to_collection(IEnumerable rows) { + BlueTable rv = null; + if (rows.Count() > 0) { + rv = new BlueTable(rows.ElementAt(0).schema); + foreach (IBlueRow r in rows) + rv.addRow(r); + } + return rv; + } + + public static IBlueTableSchema add_column(IBlueTableSchema input_schema, string new_col_name, Type new_col_type) { + List cis = new List(input_schema.columnInfo); + cis.Add(new ColumnInfo(new_col_name, cis.Count, new_col_type)); + return new BlueTableSchema(input_schema.tableName, cis.ToArray()); + } + + public static IBlueTableSchema remove_columns(IBlueTableSchema input_schema, params string[] col_names) { + Dictionary removed_col_names = new Dictionary(); + foreach (string c in col_names) + removed_col_names[c] = true; + List cis = new List(); + foreach (ColumnInfo ci in input_schema.columnInfo) + if (!removed_col_names.ContainsKey(ci.colName)) + cis.Add(new ColumnInfo(ci.colName, cis.Count, ci.colType)); + return new BlueTableSchema(cis.ToArray(), input_schema.tableName); + } + + public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) { + List rv = new List(); + foreach (string c in col_names) + rv.Add(schema[c].colNo); + return rv.ToArray(); + } + + public static string get_appended_string_val(IBlueRow r, int[] col_indices) { + System.Text.StringBuilder sb = new System.Text.StringBuilder(); + for (int i = 0; i < col_indices.Length; i++) + sb.Append(r[col_indices[i]]); + return sb.ToString(); + } + + public delegate string RowValueToString(IBlueRow r, int col_index); + public static string get_appended_string_val(IBlueRow r, int[] col_indices, RowValueToString to_string_conv) { + System.Text.StringBuilder sb = new System.Text.StringBuilder(); + for (int i = 0; i < col_indices.Length; i++) + sb.Append(to_string_conv(r, i)); + return sb.ToString(); + } + + public static IDictionary> group_together(IBlueRowCollection rows, string[] group_by_col_names) { + BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length); + return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher); + } + + public static IDictionary> group_together(IBlueRowCollection rows, string[] group_by_col_names, ref BlueRowHasher hasher) { + return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher); + } + + public static IDictionary> group_together(IBlueRowCollection rows, int[] group_by_col_nos, ref BlueRowHasher hasher) { + Dictionary> rv = new Dictionary>(); + hasher = hasher ?? new BlueRowHasher(group_by_col_nos.Length); + foreach (IBlueRow r in rows) { + int row_hash = hasher.get_hash(r, group_by_col_nos); + List this_group_rows = null; + if (!rv.TryGetValue(row_hash, out this_group_rows)) { + this_group_rows = new List(); + rv.Add(row_hash, this_group_rows); + } + this_group_rows.Add(r); + } + return rv; + } + + public static void copy_to_row(IBlueRow row, string[] data_col_names, MutableBlueRow output_row) { + foreach (string cn in data_col_names) + output_row.setValue(cn, row[cn]); + } + + /// + /// Copies the column nos specified in data_cols from row, to output_row + /// starting from index start_col_index + /// + public static void copy_to_row(IBlueRow row, int[] data_col_nos, object[] output_row, int start_col_index) { + foreach (int col_no in data_col_nos) + output_row[start_col_index++] = row[col_no]; + } + + public static object compute_min_max(object v1, object v2, bool is_max) { + if (!TypeHelper.is_dbnull(v1) && !TypeHelper.is_dbnull(v2)) { + if (TypeHelper.is_numeric_type(v1) && TypeHelper.is_numeric_type(v2)) { + double v1_num = (double)Convert.ChangeType(v1, TypeCode.Double), + v2_num = (double)Convert.ChangeType(v2, TypeCode.Double); + v1_num = Double.IsNaN(v1_num) ? (is_max ? Double.MinValue : Double.MaxValue) : v1_num; + v2_num = Double.IsNaN(v2_num) ? (is_max ? Double.MinValue : Double.MaxValue) : v2_num; + double rv = is_max ? Math.Max(v1_num, v2_num) : Math.Min(v1_num, v2_num); + return Convert.ChangeType(rv, v1.GetType()); + } else + return v1; + } else + return TypeHelper.is_dbnull(v1) ? v2 : v1; + } + + #region Sorting + + public static class ParallelSort { + public class TreeRowEffComparer : IComparer { + private int _depth_in_tree_col_no; + private int[] _aggregation_tree_col_nos; + private Pair[] _sort_map; + + public TreeRowEffComparer(int depth_in_tree_col_no, int[] aggregation_tree_col_nos, Pair[] sort_map) { + _depth_in_tree_col_no = depth_in_tree_col_no; + _aggregation_tree_col_nos = aggregation_tree_col_nos; + _sort_map = sort_map; + } + + public int Compare(IBlueRow x, IBlueRow y) { + return BlueDataUtility.sort_in_a_tree(x, y, _depth_in_tree_col_no, _aggregation_tree_col_nos, _sort_map); + } + } + + public class TabularRowComparer : IComparer { + private IEnumerable> _sort_map; + public TabularRowComparer(IEnumerable> sort_map) { _sort_map = sort_map; } + public int Compare(IBlueRow x, IBlueRow y) { return BlueDataUtility.compare_rows(x, y, _sort_map); } + } + + public class TabularRowHashPairComparer : IComparer> { + private Pair[] _sort_map; + + public TabularRowHashPairComparer(Pair[] sort_map) { _sort_map = sort_map; } + public int Compare(Pair x, Pair y) { return BlueDataUtility.compare_rows(x.first, y.first, _sort_map); } + } + + #region Public Static Methods + + public static void QuicksortSequential(IList arr, IComparer comparer) { + QuicksortSequential(arr, comparer, 0, arr.Count - 1); + } + + public static void QuicksortParallel(IList arr, IComparer comparer) { + QuicksortParallel(arr, comparer, 0, arr.Count - 1); + } + + #endregion + + #region Private Static Methods + + private static void QuicksortSequential(IList arr, IComparer comparer, int left, int right) { + if (right > left) { + int pivot = Partition(arr, comparer, left, right); + QuicksortSequential(arr, comparer, left, pivot - 1); + QuicksortSequential(arr, comparer, pivot + 1, right); + } + } + + private static void QuicksortParallel(IList arr, IComparer comparer, int left, int right) { + const int SEQUENTIAL_THRESHOLD = 64; + if (right > left) { + if (right - left < SEQUENTIAL_THRESHOLD) { + QuicksortSequential(arr, comparer, left, right); + } else { + int pivot = Partition(arr, comparer, left, right); + System.Threading.Tasks.Parallel.Invoke(new Action[] { delegate {QuicksortParallel(arr, comparer, left, pivot - 1);}, + delegate {QuicksortParallel(arr, comparer, pivot + 1, right); } + }); + } + } + } + + private static void Swap(IList arr, int i, int j) { + T tmp = arr[i]; + arr[i] = arr[j]; + arr[j] = tmp; + } + + private static int Partition(IList arr, IComparer comparer, int low, int high) { + // Simple partitioning implementation + int pivotPos = (high + low) / 2; + T pivot = arr[pivotPos]; + Swap(arr, low, pivotPos); + + int left = low; + for (int i = low + 1; i <= high; i++) { + if (comparer.Compare(arr[i], pivot) < 0) { + left++; + Swap(arr, i, left); + } + } + + Swap(arr, low, left); + return left; + } + + #endregion + } + + public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary sort_map) { + foreach (KeyValuePair p in sort_map) { + int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value); + if (comp != 0) + return comp; + } + return 0; + } + + public static int compare_rows(IBlueRow r1, IBlueRow r2, IEnumerable> sort_map) { + foreach (Pair p in sort_map) { + int comp = compare_send_dontcare_to_last(r1, p.first, r2, p.first, p.second); + if (comp != 0) + return comp; + } + return 0; + } + + public static int compare_send_dontcare_to_last(IBlueRow r1, int r1_col_no, + IBlueRow r2, int r2_col_no, + bool low_to_high) { + object o1 = BlueDataUtility.get_row_value(r1, r1_col_no, DBNull.Value), + o2 = BlueDataUtility.get_row_value(r2, r2_col_no, DBNull.Value); + return compare_send_dontcare_to_last(o1, o2, low_to_high); + } + + public static List> get_indexed_sort_map(List> sort_map, IBlueTableSchema schema) { + List> rv = new List>(); + foreach (Pair p in sort_map) + rv.Add(new Pair(schema[p.first].colNo, p.second)); + return rv; + } + + /// Comapares 2 rows in a way where nulls, nans, infinities, + /// empty strings etc are sent to the last + public static int compare_send_dontcare_to_last(IBlueRow r1, string r1_col_name, + IBlueRow r2, string r2_col_name, + bool low_to_high) { + object o1 = BlueDataUtility.get_row_value(r1, r1_col_name, DBNull.Value), + o2 = BlueDataUtility.get_row_value(r2, r2_col_name, DBNull.Value); + return compare_send_dontcare_to_last(o1, o2, low_to_high); + } + + public static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) { + if (is_dont_care_val(o1)) + return is_dont_care_val(o2) ? 0 : 1; + else + return is_dont_care_val(o2) ? -1 : (low_to_high ? ((IComparable)o1).CompareTo(o2) + : ((IComparable)o2).CompareTo(o1)); + } + + public static bool is_dont_care_val(object o) { + return o == null || o == DBNull.Value || + o is string && String.IsNullOrEmpty((string)o) || + o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) || + o is float && (Single.IsInfinity((float)o) || Single.IsNaN((float)o)) || + o is Int32 && (Int32.MaxValue.Equals(o) || Int32.MinValue.Equals(o)); + } + + #endregion + + #region Tree Aggregation + + public static int[] convert_aggregation_tree(string[] aggregation_tree_col_names, IBlueTableSchema schema) { + List rv = new List(); + foreach (string col_name in aggregation_tree_col_names) + rv.Add(schema[col_name].colNo); + return rv.ToArray(); + } + + public static bool is_leaf_node(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) { + return BlueDataUtility.get_row_value(r, depth_in_tree_col_no, -1) == (aggregation_tree.Count() - 1); + } + + public static bool is_leaf_node(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) { + return BlueDataUtility.get_row_value(r, depth_in_tree_col_name, -1) == (aggregation_tree.Count() - 1); + } + + public static string generate_aggregation_based_id(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) { + int aggregation_priority = BlueDataUtility.get_row_value(r, depth_in_tree_col_no, -1); + string rv = String.Empty; + for (int p = 0; p <= aggregation_priority; p++) + rv += (rv == String.Empty ? String.Empty : ":") + String.Format("{0},{1}", p, BlueDataUtility.get_row_value(r, aggregation_tree[p], DBNull.Value)); + return rv; + } + + /// Generates a string based id for a row in a tree + /// by appending the values of the nodes above it + public static string generate_aggregation_based_id(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) { + int aggregation_priority = BlueDataUtility.get_row_value(r, depth_in_tree_col_name, -1); + string rv = String.Empty; + for (int p = 0; p <= aggregation_priority; p++) + rv += (rv == String.Empty ? String.Empty : ":") + + String.Format("{0},{1}", p, BlueDataUtility.get_row_value(r, aggregation_tree.ElementAt(p), DBNull.Value)); + return rv; + } + /// + /// Given an aggregation hierarchy tree, figures if the first + /// arg is a parent of the second arg. + /// + public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, string depth_in_tree_col_name, string[] aggregation_tree) { + System.Diagnostics.Debug.Assert(parent_row.Contains(depth_in_tree_col_name) && child_row.Contains(depth_in_tree_col_name)); + int parent_depth = BlueDataUtility.get_row_value(parent_row, depth_in_tree_col_name, Int32.MaxValue), + child_depth = BlueDataUtility.get_row_value(child_row, depth_in_tree_col_name, Int32.MaxValue); + if (parent_depth < child_depth) { + for (int p = 0; p <= parent_depth; p++) { + string col_name = aggregation_tree.ElementAt(p); + object parent_val = BlueDataUtility.get_row_value(parent_row, col_name, DBNull.Value), + child_val = BlueDataUtility.get_row_value(child_row, col_name, DBNull.Value); + if (!parent_val.Equals(child_val)) + return false; + } + return true; + } else + return false; + } + + public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, int depth_in_tree_col_no, int[] aggregation_tree) { + int parent_depth = BlueDataUtility.get_row_value(parent_row, depth_in_tree_col_no, Int32.MaxValue), + child_depth = BlueDataUtility.get_row_value(child_row, depth_in_tree_col_no, Int32.MaxValue); + if (parent_depth < child_depth) { + for (int p = 0; p <= parent_depth; p++) { + int col_no = aggregation_tree[p]; + object parent_val = BlueDataUtility.get_row_value(parent_row, col_no, DBNull.Value), + child_val = BlueDataUtility.get_row_value(child_row, col_no, DBNull.Value); + if (!parent_val.Equals(child_val)) + return false; + } + return true; + } else + return false; + } + + public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, string depth_in_tree_col_name, string[] aggregation_tree, Dictionary sort_map) { + if (BlueDataUtility.is_parent(r1, r2, depth_in_tree_col_name, aggregation_tree)) + return -1; + else if (BlueDataUtility.is_parent(r2, r1, depth_in_tree_col_name, aggregation_tree)) + return 1; + else if (get_row_value(r1, "id", Int32.MaxValue).Equals(get_row_value(r2, "id", Int32.MaxValue))) + return 0; + else { + for (int p = 0; p < aggregation_tree.Count(); p++) { + /// sort_map is used to leaf nodes + if (p == aggregation_tree.Count() - 1) { + /// sorting for leaf nodes is special, since they can be + /// sorted by data columns + System.Diagnostics.Debug.Assert(BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree)); + int leaf_comp = compare_rows(r1, r2, sort_map); + if (leaf_comp != 0) + return leaf_comp; + } + /// sort_map is used to find sorting order + string aggregation_col_name = aggregation_tree.ElementAt(p); + bool low_to_high = sort_map.TryGetValue(aggregation_col_name, out low_to_high) ? low_to_high : true; + int comp = BlueDataUtility.compare_send_dontcare_to_last(r1, aggregation_col_name, r2, aggregation_col_name, low_to_high); + if (comp != 0) + return comp; + } + return BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree) ? + compare_rows(r1, r2, sort_map) : 0; + } + } + + public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, int depth_in_tree_col_no, int[] aggregation_tree, Pair[] sort_map) { + if (BlueDataUtility.is_parent(r1, r2, depth_in_tree_col_no, aggregation_tree)) + return -1; + else if (BlueDataUtility.is_parent(r2, r1, depth_in_tree_col_no, aggregation_tree)) + return 1; + else if (get_row_value(r1, "id", Int32.MaxValue).Equals(get_row_value(r2, "id", Int32.MaxValue))) + return 0; + else { + for (int p = 0; p < aggregation_tree.Count(); p++) { + /// sort_map is used to leaf nodes + if (p == aggregation_tree.Count() - 1) { + /// sorting for leaf nodes is special, since they can be + /// sorted by data columns + //System.Diagnostics.Debug.Assert(BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree)); + int leaf_comp = compare_rows(r1, r2, sort_map); + if (leaf_comp != 0) + return leaf_comp; + } + /// sort_map is used to find sorting order + int aggregation_col_no = aggregation_tree[p]; + bool low_to_high = true;//sort_map.TryGetValue(aggregation_col_name, out low_to_high) ? low_to_high : true; + int comp = BlueDataUtility.compare_send_dontcare_to_last(r1, aggregation_col_no, r2, aggregation_col_no, low_to_high); + if (comp != 0) + return comp; + } + return BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_no, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_no, aggregation_tree) ? + compare_rows(r1, r2, sort_map) : 0; + } + } + + #endregion + + #region Column accessors + + public static T get_array_value(Array r, int col_no, T error_val) { + try { + object rv = (r != null && col_no > -1 && col_no < r.Length) ? r.GetValue(col_no) : DBNull.Value; + if (rv == null || rv == DBNull.Value) + return error_val; + else + return (T)Convert.ChangeType(rv, typeof(T)); + } catch { + return error_val; + } + } + + public static DateTime get_array_datetime_value(Array r, int col_no, DateTime error_val) { + try { + object rv = (r != null && col_no > -1 && col_no < r.Length) ? r.GetValue(col_no) : error_val; + return _get_datetime_value(rv, error_val); + } catch { + return error_val; + } + } + + public static T get_row_value(IBlueRow r, int col_no, T error_val) { + try { + object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : DBNull.Value; + if (rv == null || rv == DBNull.Value) + return error_val; + else + return (T)Convert.ChangeType(rv, typeof(T)); + } catch { + return error_val; + } + } + + public static T get_row_value(IBlueRow r, string col_name, T error_val) { + try { + object rv = (r != null && r.Contains(col_name)) ? r[col_name] : DBNull.Value; + if (rv == null || rv == DBNull.Value) + return error_val; + else + return (T)Convert.ChangeType(rv, typeof(T)); + } catch { + return error_val; + } + } + + public static T get_row_enum_value(IBlueRow r, string col_name, T error_val) where T : struct { + try { + object rv = r.Contains(col_name) ? r[col_name] : DBNull.Value; + return _get_row_enum_value(rv, error_val); + } catch { + return error_val; + } + } + + public static T get_row_enum_value(IBlueRow r, int col_no, T error_val) where T : struct { + try { + object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : DBNull.Value; + return _get_row_enum_value(rv, error_val); + } catch { + return error_val; + } + } + + private static T _get_row_enum_value(object o, T error_val) where T : struct { + try { + if (o == null || o == DBNull.Value) + return error_val; + else if (o is string) { + T result = error_val; + if (Enum.TryParse((string)o, true, out result)) return result; + return error_val; + } else if (o is Int32 || o is Int16 || o is UInt16 || o is UInt32) + return (T)o; + else if (o is T) + return (T)o; + else + return error_val; + } catch { + return error_val; + } + } + + public static DateTime get_row_datetime_value(IBlueRow r, string col_name, DateTime error_val) { + try { + object rv = (r != null && r.Contains(col_name)) ? r[col_name] : error_val; + return _get_datetime_value(rv, error_val); + } catch { + return error_val; + } + } + + public static DateTime get_row_datetime_value(IBlueRow r, int col_no, DateTime error_val) { + try { + object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : error_val; + return _get_datetime_value(rv, error_val); + } catch { + return error_val; + } + } + + private static DateTime _get_datetime_value(object v, DateTime error_val) { + try { + if (v == null || v == DBNull.Value) + return error_val; + else { + if (v is DateTime) + return (DateTime)v; + else if (v is string) { + DateTime rv_dt = error_val; + DateTime.TryParse((string)v, out rv_dt); + return rv_dt; + } else { + double oa_date = (double)Convert.ChangeType(v, typeof(Double)); + return DateTime.FromOADate(oa_date); + } + } + } catch { + return error_val; + } + } + + #endregion + + #region CSV methods + /// + /// Reads a schema file name, and return the constructed schema + /// + public static IBlueTableSchema createSchemaFromCSVSchemaFile(string schemaFileName) { + StreamReader sr = new StreamReader(schemaFileName); + IBlueTableSchema retVal = createSchemaFromCSVSchemaFile(sr); + sr.Close(); + return retVal; + } + + /// + /// Builds a schema from a schema file. Note that the stream is not closed, so + /// the calling code should do so + /// + public static IBlueTableSchema createSchemaFromCSVSchemaFile(TextReader sr) { + string tableName = sr.ReadLine(); + ArrayList t = new ArrayList(); + string l = ""; + while ((l = sr.ReadLine()) != null) + t.Add(l.Trim()); + string[] cols = t.ToArray(typeof(string)) as String[]; + ColumnInfo[] cis = new ColumnInfo[cols.Length]; + for (int i = 0; i < cols.Length; i++) { + string[] colTags = cols[i].Split('/'); + cis[i] = new ColumnInfo(colTags[0], i, TypeInstanceLoader.loadType(colTags[1])); + } + IBlueTableSchema retVal = new BlueTableSchema(cis, tableName); + if (retVal["id"] == null) { + ArrayList temp = new ArrayList(retVal.columnInfo); + temp.Add(new ColumnInfo("id", temp.Count, typeof(System.Int32))); + retVal = new BlueTableSchema(retVal.tableName, + temp.ToArray(typeof(ColumnInfo)) as ColumnInfo[]); + } + return retVal; + } + + /// + /// Creates an IBlueRow from a CSV data file row; if the string already contains an id, + /// the id argument is ignored + /// + public static IBlueRow createRowFromCSVFileRow(string fileRow, IBlueTableSchema schema, int id) { + string[] toks = CSVParser.parse(fileRow); + ArrayList rowList = new ArrayList(); + if (toks.Length == schema.columnNames.Length) { + for (int i = 0; i < toks.Length; i++) + rowList.Add(myConvertTo(schema[i].colType, toks[i])); + } else { + int idIndex = schema["id"].colNo; + for (int i = 0; i < toks.Length; i++) + rowList.Add(myConvertTo(schema[i < idIndex ? i : i + 1].colType, toks[i])); + rowList.Insert(idIndex, id); + } + return new BlueRow(schema, rowList.ToArray()); + } + + /// + /// Returns the CSV data string corresponding to the row + /// + public static string toCSVDataString(IBlueRow r) { + string retVal = ""; + for (int i = 0; i < r.schema.columnNames.Length; i++) + retVal += + r.schema[i].colType.Equals(typeof(string)) ? + "\"" + r[i].ToString().Replace("\"", "\"\"") + "\"" : + r[i].ToString(); + return retVal.Remove(retVal.Length - 1, 1); + } + + public static string[] toCSVSchemaFileStrings(IBlueTableSchema schema) { + string[] retVal = new string[2]; + retVal[0] = schema.tableName; + foreach (ColumnInfo ci in schema.columnInfo) + retVal[1] += ci.colName + "/" + ci.colType.ToString() + " "; + retVal[1] = retVal[1].Remove(retVal[1].Length - 1, 1); + return retVal; + } + + /// + /// Converts a CSV data value to its original type. + /// + private static object myConvertTo(Type t, string val) { + val = val.Trim(); + if (t.Equals(typeof(System.String))) + return val; + else if (val != "") { + if (t.Equals(typeof(System.Char))) + return Convert.ToChar(val); + if (t.Equals(typeof(System.Int16))) + return Convert.ToInt16(val); + if (t.Equals(typeof(System.Int32))) + return Convert.ToInt32(val); + if (t.Equals(typeof(System.Int64))) + return Convert.ToInt64(val); + if (t.Equals(typeof(System.Double))) + return Convert.ToDouble(val); + if (t.Equals(typeof(System.Single))) + return Convert.ToSingle(val); + if (t.Equals(typeof(System.DateTime))) + return myToDateTime(val); + } + return DBNull.Value; + } + + private static object myToDateTime(string val) { + try { + return Convert.ToDateTime(val); + } catch (FormatException) //datetime is not stored in one of the string formats + { + return val.IndexOf('.') > -1 ? + DateTime.FromOADate(Convert.ToDouble(val)) : new DateTime(Convert.ToInt64(val)); + } + } + + public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) => + createRowsFromCSVFileWithoutTypes(fileAddress, out var _); + + public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress, out IBlueTableSchema schema) { + StreamReader sr = new StreamReader(fileAddress); + string cols = sr.ReadLine(); + string[] columns = cols.Split(','); + ColumnInfo[] ci = new ColumnInfo[columns.Length + 1]; + ci[0] = new ColumnInfo("id", 0, typeof(int)); + for (int i = 1; i < ci.Length; i++) + ci[i] = new ColumnInfo(columns[i - 1], i, typeof(string)); + schema = new BlueTableSchema(ci, "csvTable"); + BlueTable result = new BlueTable(schema); + int id = 0; + while (!sr.EndOfStream) + result.addRow(createRowFromCSVFileRow(sr.ReadLine(), schema, id++)); + sr.Close(); + return result; + } + + #endregion + + #region Magic + /// + /// Uses a cached compiled expression to implement a truly generic add function. Will throw if given types that cannot be added. + /// + internal class MagicAdder { + static Func add; + static MagicAdder() { + ParameterExpression left = Expression.Parameter(typeof(T), "a"); + ParameterExpression right = Expression.Parameter(typeof(T), "b"); + add = (Func)Expression.Lambda(Expression.Add(left, right), left, right).Compile(); + } + + public static T Add(T a, T b) { + return add(a, b); + } + } + #endregion + + #region Misc + + public static void swap(ref T t1, ref T t2) { + T temp = t1; + t1 = t2; + t2 = temp; + } + + #endregion + } +} diff --git a/src/DataStreamSuite/DataStreamPublisher/Program.cs b/src/DataStreamSuite/DataStreamPublisher/Program.cs index d25875479..930bc1ec2 100644 --- a/src/DataStreamSuite/DataStreamPublisher/Program.cs +++ b/src/DataStreamSuite/DataStreamPublisher/Program.cs @@ -1,156 +1,156 @@ -using BCG; -using BCG.Utilities.General; - -using bts; -using bts.api.core; -using bts.utils; - -using Bts.DataStreamExpressions; -using Bts.DataStreamPublisher; -using Bts.PubSub; - -using CommandLine; - -using mbts.config; - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text.Json; -using System.Threading.Tasks; - -await Parser.Default.ParseArguments(args) - .MapResult(RunAsync, errs => Task.FromResult(1)); - -async Task RunAsync(DataStreamPublisherConfig cfg) { - try { - cfg.MergeWithConfigFile(); - cfg.MergeWith(DataStreamPublisherConfig.Default()); - PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); - Configuration.initialize(cfg.Sparkd); - Log.Gen.info("Starting DataStreamPublisher"); - var service_ctx = ServiceContext.create(cfg.Sparkd); - await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty); - if (cfg.SimpleEchoClient ?? false) { - start_echo_client(); - } else { - _publish_streams(cfg.Publishers ?? new(0)); - } - } catch (Exception ex) { - // Write the error to console first, in case the logging system is compromised. - var error_msg = $"DataStreamPublisher threw exception:\n{ex}"; - Console.Error.WriteLine(error_msg); - Console.Error.Flush(); - bts.Log.Gen.error(error_msg); - return 1; - } - return 0; -} - -static ViewStream? build_stream(string? stream_expression) { - if (stream_expression is null) { - return null; - } - try { - bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}"); - var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); - var rv = exp.getView(new Dictionary(0)); - return rv; - - } catch (Exception ex) { - bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}"); - return null; - } -} - -static void start_echo_client() { - Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue"); - Console.ReadLine(); - SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345, 1000, 1000); - client.start(); - while (true) { - System.Threading.Thread.Sleep(1000); - } - // await SimpleTCPClient.start("127.0.0.1", 12345); - // Task.Delay(1000000000).Wait(); -} - -void _publish_streams(IEnumerable publishers) { - PubTableBroker pb_broker = new PubTableBroker(); - RemoteQueryManager csm = new RemoteQueryManager(); - var csvPublishers = new List(); - - foreach (var publisher in publishers) { - if (Directory.Exists(publisher.CsvDir)) { - csvPublishers.Add(new(publisher.CsvDir, pb_broker)); - } - } - - var viewPublishers = publishers - .SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher)) - .ToList(); - - while (true) { - System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1)); - ViewStreamDatabase.instance.doWork(); - csm.doWork(); - MasterWorker.getInstance(false).doWork(); - ManagedSubscriber.instance.doWork(); - } - - ViewPublisher? createViewPublisher(PublishedStream ps) { - Log.Gen.info( - $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}"); - if (string.IsNullOrEmpty(ps.Name)) { - Log.Gen.warn( - $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}"); - return null; - } - var vs = build_stream(ps.Expression); - if (vs is null) { - Log.Gen.warn( - $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}"); - return null; - } - var vpub = new ViewPublisher(ps.Name, pb_broker, vs); - vs.onChange += on_change; - return vpub; - } -} - -void on_change(object obj) { - bts.Log.Gen.info($"DSPublisher on_change dp: {obj}"); -} - -public class CSVPublisher : IWorker { - private IDictionary _pubs = new Dictionary(); - private readonly string _csv_directory; - private PubTableBroker _pub_table_broker; - - public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) { - _csv_directory = csv_directory; - _pub_table_broker = pub_table_broker; - MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); - } - public void doWork() { - var file_names = Directory.GetFiles(_csv_directory); - foreach (var fn in file_names) { - if (!_pubs.ContainsKey(fn)) { - _handle_new_file(fn); - } - } - } - - private void _handle_new_file(string file_name) { - if (file_name.EndsWith(".csv")) { - ViewStream vs = new CSVReaderStream(file_name); - vs = new SchemaGuesserStream(vs); - var toks = file_name.Split('\\'); - string table_name = toks.Last(); - table_name = table_name.Remove(table_name.Length - ".csv".Length); - ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs); - _pubs.Add(file_name, vp); - } - } -} +using BCG; +using BCG.Utilities.General; + +using bts; +using bts.api.core; +using bts.utils; + +using Bts.DataStreamExpressions; +using Bts.DataStreamPublisher; +using Bts.PubSub; + +using CommandLine; + +using mbts.config; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; + +await Parser.Default.ParseArguments(args) + .MapResult(RunAsync, errs => Task.FromResult(1)); + +async Task RunAsync(DataStreamPublisherConfig cfg) { + try { + cfg.MergeWithConfigFile(); + cfg.MergeWith(DataStreamPublisherConfig.Default()); + PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); + Configuration.initialize(cfg.Sparkd); + Log.Gen.info("Starting DataStreamPublisher"); + var service_ctx = ServiceContext.create(cfg.Sparkd); + await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty); + if (cfg.SimpleEchoClient ?? false) { + start_echo_client(); + } else { + _publish_streams(cfg.Publishers ?? new(0)); + } + } catch (Exception ex) { + // Write the error to console first, in case the logging system is compromised. + var error_msg = $"DataStreamPublisher threw exception:\n{ex}"; + Console.Error.WriteLine(error_msg); + Console.Error.Flush(); + bts.Log.Gen.error(error_msg); + return 1; + } + return 0; +} + +static ViewStream? build_stream(string? stream_expression) { + if (stream_expression is null) { + return null; + } + try { + bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}"); + var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); + var rv = exp.getView(new Dictionary(0)); + return rv; + + } catch (Exception ex) { + bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}"); + return null; + } +} + +static void start_echo_client() { + Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue"); + Console.ReadLine(); + SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345, 1000, 1000); + client.start(); + while (true) { + System.Threading.Thread.Sleep(1000); + } + // await SimpleTCPClient.start("127.0.0.1", 12345); + // Task.Delay(1000000000).Wait(); +} + +void _publish_streams(IEnumerable publishers) { + PubTableBroker pb_broker = new PubTableBroker(); + RemoteQueryManager csm = new RemoteQueryManager(); + var csvPublishers = new List(); + + foreach (var publisher in publishers) { + if (Directory.Exists(publisher.CsvDir)) { + csvPublishers.Add(new(publisher.CsvDir, pb_broker)); + } + } + + var viewPublishers = publishers + .SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher)) + .ToList(); + + while (true) { + System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1)); + ViewStreamDatabase.instance.doWork(); + csm.doWork(); + MasterWorker.getInstance(false).doWork(); + ManagedSubscriber.instance.doWork(); + } + + ViewPublisher? createViewPublisher(PublishedStream ps) { + Log.Gen.info( + $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}"); + if (string.IsNullOrEmpty(ps.Name)) { + Log.Gen.warn( + $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}"); + return null; + } + var vs = build_stream(ps.Expression); + if (vs is null) { + Log.Gen.warn( + $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}"); + return null; + } + var vpub = new ViewPublisher(ps.Name, pb_broker, vs); + vs.onChange += on_change; + return vpub; + } +} + +void on_change(object obj) { + bts.Log.Gen.info($"DSPublisher on_change dp: {obj}"); +} + +public class CSVPublisher : IWorker { + private IDictionary _pubs = new Dictionary(); + private readonly string _csv_directory; + private PubTableBroker _pub_table_broker; + + public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) { + _csv_directory = csv_directory; + _pub_table_broker = pub_table_broker; + MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); + } + public void doWork() { + var file_names = Directory.GetFiles(_csv_directory); + foreach (var fn in file_names) { + if (!_pubs.ContainsKey(fn)) { + _handle_new_file(fn); + } + } + } + + private void _handle_new_file(string file_name) { + if (file_name.EndsWith(".csv")) { + ViewStream vs = new CSVReaderStream(file_name); + vs = new SchemaGuesserStream(vs); + var toks = file_name.Split('\\'); + string table_name = toks.Last(); + table_name = table_name.Remove(table_name.Length - ".csv".Length); + ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs); + _pubs.Add(file_name, vp); + } + } +} diff --git a/src/DataStreamSuite/PubSubServer/Program.cs b/src/DataStreamSuite/PubSubServer/Program.cs index 7a978900b..9df1c2c79 100644 --- a/src/DataStreamSuite/PubSubServer/Program.cs +++ b/src/DataStreamSuite/PubSubServer/Program.cs @@ -1,76 +1,76 @@ -using BCG; - -using bts; - -using Bts.PubSub; -using Bts.PubSubServer; - -using CommandLine; - -using mbts.config; - -using Spark.Core; - -using System; -using System.Threading; -using System.Threading.Tasks; - -Parser.Default.ParseArguments(args) - .MapResult(Run, errs => 1); - -int Run(PubSubServerConfig cfg) { - try { - cfg.MergeWithConfigFile(); - cfg.MergeWith(PubSubServerConfig.Default()); - PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); - Configuration.initialize(cfg.Sparkd); - return cfg.SimpleEchoServer ?? false ? - RunEchoServer(cfg.PubSub.ServerPort.assert_value()) : - RunPubSubServer(); - } catch (Exception ex) { - // Write the error to console first, in case the logging system is compromised. - var error_msg = $"PubSubServer threw exception:\n{ex}"; - Console.Error.WriteLine(error_msg); - Console.Error.Flush(); - Log.Gen.error(error_msg); - return 1; - } -} - -int RunPubSubServer() { - var config = PubSubConfig.Instance; - long currentMemory = 0; - int defaultWaitTime = 100; - bool gced = false; - Task.Run(() => { - while (true) { - gced = false; - currentMemory = GC.GetTotalMemory(false); - if (config.GcActivationThreshold > 0 && currentMemory > config.GcActivationThreshold) { - Console.WriteLine($"GCing memory, mem before GC = {currentMemory}"); - GC.Collect(); - gced = true; - } - Thread.Sleep(gced ? config.WaitPeriodAfterGc.assert_value() : defaultWaitTime); - } - }); - new PubSubMessageReactor(config.ServerPort.assert_value()).start(); - return 0; -} - -int RunEchoServer(short port) { - Console.WriteLine("Running as TCP Echo server, Hit enter to continue"); - Console.ReadLine(); - Console.WriteLine($"ready to listen to port: {port}"); - var sts = new SimpleTCPServer(port, true); - sts.init(); - Console.CancelKeyPress += OnCancelKeyPress; - while (true) { - Thread.Sleep(TimeSpan.FromSeconds(3)); - } - - void OnCancelKeyPress(object? sender, ConsoleCancelEventArgs e) { - Log.Gen.error("Cancel key press. Disposing echo server"); - sts.Dispose(); - } -} +using BCG; + +using bts; + +using Bts.PubSub; +using Bts.PubSubServer; + +using CommandLine; + +using mbts.config; + +using Spark.Core; + +using System; +using System.Threading; +using System.Threading.Tasks; + +Parser.Default.ParseArguments(args) + .MapResult(Run, errs => 1); + +int Run(PubSubServerConfig cfg) { + try { + cfg.MergeWithConfigFile(); + cfg.MergeWith(PubSubServerConfig.Default()); + PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); + Configuration.initialize(cfg.Sparkd); + return cfg.SimpleEchoServer ?? false ? + RunEchoServer(cfg.PubSub.ServerPort.assert_value()) : + RunPubSubServer(); + } catch (Exception ex) { + // Write the error to console first, in case the logging system is compromised. + var error_msg = $"PubSubServer threw exception:\n{ex}"; + Console.Error.WriteLine(error_msg); + Console.Error.Flush(); + Log.Gen.error(error_msg); + return 1; + } +} + +int RunPubSubServer() { + var config = PubSubConfig.Instance; + long currentMemory = 0; + int defaultWaitTime = 100; + bool gced = false; + Task.Run(() => { + while (true) { + gced = false; + currentMemory = GC.GetTotalMemory(false); + if (config.GcActivationThreshold > 0 && currentMemory > config.GcActivationThreshold) { + Console.WriteLine($"GCing memory, mem before GC = {currentMemory}"); + GC.Collect(); + gced = true; + } + Thread.Sleep(gced ? config.WaitPeriodAfterGc.assert_value() : defaultWaitTime); + } + }); + new PubSubMessageReactor(config.ServerPort.assert_value()).start(); + return 0; +} + +int RunEchoServer(short port) { + Console.WriteLine("Running as TCP Echo server, Hit enter to continue"); + Console.ReadLine(); + Console.WriteLine($"ready to listen to port: {port}"); + var sts = new SimpleTCPServer(port, true); + sts.init(); + Console.CancelKeyPress += OnCancelKeyPress; + while (true) { + Thread.Sleep(TimeSpan.FromSeconds(3)); + } + + void OnCancelKeyPress(object? sender, ConsoleCancelEventArgs e) { + Log.Gen.error("Cancel key press. Disposing echo server"); + sts.Dispose(); + } +}