2563 lines
101 KiB
C#
2563 lines
101 KiB
C#
using System;
|
||
using System.Data;
|
||
using System.Collections;
|
||
using System.Collections.Generic;
|
||
using System.Reflection;
|
||
using System.IO;
|
||
using System.Diagnostics;
|
||
using System.Linq;
|
||
|
||
//using BCG.Utilities;
|
||
using BCG.Utilities.General;
|
||
using System.Linq.Expressions;
|
||
//using mbts.spark;
|
||
using mbts;
|
||
using BCG.Extensions;
|
||
using System.Text.Json;
|
||
using System.Text.Json.Nodes;
|
||
using Newtonsoft.Json;
|
||
using BCG;
|
||
using System.Runtime.CompilerServices;
|
||
using bts;
|
||
using Spark.Core;
|
||
using System.Numerics;
|
||
|
||
namespace bts {
|
||
|
||
// public class CompositeRow : IBlueRow {
|
||
// private IBlueRow _lrow, _rrow;
|
||
// private BlueRow _out_row;
|
||
// public CompositeRow(IBlueRow lrow, IBlueRow rrow) {
|
||
// _lrow = lrow;
|
||
// _rrow = rrow;
|
||
// IBlueTableSchema schema = _lrow.schema;
|
||
// List<object> vals = new List<object>();
|
||
// foreach (var o in _lrow) {
|
||
// vals.Add(o);
|
||
// }
|
||
// foreach (ColumnInfo ci in _rrow.schema.columnInfo) {
|
||
// if (ci.colName == "id") {
|
||
// continue;
|
||
// }
|
||
// vals.Add(_rrow[ci.colNo]);
|
||
// schema = schema.add_column(ci.type_code(), ci.colName);
|
||
// }
|
||
// _out_row = new BlueRow(schema, vals.ToArray());
|
||
// }
|
||
// public IBlueTableSchema schema {
|
||
// get { return _out_row.schema; }
|
||
// }
|
||
// public object? this[int index] {
|
||
// get { return _out_row[index]; }
|
||
// }
|
||
// public object? this[string colName] {
|
||
// get {
|
||
// return _out_row[colName];
|
||
// }
|
||
// }
|
||
|
||
// }
|
||
}
|
||
|
||
namespace BCG {
|
||
public static class BlueDataUtility {
|
||
public static IBlueRow create_composite_row(IBlueRow lr, IBlueRow rr, int id) {
|
||
IBlueTableSchema schema = lr.schema;
|
||
List<object> vals = new List<object>();
|
||
foreach (ColumnInfo ci in lr.schema.columnInfo) {
|
||
if (ci.colName == "id") {
|
||
vals.Add(id);
|
||
}
|
||
vals.Add(lr[ci.colName]);
|
||
}
|
||
foreach (ColumnInfo ci in rr.schema.columnInfo) {
|
||
if (ci.colName == "id") {
|
||
continue;
|
||
}
|
||
schema = schema.add_column(ci.type_code(), ci.colName);
|
||
vals.Add(rr[ci.colName]);
|
||
}
|
||
IBlueRow r = new BlueRow(schema, vals.ToArray());
|
||
return r;
|
||
}
|
||
#region Extensions
|
||
public static bool is_null(this IBlueRow r, string col_name) {
|
||
object col_val = r[col_name];
|
||
return col_val == null || DBNull.Value.Equals(col_val);
|
||
}
|
||
|
||
public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) {
|
||
return BlueDataUtility.get_row_datetime_value(r, col_name, err_value);
|
||
}
|
||
public static DateTime get_datetime(this IBlueRow r, int col_no, DateTime err_value = default(DateTime)) {
|
||
return BlueDataUtility.get_row_datetime_value(r, col_no, err_value);
|
||
}
|
||
|
||
public static double? GetFloat64(this IBlueRow r, int colNo) =>
|
||
GetFloat64(r[colNo]);
|
||
|
||
public static double? GetFloat64(this IBlueRow r, string colName) =>
|
||
GetFloat64(r[colName]);
|
||
|
||
public static double? GetFloat64(object obj) {
|
||
if (obj is double d) {
|
||
return d;
|
||
}
|
||
if (obj is int i32) {
|
||
return i32;
|
||
}
|
||
if (obj is long i64) {
|
||
return i64;
|
||
}
|
||
if (obj is string s) {
|
||
return double.TryParse(s, out d) ? d : null;
|
||
}
|
||
if (obj is uint u32) {
|
||
return u32;
|
||
}
|
||
if (obj is ulong u64) {
|
||
return u64;
|
||
}
|
||
if (obj is short i16) {
|
||
return i16;
|
||
}
|
||
if (obj is ushort u16) {
|
||
return u16;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
public static int? GetInt32(this IBlueRow r, int colNo) =>
|
||
GetInt32(r[colNo]);
|
||
|
||
public static int? GetInt32(this IBlueRow r, string colName) =>
|
||
GetInt32(r[colName]);
|
||
|
||
public static int? GetInt32(object obj) {
|
||
checked {
|
||
if (obj is double d) {
|
||
return (int)d;
|
||
}
|
||
if (obj is int i32) {
|
||
return i32;
|
||
}
|
||
if (obj is long i64) {
|
||
return (int)i64;
|
||
}
|
||
if (obj is string s) {
|
||
return double.TryParse(s, out d) ? (int)d : null;
|
||
}
|
||
if (obj is uint u32) {
|
||
return (int)u32;
|
||
}
|
||
if (obj is ulong u64) {
|
||
return (int)u64;
|
||
}
|
||
if (obj is short i16) {
|
||
return i16;
|
||
}
|
||
if (obj is ushort u16) {
|
||
return u16;
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
public static long? GetInt64(this IBlueRow r, int colNo) =>
|
||
GetInt64(r[colNo]);
|
||
|
||
public static long? GetInt64(this IBlueRow r, string colName) =>
|
||
GetInt64(r[colName]);
|
||
|
||
public static long? GetInt64(object obj) {
|
||
checked {
|
||
if (obj is double d) {
|
||
return (long)d;
|
||
}
|
||
if (obj is int i32) {
|
||
return i32;
|
||
}
|
||
if (obj is long i64) {
|
||
return i64;
|
||
}
|
||
if (obj is string s) {
|
||
return double.TryParse(s, out d) ? (long)d : null;
|
||
}
|
||
if (obj is uint u32) {
|
||
return u32;
|
||
}
|
||
if (obj is ulong u64) {
|
||
return (long)u64;
|
||
}
|
||
if (obj is short i16) {
|
||
return i16;
|
||
}
|
||
if (obj is ushort u16) {
|
||
return u16;
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
public static DateTime? GetUnixEpochDateTime(this IBlueRow row, string colName) =>
|
||
GetUnixEpochDateTime(row[colName]);
|
||
|
||
public static DateTime? GetUnixEpochDateTime(this IBlueRow row, int colNo) =>
|
||
GetUnixEpochDateTime(row[colNo]);
|
||
|
||
public static DateTime? GetUnixEpochDateTime(object obj) {
|
||
var tsNanos = GetInt64(obj);
|
||
return tsNanos.HasValue ? DateTime.UnixEpoch.Add(new TimeSpan(tsNanos.Value / 100)) : null;
|
||
}
|
||
|
||
public static DateTime? GetDateTime(object obj) {
|
||
var res = _get_datetime_value(obj, DateTime.MinValue);
|
||
if (res == DateTime.MinValue) {
|
||
return null;
|
||
}
|
||
return res;
|
||
}
|
||
|
||
public static T get_enum<T>(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct {
|
||
return BlueDataUtility.get_row_enum_value<T>(r, col_name, err_value);
|
||
}
|
||
public static T get_enum<T>(this IBlueRow r, int col_no, T err_value = default(T)) where T : struct {
|
||
return BlueDataUtility.get_row_enum_value<T>(r, col_no, err_value);
|
||
}
|
||
public static T get<T>(this IBlueRow r, string col_name, T err_value = default(T)) {
|
||
return get_row_value(r, col_name, err_value);
|
||
}
|
||
public static T get<T>(this IBlueRow r, int col_no, T err_value = default(T)) {
|
||
return get_row_value(r, col_no, err_value);
|
||
}
|
||
public static IEnumerable<T> get_column_values<T>(this IBlueRowCollection rows, string col) {
|
||
List<T> rv = new List<T>();
|
||
foreach (IBlueRow r in rows)
|
||
rv.Add(r.get<T>(col));
|
||
return rv;
|
||
}
|
||
public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col) {
|
||
List<T> rv = new List<T>();
|
||
foreach (IBlueRow r in rows)
|
||
rv.Add(r.get<T>(col));
|
||
return rv;
|
||
}
|
||
public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col, int start_index, int count) {
|
||
List<T> rv = new List<T>();
|
||
int i = 0;
|
||
int n_copied = 0;
|
||
foreach (IBlueRow r in rows)
|
||
if (i++ >= start_index && n_copied < count) {
|
||
rv.Add(r.get<T>(col));
|
||
n_copied++;
|
||
}
|
||
return rv;
|
||
}
|
||
|
||
public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
|
||
Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
|
||
foreach (IBlueRow r in rows) {
|
||
var gp_val = r.get<T>(gp_col);
|
||
List<IBlueRow> gp_val_list = null;
|
||
if (!rv.TryGetValue(gp_val, out gp_val_list)) {
|
||
gp_val_list = new List<IBlueRow>();
|
||
rv.Add(gp_val, gp_val_list);
|
||
}
|
||
gp_val_list.Add(r);
|
||
}
|
||
return rv.ToDictionary(kv => kv.Key, kv => kv.Value as IEnumerable<IBlueRow>);
|
||
}
|
||
|
||
// this function takes in a key value table and converts it into a relational table, converting the key names into columns.
|
||
// The values are returned as string values
|
||
// doc_col is the column specifyng the doc name (following mongoDB's nomenclature, one set of key value rows ar a 'doc')
|
||
public static IBlueRowCollection keyvalue_to_relational<T>(IBlueRowCollection kv_rows, string doc_col, string key_col, string val_col) {
|
||
HashSet<string> keys = new HashSet<string>();
|
||
foreach (string col in get_column_values<string>(kv_rows, key_col)) {
|
||
keys.Add(col);
|
||
}
|
||
if (keys.Contains("id")) {
|
||
keys.Remove("id");
|
||
}
|
||
//unique_cols.Add(doc_col)
|
||
// all the keys become cols
|
||
var schema_cols = keys.Select(c => Tuple.Create(c, TypeCode.String)).ToList();
|
||
//schema of the final table needs doc and id
|
||
schema_cols.Insert(0, Tuple.Create(doc_col, BlueDataUtility.get_typecode(typeof(T))));
|
||
schema_cols.Insert(0, Tuple.Create("id", TypeCode.Int32));
|
||
var schema = BlueDataUtility.form_schema("relational", schema_cols.ToArray());
|
||
BlueTable rv = new BlueTable(schema);
|
||
var gp_map = group_by<T>(kv_rows, doc_col);
|
||
foreach (var kv in gp_map) {
|
||
MutableBlueRow nr = new MutableBlueRow(schema);
|
||
nr.setValue("id", rv.Count);
|
||
nr.setValue(doc_col, kv.Key);
|
||
foreach (var r in kv.Value) {
|
||
var col = r.get<string>(key_col);
|
||
if (!keys.Contains(col))
|
||
continue;
|
||
var val = r.get<string>(val_col);
|
||
nr.setValue(col, val);
|
||
}
|
||
rv.addRow(nr);
|
||
}
|
||
return rv;
|
||
}
|
||
|
||
public static IBlueRowCollection where(this IBlueRowCollection rows, Func<IBlueRow, bool> where_func) {
|
||
if (rows == null)
|
||
return null;
|
||
else {
|
||
BlueTable rv = new BlueTable(rows.schema);
|
||
foreach (IBlueRow r in rows)
|
||
if (where_func(r))
|
||
rv.addRow(r);
|
||
return rv;
|
||
}
|
||
}
|
||
|
||
public static List<IBlueRow> to_list(this IBlueRowCollection rows) {
|
||
List<IBlueRow> rv = new List<IBlueRow>();
|
||
foreach (IBlueRow r in rows)
|
||
rv.Add(r);
|
||
return rv;
|
||
}
|
||
|
||
public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
|
||
return rows.to_list();
|
||
}
|
||
|
||
public static IDictionary<int, IBlueRow> to_map(this IBlueRowCollection rows) {
|
||
Dictionary<int, IBlueRow> rv = new Dictionary<int, IBlueRow>();
|
||
foreach (IBlueRow r in rows) {
|
||
var id = r.get<int>("id");
|
||
rv.Add(id, r);
|
||
}
|
||
return rv;
|
||
}
|
||
|
||
public static bool contains_column(this IBlueTableSchema input_schema, string col) {
|
||
return input_schema.columnNames.Contains(col);
|
||
}
|
||
|
||
public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) {
|
||
List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
|
||
foreach (Tuple<Type, string> c in cols)
|
||
cis.Add(new ColumnInfo(c.Item2, cis.Count, c.Item1));
|
||
return new BlueTableSchema(input_schema.tableName, cis.ToArray());
|
||
}
|
||
|
||
public static IBlueTableSchema add_column(this IBlueTableSchema input_schema, TypeCode type_arg, string col_name) {
|
||
Tuple<Type, string> tup = new Tuple<Type, string>(_typecode_to_type[type_arg], col_name);
|
||
return add_columns(input_schema, tup);
|
||
}
|
||
|
||
public static IBlueRowCollection to_collection(this IBlueRow r) {
|
||
if (r == null)
|
||
return null;
|
||
else {
|
||
BlueTable rv = new BlueTable(r.schema);
|
||
rv.addRow(r);
|
||
return rv;
|
||
}
|
||
}
|
||
|
||
public static DataPacket build_dp(IBlueRowCollection new_rows, DataPacket last_fired_dp) {
|
||
IBlueRowCollection added_rows = new_rows;
|
||
IBlueRowCollection deleted_rows = last_fired_dp == null ? new BlueTable(new_rows.schema) : last_fired_dp.newRows;
|
||
return new DataPacket(new_rows, added_rows, deleted_rows);
|
||
}
|
||
|
||
public static DataPacket build_dp_inc(IBlueRowCollection new_rows, DataPacket last_fired_dp, bool deep_row_comapre = false) {
|
||
if (last_fired_dp == null) {
|
||
IBlueRowCollection added_rows = new_rows;
|
||
IBlueRowCollection deleted_rows = new BlueTable(new_rows.schema);
|
||
return new DataPacket(new_rows, added_rows, deleted_rows);
|
||
} else {
|
||
IBlueRowCollection added_rows = null;
|
||
IBlueRowCollection deleted_rows = null;
|
||
if (deep_row_comapre)
|
||
compute_differences_fast(new_rows, last_fired_dp.newRows, out deleted_rows, out added_rows);
|
||
else
|
||
compute_differences_id_based(new_rows, last_fired_dp.newRows, out deleted_rows, out added_rows);
|
||
return new DataPacket(new_rows, added_rows, deleted_rows);
|
||
}
|
||
}
|
||
|
||
public static IBlueRowCollection adapt_for_sql_storage(this IBlueRowCollection rows,
|
||
bool handle_datetime_min = false,
|
||
bool handle_double_nan = false) {
|
||
if (rows != null) {
|
||
BlueTable rv = new BlueTable(rows.schema);
|
||
foreach (IBlueRow r in rows) {
|
||
MutableBlueRow mr = new MutableBlueRow(r);
|
||
rv.addRow(mr);
|
||
foreach (ColumnInfo ci in rows.schema.columnInfo) {
|
||
if (handle_datetime_min && ci.colType == typeof(DateTime) && DateTime.MinValue.Equals(r[ci.colName]))
|
||
mr.setValue(ci.colName, DBNull.Value);
|
||
if (handle_double_nan && ci.colType == typeof(double) && Double.IsNaN(r.get<double>(ci.colName)))
|
||
mr.setValue(ci.colName, DBNull.Value);
|
||
}
|
||
}
|
||
return rv;
|
||
} else
|
||
return null;
|
||
}
|
||
|
||
#endregion
|
||
|
||
#region json
|
||
|
||
public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
|
||
Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
|
||
return dictionary;
|
||
}
|
||
public static IBlueRow convert_json_to_row(string json_string, int id = 0) {
|
||
Dictionary<string, object> dictionary = JsonConvert.DeserializeObject<Dictionary<string, object>>(json_string);
|
||
if (!dictionary.ContainsKey("id")) {
|
||
dictionary.Add("id", id);
|
||
}
|
||
return new BlueRow(dictionary);
|
||
}
|
||
|
||
public static List<JsonNode> json_to_json_node_list(string jsonString) {
|
||
var jsonNodeList = new List<JsonNode>();
|
||
bool noe = String.IsNullOrEmpty(jsonString);
|
||
if (noe)
|
||
return jsonNodeList;
|
||
try {
|
||
// Parse the JSON string
|
||
JsonArray jsonArray = JsonNode.Parse(jsonString) as JsonArray;
|
||
|
||
if (jsonArray == null) {
|
||
Console.WriteLine("The JSON string must represent a list.");
|
||
return jsonNodeList;
|
||
}
|
||
|
||
// Iterate through each JsonNode in the JsonArray
|
||
foreach (JsonNode jsonNode in jsonArray) {
|
||
jsonNodeList.Add(jsonNode);
|
||
}
|
||
} catch (System.Text.Json.JsonException ex) {
|
||
Console.WriteLine($"Invalid JSON: {ex.Message}");
|
||
}
|
||
|
||
return jsonNodeList;
|
||
}
|
||
|
||
#endregion
|
||
|
||
public static void print_table(this IBlueRowCollection table, Func<string, int> get_col_width) {
|
||
var rv = get_printed_rows(table, get_col_width);
|
||
Console.WriteLine(rv);
|
||
}
|
||
|
||
public static string get_printed_rows(this IBlueRowCollection table, Func<string, int> get_col_width) {
|
||
// Print column headers
|
||
string rv = String.Empty;// = $"{table.schema.tableName}______________________________________";
|
||
foreach (var col_name in table.schema.columnNames) {
|
||
int col_width = get_col_width(col_name);
|
||
rv += $"{col_name.PadRight(col_width)} ";
|
||
}
|
||
rv += "\n";
|
||
//rv += "______________________________________";
|
||
|
||
// Print table data
|
||
foreach (IBlueRow row in table) {
|
||
foreach (string col_name in table.schema.columnNames) {
|
||
int col_width = get_col_width(col_name);
|
||
object row_val = row[col_name];
|
||
string row_val_str = row_val.ToString();
|
||
if (row_val != null) {
|
||
rv += $"{row_val_str.PadRight(col_width)} ";
|
||
}
|
||
}
|
||
rv += "\n";
|
||
}
|
||
return rv;
|
||
}
|
||
#region Schema types
|
||
// these are the only types allowed in schemas
|
||
private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
|
||
{
|
||
{ TypeCode.Boolean, typeof(bool) }, //3
|
||
{ TypeCode.Byte, typeof(byte) }, //6
|
||
{ TypeCode.Char, typeof(char) }, //4
|
||
// TODO
|
||
{ TypeCode.DateTime, typeof(DateTime) },//16
|
||
{ TypeCode.Decimal, typeof(decimal) }, //15
|
||
{ TypeCode.Double, typeof(double) }, //14
|
||
//{ TypeCode.Empty, null },
|
||
{ TypeCode.Int16, typeof(short) }, //7
|
||
{ TypeCode.Int32, typeof(int) }, //9
|
||
{ TypeCode.Int64, typeof(long) }, //11
|
||
{ TypeCode.SByte, typeof(sbyte) }, //5
|
||
{ TypeCode.Single, typeof(Single) }, //13
|
||
{ TypeCode.String, typeof(string) }, //18
|
||
{ TypeCode.UInt16, typeof(UInt16) }, //8
|
||
{ TypeCode.UInt32, typeof(UInt32) }, //10
|
||
{ TypeCode.UInt64, typeof(UInt64) } //12
|
||
};
|
||
|
||
public static IEnumerable<TypeCode> valid_column_typecodes() {
|
||
return _typecode_to_type.Keys;
|
||
}
|
||
|
||
public static IEnumerable<Type> valid_column_types() {
|
||
return _typecode_to_type.Values;
|
||
}
|
||
|
||
public static bool is_valid_col_type(Type t) {
|
||
TypeCode type_code = get_typecode(t);
|
||
return type_code != TypeCode.Empty;
|
||
}
|
||
|
||
public static TypeCode get_typecode(Type t) {
|
||
foreach (var kv in _typecode_to_type) {
|
||
if (kv.Value == t)
|
||
return kv.Key;
|
||
}
|
||
//bts.Log.assert_d(false, $"type: {t} not found");
|
||
return TypeCode.Empty;
|
||
}
|
||
|
||
public static Type get_type(TypeCode type_code) {
|
||
Type rv = null;
|
||
if (_typecode_to_type.TryGetValue(type_code, out rv)) {
|
||
return rv;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
public static TypeCode type_code(this ColumnInfo ci) {
|
||
return get_typecode(ci.colType);
|
||
}
|
||
|
||
#endregion
|
||
|
||
public static Tuple<Type, object> guess_type(string value) {
|
||
if (string.IsNullOrEmpty(value)) {
|
||
return Tuple.Create(typeof(string), (object)value);
|
||
}
|
||
|
||
int val = 0;
|
||
// Try to parse as integer
|
||
if (int.TryParse(value, out val)) {
|
||
return Tuple.Create(typeof(int), (object)val);
|
||
}
|
||
|
||
double dval = 0.0;
|
||
// Try to parse as double
|
||
if (double.TryParse(value, out dval)) {
|
||
return Tuple.Create(typeof(double), (object)dval);
|
||
}
|
||
|
||
DateTime dt;
|
||
// Try to parse as DateTime
|
||
if (DateTime.TryParse(value, out dt)) {
|
||
return Tuple.Create(typeof(DateTime), (object)dt);
|
||
}
|
||
|
||
bool bval;
|
||
// Try to parse as boolean
|
||
if (bool.TryParse(value, out bval)) {
|
||
return Tuple.Create(typeof(bool), (object)bval);
|
||
}
|
||
|
||
// If none of the above, return string type
|
||
return Tuple.Create(typeof(string), (object)value);
|
||
}
|
||
|
||
public static IBlueTableSchema form_schema(string table_name, params Tuple<string, TypeCode>[] cols) {
|
||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||
foreach (Tuple<string, TypeCode> c in cols)
|
||
cis.Add(new ColumnInfo(c.Item1, cis.Count, _typecode_to_type[c.Item2]));
|
||
if (cis.FirstOrDefault(ci => ci.colName == "id") == null)
|
||
cis.Add(new ColumnInfo("id", cis.Count, typeof(int)));
|
||
return new BlueTableSchema(table_name, cis.ToArray());
|
||
}
|
||
|
||
public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) {
|
||
// just use one row to guess
|
||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||
foreach (IBlueRow r in rows) {
|
||
foreach (var ci in r.schema.columnInfo) {
|
||
if (ci.colType != typeof(System.String)) {
|
||
cis.Add(ci);
|
||
continue;
|
||
}
|
||
var (type, _) = guess_type(r[ci.colName] as string);
|
||
cis.Add(new ColumnInfo(ci.colName, ci.colNo, type));
|
||
}
|
||
return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
|
||
}
|
||
return null;
|
||
}
|
||
|
||
// converts a row where all columns are string types to a schema where the best types are guesed
|
||
public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) {
|
||
IBlueTableSchema typed_schema = get_typed_schema(string_rows);
|
||
var rv = new BlueTable(typed_schema);
|
||
foreach (IBlueRow r in string_rows) {
|
||
var typed_row = get_typed_row(r, typed_schema);
|
||
rv.addRow(typed_row);
|
||
}
|
||
return rv;
|
||
|
||
static IBlueRow get_typed_row(IBlueRow string_row, IBlueTableSchema typed_schema) {
|
||
var rv = new MutableBlueRow(typed_schema);
|
||
foreach (var ci in typed_schema.columnInfo) {
|
||
var typed_val = convert(ci.colType, string_row[ci.colName].ToString());
|
||
rv.setValue(ci.colName, typed_val);
|
||
}
|
||
return rv;
|
||
}
|
||
|
||
static object convert(Type t, string value) {
|
||
// Attempt to convert the string to the specified type
|
||
try {
|
||
return Convert.ChangeType(value, t);
|
||
} catch (Exception) {
|
||
if (t == typeof(double)) {
|
||
// some doubles come with currency and ',' we clean that up here (there might be better ways to do this)
|
||
string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray());
|
||
cleaned_val = cleaned_val.Replace('−', '-');
|
||
double rv_d;
|
||
if (Double.TryParse(cleaned_val, out rv_d))
|
||
return rv_d;
|
||
else
|
||
return null;
|
||
}
|
||
return null;
|
||
// Handle conversion failure gracefully
|
||
// throw new ArgumentException("Failed to convert the string to the specified type.");
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Gets the Hashcode of the row with the boolean indicating whether the order of the rows should
|
||
/// be alphabetic or not
|
||
/// </summary>
|
||
/// <param name="row"></param>
|
||
/// <param name="isColAlphabetically"></param>
|
||
/// <returns></returns>
|
||
public static int getRowHashCode(IBlueRow row, bool isColAlphabetically) {
|
||
string retVal = "";
|
||
if (isColAlphabetically) {
|
||
string[] colNames = row.schema.columnNames;
|
||
Array.Sort(colNames);
|
||
foreach (string col in colNames)
|
||
retVal += row[col].ToString();
|
||
} else
|
||
foreach (object o in row)
|
||
retVal += o.ToString();
|
||
return retVal.GetHashCode();
|
||
}
|
||
|
||
/// <summary>
|
||
/// Finds the hashcode only inluding the columns (in the order) listed in the column names argument
|
||
/// </summary>
|
||
/// <param name="row"></param>
|
||
/// <param name="colNames"></param>
|
||
/// <returns></returns>
|
||
public static int getRowHashCode(IBlueRow row, params string[] colNames) {
|
||
string retVal = "";
|
||
foreach (string colName in colNames)
|
||
retVal += row[colName].ToString();
|
||
return retVal.GetHashCode();
|
||
}
|
||
|
||
/// <summary>
|
||
/// Converts a IBlueRowCollection to a .Net DataTable
|
||
/// </summary>
|
||
/// <param name="rows"></param>
|
||
/// <returns></returns>
|
||
public static DataTable toDataTable(IBlueRowCollection rows) {
|
||
DataTable retTable = getTableFromSchema(rows.schema);
|
||
retTable.BeginLoadData();
|
||
foreach (IBlueRow row in rows)
|
||
retTable.Rows.Add(toDataRow(row, retTable));
|
||
retTable.EndLoadData();
|
||
return retTable;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Converts to a DataRow, the preconditioner being that order of the columns should be same
|
||
/// in the schema of the row and the table
|
||
/// </summary>
|
||
/// <param name="row"></param>
|
||
/// <param name="table"></param>
|
||
public static DataRow toDataRow(IBlueRow row, DataTable table) {
|
||
DataRow retRow = table.NewRow();
|
||
foreach (string colName in row.schema.columnNames)
|
||
retRow[colName] = row[colName];
|
||
return retRow;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Returns the IBlueTableSchema corresponding to the row
|
||
/// </summary>
|
||
/// <param name="row">
|
||
/// DataRow whose schmea is desired
|
||
/// </param>
|
||
/// <returns></returns>
|
||
public static IBlueTableSchema getSchema(DataRow row) { return getSchema(row.Table); }
|
||
|
||
/// <summary>
|
||
/// Returns the IBlueTableSchema of the DataTable
|
||
/// </summary>
|
||
/// <param name="table">
|
||
/// table whos Schema is desired
|
||
/// </param>
|
||
/// <returns></returns>
|
||
public static IBlueTableSchema getSchema(DataTable table) {
|
||
ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
|
||
foreach (DataColumn col in table.Columns)
|
||
cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
|
||
IBlueTableSchema rv = new BlueTableSchema(cis, table.TableName);
|
||
rv = rv.add_id_col_if_missing();
|
||
return rv;
|
||
}
|
||
public static IBlueTableSchema add_id_col_if_missing(this IBlueTableSchema schema) {
|
||
IBlueTableSchema rv = schema;
|
||
if (!schema.columnNames.Contains("id")) {
|
||
rv = schema.add_columns(Tuple.Create(typeof(int), "id"));
|
||
}
|
||
return rv;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Constructs the schema from an object, using the SchemaMapping attribute. Only those
|
||
/// properties are converted into schmea columns that have a get accessor
|
||
/// </summary>
|
||
/// <param name="o"></param>
|
||
/// <returns></returns>
|
||
public static IBlueTableSchema getSchema(object o) {
|
||
return getSchema(o.GetType());
|
||
}
|
||
|
||
|
||
/// <summary>
|
||
/// Builds a new schema based on an existing one + a set of new columns
|
||
/// </summary>
|
||
/// <param name="schema">Base schema</param>
|
||
/// <param name="cis">Columns to add</param>
|
||
public static IBlueTableSchema addColumnsToSchema(IBlueTableSchema schema, params ColumnInfo[] cis) {
|
||
ArrayList rv = new ArrayList(schema.columnInfo);
|
||
rv.AddRange(cis);
|
||
return new BlueTableSchema(schema.tableName, rv.ToArray(typeof(ColumnInfo)) as ColumnInfo[]);
|
||
}
|
||
|
||
/// <summary>
|
||
/// appends the cols of s2, to s1, subtracting the id col
|
||
/// </summary>
|
||
/// <param name="s1"></param>
|
||
/// <param name="appendedSchema"></param>
|
||
/// <returns></returns>
|
||
public static IBlueTableSchema appendSchemas(IBlueTableSchema s1, IBlueTableSchema appendedSchema) {
|
||
List<ColumnInfo> cis = new List<ColumnInfo>(s1.columnInfo);
|
||
foreach (ColumnInfo c in appendedSchema.columnInfo)
|
||
if (c.colName != "id")
|
||
cis.Add(new ColumnInfo(c.colName, cis.Count, c.colType));
|
||
return new BlueTableSchema(s1.tableName, cis.ToArray());
|
||
}
|
||
|
||
|
||
public static IBlueTableSchema getSchema(Type t) {
|
||
return getSchema(t, false);
|
||
}
|
||
|
||
|
||
public static IBlueTableSchema getSchema(Type t, bool createIdField) {
|
||
return getSchema(t, t.Name, createIdField);
|
||
}
|
||
public static IBlueTableSchema getSchema(Type t, string tableName, bool createIdField) {
|
||
ArrayList colInfos = new ArrayList();
|
||
int i = 0;
|
||
foreach (PropertyInfo pi in t.GetProperties())
|
||
if (myIsActiveProperty(pi))
|
||
colInfos.Add(new ColumnInfo(pi.Name, i++,
|
||
is_valid_col_type(pi.PropertyType) ?
|
||
pi.PropertyType : typeof(String)));
|
||
foreach (FieldInfo fi in t.GetFields())
|
||
if (myIsActiveField(fi))
|
||
colInfos.Add(new ColumnInfo(fi.Name, i++,
|
||
is_valid_col_type(fi.FieldType) ? fi.FieldType : typeof(String)));
|
||
|
||
if (createIdField)
|
||
colInfos.Add(new ColumnInfo("id", i++, typeof(int)));
|
||
|
||
return new BlueTableSchema(colInfos.ToArray(typeof(ColumnInfo)) as ColumnInfo[],
|
||
tableName);
|
||
}
|
||
|
||
public static IBlueTableSchema getSchema(Type t, string tableName) {
|
||
return new BlueTableSchema(getSchema(t).columnInfo, tableName);
|
||
}
|
||
|
||
|
||
public static IBlueRow createRowFromType(IBlueTableSchema schema, object o) {
|
||
return createRowFromType(schema, o, null);
|
||
}
|
||
|
||
//creates a row from the given type, and if idVal is not null, using the given id
|
||
public static IBlueRow createRowFromType(IBlueTableSchema schema, object o, object idVal) {
|
||
object[] rowArray = new object[schema.columnInfo.Length];
|
||
Type t = o.GetType();
|
||
foreach (PropertyInfo pi in t.GetProperties())
|
||
if (myIsActiveProperty(pi))
|
||
rowArray[schema[pi.Name].colNo] =
|
||
is_valid_col_type(pi.PropertyType) ?
|
||
pi.GetValue(o, null) : pi.GetValue(o, null).ToString();
|
||
foreach (FieldInfo fi in t.GetFields())
|
||
if (myIsActiveField(fi))
|
||
rowArray[schema[fi.Name].colNo] =
|
||
is_valid_col_type(fi.FieldType) ?
|
||
fi.GetValue(o) : fi.GetValue(o).ToString();
|
||
if (idVal != null)
|
||
rowArray[schema["id"].colNo] = idVal;
|
||
return new BlueRow(schema, rowArray);
|
||
}
|
||
|
||
public static MutableBlueRow create_mutable_row(IBlueTableSchema schema, object o, object idVal,
|
||
Func<Type, object, object> type_converter) {
|
||
object[] rowArray = new object[schema.columnInfo.Length];
|
||
Type t = o.GetType();
|
||
foreach (PropertyInfo pi in t.GetProperties())
|
||
if (myIsActiveProperty(pi))
|
||
rowArray[schema[pi.Name].colNo] =
|
||
is_valid_col_type(pi.PropertyType) ?
|
||
pi.GetValue(o, null) : type_converter(pi.PropertyType, pi.GetValue(o, null));
|
||
foreach (FieldInfo fi in t.GetFields())
|
||
if (myIsActiveField(fi))
|
||
rowArray[schema[fi.Name].colNo] =
|
||
is_valid_col_type(fi.FieldType) ?
|
||
fi.GetValue(o) : type_converter(fi.FieldType, fi.GetValue(o));
|
||
if (idVal != null)
|
||
rowArray[schema["id"].colNo] = idVal;
|
||
return new MutableBlueRow(schema, rowArray);
|
||
}
|
||
|
||
|
||
public static IBlueTableSchema get_schema(Type t, string tableName, bool createIdField, Func<Type, Type> type_converter) {
|
||
List<ColumnInfo> colInfos = new List<ColumnInfo>();
|
||
int i = 0;
|
||
if (createIdField)
|
||
colInfos.Add(new ColumnInfo("id", i++, typeof(int)));
|
||
|
||
foreach (PropertyInfo pi in t.GetProperties())
|
||
colInfos.Add(new ColumnInfo(pi.Name, i++, type_converter(pi.PropertyType)));
|
||
foreach (FieldInfo fi in t.GetFields())
|
||
colInfos.Add(new ColumnInfo(fi.Name, i++, type_converter(fi.FieldType)));
|
||
return new BlueTableSchema(colInfos.ToArray(), tableName);
|
||
}
|
||
|
||
public static MutableBlueRow create_mutable_row_not_slow(IBlueTableSchema schema, FieldInfo[] fields, PropertyInfo[] props,
|
||
object o, object idVal, Func<Type, object, object> data_converter) {
|
||
object[] rowArray = new object[schema.columnInfo.Length];
|
||
if (props != null) {
|
||
foreach (PropertyInfo pi in props)
|
||
rowArray[schema[pi.Name].colNo] = data_converter(pi.PropertyType, pi.GetValue(o, null));
|
||
}
|
||
if (fields != null) {
|
||
foreach (FieldInfo fi in fields)
|
||
rowArray[schema[fi.Name].colNo] = data_converter(fi.FieldType, fi.GetValue(o));
|
||
}
|
||
if (idVal != null)
|
||
rowArray[schema["id"].colNo] = idVal;
|
||
return new MutableBlueRow(schema, rowArray);
|
||
}
|
||
|
||
public static T createTypeFromRow<T>(IBlueRow row) {
|
||
return (T)createTypeFromRow(row, typeof(T));
|
||
}
|
||
|
||
public static object createTypeFromRow(IBlueRow row, Type t) {
|
||
object outValue = Activator.CreateInstance(t);
|
||
|
||
PropertyInfo[] properties = t.GetProperties();
|
||
|
||
foreach (PropertyInfo pi in properties) {
|
||
if (myIsActiveProperty(pi))
|
||
if (row.Contains(pi.Name))
|
||
if (row[pi.Name].Equals(DBNull.Value) && !pi.PropertyType.Equals(typeof(Object)))
|
||
pi.SetValue(outValue, null, null);
|
||
else
|
||
pi.SetValue(outValue, row[pi.Name], null);
|
||
}
|
||
FieldInfo[] fields = t.GetFields();
|
||
foreach (FieldInfo fi in fields) {
|
||
if (myIsActiveField(fi))
|
||
if (row.Contains(fi.Name))
|
||
if (row[fi.Name].Equals(DBNull.Value) && !fi.FieldType.Equals(typeof(Object)))
|
||
fi.SetValue(outValue, null);
|
||
else
|
||
fi.SetValue(outValue, row[fi.Name]);
|
||
}
|
||
|
||
return outValue;
|
||
|
||
}
|
||
|
||
//these functions should originally have been made public
|
||
//but changing that would change a bunch of code
|
||
public static bool isActiveProperty(PropertyInfo pi) {
|
||
return myIsActiveProperty(pi);
|
||
}
|
||
|
||
public static bool isActiveField(FieldInfo fi) {
|
||
return myIsActiveField(fi);
|
||
}
|
||
|
||
private static bool myIsActiveProperty(PropertyInfo pi) {
|
||
if (pi.GetGetMethod() == null)
|
||
return false;
|
||
foreach (Attribute att in pi.GetCustomAttributes(true))
|
||
if (att is SchemaVisibility && !(att as SchemaVisibility).visible)
|
||
return false;
|
||
return true;
|
||
}
|
||
|
||
private static bool myIsActiveField(FieldInfo fi) {
|
||
if (!fi.IsPublic)
|
||
return false;
|
||
foreach (Attribute att in fi.GetCustomAttributes(true))
|
||
if (att is SchemaVisibility && !(att as SchemaVisibility).visible)
|
||
return false;
|
||
return true;
|
||
}
|
||
|
||
private static bool myIsActiveMember(MemberInfo mi) {
|
||
if (!(mi is FieldInfo) ||
|
||
!(mi is PropertyInfo) ||
|
||
(mi is PropertyInfo && (mi as PropertyInfo).GetGetMethod() == null) ||
|
||
(mi is FieldInfo && !(mi as FieldInfo).IsPublic))
|
||
return false;
|
||
foreach (Attribute att in mi.GetCustomAttributes(true))
|
||
if (att is SchemaVisibility && !(att as SchemaVisibility).visible)
|
||
return false;
|
||
return true;
|
||
}
|
||
|
||
private static bool myDoesRowExistsInTable(IBlueRowCollection brc2, string[] columns, object[] oarray) {
|
||
bool rowExists = false;
|
||
foreach (IBlueRow br in brc2) {
|
||
bool allElementsSame = true;
|
||
for (int i = 0; i < columns.Length; i++) {
|
||
string col = columns[i];
|
||
object o1 = br[col];
|
||
object o2 = oarray[i];
|
||
allElementsSame = (o1.Equals(o2) == false) ? false : allElementsSame;
|
||
}
|
||
if (allElementsSame == true) {
|
||
rowExists = true;
|
||
break;
|
||
}
|
||
}
|
||
return rowExists;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Constructs a DataTable from the IBlueTableSchema
|
||
/// </summary>
|
||
/// <param name="schema">
|
||
/// Schema whose table needs to be created
|
||
/// </param>
|
||
/// <returns></returns>
|
||
public static DataTable getTableFromSchema(IBlueTableSchema schema) {
|
||
DataTable retTable = new DataTable(schema.tableName);
|
||
foreach (ColumnInfo ci in schema.columnInfo)
|
||
retTable.Columns.Add(ci.colName, ci.colType);
|
||
return retTable;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Constructs a BlueRow from a DataRow.
|
||
/// </summary>
|
||
/// <param name="row">source row</param>
|
||
/// <returns></returns>
|
||
public static IBlueRow toBlueRow(DataRow row) {
|
||
return new BlueRow(getSchema(row), row);
|
||
}
|
||
|
||
public static IBlueRowCollection toIBlueRowCollection(DataTable dt) {
|
||
IBlueTableSchema schema = getSchema(dt);
|
||
BlueTable rv = new BlueTable(schema);
|
||
foreach (DataRow dr in dt.Rows) {
|
||
MutableBlueRow nr = new MutableBlueRow(schema);
|
||
foreach (DataColumn col in dr.Table.Columns)
|
||
nr.setValue(col.ColumnName, dr[col.ColumnName]);
|
||
nr.setValue("id", rv.Count);
|
||
rv.addRow(nr);
|
||
}
|
||
return rv;
|
||
}
|
||
|
||
public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) {
|
||
BlueTable retVal = null;
|
||
foreach (DataRow dr in rows) {
|
||
IBlueRow r = toBlueRow(dr);
|
||
if (retVal == null)
|
||
retVal = new BlueTable(r.schema);
|
||
retVal.addRow(r);
|
||
}
|
||
return retVal;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Returns the nth row of the IBlueRowCollection. This method could be used
|
||
/// for any of the nonRandom-Access-IBlueRowCollection-Implementations, as its
|
||
/// O(n)
|
||
/// </summary>
|
||
/// <param name="rows"></param>
|
||
/// <param name="index"></param>
|
||
/// <returns></returns>
|
||
public static IBlueRow extractRow(IBlueRowCollection rows, int index) {
|
||
int i = 0;
|
||
foreach (IBlueRow row in rows)
|
||
if (i++ == index)
|
||
return row;
|
||
return null;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Computes the equality fo the 2 rows. The rows might have different columns orderings
|
||
/// </summary>
|
||
/// <param name="r1"></param>
|
||
/// <param name="r2"></param>
|
||
/// <returns></returns>
|
||
|
||
|
||
/// <summary>
|
||
/// Computes a string equivelant for the row, using the columns provided
|
||
/// in the argument
|
||
/// </summary>
|
||
/// <param name="row"></param>
|
||
/// <param name="colNames"></param>
|
||
/// <returns></returns>
|
||
public static string rowToString(IBlueRow row, params string[] colNames) {
|
||
string retVal = "";
|
||
foreach (string colName in colNames)
|
||
retVal += (row[colName] != null ? row[colName].ToString().Trim() : "NULL") + "\n";
|
||
return retVal;
|
||
}
|
||
|
||
public static string rowToString(IBlueRow row) { return rowToString(row, row.schema.columnNames); }
|
||
|
||
public static string rowCollectionToString(IBlueRowCollection rowColl) {
|
||
if (rowColl == null)
|
||
return "Null IBlueRowCollection";
|
||
else {
|
||
string retVal = "";
|
||
foreach (IBlueRow r in rowColl)
|
||
retVal += retVal == "" ? rowToString(r) : "\n" + rowToString(r);
|
||
return retVal;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Compares 2 rows, using the column names in the order provided, ie the column name with
|
||
/// the lower index in the array has higher priority. Also a null is always assumed
|
||
/// to be less than a non-null
|
||
/// </summary>
|
||
/// <param name="r1"></param>
|
||
/// <param name="r2"></param>
|
||
/// <param name="col_names"></param>
|
||
/// <returns></returns>
|
||
public static int compare_rows(IBlueRow r1,
|
||
IBlueRow r2,
|
||
params string[] col_names) {
|
||
int comp_res = 0;
|
||
foreach (string cn in col_names) {
|
||
object val1 = r1[cn];
|
||
object val2 = r2[cn];
|
||
comp_res = _compare_row_cols(val1, val2);
|
||
if (comp_res != 0)
|
||
break;
|
||
}
|
||
return comp_res;
|
||
}
|
||
|
||
private static int _compare_row_cols(object v1, object v2) {
|
||
if (_is_null(v1))
|
||
return _is_null(v2) ? 0 : -1;
|
||
else {
|
||
if (_is_null(v2))
|
||
return 1;
|
||
else {
|
||
if (v1 is IComparable && v2 is IComparable)
|
||
return (v1 as IComparable).CompareTo(v2);
|
||
else if (v1 is byte[] && v2 is byte[])
|
||
return _compare_bytes(v1 as byte[], v2 as byte[]) ? 0 : 1;
|
||
else
|
||
return 1;
|
||
}
|
||
}
|
||
}
|
||
|
||
private static bool _compare_bytes(byte[] v1, byte[] v2) {
|
||
if (v1.Length != v2.Length)
|
||
return false;
|
||
else
|
||
for (int i = 0; i < v1.Length; i++)
|
||
if (v1[i] != v2[i])
|
||
return false;
|
||
return true;
|
||
}
|
||
|
||
private static bool _is_null(object o) { return o == null || o == DBNull.Value; }
|
||
/// <summary>
|
||
/// Creates a new IBlueRow with a schema matching the schema provided in 'new schema'
|
||
/// </summary>
|
||
/// <param name="inRow">a row with the values to be converted</param>
|
||
/// <param name="newSchema">the desired schema for the output row</param>
|
||
/// <returns></returns>
|
||
public static IBlueRow changeSchema(IBlueRow inRow, IBlueTableSchema newSchema) {
|
||
object[] outputValues = new object[newSchema.columnInfo.Length];
|
||
|
||
for (int i = 0; i < outputValues.Length; ++i)
|
||
outputValues[i] = inRow[newSchema.columnNames[i]];
|
||
|
||
return new BlueRow(newSchema, outputValues);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Combines input tables into a single output table, with different ID's
|
||
/// this will result in new rows being created, which is kind of expensive
|
||
/// </summary>
|
||
/// <param name="tables"></param>
|
||
/// <returns></returns>
|
||
public static IBlueRowCollection combineTables(params IBlueRowCollection[] tables) {
|
||
//get some base cases out of the way
|
||
if (tables.Length == 0)
|
||
return null;
|
||
if (tables.Length == 1)
|
||
return tables[0];
|
||
|
||
BlueTable result = new BlueTable(tables[0].schema);
|
||
int idToUseNext = 0;
|
||
int numCols = tables[0].schema.columnInfo.Length;
|
||
int idCol = tables[0].schema["id"].colNo;
|
||
IBlueRowCollection previousTable = null;
|
||
foreach (IBlueRowCollection table in tables) {
|
||
if (previousTable != null)
|
||
if (!compareSchemas(previousTable.schema, table.schema))
|
||
throw new ArgumentException("Error, tables must have the same schema to be combined");
|
||
|
||
foreach (IBlueRow r in table) {
|
||
object[] values = new object[numCols];
|
||
r.CopyTo(values, 0);
|
||
values[idCol] = idToUseNext++;
|
||
result.addRow(new BlueRow(table.schema, values));
|
||
}
|
||
|
||
previousTable = table;
|
||
|
||
}
|
||
|
||
return result;
|
||
}
|
||
/// <summary>
|
||
/// Compares two rows (having the same schema) and returns the
|
||
/// columns whose values dont match
|
||
/// </summary>
|
||
/// <param name="r1"></param>
|
||
/// <param name="r2"></param>
|
||
/// <returns></returns>
|
||
public static string[] compareRows(IBlueRow r1, IBlueRow r2) {
|
||
ArrayList retVal = new ArrayList();
|
||
foreach (string col in r1.schema.columnNames)
|
||
if (!r1[col].Equals(r2[col]))
|
||
retVal.Add(col);
|
||
return retVal.ToArray(typeof(string)) as string[];
|
||
}
|
||
|
||
public static BlueTable cloneRowCollection(IBlueRowCollection rows) {
|
||
BlueTable retVal = new BlueTable(rows.schema);
|
||
foreach (IBlueRow r in rows)
|
||
retVal.addRow(new BlueRow(r));
|
||
return retVal;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Creates a new row with the values of the row passed, except that the newValues are forced into the
|
||
/// returned row, in places specified by the indexes (makes any sense ?)
|
||
/// </summary>
|
||
/// <param name="r"></param>
|
||
/// <param name="newValues"></param>
|
||
/// <param name="indexes"></param>
|
||
/// <returns></returns>
|
||
public static IBlueRow getNewReplacedRow(IBlueRow r, object[] newValues, int[] indexes) {
|
||
if (!newValues.Length.Equals(indexes.Length))
|
||
throw new Exception("Debug Assert");
|
||
|
||
object[] rowArray = new object[r.Count];
|
||
r.CopyTo(rowArray, 0);
|
||
for (int i = 0; i < indexes.Length; i++)
|
||
rowArray[indexes[i]] = newValues[i];
|
||
return new BlueRow(r.schema, rowArray);
|
||
}
|
||
|
||
public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, int index) {
|
||
return getNewReplacedRow(r, new object[] { newValue }, new int[] { index });
|
||
}
|
||
|
||
public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, string colName) {
|
||
return getNewReplacedRow(r, newValue, r.schema[colName].colNo);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Computes the max row, based on the cols provided. If the collection is empty, null is returned
|
||
/// </summary>
|
||
/// <param name="rows"></param>
|
||
/// <param name="cols"></param>
|
||
/// <returns></returns>
|
||
public static IBlueRow computeMaxRow(IBlueRowCollection rows, params string[] cols) {
|
||
IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null;
|
||
foreach (IBlueRow r in rows)
|
||
retVal = compare_rows(r, retVal, cols) > 0 ? r : retVal;
|
||
return retVal;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Computes the max row, based on the cols provided. If the collection is empty, null is returned
|
||
/// </summary>
|
||
/// <param name="rows"></param>
|
||
/// <param name="cols"></param>
|
||
/// <returns></returns>
|
||
public static IBlueRow computeMinRow(IBlueRowCollection rows, params string[] cols) {
|
||
IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null;
|
||
foreach (IBlueRow r in rows)
|
||
retVal = compare_rows(r, retVal, cols) < 0 ? r : retVal;
|
||
return retVal;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Computes the sum of rows, based on the cols provided. If the collection is empty, null is returned
|
||
/// </summary>
|
||
/// <param name="rows"></param>
|
||
/// <param name="cols"></param>
|
||
/// <returns></returns>
|
||
public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) {
|
||
if (rows.Count == 0) return null;
|
||
|
||
IBlueRow retVal = BlueDataUtility.extractRow(rows, 0);
|
||
|
||
object[] sums = new object[cols.Length];
|
||
int[] idxs = new int[cols.Length];
|
||
for (int i = 0; i < cols.Length; i++) {
|
||
sums[i] = 0.0;
|
||
idxs[i] = rows.schema[cols[i]].colNo;
|
||
}
|
||
|
||
foreach (IBlueRow r in rows) {
|
||
for (int i = 0; i < cols.Length; i++) {
|
||
double dval = BlueDataUtility.get_row_value<double>(r, cols[i], 0.0);
|
||
//r[cols[i]] == null || r[cols[i]] is DBNull ? 0.0 : (double)r[cols[i]];
|
||
sums[i] = (double)sums[i] + dval;
|
||
}
|
||
}
|
||
|
||
return BlueDataUtility.getNewReplacedRow(retVal, sums, idxs);
|
||
}
|
||
|
||
|
||
public static IBlueRow computeSum<T>(IBlueRowCollection rows, params string[] cols) {
|
||
if (rows.Count == 0) return null;
|
||
IBlueRow retVal = BlueDataUtility.extractRow(rows, 0);
|
||
if (rows.Count == 1) return retVal;
|
||
|
||
object[] sums = new object[cols.Length];
|
||
|
||
int[] idxs = new int[cols.Length];
|
||
for (int i = 0; i < cols.Length; i++) {
|
||
sums[i] = default(T);
|
||
idxs[i] = rows.schema[cols[i]].colNo;
|
||
}
|
||
|
||
foreach (IBlueRow r in rows) {
|
||
for (int i = 0; i < cols.Length; i++) {
|
||
T tval = r[cols[i]] == null || r[cols[i]] is DBNull ? default(T) : (T)r[cols[i]];
|
||
sums[i] = MagicAdder<T>.Add((T)sums[i], tval);
|
||
}
|
||
}
|
||
|
||
return BlueDataUtility.getNewReplacedRow(retVal, sums, idxs);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Uses the ID to identify the presence of a row in a collection
|
||
/// </summary>
|
||
/// <param name="table"></param>
|
||
/// <param name="row"></param>
|
||
/// <returns></returns>
|
||
public static bool rowPresent(IBlueRowCollection table, IBlueRow row) {
|
||
return table.containsID((int)row["id"]);
|
||
}
|
||
|
||
public static void computeDifferences(IBlueRowCollection newRows,
|
||
IBlueRowCollection oldRows,
|
||
out IBlueRowCollection dRows,
|
||
out IBlueRowCollection aRows) {
|
||
computeDifferences(
|
||
newRows,
|
||
oldRows,
|
||
newRows.schema.columnNames,
|
||
out dRows,
|
||
out aRows);
|
||
}
|
||
|
||
public static bool compute_equality(IBlueRowCollection rows1, IBlueRowCollection rows2) {
|
||
IBlueRowCollection d_rows, a_rows;
|
||
compute_differences_fast(rows2, rows1, out d_rows, out a_rows);
|
||
return d_rows.Count == 0 && a_rows.Count == 0;
|
||
}
|
||
|
||
public static bool compute_equality(ColumnInfo ci1, ColumnInfo ci2) {
|
||
return (ci1.colNo == ci2.colNo) && (ci1.colName == ci2.colName) && (ci1.colType == ci2.colType);
|
||
}
|
||
|
||
public static bool compute_equality(IBlueTableSchema s1, IBlueTableSchema s2) {
|
||
if (s1.tableName != s1.tableName)
|
||
return false;
|
||
if (s1.columnInfo.Length != s2.columnInfo.Length)
|
||
return false;
|
||
for (int i = 0; i < s1.columnInfo.Length; i++) {
|
||
ColumnInfo ci1 = s2.columnInfo[i], ci2 = s2.columnInfo[i];
|
||
if (!compute_equality(ci1, ci2))
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
public static bool compute_equality(IBlueRow r1, IBlueRow r2) {
|
||
return compute_row_equality(r1, r2, r1.schema.columnNames);
|
||
}
|
||
|
||
public static bool compute_row_equality(IBlueRow r1,
|
||
IBlueRow r2,
|
||
params string[] col_names) {
|
||
return compare_rows(r1, r2, col_names) == 0 ? true : false;
|
||
}
|
||
|
||
public static string to_string(ColumnInfo ci) {
|
||
return ci.colNo + ";" + ci.colName + ";" + ci.colType;
|
||
}
|
||
|
||
public static string to_string(this IBlueRow r) {
|
||
string rv = String.Empty;
|
||
foreach (ColumnInfo c in r.schema.columnInfo) {
|
||
rv += $"{c.colName}: {r[c.colName]}\n";
|
||
}
|
||
return rv;
|
||
}
|
||
|
||
public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
|
||
out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
|
||
if (old_rows == null || old_rows.Count == 0) {
|
||
out_d_rows = new BlueTable(new_rows.schema);
|
||
out_a_rows = new BlueTable(new_rows);
|
||
return;
|
||
}
|
||
int[] col_indices = new int[new_rows.schema.columnNames.Length];
|
||
for (int i = 0; i < col_indices.Length; i++)
|
||
col_indices[i] = i;
|
||
compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows);
|
||
}
|
||
|
||
public static void compute_differences_id_based(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
|
||
out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
|
||
if (old_rows == null || old_rows.Count == 0) {
|
||
out_d_rows = new BlueTable(new_rows.schema);
|
||
out_a_rows = new BlueTable(new_rows);
|
||
return;
|
||
}
|
||
int[] col_indices = new int[] { new_rows.schema["id"].colNo };
|
||
compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows);
|
||
}
|
||
|
||
public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, int[] col_indices,
|
||
out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
|
||
Func<IBlueRow, IBlueRow, bool> rows_equal = (r1, r2) => {
|
||
foreach (int col_no in col_indices)
|
||
if (!(r1[col_no] ?? DBNull.Value).Equals(r2[col_no] ?? DBNull.Value))
|
||
return false;
|
||
return true;
|
||
};
|
||
|
||
BlueTable del_rows = new BlueTable(new_rows.schema), add_rows = new BlueTable(new_rows.schema);
|
||
// adds + changes
|
||
foreach (IBlueRow r in new_rows)
|
||
if (!BlueDataUtility.rowPresent(old_rows, r))
|
||
add_rows.addRow(r);
|
||
else if (!rows_equal(r, old_rows[(int)r["id"]])) {
|
||
add_rows.addRow(r);
|
||
del_rows.addRow(old_rows[(int)r["id"]]);
|
||
}
|
||
// deletes
|
||
foreach (IBlueRow r in old_rows)
|
||
if (!BlueDataUtility.rowPresent(new_rows, r))
|
||
del_rows.addRow(r);
|
||
out_d_rows = del_rows;
|
||
out_a_rows = add_rows;
|
||
}
|
||
|
||
|
||
public static void computeDifferences(IBlueRowCollection c1,
|
||
IBlueRowCollection c2,
|
||
string[] columns,
|
||
out IBlueRowCollection dRows,
|
||
out IBlueRowCollection aRows) {
|
||
dRows = new BlueTable(c1.schema);
|
||
aRows = new BlueTable(c1.schema);
|
||
|
||
// adds + changes
|
||
foreach (IBlueRow nr in c1)
|
||
if (!BlueDataUtility.rowPresent(c2, nr))
|
||
(aRows as BlueTable).addRow(nr);
|
||
else if (BlueDataUtility.compare_rows(nr, c2[(int)
|
||
nr["id"]], columns) != 0) {
|
||
(aRows as BlueTable).addRow(nr);
|
||
(dRows as BlueTable).addRow(c2[(int)nr["id"]]);
|
||
|
||
}
|
||
|
||
// deletes
|
||
foreach (IBlueRow nr in c2)
|
||
if (!BlueDataUtility.rowPresent(c1, nr))
|
||
(dRows as BlueTable).addRow(nr);
|
||
}
|
||
|
||
//brcDiff = brc1 - brc2
|
||
public static void diff(IBlueRowCollection brc1, IBlueRowCollection brc2, string[] columns, out IBlueRowCollection brcDiff) {
|
||
brcDiff = new BlueTable(brc1.schema);
|
||
foreach (IBlueRow br in brc1) {
|
||
object[] oarray = new object[columns.Length];
|
||
for (int i = 0; i < columns.Length; i++) {
|
||
string col = columns[i];
|
||
oarray[i] = br[col];
|
||
}
|
||
if (myDoesRowExistsInTable(brc2, columns, oarray) == false)
|
||
(brcDiff as BlueTable).addRow(br);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Abosrbs the deletes and adds into the BlueTable
|
||
/// </summary>
|
||
/// <param name="t"></param>
|
||
/// <param name="dels"></param>
|
||
/// <param name="adds"></param>
|
||
public static void absorbChanges(BlueTable t, IBlueRowCollection dels, IBlueRowCollection adds) {
|
||
foreach (IBlueRow r in dels)
|
||
if (!t.deleteRow(r))
|
||
throw new Exception(String.Format("{0} does not have row with id {1} to delete", t.schema.tableName, r["id"]));
|
||
|
||
foreach (IBlueRow r in adds)
|
||
t.addRow(r);
|
||
}
|
||
|
||
public static IBlueRowCollection toBlueRowCollection(IBlueRow r) {
|
||
BlueTable rv = new BlueTable(r.schema);
|
||
rv.addRow(r);
|
||
return rv;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Converts an ICollection of IBlueRows to IBlueRowCollection. If there are no rows in the
|
||
/// collection, a null is returned
|
||
/// </summary>
|
||
/// <param name="col"></param>
|
||
/// <returns></returns>
|
||
public static IBlueRowCollection toBlueRowCollection(ICollection col) {
|
||
BlueTable retVal = null;
|
||
foreach (IBlueRow r in col) {
|
||
retVal = retVal ?? new BlueTable(r.schema);
|
||
retVal.addRow(r);
|
||
}
|
||
return retVal;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Computes the rows that are truely deleted, added or changed (with respect to relevant columns)
|
||
/// </summary>
|
||
/// <param name="dels">all the deleted rows in a certain callback</param>
|
||
/// <param name="adds">all the added rows in a certain callback</param>
|
||
/// <param name="truelyDels">the rows that are truely deleted or changed with respect to the relevant columns</param>
|
||
/// <param name="truelyAdds">the rows that are truely added or changed with respect to the relevant columns</param>
|
||
/// <param name="cols"></param>
|
||
public static void getTrueDeletesAndAdds(IBlueRowCollection dels,
|
||
IBlueRowCollection adds,
|
||
out IBlueRowCollection truelyDels,
|
||
out IBlueRowCollection truelyAdds,
|
||
string[] cols) {
|
||
truelyDels = new BlueTable(dels);
|
||
truelyAdds = new BlueTable(adds);
|
||
foreach (IBlueRow r in dels) {
|
||
if (adds.containsID((int)r["id"]) &&
|
||
BlueDataUtility.compute_row_equality(adds[(int)r["id"]], r, cols)) {
|
||
(truelyDels as BlueTable).deleteRow((int)r["id"]);
|
||
(truelyAdds as BlueTable).deleteRow((int)r["id"]);
|
||
}
|
||
}
|
||
}
|
||
|
||
public static IBlueRowCollection getSerializableCollection(IBlueRowCollection coll) {
|
||
BlueTable retVal = new OrderedBlueTable(coll.schema);
|
||
foreach (IBlueRow r in coll)
|
||
retVal.addRow(new BlueRow(r));
|
||
return retVal;
|
||
}
|
||
|
||
/// <summary>
|
||
/// extracts out a column from a table
|
||
/// </summary>
|
||
/// <param name="rows"></param>
|
||
/// <param name="colName"></param>
|
||
/// <returns></returns>
|
||
public static ArrayList extractColumn(IBlueRowCollection rows, string colName) {
|
||
ArrayList rv = new ArrayList();
|
||
foreach (IBlueRow r in rows)
|
||
rv.Add(r[colName]);
|
||
return rv;
|
||
}
|
||
|
||
/// <summary>
|
||
/// applies the deletedRows and addedRows to the oldState and asserts if it does not
|
||
/// match the newRows. for an old state with 10 K rows, and all 10 K rows being deleted and
|
||
/// added it takes ~ 35 ms to execute this method.
|
||
/// </summary>
|
||
/// <param name="oldState"></param>
|
||
/// <param name="newRows"></param>
|
||
/// <param name="delRows"></param>
|
||
/// <param name="addedRows"></param>
|
||
public static void checkConsistency(IBlueRowCollection oldState,
|
||
IBlueRowCollection newRows,
|
||
IBlueRowCollection delRows,
|
||
IBlueRowCollection addedRows) {
|
||
BlueTable calc = new BlueTable(oldState);
|
||
Debug.Assert(oldState != null && newRows != null && delRows != null && addedRows != null);
|
||
|
||
foreach (IBlueRow r in delRows) {
|
||
Debug.Assert(calc.containsID((int)r["id"]), "deleted row's id = " + r["id"] + " does not exist");
|
||
calc.deleteRow(r);
|
||
}
|
||
|
||
foreach (IBlueRow r in addedRows) {
|
||
Debug.Assert(!calc.containsID((int)r["id"]), "added rows's id = " + r["id"].ToString() + " already exists");
|
||
calc.addRow(r);
|
||
}
|
||
|
||
Debug.Assert(calc.Count == newRows.Count, "row count should be the same. Counts: calc=" + calc.Count + "; newRows=" + newRows.Count);
|
||
foreach (IBlueRow r in calc)
|
||
Debug.Assert(newRows.containsID((int)r["id"]), r["id"] + " is not found");
|
||
|
||
}
|
||
|
||
/// <summary>
|
||
/// Extracts values from an IBlueRow specified by the columnNames arg
|
||
/// </summary>
|
||
/// <param name="r"></param>
|
||
/// <param name="colNames"></param>
|
||
/// <returns></returns>
|
||
public static object[] getSelectedValues(IBlueRow r, params string[] colNames) {
|
||
object[] retVal = new object[colNames.Length];
|
||
for (int i = 0; i < colNames.Length; i++)
|
||
retVal[i] = r[colNames[i]];
|
||
return retVal;
|
||
}
|
||
|
||
|
||
/// <summary>
|
||
/// From an IBlueRowCollection, for a set of columns, extracts a row with the same values as
|
||
/// provided in the third arg
|
||
/// </summary>
|
||
/// <param name="rows"></param>
|
||
/// <param name="keyColumns"></param>
|
||
/// <param name="colValues"></param>
|
||
/// <returns></returns>
|
||
public static IBlueRow getRowWithKey(IBlueRowCollection rows, string[] keyColumns, object[] colValues) {
|
||
foreach (IBlueRow r in rows) {
|
||
object[] vals = getSelectedValues(r, keyColumns);
|
||
if (CollectionHelpers.equal(vals, colValues))
|
||
return r;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
public static IBlueRowCollection deepCopy(IBlueRowCollection rows) {
|
||
BlueTable retVal = new BlueTable(rows.schema);
|
||
foreach (IBlueRow r in rows)
|
||
retVal.addRow(new BlueRow(r));
|
||
return retVal;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Returns the columns, sorted alphabetically. The second arg
|
||
/// is for the inclusion of id
|
||
/// </summary>
|
||
/// <param name="schema"></param>
|
||
/// <param name="includeId"></param>
|
||
/// <returns></returns>
|
||
public static int[] getColumnsInSortedOrder(IBlueTableSchema schema, bool includeId) {
|
||
ArrayList names = new ArrayList(schema.columnNames),
|
||
retVal = new ArrayList();
|
||
names.Sort();
|
||
foreach (string n in names)
|
||
if (n != "id" || includeId)
|
||
retVal.Add(schema[n].colNo);
|
||
return retVal.ToArray(typeof(int)) as int[];
|
||
}
|
||
|
||
//this class is used so that we can sort a schema by column name
|
||
//that way, prepublished tables will allways get published with schemas in the same order
|
||
//regardless of how the underlying streams end up working
|
||
private class ColumnInfoComparisonWrapper : IComparable<ColumnInfoComparisonWrapper> {
|
||
|
||
ColumnInfo myColumn;
|
||
|
||
public string ColumnName {
|
||
get { return myColumn.colName; }
|
||
//can't set it, these are only for reading columns
|
||
}
|
||
|
||
public ColumnInfo Column {
|
||
get { return myColumn; }
|
||
}
|
||
public ColumnInfoComparisonWrapper(ColumnInfo theColumn) {
|
||
myColumn = theColumn;
|
||
}
|
||
|
||
public int CompareTo(ColumnInfoComparisonWrapper otherGuy) {
|
||
return ColumnName.CompareTo(otherGuy.ColumnName);
|
||
}
|
||
}
|
||
|
||
public static IBlueTableSchema getSortedSchema(IBlueTableSchema inSchema) {
|
||
List<string> sortedColNames = new List<string>(inSchema.columnNames);
|
||
sortedColNames.Sort();
|
||
List<ColumnInfo> sortedColInfos = new List<ColumnInfo>();
|
||
int colNo = 0;
|
||
foreach (string c in sortedColNames) {
|
||
ColumnInfo ci = inSchema[c];
|
||
sortedColInfos.Add(new ColumnInfo(c, colNo++, ci.colType));
|
||
|
||
}
|
||
return new BlueTableSchema(inSchema.tableName, sortedColInfos.ToArray());
|
||
}
|
||
|
||
public static IBlueTableSchema getSortedSchema(IBlueTableSchema theSchema, out int[] sortedSchemaToOriginalSchemaMapping, out int idColumnInSortedSchema) {
|
||
|
||
//for now this function will not do anything
|
||
//that is because we don't want to break the C++ deserialization code
|
||
int numCols = theSchema.columnNames.Length;
|
||
sortedSchemaToOriginalSchemaMapping = new int[numCols];
|
||
for (int col = 0; col < theSchema.columnNames.Length; ++col) {
|
||
//the mapping is just the identity mapping
|
||
sortedSchemaToOriginalSchemaMapping[col] = col;
|
||
|
||
|
||
}
|
||
|
||
idColumnInSortedSchema = theSchema["id"].colNo;
|
||
|
||
return theSchema;
|
||
|
||
/*
|
||
//set this to -1 in case the method doesn't find an id column
|
||
idColumnInSortedSchema = 01;
|
||
List<ColumnInfoComparisonWrapper> schemaColumns = new List<ColumnInfoComparisonWrapper>();
|
||
|
||
foreach (ColumnInfo ci in theSchema.columnInfo)
|
||
schemaColumns.Add(new ColumnInfoComparisonWrapper(ci));
|
||
|
||
schemaColumns.Sort();
|
||
|
||
int numCols = theSchema.columnInfo.Length;
|
||
ColumnInfo[] outputColumns = new ColumnInfo[numCols];
|
||
sortedSchemaToOriginalSchemaMapping = new int[numCols];
|
||
|
||
for(int i = 0; i < numCols; ++i)
|
||
{
|
||
string thisColName = schemaColumns[i].ColumnName;
|
||
outputColumns[i] = new ColumnInfo(thisColName, i, schemaColumns[i].Column.colType);
|
||
sortedSchemaToOriginalSchemaMapping[i] = theSchema[thisColName].colNo;
|
||
|
||
//we cache the ID column because it'll be differentin the serialized row than in our schema
|
||
if (thisColName == "id")
|
||
idColumnInSortedSchema = i;
|
||
|
||
}
|
||
|
||
|
||
theSchema = new BlueTableSchema(theSchema.tableName,outputColumns);
|
||
return theSchema;
|
||
*/
|
||
|
||
|
||
}
|
||
|
||
|
||
|
||
|
||
/// <summary>
|
||
/// Compares 2 schemas for column-by-column match
|
||
/// </summary>
|
||
/// <param name="s1"></param>
|
||
/// <param name="s2"></param>
|
||
/// <returns>true if equal</returns>
|
||
public static bool compareSchemas(IBlueTableSchema s1, IBlueTableSchema s2) {
|
||
bool result = true;
|
||
try {
|
||
if (s1.columnInfo.Length != s2.columnInfo.Length)
|
||
result = false;
|
||
else
|
||
foreach (ColumnInfo ci1 in s1.columnInfo) {
|
||
ColumnInfo ci2 = s2[ci1.colNo];
|
||
if (ci1.colName != ci2.colName || ci1.colType != ci2.colType || ci1.colNo != ci2.colNo) {
|
||
result = false;
|
||
break;
|
||
}
|
||
}
|
||
} catch {
|
||
result = false; // the col# wasn't found in s2
|
||
}
|
||
return result;
|
||
}
|
||
|
||
public static void validate_row_schema(IBlueRow r) {
|
||
foreach (var ci in r.schema.columnInfo) {
|
||
object rval = r[ci.colName];
|
||
if (rval == null)
|
||
continue;
|
||
bts.Log.assert(rval.isDBNull() || rval.GetType().Equals(ci.colType));
|
||
}
|
||
}
|
||
|
||
public static void validate_table_schema(IBlueRowCollection rows) {
|
||
foreach (IBlueRow r in rows) {
|
||
validate_row_schema(r);
|
||
}
|
||
}
|
||
|
||
public static IBlueRowCollection removeRows(BlueTable table, string[] cols, object[] vals) {
|
||
IBlueRowCollection dels = findMatchingRows(table, cols, vals);
|
||
table.deleteRows(dels);
|
||
return dels;
|
||
}
|
||
|
||
public static IBlueRowCollection findMatchingRows(IBlueRowCollection rows, string[] cols, object[] vals) {
|
||
BlueTable rv = new BlueTable(rows.schema);
|
||
foreach (IBlueRow r in rows) {
|
||
bool match = true;
|
||
for (int i = 0; i < cols.Length; i++)
|
||
match &= vals[i].Equals(r[cols[i]]);
|
||
if (match)
|
||
rv.addRow(r);
|
||
}
|
||
return rv;
|
||
}
|
||
|
||
public static void checkRowTypes(IBlueRowCollection rows) {
|
||
foreach (IBlueRow r in rows)
|
||
foreach (string col in r.schema.columnNames)
|
||
if (r[col] != null && r[col] != DBNull.Value)
|
||
Debug.Assert(r[col].GetType() == rows.schema[col].colType,
|
||
String.Format("col {0} in table {1} is of type {2} in schema but is a {3} in the row", col, rows.schema.tableName, r.schema[col].colType.ToString(), r[col].GetType().ToString()));
|
||
}
|
||
|
||
public static void addRows(ref BlueTable table, ICollection rows) {
|
||
foreach (IBlueRow r in rows)
|
||
table.addRow(r);
|
||
}
|
||
|
||
public static void removeRows(ref BlueTable table, ICollection rows) {
|
||
foreach (IBlueRow r in rows)
|
||
table.deleteRow(r);
|
||
}
|
||
|
||
public static void sortOutPacket(IBlueRowCollection dels, IBlueRowCollection adds, string[] keyCols,
|
||
out IBlueRowCollection addedRows,
|
||
out IBlueRowCollection deletedRows,
|
||
out IBlueRowCollection changedRows) {
|
||
addedRows = new BlueTable(dels.schema);
|
||
deletedRows = new BlueTable(dels.schema);
|
||
changedRows = new BlueTable(dels.schema);
|
||
|
||
KeysBasedTable2 keyedDels = new KeysBasedTable2(dels.schema, keyCols),
|
||
keyedAdds = new KeysBasedTable2(dels.schema, keyCols);
|
||
|
||
foreach (IBlueRow r in dels)
|
||
keyedDels.update(r);
|
||
foreach (IBlueRow r in adds)
|
||
keyedAdds.update(r);
|
||
|
||
foreach (IBlueRow r in keyedDels) {
|
||
object[] key = BlueDataUtility.getSelectedValues(r, keyCols);
|
||
if (keyedAdds.getRow(key) != null) {
|
||
///change
|
||
(changedRows as BlueTable).addRow(r);
|
||
} else {
|
||
///delete
|
||
(deletedRows as BlueTable).addRow(r);
|
||
}
|
||
}
|
||
|
||
foreach (IBlueRow r in keyedAdds) {
|
||
object[] key = BlueDataUtility.getSelectedValues(r, keyCols);
|
||
if (keyedDels.getRow(key) == null) {
|
||
(addedRows as BlueTable).addRow(r);
|
||
}
|
||
}
|
||
}
|
||
|
||
public static int HashExpStrike(DateTime exp, double strike) {
|
||
return ((DateTime)exp).ToShortDateString().GetHashCode() ^ strike.GetHashCode();
|
||
}
|
||
|
||
public static IEnumerable<IBlueRow> collection_to_ienum(IBlueRowCollection rows) {
|
||
List<IBlueRow> rv = new List<IBlueRow>();
|
||
foreach (IBlueRow r in rows)
|
||
rv.Add(r);
|
||
return rv;
|
||
}
|
||
|
||
public static IBlueRowCollection ienum_to_collection(IEnumerable<IBlueRow> rows) {
|
||
BlueTable rv = null;
|
||
if (rows.Count() > 0) {
|
||
rv = new BlueTable(rows.ElementAt(0).schema);
|
||
foreach (IBlueRow r in rows)
|
||
rv.addRow(r);
|
||
}
|
||
return rv;
|
||
}
|
||
|
||
public static IBlueTableSchema add_column(IBlueTableSchema input_schema, string new_col_name, Type new_col_type) {
|
||
List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
|
||
cis.Add(new ColumnInfo(new_col_name, cis.Count, new_col_type));
|
||
return new BlueTableSchema(input_schema.tableName, cis.ToArray());
|
||
}
|
||
|
||
public static IBlueTableSchema remove_columns(IBlueTableSchema input_schema, params string[] col_names) {
|
||
Dictionary<string, bool> removed_col_names = new Dictionary<string, bool>();
|
||
foreach (string c in col_names)
|
||
removed_col_names[c] = true;
|
||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||
foreach (ColumnInfo ci in input_schema.columnInfo)
|
||
if (!removed_col_names.ContainsKey(ci.colName))
|
||
cis.Add(new ColumnInfo(ci.colName, cis.Count, ci.colType));
|
||
return new BlueTableSchema(cis.ToArray(), input_schema.tableName);
|
||
}
|
||
|
||
public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) {
|
||
List<int> rv = new List<int>();
|
||
foreach (string c in col_names) {
|
||
Log.assert(schema.contains_column(c), $"Could not find col: {c} in table: {schema.tableName}");
|
||
rv.Add(schema[c].colNo);
|
||
}
|
||
return rv.ToArray();
|
||
}
|
||
|
||
public static string get_appended_string_val(IBlueRow r, int[] col_indices) {
|
||
System.Text.StringBuilder sb = new System.Text.StringBuilder();
|
||
for (int i = 0; i < col_indices.Length; i++)
|
||
sb.Append(r[col_indices[i]]);
|
||
return sb.ToString();
|
||
}
|
||
|
||
public delegate string RowValueToString(IBlueRow r, int col_index);
|
||
public static string get_appended_string_val(IBlueRow r, int[] col_indices, RowValueToString to_string_conv) {
|
||
System.Text.StringBuilder sb = new System.Text.StringBuilder();
|
||
for (int i = 0; i < col_indices.Length; i++)
|
||
sb.Append(to_string_conv(r, i));
|
||
return sb.ToString();
|
||
}
|
||
|
||
public static string id_string(IBlueRowCollection table) {
|
||
string rv = String.Empty;
|
||
foreach (IBlueRow r in table)
|
||
rv += $"{r["id"]} ";
|
||
return rv;
|
||
}
|
||
|
||
public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, string[] group_by_col_names) {
|
||
BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length);
|
||
return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher);
|
||
}
|
||
|
||
public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, string[] group_by_col_names, ref BlueRowHasher hasher) {
|
||
return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher);
|
||
}
|
||
|
||
public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, int[] group_by_col_nos, ref BlueRowHasher hasher) {
|
||
Dictionary<int, List<IBlueRow>> rv = new Dictionary<int, List<IBlueRow>>();
|
||
hasher = hasher ?? new BlueRowHasher(group_by_col_nos.Length);
|
||
foreach (IBlueRow r in rows) {
|
||
int row_hash = hasher.get_hash(r, group_by_col_nos);
|
||
List<IBlueRow> this_group_rows = null;
|
||
if (!rv.TryGetValue(row_hash, out this_group_rows)) {
|
||
this_group_rows = new List<IBlueRow>();
|
||
rv.Add(row_hash, this_group_rows);
|
||
}
|
||
this_group_rows.Add(r);
|
||
}
|
||
return rv;
|
||
}
|
||
|
||
public static void copy_to_row(IBlueRow row, string[] data_col_names, MutableBlueRow output_row) {
|
||
foreach (string cn in data_col_names)
|
||
output_row.setValue(cn, row[cn]);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Copies the column nos specified in data_cols from row, to output_row
|
||
/// starting from index start_col_index
|
||
/// </summary>
|
||
public static void copy_to_row(IBlueRow row, int[] data_col_nos, object[] output_row, int start_col_index) {
|
||
foreach (int col_no in data_col_nos)
|
||
output_row[start_col_index++] = row[col_no];
|
||
}
|
||
|
||
public static object compute_min_max(object v1, object v2, bool is_max) {
|
||
if (!TypeHelper.is_dbnull(v1) && !TypeHelper.is_dbnull(v2)) {
|
||
if (TypeHelper.is_numeric_type(v1) && TypeHelper.is_numeric_type(v2)) {
|
||
double v1_num = (double)Convert.ChangeType(v1, TypeCode.Double),
|
||
v2_num = (double)Convert.ChangeType(v2, TypeCode.Double);
|
||
v1_num = Double.IsNaN(v1_num) ? (is_max ? Double.MinValue : Double.MaxValue) : v1_num;
|
||
v2_num = Double.IsNaN(v2_num) ? (is_max ? Double.MinValue : Double.MaxValue) : v2_num;
|
||
double rv = is_max ? Math.Max(v1_num, v2_num) : Math.Min(v1_num, v2_num);
|
||
return Convert.ChangeType(rv, v1.GetType());
|
||
} else
|
||
return v1;
|
||
} else
|
||
return TypeHelper.is_dbnull(v1) ? v2 : v1;
|
||
}
|
||
|
||
#region Sorting
|
||
|
||
public static class ParallelSort {
|
||
public class TreeRowEffComparer : IComparer<IBlueRow> {
|
||
private int _depth_in_tree_col_no;
|
||
private int[] _aggregation_tree_col_nos;
|
||
private Pair<int, bool>[] _sort_map;
|
||
|
||
public TreeRowEffComparer(int depth_in_tree_col_no, int[] aggregation_tree_col_nos, Pair<int, bool>[] sort_map) {
|
||
_depth_in_tree_col_no = depth_in_tree_col_no;
|
||
_aggregation_tree_col_nos = aggregation_tree_col_nos;
|
||
_sort_map = sort_map;
|
||
}
|
||
|
||
public int Compare(IBlueRow x, IBlueRow y) {
|
||
return BlueDataUtility.sort_in_a_tree(x, y, _depth_in_tree_col_no, _aggregation_tree_col_nos, _sort_map);
|
||
}
|
||
}
|
||
|
||
public class TabularRowComparer : IComparer<IBlueRow> {
|
||
private IEnumerable<Pair<int, bool>> _sort_map;
|
||
public TabularRowComparer(IEnumerable<Pair<int, bool>> sort_map) { _sort_map = sort_map; }
|
||
public int Compare(IBlueRow x, IBlueRow y) { return BlueDataUtility.compare_rows(x, y, _sort_map); }
|
||
}
|
||
|
||
public class TabularRowHashPairComparer : IComparer<Pair<IBlueRow, int>> {
|
||
private Pair<int, bool>[] _sort_map;
|
||
|
||
public TabularRowHashPairComparer(Pair<int, bool>[] sort_map) { _sort_map = sort_map; }
|
||
public int Compare(Pair<IBlueRow, int> x, Pair<IBlueRow, int> y) { return BlueDataUtility.compare_rows(x.first, y.first, _sort_map); }
|
||
}
|
||
|
||
#region Public Static Methods
|
||
|
||
public static void QuicksortSequential<T>(IList<T> arr, IComparer<T> comparer) {
|
||
QuicksortSequential(arr, comparer, 0, arr.Count - 1);
|
||
}
|
||
|
||
public static void QuicksortParallel<T>(IList<T> arr, IComparer<T> comparer) {
|
||
QuicksortParallel(arr, comparer, 0, arr.Count - 1);
|
||
}
|
||
|
||
#endregion
|
||
|
||
#region Private Static Methods
|
||
|
||
private static void QuicksortSequential<T>(IList<T> arr, IComparer<T> comparer, int left, int right) {
|
||
if (right > left) {
|
||
int pivot = Partition(arr, comparer, left, right);
|
||
QuicksortSequential(arr, comparer, left, pivot - 1);
|
||
QuicksortSequential(arr, comparer, pivot + 1, right);
|
||
}
|
||
}
|
||
|
||
private static void QuicksortParallel<T>(IList<T> arr, IComparer<T> comparer, int left, int right) {
|
||
const int SEQUENTIAL_THRESHOLD = 64;
|
||
if (right > left) {
|
||
if (right - left < SEQUENTIAL_THRESHOLD) {
|
||
QuicksortSequential(arr, comparer, left, right);
|
||
} else {
|
||
int pivot = Partition(arr, comparer, left, right);
|
||
System.Threading.Tasks.Parallel.Invoke(new Action[] { delegate {QuicksortParallel(arr, comparer, left, pivot - 1);},
|
||
delegate {QuicksortParallel(arr, comparer, pivot + 1, right); }
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
private static void Swap<T>(IList<T> arr, int i, int j) {
|
||
T tmp = arr[i];
|
||
arr[i] = arr[j];
|
||
arr[j] = tmp;
|
||
}
|
||
|
||
private static int Partition<T>(IList<T> arr, IComparer<T> comparer, int low, int high) {
|
||
// Simple partitioning implementation
|
||
int pivotPos = (high + low) / 2;
|
||
T pivot = arr[pivotPos];
|
||
Swap(arr, low, pivotPos);
|
||
|
||
int left = low;
|
||
for (int i = low + 1; i <= high; i++) {
|
||
if (comparer.Compare(arr[i], pivot) < 0) {
|
||
left++;
|
||
Swap(arr, i, left);
|
||
}
|
||
}
|
||
|
||
Swap(arr, low, left);
|
||
return left;
|
||
}
|
||
|
||
#endregion
|
||
}
|
||
|
||
public static IBlueRow find_max(IEnumerable<IBlueRow> rows, params string[] cols_to_compare) {
|
||
if (rows == null || rows.Count() == 0)
|
||
return null;
|
||
IBlueRow max_row = rows.ElementAt(0);
|
||
foreach (IBlueRow r in rows) {
|
||
int c = compare_rows(r, max_row, cols_to_compare);
|
||
if (c > 0)
|
||
max_row = r;
|
||
}
|
||
return max_row;
|
||
}
|
||
|
||
public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) {
|
||
foreach (KeyValuePair<string, bool> p in sort_map) {
|
||
int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value);
|
||
if (comp != 0)
|
||
return comp;
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
public static int compare_rows(IBlueRow r1, IBlueRow r2, IEnumerable<Pair<int, bool>> sort_map) {
|
||
foreach (Pair<int, bool> p in sort_map) {
|
||
int comp = compare_send_dontcare_to_last(r1, p.first, r2, p.first, p.second);
|
||
if (comp != 0)
|
||
return comp;
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
public static int compare_send_dontcare_to_last(IBlueRow r1, int r1_col_no,
|
||
IBlueRow r2, int r2_col_no,
|
||
bool low_to_high) {
|
||
object o1 = BlueDataUtility.get_row_value<object>(r1, r1_col_no, DBNull.Value),
|
||
o2 = BlueDataUtility.get_row_value<object>(r2, r2_col_no, DBNull.Value);
|
||
return compare_send_dontcare_to_last(o1, o2, low_to_high);
|
||
}
|
||
|
||
public static List<Pair<int, bool>> get_indexed_sort_map(List<Pair<string, bool>> sort_map, IBlueTableSchema schema) {
|
||
List<Pair<int, bool>> rv = new List<Pair<int, bool>>();
|
||
foreach (Pair<string, bool> p in sort_map)
|
||
rv.Add(new Pair<int, bool>(schema[p.first].colNo, p.second));
|
||
return rv;
|
||
}
|
||
|
||
/// Comapares 2 rows in a way where nulls, nans, infinities,
|
||
/// empty strings etc are sent to the last
|
||
public static int compare_send_dontcare_to_last(IBlueRow r1, string r1_col_name,
|
||
IBlueRow r2, string r2_col_name,
|
||
bool low_to_high) {
|
||
object o1 = BlueDataUtility.get_row_value<object>(r1, r1_col_name, DBNull.Value),
|
||
o2 = BlueDataUtility.get_row_value<object>(r2, r2_col_name, DBNull.Value);
|
||
return compare_send_dontcare_to_last(o1, o2, low_to_high);
|
||
}
|
||
|
||
public static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) {
|
||
if (is_dont_care_val(o1))
|
||
return is_dont_care_val(o2) ? 0 : 1;
|
||
else
|
||
return is_dont_care_val(o2) ? -1 : (low_to_high ? ((IComparable)o1).CompareTo(o2)
|
||
: ((IComparable)o2).CompareTo(o1));
|
||
}
|
||
|
||
public static bool is_dont_care_val(object o) {
|
||
return o == null || o == DBNull.Value ||
|
||
o is string && String.IsNullOrEmpty((string)o) ||
|
||
o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) ||
|
||
o is float && (Single.IsInfinity((float)o) || Single.IsNaN((float)o)) ||
|
||
o is Int32 && (Int32.MaxValue.Equals(o) || Int32.MinValue.Equals(o));
|
||
}
|
||
|
||
#endregion
|
||
|
||
#region Tree Aggregation
|
||
|
||
public static int[] convert_aggregation_tree(string[] aggregation_tree_col_names, IBlueTableSchema schema) {
|
||
List<int> rv = new List<int>();
|
||
foreach (string col_name in aggregation_tree_col_names)
|
||
rv.Add(schema[col_name].colNo);
|
||
return rv.ToArray();
|
||
}
|
||
|
||
public static bool is_leaf_node(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) {
|
||
return BlueDataUtility.get_row_value<int>(r, depth_in_tree_col_no, -1) == (aggregation_tree.Count() - 1);
|
||
}
|
||
|
||
public static bool is_leaf_node(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) {
|
||
return BlueDataUtility.get_row_value<int>(r, depth_in_tree_col_name, -1) == (aggregation_tree.Count() - 1);
|
||
}
|
||
|
||
public static string generate_aggregation_based_id(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) {
|
||
int aggregation_priority = BlueDataUtility.get_row_value<int>(r, depth_in_tree_col_no, -1);
|
||
string rv = String.Empty;
|
||
for (int p = 0; p <= aggregation_priority; p++)
|
||
rv += (rv == String.Empty ? String.Empty : ":") + String.Format("{0},{1}", p, BlueDataUtility.get_row_value<object>(r, aggregation_tree[p], DBNull.Value));
|
||
return rv;
|
||
}
|
||
|
||
/// Generates a string based id for a row in a tree
|
||
/// by appending the values of the nodes above it
|
||
public static string generate_aggregation_based_id(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) {
|
||
int aggregation_priority = BlueDataUtility.get_row_value<int>(r, depth_in_tree_col_name, -1);
|
||
string rv = String.Empty;
|
||
for (int p = 0; p <= aggregation_priority; p++)
|
||
rv += (rv == String.Empty ? String.Empty : ":") +
|
||
String.Format("{0},{1}", p, BlueDataUtility.get_row_value<object>(r, aggregation_tree.ElementAt(p), DBNull.Value));
|
||
return rv;
|
||
}
|
||
/// <summary>
|
||
/// Given an aggregation hierarchy tree, figures if the first
|
||
/// arg is a parent of the second arg.
|
||
/// </summary>
|
||
public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, string depth_in_tree_col_name, string[] aggregation_tree) {
|
||
System.Diagnostics.Debug.Assert(parent_row.Contains(depth_in_tree_col_name) && child_row.Contains(depth_in_tree_col_name));
|
||
int parent_depth = BlueDataUtility.get_row_value<int>(parent_row, depth_in_tree_col_name, Int32.MaxValue),
|
||
child_depth = BlueDataUtility.get_row_value<int>(child_row, depth_in_tree_col_name, Int32.MaxValue);
|
||
if (parent_depth < child_depth) {
|
||
for (int p = 0; p <= parent_depth; p++) {
|
||
string col_name = aggregation_tree.ElementAt(p);
|
||
object parent_val = BlueDataUtility.get_row_value<object>(parent_row, col_name, DBNull.Value),
|
||
child_val = BlueDataUtility.get_row_value<object>(child_row, col_name, DBNull.Value);
|
||
if (!parent_val.Equals(child_val))
|
||
return false;
|
||
}
|
||
return true;
|
||
} else
|
||
return false;
|
||
}
|
||
|
||
public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, int depth_in_tree_col_no, int[] aggregation_tree) {
|
||
int parent_depth = BlueDataUtility.get_row_value<int>(parent_row, depth_in_tree_col_no, Int32.MaxValue),
|
||
child_depth = BlueDataUtility.get_row_value<int>(child_row, depth_in_tree_col_no, Int32.MaxValue);
|
||
if (parent_depth < child_depth) {
|
||
for (int p = 0; p <= parent_depth; p++) {
|
||
int col_no = aggregation_tree[p];
|
||
object parent_val = BlueDataUtility.get_row_value<object>(parent_row, col_no, DBNull.Value),
|
||
child_val = BlueDataUtility.get_row_value<object>(child_row, col_no, DBNull.Value);
|
||
if (!parent_val.Equals(child_val))
|
||
return false;
|
||
}
|
||
return true;
|
||
} else
|
||
return false;
|
||
}
|
||
|
||
public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, string depth_in_tree_col_name, string[] aggregation_tree, Dictionary<string, bool> sort_map) {
|
||
if (BlueDataUtility.is_parent(r1, r2, depth_in_tree_col_name, aggregation_tree))
|
||
return -1;
|
||
else if (BlueDataUtility.is_parent(r2, r1, depth_in_tree_col_name, aggregation_tree))
|
||
return 1;
|
||
else if (get_row_value<int>(r1, "id", Int32.MaxValue).Equals(get_row_value<int>(r2, "id", Int32.MaxValue)))
|
||
return 0;
|
||
else {
|
||
for (int p = 0; p < aggregation_tree.Count(); p++) {
|
||
/// sort_map is used to leaf nodes
|
||
if (p == aggregation_tree.Count() - 1) {
|
||
/// sorting for leaf nodes is special, since they can be
|
||
/// sorted by data columns
|
||
System.Diagnostics.Debug.Assert(BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree));
|
||
int leaf_comp = compare_rows(r1, r2, sort_map);
|
||
if (leaf_comp != 0)
|
||
return leaf_comp;
|
||
}
|
||
/// sort_map is used to find sorting order
|
||
string aggregation_col_name = aggregation_tree.ElementAt(p);
|
||
bool low_to_high = sort_map.TryGetValue(aggregation_col_name, out low_to_high) ? low_to_high : true;
|
||
int comp = BlueDataUtility.compare_send_dontcare_to_last(r1, aggregation_col_name, r2, aggregation_col_name, low_to_high);
|
||
if (comp != 0)
|
||
return comp;
|
||
}
|
||
return BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree) ?
|
||
compare_rows(r1, r2, sort_map) : 0;
|
||
}
|
||
}
|
||
|
||
public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, int depth_in_tree_col_no, int[] aggregation_tree, Pair<int, bool>[] sort_map) {
|
||
if (BlueDataUtility.is_parent(r1, r2, depth_in_tree_col_no, aggregation_tree))
|
||
return -1;
|
||
else if (BlueDataUtility.is_parent(r2, r1, depth_in_tree_col_no, aggregation_tree))
|
||
return 1;
|
||
else if (get_row_value<int>(r1, "id", Int32.MaxValue).Equals(get_row_value<int>(r2, "id", Int32.MaxValue)))
|
||
return 0;
|
||
else {
|
||
for (int p = 0; p < aggregation_tree.Count(); p++) {
|
||
/// sort_map is used to leaf nodes
|
||
if (p == aggregation_tree.Count() - 1) {
|
||
/// sorting for leaf nodes is special, since they can be
|
||
/// sorted by data columns
|
||
//System.Diagnostics.Debug.Assert(BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree));
|
||
int leaf_comp = compare_rows(r1, r2, sort_map);
|
||
if (leaf_comp != 0)
|
||
return leaf_comp;
|
||
}
|
||
/// sort_map is used to find sorting order
|
||
int aggregation_col_no = aggregation_tree[p];
|
||
bool low_to_high = true;//sort_map.TryGetValue(aggregation_col_name, out low_to_high) ? low_to_high : true;
|
||
int comp = BlueDataUtility.compare_send_dontcare_to_last(r1, aggregation_col_no, r2, aggregation_col_no, low_to_high);
|
||
if (comp != 0)
|
||
return comp;
|
||
}
|
||
return BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_no, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_no, aggregation_tree) ?
|
||
compare_rows(r1, r2, sort_map) : 0;
|
||
}
|
||
}
|
||
|
||
#endregion
|
||
|
||
#region Column accessors
|
||
|
||
public static T get_array_value<T>(Array r, int col_no, T error_val) {
|
||
try {
|
||
object rv = (r != null && col_no > -1 && col_no < r.Length) ? r.GetValue(col_no) : DBNull.Value;
|
||
if (rv == null || rv == DBNull.Value)
|
||
return error_val;
|
||
else
|
||
return (T)Convert.ChangeType(rv, typeof(T));
|
||
} catch {
|
||
return error_val;
|
||
}
|
||
}
|
||
|
||
public static DateTime get_array_datetime_value(Array r, int col_no, DateTime error_val) {
|
||
try {
|
||
object rv = (r != null && col_no > -1 && col_no < r.Length) ? r.GetValue(col_no) : error_val;
|
||
return _get_datetime_value(rv, error_val);
|
||
} catch {
|
||
return error_val;
|
||
}
|
||
}
|
||
|
||
public static T get_row_value<T>(IBlueRow r, int col_no, T error_val) {
|
||
try {
|
||
object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : DBNull.Value;
|
||
if (rv == null || rv == DBNull.Value)
|
||
return error_val;
|
||
else
|
||
return (T)Convert.ChangeType(rv, typeof(T));
|
||
} catch {
|
||
return error_val;
|
||
}
|
||
}
|
||
|
||
public static T get_row_value<T>(IBlueRow r, string col_name, T error_val) {
|
||
try {
|
||
object rv = (r != null && r.Contains(col_name)) ? r[col_name] : DBNull.Value;
|
||
if (rv == null || rv == DBNull.Value)
|
||
return error_val;
|
||
else
|
||
return (T)Convert.ChangeType(rv, typeof(T));
|
||
} catch {
|
||
return error_val;
|
||
}
|
||
}
|
||
|
||
public static T get_row_enum_value<T>(IBlueRow r, string col_name, T error_val) where T : struct {
|
||
try {
|
||
object rv = r.Contains(col_name) ? r[col_name] : DBNull.Value;
|
||
return _get_row_enum_value<T>(rv, error_val);
|
||
} catch {
|
||
return error_val;
|
||
}
|
||
}
|
||
|
||
public static T get_row_enum_value<T>(IBlueRow r, int col_no, T error_val) where T : struct {
|
||
try {
|
||
object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : DBNull.Value;
|
||
return _get_row_enum_value<T>(rv, error_val);
|
||
} catch {
|
||
return error_val;
|
||
}
|
||
}
|
||
|
||
private static T _get_row_enum_value<T>(object o, T error_val) where T : struct {
|
||
try {
|
||
if (o == null || o == DBNull.Value)
|
||
return error_val;
|
||
else if (o is string) {
|
||
T result = error_val;
|
||
if (Enum.TryParse<T>((string)o, true, out result)) return result;
|
||
return error_val;
|
||
} else if (o is Int32 || o is Int16 || o is UInt16 || o is UInt32)
|
||
return (T)o;
|
||
else if (o is T)
|
||
return (T)o;
|
||
else
|
||
return error_val;
|
||
} catch {
|
||
return error_val;
|
||
}
|
||
}
|
||
|
||
public static DateTime get_row_datetime_value(IBlueRow r, string col_name, DateTime error_val) {
|
||
try {
|
||
object rv = (r != null && r.Contains(col_name)) ? r[col_name] : error_val;
|
||
return _get_datetime_value(rv, error_val);
|
||
} catch {
|
||
return error_val;
|
||
}
|
||
}
|
||
|
||
public static DateTime get_row_datetime_value(IBlueRow r, int col_no, DateTime error_val) {
|
||
try {
|
||
object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : error_val;
|
||
return _get_datetime_value(rv, error_val);
|
||
} catch {
|
||
return error_val;
|
||
}
|
||
}
|
||
|
||
private static DateTime _get_datetime_value(object v, DateTime error_val) {
|
||
try {
|
||
if (v == null || v == DBNull.Value)
|
||
return error_val;
|
||
|
||
if (v is DateTime) {
|
||
return (DateTime)v;
|
||
} else if (v is string) {
|
||
DateTime rv_dt;
|
||
|
||
// Try parsing the string as a DateTime
|
||
if (DateTime.TryParse((string)v,
|
||
System.Globalization.CultureInfo.InvariantCulture,
|
||
System.Globalization.DateTimeStyles.RoundtripKind,
|
||
out rv_dt)) {
|
||
return rv_dt;
|
||
} else {
|
||
return error_val; // Return the error value if parsing fails
|
||
}
|
||
} else {
|
||
// Assume the input can be converted to a double and use FromOADate
|
||
double oa_date = (double)Convert.ChangeType(v, typeof(Double));
|
||
return DateTime.FromOADate(oa_date);
|
||
}
|
||
} catch {
|
||
return error_val;
|
||
}
|
||
}
|
||
|
||
public static TimeSpan? get_timespan(this IBlueRow r, int col_no) =>
|
||
get_timespan(r[col_no]);
|
||
|
||
public static TimeSpan? get_timespan(this IBlueRow r, string col_name) =>
|
||
get_timespan(r[col_name]);
|
||
|
||
public static TimeSpan? get_timespan(object obj) {
|
||
if (obj is TimeSpan ts) {
|
||
return ts;
|
||
}
|
||
var str = obj as string ?? obj?.ToString();
|
||
if (str is null) {
|
||
return null;
|
||
}
|
||
if (TimeSpan.TryParse(str, out ts)) {
|
||
return ts;
|
||
}
|
||
if (str.StartsWith("PT")) {
|
||
try {
|
||
// Parse ISO 8601 duration
|
||
return System.Xml.XmlConvert.ToTimeSpan(str);
|
||
} catch {
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
#endregion
|
||
|
||
#region CSV methods
|
||
/// <summary>
|
||
/// Reads a schema file name, and return the constructed schema
|
||
/// </summary>
|
||
public static IBlueTableSchema createSchemaFromCSVSchemaFile(string schemaFileName) {
|
||
StreamReader sr = new StreamReader(schemaFileName);
|
||
IBlueTableSchema retVal = createSchemaFromCSVSchemaFile(sr);
|
||
sr.Close();
|
||
return retVal;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Builds a schema from a schema file. Note that the stream is not closed, so
|
||
/// the calling code should do so
|
||
/// </summary>
|
||
public static IBlueTableSchema createSchemaFromCSVSchemaFile(TextReader sr) {
|
||
string tableName = sr.ReadLine();
|
||
ArrayList t = new ArrayList();
|
||
string l = "";
|
||
while ((l = sr.ReadLine()) != null)
|
||
t.Add(l.Trim());
|
||
string[] cols = t.ToArray(typeof(string)) as String[];
|
||
ColumnInfo[] cis = new ColumnInfo[cols.Length];
|
||
for (int i = 0; i < cols.Length; i++) {
|
||
string[] colTags = cols[i].Split('/');
|
||
cis[i] = new ColumnInfo(colTags[0], i, TypeInstanceLoader.loadType(colTags[1]));
|
||
}
|
||
IBlueTableSchema retVal = new BlueTableSchema(cis, tableName);
|
||
if (retVal["id"] == null) {
|
||
ArrayList temp = new ArrayList(retVal.columnInfo);
|
||
temp.Add(new ColumnInfo("id", temp.Count, typeof(System.Int32)));
|
||
retVal = new BlueTableSchema(retVal.tableName,
|
||
temp.ToArray(typeof(ColumnInfo)) as ColumnInfo[]);
|
||
}
|
||
return retVal;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Creates an IBlueRow from a CSV data file row; if the string already contains an id,
|
||
/// the id argument is ignored
|
||
/// </summary>
|
||
public static IBlueRow createRowFromCSVFileRow(string fileRow, IBlueTableSchema schema, int id) {
|
||
string[] toks = CSVParser.parse(fileRow);
|
||
ArrayList rowList = new ArrayList();
|
||
if (toks.Length == schema.columnNames.Length) {
|
||
for (int i = 0; i < toks.Length; i++)
|
||
rowList.Add(myConvertTo(schema[i].colType, toks[i]));
|
||
} else {
|
||
int idIndex = schema["id"].colNo;
|
||
for (int i = 0; i < toks.Length; i++)
|
||
rowList.Add(myConvertTo(schema[i < idIndex ? i : i + 1].colType, toks[i]));
|
||
rowList.Insert(idIndex, id);
|
||
}
|
||
return new BlueRow(schema, rowList.ToArray());
|
||
}
|
||
|
||
/// <summary>
|
||
/// Returns the CSV data string corresponding to the row
|
||
/// </summary>
|
||
public static string toCSVDataString(IBlueRow r) {
|
||
string retVal = "";
|
||
for (int i = 0; i < r.schema.columnNames.Length; i++)
|
||
retVal +=
|
||
r.schema[i].colType.Equals(typeof(string)) ?
|
||
"\"" + r[i].ToString().Replace("\"", "\"\"") + "\"" :
|
||
r[i].ToString();
|
||
return retVal.Remove(retVal.Length - 1, 1);
|
||
}
|
||
|
||
public static string[] toCSVSchemaFileStrings(IBlueTableSchema schema) {
|
||
string[] retVal = new string[2];
|
||
retVal[0] = schema.tableName;
|
||
foreach (ColumnInfo ci in schema.columnInfo)
|
||
retVal[1] += ci.colName + "/" + ci.colType.ToString() + " ";
|
||
retVal[1] = retVal[1].Remove(retVal[1].Length - 1, 1);
|
||
return retVal;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Converts a CSV data value to its original type.
|
||
/// </summary>
|
||
private static object myConvertTo(Type t, string val) {
|
||
val = val.Trim();
|
||
if (t.Equals(typeof(System.String)))
|
||
return val;
|
||
else if (val != "") {
|
||
if (t.Equals(typeof(System.Char)))
|
||
return Convert.ToChar(val);
|
||
if (t.Equals(typeof(System.Int16)))
|
||
return Convert.ToInt16(val);
|
||
if (t.Equals(typeof(System.Int32)))
|
||
return Convert.ToInt32(val);
|
||
if (t.Equals(typeof(System.Int64)))
|
||
return Convert.ToInt64(val);
|
||
if (t.Equals(typeof(System.Double)))
|
||
return Convert.ToDouble(val);
|
||
if (t.Equals(typeof(System.Single)))
|
||
return Convert.ToSingle(val);
|
||
if (t.Equals(typeof(System.DateTime)))
|
||
return myToDateTime(val);
|
||
}
|
||
return DBNull.Value;
|
||
}
|
||
|
||
private static object myToDateTime(string val) {
|
||
try {
|
||
return Convert.ToDateTime(val);
|
||
} catch (FormatException) //datetime is not stored in one of the string formats
|
||
{
|
||
return val.IndexOf('.') > -1 ?
|
||
DateTime.FromOADate(Convert.ToDouble(val)) : new DateTime(Convert.ToInt64(val));
|
||
}
|
||
}
|
||
|
||
public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) =>
|
||
createRowsFromCSVFileWithoutTypes(fileAddress, out var _);
|
||
|
||
public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress, out IBlueTableSchema schema) {
|
||
StreamReader sr = new StreamReader(fileAddress);
|
||
string cols = sr.ReadLine();
|
||
string[] columns = cols.Split(',');
|
||
ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
|
||
ci[0] = new ColumnInfo("id", 0, typeof(int));
|
||
for (int i = 1; i < ci.Length; i++)
|
||
ci[i] = new ColumnInfo(columns[i - 1], i, typeof(string));
|
||
schema = new BlueTableSchema(ci, "csvTable");
|
||
BlueTable result = new BlueTable(schema);
|
||
int id = 0;
|
||
while (!sr.EndOfStream)
|
||
result.addRow(createRowFromCSVFileRow(sr.ReadLine(), schema, id++));
|
||
sr.Close();
|
||
return result;
|
||
}
|
||
|
||
#endregion
|
||
|
||
#region Magic
|
||
/// <summary>
|
||
/// Uses a cached compiled expression to implement a truly generic add function. Will throw if given types that cannot be added.
|
||
/// </summary>
|
||
internal class MagicAdder<T> {
|
||
static Func<T, T, T> add;
|
||
static MagicAdder() {
|
||
ParameterExpression left = Expression.Parameter(typeof(T), "a");
|
||
ParameterExpression right = Expression.Parameter(typeof(T), "b");
|
||
add = (Func<T, T, T>)Expression.Lambda(Expression.Add(left, right), left, right).Compile();
|
||
}
|
||
|
||
public static T Add(T a, T b) {
|
||
return add(a, b);
|
||
}
|
||
}
|
||
#endregion
|
||
|
||
#region Misc
|
||
|
||
public static void swap<T>(ref T t1, ref T t2) {
|
||
T temp = t1;
|
||
t1 = t2;
|
||
t2 = temp;
|
||
}
|
||
|
||
#endregion
|
||
}
|
||
}
|