using System;
using System.Data;
using System.Collections;
using System.Collections.Generic;
using System.Reflection;
using System.IO;
using System.Diagnostics;
using System.Linq;

//using BCG.Utilities;
using BCG.Utilities.General;
using System.Linq.Expressions;
//using mbts.spark;
using mbts;
using BCG.Extensions;
using System.Text.Json;
using System.Text.Json.Nodes;
using Newtonsoft.Json;
using BCG;
using System.Runtime.CompilerServices;
using bts;
using Spark.Core;
using System.Numerics;

namespace bts {

   // public class CompositeRow : IBlueRow {
   //    private IBlueRow _lrow, _rrow;
   //    private BlueRow _out_row;
   //    public CompositeRow(IBlueRow lrow, IBlueRow rrow) {
   //       _lrow = lrow;
   //       _rrow = rrow;
   //       IBlueTableSchema schema = _lrow.schema;
   //       List<object> vals = new List<object>();
   //       foreach (var o in _lrow) {
   //          vals.Add(o);
   //       }
   //       foreach (ColumnInfo ci in _rrow.schema.columnInfo) {
   //          if (ci.colName == "id") {
   //             continue;
   //          }
   //          vals.Add(_rrow[ci.colNo]);
   //          schema = schema.add_column(ci.type_code(), ci.colName);
   //       }
   //       _out_row = new BlueRow(schema, vals.ToArray());
   //    }
   //    public IBlueTableSchema schema {
   //       get { return _out_row.schema; }
   //    }
   //    public object? this[int index] {
   //       get { return _out_row[index]; }
   //    }
   //    public object? this[string colName] {
   //       get {
   //          return _out_row[colName];
   //       }
   //    }

   // }
}

namespace BCG {
   public static class BlueDataUtility {
      public static IBlueRow create_composite_row(IBlueRow lr, IBlueRow rr, int id) {
         IBlueTableSchema schema = lr.schema;
         List<object> vals = new List<object>();
         foreach (ColumnInfo ci in lr.schema.columnInfo) {
            if (ci.colName == "id") {
               vals.Add(id);
            }
            vals.Add(lr[ci.colName]);
         }
         foreach (ColumnInfo ci in rr.schema.columnInfo) {
            if (ci.colName == "id") {
               continue;
            }
            schema = schema.add_column(ci.type_code(), ci.colName);
            vals.Add(rr[ci.colName]);
         }
         IBlueRow r = new BlueRow(schema, vals.ToArray());
         return r;
      }
      #region Extensions
      public static bool is_null(this IBlueRow r, string col_name) {
         object col_val = r[col_name];
         return col_val == null || DBNull.Value.Equals(col_val);
      }

      public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) {
         return BlueDataUtility.get_row_datetime_value(r, col_name, err_value);
      }
      public static DateTime get_datetime(this IBlueRow r, int col_no, DateTime err_value = default(DateTime)) {
         return BlueDataUtility.get_row_datetime_value(r, col_no, err_value);
      }

      public static double? GetFloat64(this IBlueRow r, int colNo) =>
         GetFloat64(r[colNo]);

      public static double? GetFloat64(this IBlueRow r, string colName) =>
         GetFloat64(r[colName]);

      public static double? GetFloat64(object obj) {
         if (obj is double d) {
            return d;
         }
         if (obj is int i32) {
            return i32;
         }
         if (obj is long i64) {
            return i64;
         }
         if (obj is string s) {
            return double.TryParse(s, out d) ? d : null;
         }
         if (obj is uint u32) {
            return u32;
         }
         if (obj is ulong u64) {
            return u64;
         }
         if (obj is short i16) {
            return i16;
         }
         if (obj is ushort u16) {
            return u16;
         }
         return null;
      }

      public static int? GetInt32(this IBlueRow r, int colNo) =>
         GetInt32(r[colNo]);

      public static int? GetInt32(this IBlueRow r, string colName) =>
         GetInt32(r[colName]);

      public static int? GetInt32(object obj) {
         checked {
            if (obj is double d) {
               return (int)d;
            }
            if (obj is int i32) {
               return i32;
            }
            if (obj is long i64) {
               return (int)i64;
            }
            if (obj is string s) {
               return double.TryParse(s, out d) ? (int)d : null;
            }
            if (obj is uint u32) {
               return (int)u32;
            }
            if (obj is ulong u64) {
               return (int)u64;
            }
            if (obj is short i16) {
               return i16;
            }
            if (obj is ushort u16) {
               return u16;
            }
         }
         return null;
      }

      public static long? GetInt64(this IBlueRow r, int colNo) =>
         GetInt64(r[colNo]);

      public static long? GetInt64(this IBlueRow r, string colName) =>
         GetInt64(r[colName]);

      public static long? GetInt64(object obj) {
         checked {
            if (obj is double d) {
               return (long)d;
            }
            if (obj is int i32) {
               return i32;
            }
            if (obj is long i64) {
               return i64;
            }
            if (obj is string s) {
               return double.TryParse(s, out d) ? (long)d : null;
            }
            if (obj is uint u32) {
               return u32;
            }
            if (obj is ulong u64) {
               return (long)u64;
            }
            if (obj is short i16) {
               return i16;
            }
            if (obj is ushort u16) {
               return u16;
            }
         }
         return null;
      }

      public static DateTime? GetUnixEpochDateTime(this IBlueRow row, string colName) =>
         GetUnixEpochDateTime(row[colName]);

      public static DateTime? GetUnixEpochDateTime(this IBlueRow row, int colNo) =>
         GetUnixEpochDateTime(row[colNo]);

      public static DateTime? GetUnixEpochDateTime(object obj) {
         var tsNanos = GetInt64(obj);
         return tsNanos.HasValue ? DateTime.UnixEpoch.Add(new TimeSpan(tsNanos.Value / 100)) : null;
      }

      public static DateTime? GetDateTime(object obj) {
         var res = _get_datetime_value(obj, DateTime.MinValue);
         if (res == DateTime.MinValue) {
            return null;
         }
         return res;
      }

      public static T get_enum<T>(this IBlueRow r, string col_name, T err_value = default(T)) where T : struct {
         return BlueDataUtility.get_row_enum_value<T>(r, col_name, err_value);
      }
      public static T get_enum<T>(this IBlueRow r, int col_no, T err_value = default(T)) where T : struct {
         return BlueDataUtility.get_row_enum_value<T>(r, col_no, err_value);
      }
      public static T get<T>(this IBlueRow r, string col_name, T err_value = default(T)) {
         return get_row_value(r, col_name, err_value);
      }
      public static T get<T>(this IBlueRow r, int col_no, T err_value = default(T)) {
         return get_row_value(r, col_no, err_value);
      }
      public static IEnumerable<T> get_column_values<T>(this IBlueRowCollection rows, string col) {
         List<T> rv = new List<T>();
         foreach (IBlueRow r in rows)
            rv.Add(r.get<T>(col));
         return rv;
      }
      public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col) {
         List<T> rv = new List<T>();
         foreach (IBlueRow r in rows)
            rv.Add(r.get<T>(col));
         return rv;
      }
      public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col, int start_index, int count) {
         List<T> rv = new List<T>();
         int i = 0;
         int n_copied = 0;
         foreach (IBlueRow r in rows)
            if (i++ >= start_index && n_copied < count) {
               rv.Add(r.get<T>(col));
               n_copied++;
            }
         return rv;
      }

      public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
         Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
         foreach (IBlueRow r in rows) {
            var gp_val = r.get<T>(gp_col);
            List<IBlueRow> gp_val_list = null;
            if (!rv.TryGetValue(gp_val, out gp_val_list)) {
               gp_val_list = new List<IBlueRow>();
               rv.Add(gp_val, gp_val_list);
            }
            gp_val_list.Add(r);
         }
         return rv.ToDictionary(kv => kv.Key, kv => kv.Value as IEnumerable<IBlueRow>);
      }

      // this function takes in a key value table and converts it into a relational table, converting the key names into columns.
      // The values are returned as string values
      // doc_col is the column specifyng the doc name (following mongoDB's nomenclature, one set of key value rows ar a 'doc')
      public static IBlueRowCollection keyvalue_to_relational<T>(IBlueRowCollection kv_rows, string doc_col, string key_col, string val_col) {
         HashSet<string> keys = new HashSet<string>();
         foreach (string col in get_column_values<string>(kv_rows, key_col)) {
            keys.Add(col);
         }
         if (keys.Contains("id")) {
            keys.Remove("id");
         }
         //unique_cols.Add(doc_col)
         // all the keys become cols
         var schema_cols = keys.Select(c => Tuple.Create(c, TypeCode.String)).ToList();
         //schema of the final table needs doc and id
         schema_cols.Insert(0, Tuple.Create(doc_col, BlueDataUtility.get_typecode(typeof(T))));
         schema_cols.Insert(0, Tuple.Create("id", TypeCode.Int32));
         var schema = BlueDataUtility.form_schema("relational", schema_cols.ToArray());
         BlueTable rv = new BlueTable(schema);
         var gp_map = group_by<T>(kv_rows, doc_col);
         foreach (var kv in gp_map) {
            MutableBlueRow nr = new MutableBlueRow(schema);
            nr.setValue("id", rv.Count);
            nr.setValue(doc_col, kv.Key);
            foreach (var r in kv.Value) {
               var col = r.get<string>(key_col);
               if (!keys.Contains(col))
                  continue;
               var val = r.get<string>(val_col);
               nr.setValue(col, val);
            }
            rv.addRow(nr);
         }
         return rv;
      }

      public static IBlueRowCollection where(this IBlueRowCollection rows, Func<IBlueRow, bool> where_func) {
         if (rows == null)
            return null;
         else {
            BlueTable rv = new BlueTable(rows.schema);
            foreach (IBlueRow r in rows)
               if (where_func(r))
                  rv.addRow(r);
            return rv;
         }
      }

      public static List<IBlueRow> to_list(this IBlueRowCollection rows) {
         List<IBlueRow> rv = new List<IBlueRow>();
         foreach (IBlueRow r in rows)
            rv.Add(r);
         return rv;
      }

      public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
         return rows.to_list();
      }

      public static IDictionary<int, IBlueRow> to_map(this IBlueRowCollection rows) {
         Dictionary<int, IBlueRow> rv = new Dictionary<int, IBlueRow>();
         foreach (IBlueRow r in rows) {
            var id = r.get<int>("id");
            rv.Add(id, r);
         }
         return rv;
      }

      public static bool contains_column(this IBlueTableSchema input_schema, string col) {
         return input_schema.columnNames.Contains(col);
      }

      public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) {
         List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
         foreach (Tuple<Type, string> c in cols)
            cis.Add(new ColumnInfo(c.Item2, cis.Count, c.Item1));
         return new BlueTableSchema(input_schema.tableName, cis.ToArray());
      }

      public static IBlueTableSchema add_column(this IBlueTableSchema input_schema, TypeCode type_arg, string col_name) {
         Tuple<Type, string> tup = new Tuple<Type, string>(_typecode_to_type[type_arg], col_name);
         return add_columns(input_schema, tup);
      }

      public static IBlueRowCollection to_collection(this IBlueRow r) {
         if (r == null)
            return null;
         else {
            BlueTable rv = new BlueTable(r.schema);
            rv.addRow(r);
            return rv;
         }
      }

      public static DataPacket build_dp(IBlueRowCollection new_rows, DataPacket last_fired_dp) {
         IBlueRowCollection added_rows = new_rows;
         IBlueRowCollection deleted_rows = last_fired_dp == null ? new BlueTable(new_rows.schema) : last_fired_dp.newRows;
         return new DataPacket(new_rows, added_rows, deleted_rows);
      }

      public static DataPacket build_dp_inc(IBlueRowCollection new_rows, DataPacket last_fired_dp, bool deep_row_comapre = false) {
         if (last_fired_dp == null) {
            IBlueRowCollection added_rows = new_rows;
            IBlueRowCollection deleted_rows = new BlueTable(new_rows.schema);
            return new DataPacket(new_rows, added_rows, deleted_rows);
         } else {
            IBlueRowCollection added_rows = null;
            IBlueRowCollection deleted_rows = null;
            if (deep_row_comapre)
               compute_differences_fast(new_rows, last_fired_dp.newRows, out deleted_rows, out added_rows);
            else
               compute_differences_id_based(new_rows, last_fired_dp.newRows, out deleted_rows, out added_rows);
            return new DataPacket(new_rows, added_rows, deleted_rows);
         }
      }

      public static IBlueRowCollection adapt_for_sql_storage(this IBlueRowCollection rows,
                                                             bool handle_datetime_min = false,
                                                             bool handle_double_nan = false) {
         if (rows != null) {
            BlueTable rv = new BlueTable(rows.schema);
            foreach (IBlueRow r in rows) {
               MutableBlueRow mr = new MutableBlueRow(r);
               rv.addRow(mr);
               foreach (ColumnInfo ci in rows.schema.columnInfo) {
                  if (handle_datetime_min && ci.colType == typeof(DateTime) && DateTime.MinValue.Equals(r[ci.colName]))
                     mr.setValue(ci.colName, DBNull.Value);
                  if (handle_double_nan && ci.colType == typeof(double) && Double.IsNaN(r.get<double>(ci.colName)))
                     mr.setValue(ci.colName, DBNull.Value);
               }
            }
            return rv;
         } else
            return null;
      }

      #endregion

      #region json

      public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
         Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
         return dictionary;
      }
      public static IBlueRow convert_json_to_row(string json_string, int id = 0) {
         Dictionary<string, object> dictionary = JsonConvert.DeserializeObject<Dictionary<string, object>>(json_string);
         if (!dictionary.ContainsKey("id")) {
            dictionary.Add("id", id);
         }
         return new BlueRow(dictionary);
      }

      public static List<JsonNode> json_to_json_node_list(string jsonString) {
         var jsonNodeList = new List<JsonNode>();
         bool noe = String.IsNullOrEmpty(jsonString);
         if (noe)
            return jsonNodeList;
         try {
            // Parse the JSON string
            JsonArray jsonArray = JsonNode.Parse(jsonString) as JsonArray;

            if (jsonArray == null) {
               Console.WriteLine("The JSON string must represent a list.");
               return jsonNodeList;
            }

            // Iterate through each JsonNode in the JsonArray
            foreach (JsonNode jsonNode in jsonArray) {
               jsonNodeList.Add(jsonNode);
            }
         } catch (System.Text.Json.JsonException ex) {
            Console.WriteLine($"Invalid JSON: {ex.Message}");
         }

         return jsonNodeList;
      }

      #endregion

      public static void print_table(this IBlueRowCollection table, Func<string, int> get_col_width) {
         var rv = get_printed_rows(table, get_col_width);
         Console.WriteLine(rv);
      }

      public static string get_printed_rows(this IBlueRowCollection table, Func<string, int> get_col_width) {
         // Print column headers
         string rv = String.Empty;// = $"{table.schema.tableName}______________________________________";
         foreach (var col_name in table.schema.columnNames) {
            int col_width = get_col_width(col_name);
            rv += $"{col_name.PadRight(col_width)}  ";
         }
         rv += "\n";
         //rv += "______________________________________";

         // Print table data
         foreach (IBlueRow row in table) {
            foreach (string col_name in table.schema.columnNames) {
               int col_width = get_col_width(col_name);
               object row_val = row[col_name];
               string row_val_str = row_val.ToString();
               if (row_val != null) {
                  rv += $"{row_val_str.PadRight(col_width)} ";
               }
            }
            rv += "\n";
         }
         return rv;
      }
      #region Schema types
      // these are the only types allowed in schemas
      private static IReadOnlyDictionary<TypeCode, Type> _typecode_to_type = new Dictionary<TypeCode, Type>
      {
         { TypeCode.Boolean, typeof(bool) }, //3
         { TypeCode.Byte, typeof(byte) },    //6
         { TypeCode.Char, typeof(char) },    //4
         // TODO
         { TypeCode.DateTime, typeof(DateTime) },//16
         { TypeCode.Decimal, typeof(decimal) }, //15
         { TypeCode.Double, typeof(double) },   //14
         //{ TypeCode.Empty, null },
         { TypeCode.Int16, typeof(short) },     //7
         { TypeCode.Int32, typeof(int) },       //9
         { TypeCode.Int64, typeof(long) },      //11
         { TypeCode.SByte, typeof(sbyte) },     //5
         { TypeCode.Single, typeof(Single) },   //13   
         { TypeCode.String, typeof(string) },   //18
         { TypeCode.UInt16, typeof(UInt16) },    //8
         { TypeCode.UInt32, typeof(UInt32) },    //10
         { TypeCode.UInt64, typeof(UInt64) }    //12
      };

      public static IEnumerable<TypeCode> valid_column_typecodes() {
         return _typecode_to_type.Keys;
      }

      public static IEnumerable<Type> valid_column_types() {
         return _typecode_to_type.Values;
      }

      public static bool is_valid_col_type(Type t) {
         TypeCode type_code = get_typecode(t);
         return type_code != TypeCode.Empty;
      }

      public static TypeCode get_typecode(Type t) {
         foreach (var kv in _typecode_to_type) {
            if (kv.Value == t)
               return kv.Key;
         }
         //bts.Log.assert_d(false, $"type: {t} not found");
         return TypeCode.Empty;
      }

      public static Type get_type(TypeCode type_code) {
         Type rv = null;
         if (_typecode_to_type.TryGetValue(type_code, out rv)) {
            return rv;
         }
         return null;
      }

      public static TypeCode type_code(this ColumnInfo ci) {
         return get_typecode(ci.colType);
      }

      #endregion

      public static Tuple<Type, object> guess_type(string value) {
         if (string.IsNullOrEmpty(value)) {
            return Tuple.Create(typeof(string), (object)value);
         }

         int val = 0;
         // Try to parse as integer
         if (int.TryParse(value, out val)) {
            return Tuple.Create(typeof(int), (object)val);
         }

         double dval = 0.0;
         // Try to parse as double
         if (double.TryParse(value, out dval)) {
            return Tuple.Create(typeof(double), (object)dval);
         }

         DateTime dt;
         // Try to parse as DateTime
         if (DateTime.TryParse(value, out dt)) {
            return Tuple.Create(typeof(DateTime), (object)dt);
         }

         bool bval;
         // Try to parse as boolean
         if (bool.TryParse(value, out bval)) {
            return Tuple.Create(typeof(bool), (object)bval);
         }

         // If none of the above, return string type
         return Tuple.Create(typeof(string), (object)value);
      }

      public static IBlueTableSchema form_schema(string table_name, params Tuple<string, TypeCode>[] cols) {
         List<ColumnInfo> cis = new List<ColumnInfo>();
         foreach (Tuple<string, TypeCode> c in cols)
            cis.Add(new ColumnInfo(c.Item1, cis.Count, _typecode_to_type[c.Item2]));
         if (cis.FirstOrDefault(ci => ci.colName == "id") == null)
            cis.Add(new ColumnInfo("id", cis.Count, typeof(int)));
         return new BlueTableSchema(table_name, cis.ToArray());
      }

      public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) {
         // just use one row to guess
         List<ColumnInfo> cis = new List<ColumnInfo>();
         foreach (IBlueRow r in rows) {
            foreach (var ci in r.schema.columnInfo) {
               if (ci.colType != typeof(System.String)) {
                  cis.Add(ci);
                  continue;
               }
               var (type, _) = guess_type(r[ci.colName] as string);
               cis.Add(new ColumnInfo(ci.colName, ci.colNo, type));
            }
            return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
         }
         return null;
      }

      // converts a row where all columns are string types to a schema where the best types are guesed
      public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) {
         IBlueTableSchema typed_schema = get_typed_schema(string_rows);
         var rv = new BlueTable(typed_schema);
         foreach (IBlueRow r in string_rows) {
            var typed_row = get_typed_row(r, typed_schema);
            rv.addRow(typed_row);
         }
         return rv;

         static IBlueRow get_typed_row(IBlueRow string_row, IBlueTableSchema typed_schema) {
            var rv = new MutableBlueRow(typed_schema);
            foreach (var ci in typed_schema.columnInfo) {
               var typed_val = convert(ci.colType, string_row[ci.colName].ToString());
               rv.setValue(ci.colName, typed_val);
            }
            return rv;
         }

         static object convert(Type t, string value) {
            // Attempt to convert the string to the specified type
            try {
               return Convert.ChangeType(value, t);
            } catch (Exception) {
               if (t == typeof(double)) {
                  // some doubles come with currency and ',' we clean that up here (there might be better ways to do this)
                  string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray());
                  cleaned_val = cleaned_val.Replace('−', '-');
                  double rv_d;
                  if (Double.TryParse(cleaned_val, out rv_d))
                     return rv_d;
                  else
                     return null;
               }
               return null;
               // Handle conversion failure gracefully
               // throw new ArgumentException("Failed to convert the string to the specified type.");
            }
         }
      }

      /// <summary>
      /// Gets the Hashcode of the row with the boolean indicating whether the order of the rows should
      /// be alphabetic or not
      /// </summary>
      /// <param name="row"></param>
      /// <param name="isColAlphabetically"></param>
      /// <returns></returns>
      public static int getRowHashCode(IBlueRow row, bool isColAlphabetically) {
         string retVal = "";
         if (isColAlphabetically) {
            string[] colNames = row.schema.columnNames;
            Array.Sort(colNames);
            foreach (string col in colNames)
               retVal += row[col].ToString();
         } else
            foreach (object o in row)
               retVal += o.ToString();
         return retVal.GetHashCode();
      }

      /// <summary>
      /// Finds the hashcode only inluding the columns (in the order) listed in the column names argument
      /// </summary>
      /// <param name="row"></param>
      /// <param name="colNames"></param>
      /// <returns></returns>
      public static int getRowHashCode(IBlueRow row, params string[] colNames) {
         string retVal = "";
         foreach (string colName in colNames)
            retVal += row[colName].ToString();
         return retVal.GetHashCode();
      }

      /// <summary>
      /// Converts a IBlueRowCollection to a .Net DataTable
      /// </summary>
      /// <param name="rows"></param>
      /// <returns></returns>
      public static DataTable toDataTable(IBlueRowCollection rows) {
         DataTable retTable = getTableFromSchema(rows.schema);
         retTable.BeginLoadData();
         foreach (IBlueRow row in rows)
            retTable.Rows.Add(toDataRow(row, retTable));
         retTable.EndLoadData();
         return retTable;
      }

      /// <summary>
      /// Converts to a DataRow, the preconditioner being that order of the columns should be same 
      /// in the schema of the row and the table
      /// </summary>
      /// <param name="row"></param>
      /// <param name="table"></param>
      public static DataRow toDataRow(IBlueRow row, DataTable table) {
         DataRow retRow = table.NewRow();
         foreach (string colName in row.schema.columnNames)
            retRow[colName] = row[colName];
         return retRow;
      }

      /// <summary>
      /// Returns the IBlueTableSchema corresponding to the row
      /// </summary>
      /// <param name="row">
      /// DataRow whose schmea is desired
      /// </param>
      /// <returns></returns>
      public static IBlueTableSchema getSchema(DataRow row) { return getSchema(row.Table); }

      /// <summary>
      /// Returns the IBlueTableSchema of the DataTable
      /// </summary>
      /// <param name="table">
      /// table whos Schema is desired
      /// </param>
      /// <returns></returns>
      public static IBlueTableSchema getSchema(DataTable table) {
         ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
         foreach (DataColumn col in table.Columns)
            cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
         IBlueTableSchema rv = new BlueTableSchema(cis, table.TableName);
         rv = rv.add_id_col_if_missing();
         return rv;
      }
      public static IBlueTableSchema add_id_col_if_missing(this IBlueTableSchema schema) {
         IBlueTableSchema rv = schema;
         if (!schema.columnNames.Contains("id")) {
            rv = schema.add_columns(Tuple.Create(typeof(int), "id"));
         }
         return rv;
      }

      /// <summary>
      /// Constructs the schema from an object, using the SchemaMapping attribute. Only those
      /// properties are converted into schmea columns that have a get accessor
      /// </summary>
      /// <param name="o"></param>
      /// <returns></returns>
      public static IBlueTableSchema getSchema(object o) {
         return getSchema(o.GetType());
      }


      /// <summary>
      /// Builds a new schema based on an existing one + a set of new columns
      /// </summary>
      /// <param name="schema">Base schema</param>
      /// <param name="cis">Columns to add</param>
      public static IBlueTableSchema addColumnsToSchema(IBlueTableSchema schema, params ColumnInfo[] cis) {
         ArrayList rv = new ArrayList(schema.columnInfo);
         rv.AddRange(cis);
         return new BlueTableSchema(schema.tableName, rv.ToArray(typeof(ColumnInfo)) as ColumnInfo[]);
      }

      /// <summary>
      /// appends the cols of s2, to s1, subtracting the id col
      /// </summary>
      /// <param name="s1"></param>
      /// <param name="appendedSchema"></param>
      /// <returns></returns>
      public static IBlueTableSchema appendSchemas(IBlueTableSchema s1, IBlueTableSchema appendedSchema) {
         List<ColumnInfo> cis = new List<ColumnInfo>(s1.columnInfo);
         foreach (ColumnInfo c in appendedSchema.columnInfo)
            if (c.colName != "id")
               cis.Add(new ColumnInfo(c.colName, cis.Count, c.colType));
         return new BlueTableSchema(s1.tableName, cis.ToArray());
      }


      public static IBlueTableSchema getSchema(Type t) {
         return getSchema(t, false);
      }


      public static IBlueTableSchema getSchema(Type t, bool createIdField) {
         return getSchema(t, t.Name, createIdField);
      }
      public static IBlueTableSchema getSchema(Type t, string tableName, bool createIdField) {
         ArrayList colInfos = new ArrayList();
         int i = 0;
         foreach (PropertyInfo pi in t.GetProperties())
            if (myIsActiveProperty(pi))
               colInfos.Add(new ColumnInfo(pi.Name, i++,
                  is_valid_col_type(pi.PropertyType) ?
                  pi.PropertyType : typeof(String)));
         foreach (FieldInfo fi in t.GetFields())
            if (myIsActiveField(fi))
               colInfos.Add(new ColumnInfo(fi.Name, i++,
                  is_valid_col_type(fi.FieldType) ? fi.FieldType : typeof(String)));

         if (createIdField)
            colInfos.Add(new ColumnInfo("id", i++, typeof(int)));

         return new BlueTableSchema(colInfos.ToArray(typeof(ColumnInfo)) as ColumnInfo[],
            tableName);
      }

      public static IBlueTableSchema getSchema(Type t, string tableName) {
         return new BlueTableSchema(getSchema(t).columnInfo, tableName);
      }


      public static IBlueRow createRowFromType(IBlueTableSchema schema, object o) {
         return createRowFromType(schema, o, null);
      }

      //creates a row from the given type, and if idVal is not null, using the given id
      public static IBlueRow createRowFromType(IBlueTableSchema schema, object o, object idVal) {
         object[] rowArray = new object[schema.columnInfo.Length];
         Type t = o.GetType();
         foreach (PropertyInfo pi in t.GetProperties())
            if (myIsActiveProperty(pi))
               rowArray[schema[pi.Name].colNo] =
                  is_valid_col_type(pi.PropertyType) ?
                  pi.GetValue(o, null) : pi.GetValue(o, null).ToString();
         foreach (FieldInfo fi in t.GetFields())
            if (myIsActiveField(fi))
               rowArray[schema[fi.Name].colNo] =
                  is_valid_col_type(fi.FieldType) ?
                  fi.GetValue(o) : fi.GetValue(o).ToString();
         if (idVal != null)
            rowArray[schema["id"].colNo] = idVal;
         return new BlueRow(schema, rowArray);
      }

      public static MutableBlueRow create_mutable_row(IBlueTableSchema schema, object o, object idVal,
                                                      Func<Type, object, object> type_converter) {
         object[] rowArray = new object[schema.columnInfo.Length];
         Type t = o.GetType();
         foreach (PropertyInfo pi in t.GetProperties())
            if (myIsActiveProperty(pi))
               rowArray[schema[pi.Name].colNo] =
                  is_valid_col_type(pi.PropertyType) ?
                  pi.GetValue(o, null) : type_converter(pi.PropertyType, pi.GetValue(o, null));
         foreach (FieldInfo fi in t.GetFields())
            if (myIsActiveField(fi))
               rowArray[schema[fi.Name].colNo] =
                  is_valid_col_type(fi.FieldType) ?
                  fi.GetValue(o) : type_converter(fi.FieldType, fi.GetValue(o));
         if (idVal != null)
            rowArray[schema["id"].colNo] = idVal;
         return new MutableBlueRow(schema, rowArray);
      }


      public static IBlueTableSchema get_schema(Type t, string tableName, bool createIdField, Func<Type, Type> type_converter) {
         List<ColumnInfo> colInfos = new List<ColumnInfo>();
         int i = 0;
         if (createIdField)
            colInfos.Add(new ColumnInfo("id", i++, typeof(int)));

         foreach (PropertyInfo pi in t.GetProperties())
            colInfos.Add(new ColumnInfo(pi.Name, i++, type_converter(pi.PropertyType)));
         foreach (FieldInfo fi in t.GetFields())
            colInfos.Add(new ColumnInfo(fi.Name, i++, type_converter(fi.FieldType)));
         return new BlueTableSchema(colInfos.ToArray(), tableName);
      }

      public static MutableBlueRow create_mutable_row_not_slow(IBlueTableSchema schema, FieldInfo[] fields, PropertyInfo[] props,
                                                               object o, object idVal, Func<Type, object, object> data_converter) {
         object[] rowArray = new object[schema.columnInfo.Length];
         if (props != null) {
            foreach (PropertyInfo pi in props)
               rowArray[schema[pi.Name].colNo] = data_converter(pi.PropertyType, pi.GetValue(o, null));
         }
         if (fields != null) {
            foreach (FieldInfo fi in fields)
               rowArray[schema[fi.Name].colNo] = data_converter(fi.FieldType, fi.GetValue(o));
         }
         if (idVal != null)
            rowArray[schema["id"].colNo] = idVal;
         return new MutableBlueRow(schema, rowArray);
      }

      public static T createTypeFromRow<T>(IBlueRow row) {
         return (T)createTypeFromRow(row, typeof(T));
      }

      public static object createTypeFromRow(IBlueRow row, Type t) {
         object outValue = Activator.CreateInstance(t);

         PropertyInfo[] properties = t.GetProperties();

         foreach (PropertyInfo pi in properties) {
            if (myIsActiveProperty(pi))
               if (row.Contains(pi.Name))
                  if (row[pi.Name].Equals(DBNull.Value) && !pi.PropertyType.Equals(typeof(Object)))
                     pi.SetValue(outValue, null, null);
                  else
                     pi.SetValue(outValue, row[pi.Name], null);
         }
         FieldInfo[] fields = t.GetFields();
         foreach (FieldInfo fi in fields) {
            if (myIsActiveField(fi))
               if (row.Contains(fi.Name))
                  if (row[fi.Name].Equals(DBNull.Value) && !fi.FieldType.Equals(typeof(Object)))
                     fi.SetValue(outValue, null);
                  else
                     fi.SetValue(outValue, row[fi.Name]);
         }

         return outValue;

      }

      //these functions should originally have been made public
      //but changing that would change a bunch of code
      public static bool isActiveProperty(PropertyInfo pi) {
         return myIsActiveProperty(pi);
      }

      public static bool isActiveField(FieldInfo fi) {
         return myIsActiveField(fi);
      }

      private static bool myIsActiveProperty(PropertyInfo pi) {
         if (pi.GetGetMethod() == null)
            return false;
         foreach (Attribute att in pi.GetCustomAttributes(true))
            if (att is SchemaVisibility && !(att as SchemaVisibility).visible)
               return false;
         return true;
      }

      private static bool myIsActiveField(FieldInfo fi) {
         if (!fi.IsPublic)
            return false;
         foreach (Attribute att in fi.GetCustomAttributes(true))
            if (att is SchemaVisibility && !(att as SchemaVisibility).visible)
               return false;
         return true;
      }

      private static bool myIsActiveMember(MemberInfo mi) {
         if (!(mi is FieldInfo) ||
            !(mi is PropertyInfo) ||
            (mi is PropertyInfo && (mi as PropertyInfo).GetGetMethod() == null) ||
            (mi is FieldInfo && !(mi as FieldInfo).IsPublic))
            return false;
         foreach (Attribute att in mi.GetCustomAttributes(true))
            if (att is SchemaVisibility && !(att as SchemaVisibility).visible)
               return false;
         return true;
      }

      private static bool myDoesRowExistsInTable(IBlueRowCollection brc2, string[] columns, object[] oarray) {
         bool rowExists = false;
         foreach (IBlueRow br in brc2) {
            bool allElementsSame = true;
            for (int i = 0; i < columns.Length; i++) {
               string col = columns[i];
               object o1 = br[col];
               object o2 = oarray[i];
               allElementsSame = (o1.Equals(o2) == false) ? false : allElementsSame;
            }
            if (allElementsSame == true) {
               rowExists = true;
               break;
            }
         }
         return rowExists;
      }

      /// <summary>
      /// Constructs a DataTable from the IBlueTableSchema
      /// </summary>
      /// <param name="schema">
      /// Schema whose table needs to be created
      /// </param>
      /// <returns></returns>
      public static DataTable getTableFromSchema(IBlueTableSchema schema) {
         DataTable retTable = new DataTable(schema.tableName);
         foreach (ColumnInfo ci in schema.columnInfo)
            retTable.Columns.Add(ci.colName, ci.colType);
         return retTable;
      }

      /// <summary>
      /// Constructs a BlueRow from a DataRow.
      /// </summary>
      /// <param name="row">source row</param>
      /// <returns></returns>
      public static IBlueRow toBlueRow(DataRow row) {
         return new BlueRow(getSchema(row), row);
      }

      public static IBlueRowCollection toIBlueRowCollection(DataTable dt) {
         IBlueTableSchema schema = getSchema(dt);
         BlueTable rv = new BlueTable(schema);
         foreach (DataRow dr in dt.Rows) {
            MutableBlueRow nr = new MutableBlueRow(schema);
            foreach (DataColumn col in dr.Table.Columns)
               nr.setValue(col.ColumnName, dr[col.ColumnName]);
            nr.setValue("id", rv.Count);
            rv.addRow(nr);
         }
         return rv;
      }

      public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) {
         BlueTable retVal = null;
         foreach (DataRow dr in rows) {
            IBlueRow r = toBlueRow(dr);
            if (retVal == null)
               retVal = new BlueTable(r.schema);
            retVal.addRow(r);
         }
         return retVal;
      }

      /// <summary>
      /// Returns the nth row of the IBlueRowCollection. This method could be used
      /// for any of the nonRandom-Access-IBlueRowCollection-Implementations, as its
      /// O(n)
      /// </summary>
      /// <param name="rows"></param>
      /// <param name="index"></param>
      /// <returns></returns>
      public static IBlueRow extractRow(IBlueRowCollection rows, int index) {
         int i = 0;
         foreach (IBlueRow row in rows)
            if (i++ == index)
               return row;
         return null;
      }

      /// <summary>
      /// Computes the equality fo the 2 rows. The rows might have different columns orderings
      /// </summary>
      /// <param name="r1"></param>
      /// <param name="r2"></param>
      /// <returns></returns>


      /// <summary>
      /// Computes a string equivelant for the row, using the columns provided
      /// in the argument
      /// </summary>
      /// <param name="row"></param>
      /// <param name="colNames"></param>
      /// <returns></returns>
      public static string rowToString(IBlueRow row, params string[] colNames) {
         string retVal = "";
         foreach (string colName in colNames)
            retVal += (row[colName] != null ? row[colName].ToString().Trim() : "NULL") + "\n";
         return retVal;
      }

      public static string rowToString(IBlueRow row) { return rowToString(row, row.schema.columnNames); }

      public static string rowCollectionToString(IBlueRowCollection rowColl) {
         if (rowColl == null)
            return "Null IBlueRowCollection";
         else {
            string retVal = "";
            foreach (IBlueRow r in rowColl)
               retVal += retVal == "" ? rowToString(r) : "\n" + rowToString(r);
            return retVal;
         }
      }

      /// <summary>
      /// Compares 2 rows, using the column names in the order provided, ie the column name with
      /// the lower index in the array has higher priority. Also a null is always assumed
      /// to be less than a non-null
      /// </summary>
      /// <param name="r1"></param>
      /// <param name="r2"></param>
      /// <param name="col_names"></param>
      /// <returns></returns>
      public static int compare_rows(IBlueRow r1,
         IBlueRow r2,
         params string[] col_names) {
         int comp_res = 0;
         foreach (string cn in col_names) {
            object val1 = r1[cn];
            object val2 = r2[cn];
            comp_res = _compare_row_cols(val1, val2);
            if (comp_res != 0)
               break;
         }
         return comp_res;
      }

      private static int _compare_row_cols(object v1, object v2) {
         if (_is_null(v1))
            return _is_null(v2) ? 0 : -1;
         else {
            if (_is_null(v2))
               return 1;
            else {
               if (v1 is IComparable && v2 is IComparable)
                  return (v1 as IComparable).CompareTo(v2);
               else if (v1 is byte[] && v2 is byte[])
                  return _compare_bytes(v1 as byte[], v2 as byte[]) ? 0 : 1;
               else
                  return 1;
            }
         }
      }

      private static bool _compare_bytes(byte[] v1, byte[] v2) {
         if (v1.Length != v2.Length)
            return false;
         else
            for (int i = 0; i < v1.Length; i++)
               if (v1[i] != v2[i])
                  return false;
         return true;
      }

      private static bool _is_null(object o) { return o == null || o == DBNull.Value; }
      /// <summary>
      /// Creates a new IBlueRow with a schema matching the schema provided in 'new schema'
      /// </summary>
      /// <param name="inRow">a row with the values to be converted</param>
      /// <param name="newSchema">the desired schema for the output row</param>
      /// <returns></returns>
      public static IBlueRow changeSchema(IBlueRow inRow, IBlueTableSchema newSchema) {
         object[] outputValues = new object[newSchema.columnInfo.Length];

         for (int i = 0; i < outputValues.Length; ++i)
            outputValues[i] = inRow[newSchema.columnNames[i]];

         return new BlueRow(newSchema, outputValues);
      }

      /// <summary>
      /// Combines input tables into a single output table, with different ID's
      /// this will result in new rows being created, which is kind of expensive
      /// </summary>
      /// <param name="tables"></param>
      /// <returns></returns>
      public static IBlueRowCollection combineTables(params IBlueRowCollection[] tables) {
         //get some base cases out of the way
         if (tables.Length == 0)
            return null;
         if (tables.Length == 1)
            return tables[0];

         BlueTable result = new BlueTable(tables[0].schema);
         int idToUseNext = 0;
         int numCols = tables[0].schema.columnInfo.Length;
         int idCol = tables[0].schema["id"].colNo;
         IBlueRowCollection previousTable = null;
         foreach (IBlueRowCollection table in tables) {
            if (previousTable != null)
               if (!compareSchemas(previousTable.schema, table.schema))
                  throw new ArgumentException("Error, tables must have the same schema to be combined");

            foreach (IBlueRow r in table) {
               object[] values = new object[numCols];
               r.CopyTo(values, 0);
               values[idCol] = idToUseNext++;
               result.addRow(new BlueRow(table.schema, values));
            }

            previousTable = table;

         }

         return result;
      }
      /// <summary>
      /// Compares two rows (having the same schema) and returns the 
      /// columns whose values dont match
      /// </summary>
      /// <param name="r1"></param>
      /// <param name="r2"></param>
      /// <returns></returns>
      public static string[] compareRows(IBlueRow r1, IBlueRow r2) {
         ArrayList retVal = new ArrayList();
         foreach (string col in r1.schema.columnNames)
            if (!r1[col].Equals(r2[col]))
               retVal.Add(col);
         return retVal.ToArray(typeof(string)) as string[];
      }

      public static BlueTable cloneRowCollection(IBlueRowCollection rows) {
         BlueTable retVal = new BlueTable(rows.schema);
         foreach (IBlueRow r in rows)
            retVal.addRow(new BlueRow(r));
         return retVal;
      }

      /// <summary>
      /// Creates a new row with the values of the row passed, except that the newValues are forced into the 
      /// returned row, in places specified by the indexes (makes any sense ?)
      /// </summary>
      /// <param name="r"></param>
      /// <param name="newValues"></param>
      /// <param name="indexes"></param>
      /// <returns></returns>
      public static IBlueRow getNewReplacedRow(IBlueRow r, object[] newValues, int[] indexes) {
         if (!newValues.Length.Equals(indexes.Length))
            throw new Exception("Debug Assert");

         object[] rowArray = new object[r.Count];
         r.CopyTo(rowArray, 0);
         for (int i = 0; i < indexes.Length; i++)
            rowArray[indexes[i]] = newValues[i];
         return new BlueRow(r.schema, rowArray);
      }

      public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, int index) {
         return getNewReplacedRow(r, new object[] { newValue }, new int[] { index });
      }

      public static IBlueRow getNewReplacedRow(IBlueRow r, object newValue, string colName) {
         return getNewReplacedRow(r, newValue, r.schema[colName].colNo);
      }

      /// <summary>
      /// Computes the max row, based on the cols provided. If the collection is empty, null is returned
      /// </summary>
      /// <param name="rows"></param>
      /// <param name="cols"></param>
      /// <returns></returns>
      public static IBlueRow computeMaxRow(IBlueRowCollection rows, params string[] cols) {
         IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null;
         foreach (IBlueRow r in rows)
            retVal = compare_rows(r, retVal, cols) > 0 ? r : retVal;
         return retVal;
      }

      /// <summary>
      /// Computes the max row, based on the cols provided. If the collection is empty, null is returned
      /// </summary>
      /// <param name="rows"></param>
      /// <param name="cols"></param>
      /// <returns></returns>
      public static IBlueRow computeMinRow(IBlueRowCollection rows, params string[] cols) {
         IBlueRow retVal = rows.Count != 0 ? BlueDataUtility.extractRow(rows, 0) : null;
         foreach (IBlueRow r in rows)
            retVal = compare_rows(r, retVal, cols) < 0 ? r : retVal;
         return retVal;
      }

      /// <summary>
      /// Computes the sum of rows, based on the cols provided. If the collection is empty, null is returned
      /// </summary>
      /// <param name="rows"></param>
      /// <param name="cols"></param>
      /// <returns></returns>
      public static IBlueRow computeSum(IBlueRowCollection rows, params string[] cols) {
         if (rows.Count == 0) return null;

         IBlueRow retVal = BlueDataUtility.extractRow(rows, 0);

         object[] sums = new object[cols.Length];
         int[] idxs = new int[cols.Length];
         for (int i = 0; i < cols.Length; i++) {
            sums[i] = 0.0;
            idxs[i] = rows.schema[cols[i]].colNo;
         }

         foreach (IBlueRow r in rows) {
            for (int i = 0; i < cols.Length; i++) {
               double dval = BlueDataUtility.get_row_value<double>(r, cols[i], 0.0);
               //r[cols[i]] == null || r[cols[i]] is DBNull ? 0.0 : (double)r[cols[i]];
               sums[i] = (double)sums[i] + dval;
            }
         }

         return BlueDataUtility.getNewReplacedRow(retVal, sums, idxs);
      }


      public static IBlueRow computeSum<T>(IBlueRowCollection rows, params string[] cols) {
         if (rows.Count == 0) return null;
         IBlueRow retVal = BlueDataUtility.extractRow(rows, 0);
         if (rows.Count == 1) return retVal;

         object[] sums = new object[cols.Length];

         int[] idxs = new int[cols.Length];
         for (int i = 0; i < cols.Length; i++) {
            sums[i] = default(T);
            idxs[i] = rows.schema[cols[i]].colNo;
         }

         foreach (IBlueRow r in rows) {
            for (int i = 0; i < cols.Length; i++) {
               T tval = r[cols[i]] == null || r[cols[i]] is DBNull ? default(T) : (T)r[cols[i]];
               sums[i] = MagicAdder<T>.Add((T)sums[i], tval);
            }
         }

         return BlueDataUtility.getNewReplacedRow(retVal, sums, idxs);
      }

      /// <summary>
      /// Uses the ID to identify the presence of a row in a collection
      /// </summary>
      /// <param name="table"></param>
      /// <param name="row"></param>
      /// <returns></returns>
      public static bool rowPresent(IBlueRowCollection table, IBlueRow row) {
         return table.containsID((int)row["id"]);
      }

      public static void computeDifferences(IBlueRowCollection newRows,
         IBlueRowCollection oldRows,
         out IBlueRowCollection dRows,
         out IBlueRowCollection aRows) {
         computeDifferences(
            newRows,
            oldRows,
            newRows.schema.columnNames,
            out dRows,
            out aRows);
      }

      public static bool compute_equality(IBlueRowCollection rows1, IBlueRowCollection rows2) {
         IBlueRowCollection d_rows, a_rows;
         compute_differences_fast(rows2, rows1, out d_rows, out a_rows);
         return d_rows.Count == 0 && a_rows.Count == 0;
      }

      public static bool compute_equality(ColumnInfo ci1, ColumnInfo ci2) {
         return (ci1.colNo == ci2.colNo) && (ci1.colName == ci2.colName) && (ci1.colType == ci2.colType);
      }

      public static bool compute_equality(IBlueTableSchema s1, IBlueTableSchema s2) {
         if (s1.tableName != s1.tableName)
            return false;
         if (s1.columnInfo.Length != s2.columnInfo.Length)
            return false;
         for (int i = 0; i < s1.columnInfo.Length; i++) {
            ColumnInfo ci1 = s2.columnInfo[i], ci2 = s2.columnInfo[i];
            if (!compute_equality(ci1, ci2))
               return false;
         }
         return true;
      }

      public static bool compute_equality(IBlueRow r1, IBlueRow r2) {
         return compute_row_equality(r1, r2, r1.schema.columnNames);
      }

      public static bool compute_row_equality(IBlueRow r1,
         IBlueRow r2,
         params string[] col_names) {
         return compare_rows(r1, r2, col_names) == 0 ? true : false;
      }

      public static string to_string(ColumnInfo ci) {
         return ci.colNo + ";" + ci.colName + ";" + ci.colType;
      }

      public static string to_string(this IBlueRow r) {
         string rv = String.Empty;
         foreach (ColumnInfo c in r.schema.columnInfo) {
            rv += $"{c.colName}: {r[c.colName]}\n";
         }
         return rv;
      }

      public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
                                                  out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
         if (old_rows == null || old_rows.Count == 0) {
            out_d_rows = new BlueTable(new_rows.schema);
            out_a_rows = new BlueTable(new_rows);
            return;
         }
         int[] col_indices = new int[new_rows.schema.columnNames.Length];
         for (int i = 0; i < col_indices.Length; i++)
            col_indices[i] = i;
         compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows);
      }

      public static void compute_differences_id_based(IBlueRowCollection new_rows, IBlueRowCollection old_rows,
                                                      out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
         if (old_rows == null || old_rows.Count == 0) {
            out_d_rows = new BlueTable(new_rows.schema);
            out_a_rows = new BlueTable(new_rows);
            return;
         }
         int[] col_indices = new int[] { new_rows.schema["id"].colNo };
         compute_differences_fast(new_rows, old_rows, col_indices, out out_d_rows, out out_a_rows);
      }

      public static void compute_differences_fast(IBlueRowCollection new_rows, IBlueRowCollection old_rows, int[] col_indices,
                                                  out IBlueRowCollection out_d_rows, out IBlueRowCollection out_a_rows) {
         Func<IBlueRow, IBlueRow, bool> rows_equal = (r1, r2) => {
            foreach (int col_no in col_indices)
               if (!(r1[col_no] ?? DBNull.Value).Equals(r2[col_no] ?? DBNull.Value))
                  return false;
            return true;
         };

         BlueTable del_rows = new BlueTable(new_rows.schema), add_rows = new BlueTable(new_rows.schema);
         // adds + changes
         foreach (IBlueRow r in new_rows)
            if (!BlueDataUtility.rowPresent(old_rows, r))
               add_rows.addRow(r);
            else if (!rows_equal(r, old_rows[(int)r["id"]])) {
               add_rows.addRow(r);
               del_rows.addRow(old_rows[(int)r["id"]]);
            }
         // deletes
         foreach (IBlueRow r in old_rows)
            if (!BlueDataUtility.rowPresent(new_rows, r))
               del_rows.addRow(r);
         out_d_rows = del_rows;
         out_a_rows = add_rows;
      }


      public static void computeDifferences(IBlueRowCollection c1,
         IBlueRowCollection c2,
         string[] columns,
         out IBlueRowCollection dRows,
         out IBlueRowCollection aRows) {
         dRows = new BlueTable(c1.schema);
         aRows = new BlueTable(c1.schema);

         // adds + changes
         foreach (IBlueRow nr in c1)
            if (!BlueDataUtility.rowPresent(c2, nr))
               (aRows as BlueTable).addRow(nr);
            else if (BlueDataUtility.compare_rows(nr, c2[(int)
               nr["id"]], columns) != 0) {
               (aRows as BlueTable).addRow(nr);
               (dRows as BlueTable).addRow(c2[(int)nr["id"]]);

            }

         // deletes
         foreach (IBlueRow nr in c2)
            if (!BlueDataUtility.rowPresent(c1, nr))
               (dRows as BlueTable).addRow(nr);
      }

      //brcDiff = brc1 - brc2
      public static void diff(IBlueRowCollection brc1, IBlueRowCollection brc2, string[] columns, out IBlueRowCollection brcDiff) {
         brcDiff = new BlueTable(brc1.schema);
         foreach (IBlueRow br in brc1) {
            object[] oarray = new object[columns.Length];
            for (int i = 0; i < columns.Length; i++) {
               string col = columns[i];
               oarray[i] = br[col];
            }
            if (myDoesRowExistsInTable(brc2, columns, oarray) == false)
               (brcDiff as BlueTable).addRow(br);
         }
      }

      /// <summary>
      /// Abosrbs the deletes and adds into the BlueTable
      /// </summary>
      /// <param name="t"></param>
      /// <param name="dels"></param>
      /// <param name="adds"></param>
      public static void absorbChanges(BlueTable t, IBlueRowCollection dels, IBlueRowCollection adds) {
         foreach (IBlueRow r in dels)
            if (!t.deleteRow(r))
               throw new Exception(String.Format("{0} does not have row with id {1} to delete", t.schema.tableName, r["id"]));

         foreach (IBlueRow r in adds)
            t.addRow(r);
      }

      public static IBlueRowCollection toBlueRowCollection(IBlueRow r) {
         BlueTable rv = new BlueTable(r.schema);
         rv.addRow(r);
         return rv;
      }

      /// <summary>
      /// Converts an ICollection of IBlueRows to IBlueRowCollection. If there are no rows in the 
      /// collection, a null is returned
      /// </summary>
      /// <param name="col"></param>
      /// <returns></returns>
      public static IBlueRowCollection toBlueRowCollection(ICollection col) {
         BlueTable retVal = null;
         foreach (IBlueRow r in col) {
            retVal = retVal ?? new BlueTable(r.schema);
            retVal.addRow(r);
         }
         return retVal;
      }

      /// <summary>
      /// Computes the rows that are truely deleted, added or changed (with respect to relevant columns)
      /// </summary>
      /// <param name="dels">all the deleted rows in a certain callback</param>
      /// <param name="adds">all the added rows in a certain callback</param>
      /// <param name="truelyDels">the rows that are truely deleted or changed with respect to the relevant columns</param>
      /// <param name="truelyAdds">the rows that are truely added or changed with respect to the relevant columns</param>
      /// <param name="cols"></param>
      public static void getTrueDeletesAndAdds(IBlueRowCollection dels,
                                                IBlueRowCollection adds,
                                                out IBlueRowCollection truelyDels,
                                                out IBlueRowCollection truelyAdds,
                                                string[] cols) {
         truelyDels = new BlueTable(dels);
         truelyAdds = new BlueTable(adds);
         foreach (IBlueRow r in dels) {
            if (adds.containsID((int)r["id"]) &&
               BlueDataUtility.compute_row_equality(adds[(int)r["id"]], r, cols)) {
               (truelyDels as BlueTable).deleteRow((int)r["id"]);
               (truelyAdds as BlueTable).deleteRow((int)r["id"]);
            }
         }
      }

      public static IBlueRowCollection getSerializableCollection(IBlueRowCollection coll) {
         BlueTable retVal = new OrderedBlueTable(coll.schema);
         foreach (IBlueRow r in coll)
            retVal.addRow(new BlueRow(r));
         return retVal;
      }

      /// <summary>
      /// extracts out a column from a table
      /// </summary>
      /// <param name="rows"></param>
      /// <param name="colName"></param>
      /// <returns></returns>
      public static ArrayList extractColumn(IBlueRowCollection rows, string colName) {
         ArrayList rv = new ArrayList();
         foreach (IBlueRow r in rows)
            rv.Add(r[colName]);
         return rv;
      }

      /// <summary>
      /// applies the deletedRows and addedRows to the oldState and asserts if it does not
      /// match the newRows. for an old state with 10 K rows, and all 10 K rows being deleted and 
      /// added it takes ~ 35 ms to execute this method.
      /// </summary>
      /// <param name="oldState"></param>
      /// <param name="newRows"></param>
      /// <param name="delRows"></param>
      /// <param name="addedRows"></param>
      public static void checkConsistency(IBlueRowCollection oldState,
                                             IBlueRowCollection newRows,
                                             IBlueRowCollection delRows,
                                             IBlueRowCollection addedRows) {
         BlueTable calc = new BlueTable(oldState);
         Debug.Assert(oldState != null && newRows != null && delRows != null && addedRows != null);

         foreach (IBlueRow r in delRows) {
            Debug.Assert(calc.containsID((int)r["id"]), "deleted row's id = " + r["id"] + " does not exist");
            calc.deleteRow(r);
         }

         foreach (IBlueRow r in addedRows) {
            Debug.Assert(!calc.containsID((int)r["id"]), "added rows's id = " + r["id"].ToString() + " already exists");
            calc.addRow(r);
         }

         Debug.Assert(calc.Count == newRows.Count, "row count should be the same. Counts: calc=" + calc.Count + "; newRows=" + newRows.Count);
         foreach (IBlueRow r in calc)
            Debug.Assert(newRows.containsID((int)r["id"]), r["id"] + " is not found");

      }

      /// <summary>
      /// Extracts values from an IBlueRow specified by the columnNames arg
      /// </summary>
      /// <param name="r"></param>
      /// <param name="colNames"></param>
      /// <returns></returns>
      public static object[] getSelectedValues(IBlueRow r, params string[] colNames) {
         object[] retVal = new object[colNames.Length];
         for (int i = 0; i < colNames.Length; i++)
            retVal[i] = r[colNames[i]];
         return retVal;
      }


      /// <summary>
      /// From an IBlueRowCollection, for a set of columns, extracts a row with the same values as 
      /// provided in the third arg
      /// </summary>
      /// <param name="rows"></param>
      /// <param name="keyColumns"></param>
      /// <param name="colValues"></param>
      /// <returns></returns>
      public static IBlueRow getRowWithKey(IBlueRowCollection rows, string[] keyColumns, object[] colValues) {
         foreach (IBlueRow r in rows) {
            object[] vals = getSelectedValues(r, keyColumns);
            if (CollectionHelpers.equal(vals, colValues))
               return r;
         }
         return null;
      }

      public static IBlueRowCollection deepCopy(IBlueRowCollection rows) {
         BlueTable retVal = new BlueTable(rows.schema);
         foreach (IBlueRow r in rows)
            retVal.addRow(new BlueRow(r));
         return retVal;
      }

      /// <summary>
      /// Returns the columns, sorted alphabetically. The second arg
      /// is for the inclusion of id
      /// </summary>
      /// <param name="schema"></param>
      /// <param name="includeId"></param>
      /// <returns></returns>
      public static int[] getColumnsInSortedOrder(IBlueTableSchema schema, bool includeId) {
         ArrayList names = new ArrayList(schema.columnNames),
                     retVal = new ArrayList();
         names.Sort();
         foreach (string n in names)
            if (n != "id" || includeId)
               retVal.Add(schema[n].colNo);
         return retVal.ToArray(typeof(int)) as int[];
      }

      //this class is used so that we can sort a schema by column name
      //that way, prepublished tables will allways get published with schemas in the same order
      //regardless of how the underlying streams end up working
      private class ColumnInfoComparisonWrapper : IComparable<ColumnInfoComparisonWrapper> {

         ColumnInfo myColumn;

         public string ColumnName {
            get { return myColumn.colName; }
            //can't set it, these are only for reading columns
         }

         public ColumnInfo Column {
            get { return myColumn; }
         }
         public ColumnInfoComparisonWrapper(ColumnInfo theColumn) {
            myColumn = theColumn;
         }

         public int CompareTo(ColumnInfoComparisonWrapper otherGuy) {
            return ColumnName.CompareTo(otherGuy.ColumnName);
         }
      }

      public static IBlueTableSchema getSortedSchema(IBlueTableSchema inSchema) {
         List<string> sortedColNames = new List<string>(inSchema.columnNames);
         sortedColNames.Sort();
         List<ColumnInfo> sortedColInfos = new List<ColumnInfo>();
         int colNo = 0;
         foreach (string c in sortedColNames) {
            ColumnInfo ci = inSchema[c];
            sortedColInfos.Add(new ColumnInfo(c, colNo++, ci.colType));

         }
         return new BlueTableSchema(inSchema.tableName, sortedColInfos.ToArray());
      }

      public static IBlueTableSchema getSortedSchema(IBlueTableSchema theSchema, out int[] sortedSchemaToOriginalSchemaMapping, out int idColumnInSortedSchema) {

         //for now this function will not do anything
         //that is because we don't want to break the C++ deserialization code
         int numCols = theSchema.columnNames.Length;
         sortedSchemaToOriginalSchemaMapping = new int[numCols];
         for (int col = 0; col < theSchema.columnNames.Length; ++col) {
            //the mapping is just the identity mapping
            sortedSchemaToOriginalSchemaMapping[col] = col;


         }

         idColumnInSortedSchema = theSchema["id"].colNo;

         return theSchema;

         /*
         //set this to -1 in case the method doesn't find an id column
         idColumnInSortedSchema = 01;
         List<ColumnInfoComparisonWrapper> schemaColumns = new List<ColumnInfoComparisonWrapper>();

         foreach (ColumnInfo ci in theSchema.columnInfo)
             schemaColumns.Add(new ColumnInfoComparisonWrapper(ci));

         schemaColumns.Sort();

         int numCols = theSchema.columnInfo.Length;
         ColumnInfo[] outputColumns = new ColumnInfo[numCols];
         sortedSchemaToOriginalSchemaMapping = new int[numCols];

         for(int i = 0; i < numCols; ++i)
         {
             string thisColName = schemaColumns[i].ColumnName;
             outputColumns[i] = new ColumnInfo(thisColName, i, schemaColumns[i].Column.colType);
             sortedSchemaToOriginalSchemaMapping[i] = theSchema[thisColName].colNo;

             //we cache the ID column because it'll be differentin the serialized row than in our schema
             if (thisColName == "id")
                 idColumnInSortedSchema = i;

         }


         theSchema  = new BlueTableSchema(theSchema.tableName,outputColumns);
         return theSchema;
         */


      }




      /// <summary>
      /// Compares 2 schemas for column-by-column match
      /// </summary>
      /// <param name="s1"></param>
      /// <param name="s2"></param>
      /// <returns>true if equal</returns>
      public static bool compareSchemas(IBlueTableSchema s1, IBlueTableSchema s2) {
         bool result = true;
         try {
            if (s1.columnInfo.Length != s2.columnInfo.Length)
               result = false;
            else
               foreach (ColumnInfo ci1 in s1.columnInfo) {
                  ColumnInfo ci2 = s2[ci1.colNo];
                  if (ci1.colName != ci2.colName || ci1.colType != ci2.colType || ci1.colNo != ci2.colNo) {
                     result = false;
                     break;
                  }
               }
         } catch {
            result = false; // the col# wasn't found in s2
         }
         return result;
      }

      public static void validate_row_schema(IBlueRow r) {
         foreach (var ci in r.schema.columnInfo) {
            object rval = r[ci.colName];
            if (rval == null)
               continue;
            bts.Log.assert(rval.isDBNull() || rval.GetType().Equals(ci.colType));
         }
      }

      public static void validate_table_schema(IBlueRowCollection rows) {
         foreach (IBlueRow r in rows) {
            validate_row_schema(r);
         }
      }

      public static IBlueRowCollection removeRows(BlueTable table, string[] cols, object[] vals) {
         IBlueRowCollection dels = findMatchingRows(table, cols, vals);
         table.deleteRows(dels);
         return dels;
      }

      public static IBlueRowCollection findMatchingRows(IBlueRowCollection rows, string[] cols, object[] vals) {
         BlueTable rv = new BlueTable(rows.schema);
         foreach (IBlueRow r in rows) {
            bool match = true;
            for (int i = 0; i < cols.Length; i++)
               match &= vals[i].Equals(r[cols[i]]);
            if (match)
               rv.addRow(r);
         }
         return rv;
      }

      public static void checkRowTypes(IBlueRowCollection rows) {
         foreach (IBlueRow r in rows)
            foreach (string col in r.schema.columnNames)
               if (r[col] != null && r[col] != DBNull.Value)
                  Debug.Assert(r[col].GetType() == rows.schema[col].colType,
                                 String.Format("col {0} in table {1} is of type {2} in schema but is a {3} in the row", col, rows.schema.tableName, r.schema[col].colType.ToString(), r[col].GetType().ToString()));
      }

      public static void addRows(ref BlueTable table, ICollection rows) {
         foreach (IBlueRow r in rows)
            table.addRow(r);
      }

      public static void removeRows(ref BlueTable table, ICollection rows) {
         foreach (IBlueRow r in rows)
            table.deleteRow(r);
      }

      public static void sortOutPacket(IBlueRowCollection dels, IBlueRowCollection adds, string[] keyCols,
                                       out IBlueRowCollection addedRows,
                                       out IBlueRowCollection deletedRows,
                                       out IBlueRowCollection changedRows) {
         addedRows = new BlueTable(dels.schema);
         deletedRows = new BlueTable(dels.schema);
         changedRows = new BlueTable(dels.schema);

         KeysBasedTable2 keyedDels = new KeysBasedTable2(dels.schema, keyCols),
                           keyedAdds = new KeysBasedTable2(dels.schema, keyCols);

         foreach (IBlueRow r in dels)
            keyedDels.update(r);
         foreach (IBlueRow r in adds)
            keyedAdds.update(r);

         foreach (IBlueRow r in keyedDels) {
            object[] key = BlueDataUtility.getSelectedValues(r, keyCols);
            if (keyedAdds.getRow(key) != null) {
               ///change
               (changedRows as BlueTable).addRow(r);
            } else {
               ///delete
               (deletedRows as BlueTable).addRow(r);
            }
         }

         foreach (IBlueRow r in keyedAdds) {
            object[] key = BlueDataUtility.getSelectedValues(r, keyCols);
            if (keyedDels.getRow(key) == null) {
               (addedRows as BlueTable).addRow(r);
            }
         }
      }

      public static int HashExpStrike(DateTime exp, double strike) {
         return ((DateTime)exp).ToShortDateString().GetHashCode() ^ strike.GetHashCode();
      }

      public static IEnumerable<IBlueRow> collection_to_ienum(IBlueRowCollection rows) {
         List<IBlueRow> rv = new List<IBlueRow>();
         foreach (IBlueRow r in rows)
            rv.Add(r);
         return rv;
      }

      public static IBlueRowCollection ienum_to_collection(IEnumerable<IBlueRow> rows) {
         BlueTable rv = null;
         if (rows.Count() > 0) {
            rv = new BlueTable(rows.ElementAt(0).schema);
            foreach (IBlueRow r in rows)
               rv.addRow(r);
         }
         return rv;
      }

      public static IBlueTableSchema add_column(IBlueTableSchema input_schema, string new_col_name, Type new_col_type) {
         List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
         cis.Add(new ColumnInfo(new_col_name, cis.Count, new_col_type));
         return new BlueTableSchema(input_schema.tableName, cis.ToArray());
      }

      public static IBlueTableSchema remove_columns(IBlueTableSchema input_schema, params string[] col_names) {
         Dictionary<string, bool> removed_col_names = new Dictionary<string, bool>();
         foreach (string c in col_names)
            removed_col_names[c] = true;
         List<ColumnInfo> cis = new List<ColumnInfo>();
         foreach (ColumnInfo ci in input_schema.columnInfo)
            if (!removed_col_names.ContainsKey(ci.colName))
               cis.Add(new ColumnInfo(ci.colName, cis.Count, ci.colType));
         return new BlueTableSchema(cis.ToArray(), input_schema.tableName);
      }

      public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) {
         List<int> rv = new List<int>();
         foreach (string c in col_names) {
            Log.assert(schema.contains_column(c), $"Could not find col: {c} in table: {schema.tableName}");
            rv.Add(schema[c].colNo);
         }
         return rv.ToArray();
      }

      public static string get_appended_string_val(IBlueRow r, int[] col_indices) {
         System.Text.StringBuilder sb = new System.Text.StringBuilder();
         for (int i = 0; i < col_indices.Length; i++)
            sb.Append(r[col_indices[i]]);
         return sb.ToString();
      }

      public delegate string RowValueToString(IBlueRow r, int col_index);
      public static string get_appended_string_val(IBlueRow r, int[] col_indices, RowValueToString to_string_conv) {
         System.Text.StringBuilder sb = new System.Text.StringBuilder();
         for (int i = 0; i < col_indices.Length; i++)
            sb.Append(to_string_conv(r, i));
         return sb.ToString();
      }

      public static string id_string(IBlueRowCollection table) {
         string rv = String.Empty;
         foreach (IBlueRow r in table)
            rv += $"{r["id"]} ";
         return rv;
      }

      public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, string[] group_by_col_names) {
         BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length);
         return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher);
      }

      public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, string[] group_by_col_names, ref BlueRowHasher hasher) {
         return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher);
      }

      public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, int[] group_by_col_nos, ref BlueRowHasher hasher) {
         Dictionary<int, List<IBlueRow>> rv = new Dictionary<int, List<IBlueRow>>();
         hasher = hasher ?? new BlueRowHasher(group_by_col_nos.Length);
         foreach (IBlueRow r in rows) {
            int row_hash = hasher.get_hash(r, group_by_col_nos);
            List<IBlueRow> this_group_rows = null;
            if (!rv.TryGetValue(row_hash, out this_group_rows)) {
               this_group_rows = new List<IBlueRow>();
               rv.Add(row_hash, this_group_rows);
            }
            this_group_rows.Add(r);
         }
         return rv;
      }

      public static void copy_to_row(IBlueRow row, string[] data_col_names, MutableBlueRow output_row) {
         foreach (string cn in data_col_names)
            output_row.setValue(cn, row[cn]);
      }

      /// <summary>
      /// Copies the column nos specified in data_cols from row, to output_row
      /// starting from index start_col_index
      /// </summary>
      public static void copy_to_row(IBlueRow row, int[] data_col_nos, object[] output_row, int start_col_index) {
         foreach (int col_no in data_col_nos)
            output_row[start_col_index++] = row[col_no];
      }

      public static object compute_min_max(object v1, object v2, bool is_max) {
         if (!TypeHelper.is_dbnull(v1) && !TypeHelper.is_dbnull(v2)) {
            if (TypeHelper.is_numeric_type(v1) && TypeHelper.is_numeric_type(v2)) {
               double v1_num = (double)Convert.ChangeType(v1, TypeCode.Double),
                        v2_num = (double)Convert.ChangeType(v2, TypeCode.Double);
               v1_num = Double.IsNaN(v1_num) ? (is_max ? Double.MinValue : Double.MaxValue) : v1_num;
               v2_num = Double.IsNaN(v2_num) ? (is_max ? Double.MinValue : Double.MaxValue) : v2_num;
               double rv = is_max ? Math.Max(v1_num, v2_num) : Math.Min(v1_num, v2_num);
               return Convert.ChangeType(rv, v1.GetType());
            } else
               return v1;
         } else
            return TypeHelper.is_dbnull(v1) ? v2 : v1;
      }

      #region Sorting

      public static class ParallelSort {
         public class TreeRowEffComparer : IComparer<IBlueRow> {
            private int _depth_in_tree_col_no;
            private int[] _aggregation_tree_col_nos;
            private Pair<int, bool>[] _sort_map;

            public TreeRowEffComparer(int depth_in_tree_col_no, int[] aggregation_tree_col_nos, Pair<int, bool>[] sort_map) {
               _depth_in_tree_col_no = depth_in_tree_col_no;
               _aggregation_tree_col_nos = aggregation_tree_col_nos;
               _sort_map = sort_map;
            }

            public int Compare(IBlueRow x, IBlueRow y) {
               return BlueDataUtility.sort_in_a_tree(x, y, _depth_in_tree_col_no, _aggregation_tree_col_nos, _sort_map);
            }
         }

         public class TabularRowComparer : IComparer<IBlueRow> {
            private IEnumerable<Pair<int, bool>> _sort_map;
            public TabularRowComparer(IEnumerable<Pair<int, bool>> sort_map) { _sort_map = sort_map; }
            public int Compare(IBlueRow x, IBlueRow y) { return BlueDataUtility.compare_rows(x, y, _sort_map); }
         }

         public class TabularRowHashPairComparer : IComparer<Pair<IBlueRow, int>> {
            private Pair<int, bool>[] _sort_map;

            public TabularRowHashPairComparer(Pair<int, bool>[] sort_map) { _sort_map = sort_map; }
            public int Compare(Pair<IBlueRow, int> x, Pair<IBlueRow, int> y) { return BlueDataUtility.compare_rows(x.first, y.first, _sort_map); }
         }

         #region Public Static Methods

         public static void QuicksortSequential<T>(IList<T> arr, IComparer<T> comparer) {
            QuicksortSequential(arr, comparer, 0, arr.Count - 1);
         }

         public static void QuicksortParallel<T>(IList<T> arr, IComparer<T> comparer) {
            QuicksortParallel(arr, comparer, 0, arr.Count - 1);
         }

         #endregion

         #region Private Static Methods

         private static void QuicksortSequential<T>(IList<T> arr, IComparer<T> comparer, int left, int right) {
            if (right > left) {
               int pivot = Partition(arr, comparer, left, right);
               QuicksortSequential(arr, comparer, left, pivot - 1);
               QuicksortSequential(arr, comparer, pivot + 1, right);
            }
         }

         private static void QuicksortParallel<T>(IList<T> arr, IComparer<T> comparer, int left, int right) {
            const int SEQUENTIAL_THRESHOLD = 64;
            if (right > left) {
               if (right - left < SEQUENTIAL_THRESHOLD) {
                  QuicksortSequential(arr, comparer, left, right);
               } else {
                  int pivot = Partition(arr, comparer, left, right);
                  System.Threading.Tasks.Parallel.Invoke(new Action[] { delegate {QuicksortParallel(arr, comparer, left, pivot - 1);},
                                                                     delegate {QuicksortParallel(arr, comparer, pivot + 1, right); }
                });
               }
            }
         }

         private static void Swap<T>(IList<T> arr, int i, int j) {
            T tmp = arr[i];
            arr[i] = arr[j];
            arr[j] = tmp;
         }

         private static int Partition<T>(IList<T> arr, IComparer<T> comparer, int low, int high) {
            // Simple partitioning implementation
            int pivotPos = (high + low) / 2;
            T pivot = arr[pivotPos];
            Swap(arr, low, pivotPos);

            int left = low;
            for (int i = low + 1; i <= high; i++) {
               if (comparer.Compare(arr[i], pivot) < 0) {
                  left++;
                  Swap(arr, i, left);
               }
            }

            Swap(arr, low, left);
            return left;
         }

         #endregion
      }

      public static IBlueRow find_max(IEnumerable<IBlueRow> rows, params string[] cols_to_compare) {
         if (rows == null || rows.Count() == 0)
            return null;
         IBlueRow max_row = rows.ElementAt(0);
         foreach (IBlueRow r in rows) {
            int c = compare_rows(r, max_row, cols_to_compare);
            if (c > 0)
               max_row = r;
         }
         return max_row;
      }

      public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) {
         foreach (KeyValuePair<string, bool> p in sort_map) {
            int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value);
            if (comp != 0)
               return comp;
         }
         return 0;
      }

      public static int compare_rows(IBlueRow r1, IBlueRow r2, IEnumerable<Pair<int, bool>> sort_map) {
         foreach (Pair<int, bool> p in sort_map) {
            int comp = compare_send_dontcare_to_last(r1, p.first, r2, p.first, p.second);
            if (comp != 0)
               return comp;
         }
         return 0;
      }

      public static int compare_send_dontcare_to_last(IBlueRow r1, int r1_col_no,
                                                      IBlueRow r2, int r2_col_no,
                                                      bool low_to_high) {
         object o1 = BlueDataUtility.get_row_value<object>(r1, r1_col_no, DBNull.Value),
               o2 = BlueDataUtility.get_row_value<object>(r2, r2_col_no, DBNull.Value);
         return compare_send_dontcare_to_last(o1, o2, low_to_high);
      }

      public static List<Pair<int, bool>> get_indexed_sort_map(List<Pair<string, bool>> sort_map, IBlueTableSchema schema) {
         List<Pair<int, bool>> rv = new List<Pair<int, bool>>();
         foreach (Pair<string, bool> p in sort_map)
            rv.Add(new Pair<int, bool>(schema[p.first].colNo, p.second));
         return rv;
      }

      /// Comapares 2 rows in a way where nulls, nans, infinities,
      /// empty strings etc are sent to the last
      public static int compare_send_dontcare_to_last(IBlueRow r1, string r1_col_name,
                                                      IBlueRow r2, string r2_col_name,
                                                      bool low_to_high) {
         object o1 = BlueDataUtility.get_row_value<object>(r1, r1_col_name, DBNull.Value),
               o2 = BlueDataUtility.get_row_value<object>(r2, r2_col_name, DBNull.Value);
         return compare_send_dontcare_to_last(o1, o2, low_to_high);
      }

      public static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) {
         if (is_dont_care_val(o1))
            return is_dont_care_val(o2) ? 0 : 1;
         else
            return is_dont_care_val(o2) ? -1 : (low_to_high ? ((IComparable)o1).CompareTo(o2)
                                                    : ((IComparable)o2).CompareTo(o1));
      }

      public static bool is_dont_care_val(object o) {
         return o == null || o == DBNull.Value ||
                  o is string && String.IsNullOrEmpty((string)o) ||
                  o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) ||
                  o is float && (Single.IsInfinity((float)o) || Single.IsNaN((float)o)) ||
                  o is Int32 && (Int32.MaxValue.Equals(o) || Int32.MinValue.Equals(o));
      }

      #endregion

      #region Tree Aggregation

      public static int[] convert_aggregation_tree(string[] aggregation_tree_col_names, IBlueTableSchema schema) {
         List<int> rv = new List<int>();
         foreach (string col_name in aggregation_tree_col_names)
            rv.Add(schema[col_name].colNo);
         return rv.ToArray();
      }

      public static bool is_leaf_node(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) {
         return BlueDataUtility.get_row_value<int>(r, depth_in_tree_col_no, -1) == (aggregation_tree.Count() - 1);
      }

      public static bool is_leaf_node(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) {
         return BlueDataUtility.get_row_value<int>(r, depth_in_tree_col_name, -1) == (aggregation_tree.Count() - 1);
      }

      public static string generate_aggregation_based_id(IBlueRow r, int depth_in_tree_col_no, int[] aggregation_tree) {
         int aggregation_priority = BlueDataUtility.get_row_value<int>(r, depth_in_tree_col_no, -1);
         string rv = String.Empty;
         for (int p = 0; p <= aggregation_priority; p++)
            rv += (rv == String.Empty ? String.Empty : ":") + String.Format("{0},{1}", p, BlueDataUtility.get_row_value<object>(r, aggregation_tree[p], DBNull.Value));
         return rv;
      }

      /// Generates a string based id for a row in a tree
      /// by appending the values of the nodes above it
      public static string generate_aggregation_based_id(IBlueRow r, string depth_in_tree_col_name, string[] aggregation_tree) {
         int aggregation_priority = BlueDataUtility.get_row_value<int>(r, depth_in_tree_col_name, -1);
         string rv = String.Empty;
         for (int p = 0; p <= aggregation_priority; p++)
            rv += (rv == String.Empty ? String.Empty : ":") +
                  String.Format("{0},{1}", p, BlueDataUtility.get_row_value<object>(r, aggregation_tree.ElementAt(p), DBNull.Value));
         return rv;
      }
      /// <summary>
      /// Given an aggregation hierarchy tree, figures if the first
      /// arg is a parent of the second arg.
      /// </summary>
      public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, string depth_in_tree_col_name, string[] aggregation_tree) {
         System.Diagnostics.Debug.Assert(parent_row.Contains(depth_in_tree_col_name) && child_row.Contains(depth_in_tree_col_name));
         int parent_depth = BlueDataUtility.get_row_value<int>(parent_row, depth_in_tree_col_name, Int32.MaxValue),
             child_depth = BlueDataUtility.get_row_value<int>(child_row, depth_in_tree_col_name, Int32.MaxValue);
         if (parent_depth < child_depth) {
            for (int p = 0; p <= parent_depth; p++) {
               string col_name = aggregation_tree.ElementAt(p);
               object parent_val = BlueDataUtility.get_row_value<object>(parent_row, col_name, DBNull.Value),
                     child_val = BlueDataUtility.get_row_value<object>(child_row, col_name, DBNull.Value);
               if (!parent_val.Equals(child_val))
                  return false;
            }
            return true;
         } else
            return false;
      }

      public static bool is_parent(IBlueRow parent_row, IBlueRow child_row, int depth_in_tree_col_no, int[] aggregation_tree) {
         int parent_depth = BlueDataUtility.get_row_value<int>(parent_row, depth_in_tree_col_no, Int32.MaxValue),
             child_depth = BlueDataUtility.get_row_value<int>(child_row, depth_in_tree_col_no, Int32.MaxValue);
         if (parent_depth < child_depth) {
            for (int p = 0; p <= parent_depth; p++) {
               int col_no = aggregation_tree[p];
               object parent_val = BlueDataUtility.get_row_value<object>(parent_row, col_no, DBNull.Value),
                     child_val = BlueDataUtility.get_row_value<object>(child_row, col_no, DBNull.Value);
               if (!parent_val.Equals(child_val))
                  return false;
            }
            return true;
         } else
            return false;
      }

      public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, string depth_in_tree_col_name, string[] aggregation_tree, Dictionary<string, bool> sort_map) {
         if (BlueDataUtility.is_parent(r1, r2, depth_in_tree_col_name, aggregation_tree))
            return -1;
         else if (BlueDataUtility.is_parent(r2, r1, depth_in_tree_col_name, aggregation_tree))
            return 1;
         else if (get_row_value<int>(r1, "id", Int32.MaxValue).Equals(get_row_value<int>(r2, "id", Int32.MaxValue)))
            return 0;
         else {
            for (int p = 0; p < aggregation_tree.Count(); p++) {
               /// sort_map is used to leaf nodes
               if (p == aggregation_tree.Count() - 1) {
                  /// sorting for leaf nodes is special, since they can be
                  /// sorted by data columns
                  System.Diagnostics.Debug.Assert(BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree));
                  int leaf_comp = compare_rows(r1, r2, sort_map);
                  if (leaf_comp != 0)
                     return leaf_comp;
               }
               /// sort_map is used to find sorting order
               string aggregation_col_name = aggregation_tree.ElementAt(p);
               bool low_to_high = sort_map.TryGetValue(aggregation_col_name, out low_to_high) ? low_to_high : true;
               int comp = BlueDataUtility.compare_send_dontcare_to_last(r1, aggregation_col_name, r2, aggregation_col_name, low_to_high);
               if (comp != 0)
                  return comp;
            }
            return BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree) ?
                     compare_rows(r1, r2, sort_map) : 0;
         }
      }

      public static int sort_in_a_tree(IBlueRow r1, IBlueRow r2, int depth_in_tree_col_no, int[] aggregation_tree, Pair<int, bool>[] sort_map) {
         if (BlueDataUtility.is_parent(r1, r2, depth_in_tree_col_no, aggregation_tree))
            return -1;
         else if (BlueDataUtility.is_parent(r2, r1, depth_in_tree_col_no, aggregation_tree))
            return 1;
         else if (get_row_value<int>(r1, "id", Int32.MaxValue).Equals(get_row_value<int>(r2, "id", Int32.MaxValue)))
            return 0;
         else {
            for (int p = 0; p < aggregation_tree.Count(); p++) {
               /// sort_map is used to leaf nodes
               if (p == aggregation_tree.Count() - 1) {
                  /// sorting for leaf nodes is special, since they can be
                  /// sorted by data columns
                  //System.Diagnostics.Debug.Assert(BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_name, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_name, aggregation_tree));
                  int leaf_comp = compare_rows(r1, r2, sort_map);
                  if (leaf_comp != 0)
                     return leaf_comp;
               }
               /// sort_map is used to find sorting order
               int aggregation_col_no = aggregation_tree[p];
               bool low_to_high = true;//sort_map.TryGetValue(aggregation_col_name, out low_to_high) ? low_to_high : true;
               int comp = BlueDataUtility.compare_send_dontcare_to_last(r1, aggregation_col_no, r2, aggregation_col_no, low_to_high);
               if (comp != 0)
                  return comp;
            }
            return BlueDataUtility.is_leaf_node(r1, depth_in_tree_col_no, aggregation_tree) && BlueDataUtility.is_leaf_node(r2, depth_in_tree_col_no, aggregation_tree) ?
                     compare_rows(r1, r2, sort_map) : 0;
         }
      }

      #endregion

      #region Column accessors

      public static T get_array_value<T>(Array r, int col_no, T error_val) {
         try {
            object rv = (r != null && col_no > -1 && col_no < r.Length) ? r.GetValue(col_no) : DBNull.Value;
            if (rv == null || rv == DBNull.Value)
               return error_val;
            else
               return (T)Convert.ChangeType(rv, typeof(T));
         } catch {
            return error_val;
         }
      }

      public static DateTime get_array_datetime_value(Array r, int col_no, DateTime error_val) {
         try {
            object rv = (r != null && col_no > -1 && col_no < r.Length) ? r.GetValue(col_no) : error_val;
            return _get_datetime_value(rv, error_val);
         } catch {
            return error_val;
         }
      }

      public static T get_row_value<T>(IBlueRow r, int col_no, T error_val) {
         try {
            object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : DBNull.Value;
            if (rv == null || rv == DBNull.Value)
               return error_val;
            else
               return (T)Convert.ChangeType(rv, typeof(T));
         } catch {
            return error_val;
         }
      }

      public static T get_row_value<T>(IBlueRow r, string col_name, T error_val) {
         try {
            object rv = (r != null && r.Contains(col_name)) ? r[col_name] : DBNull.Value;
            if (rv == null || rv == DBNull.Value)
               return error_val;
            else
               return (T)Convert.ChangeType(rv, typeof(T));
         } catch {
            return error_val;
         }
      }

      public static T get_row_enum_value<T>(IBlueRow r, string col_name, T error_val) where T : struct {
         try {
            object rv = r.Contains(col_name) ? r[col_name] : DBNull.Value;
            return _get_row_enum_value<T>(rv, error_val);
         } catch {
            return error_val;
         }
      }

      public static T get_row_enum_value<T>(IBlueRow r, int col_no, T error_val) where T : struct {
         try {
            object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : DBNull.Value;
            return _get_row_enum_value<T>(rv, error_val);
         } catch {
            return error_val;
         }
      }

      private static T _get_row_enum_value<T>(object o, T error_val) where T : struct {
         try {
            if (o == null || o == DBNull.Value)
               return error_val;
            else if (o is string) {
               T result = error_val;
               if (Enum.TryParse<T>((string)o, true, out result)) return result;
               return error_val;
            } else if (o is Int32 || o is Int16 || o is UInt16 || o is UInt32)
               return (T)o;
            else if (o is T)
               return (T)o;
            else
               return error_val;
         } catch {
            return error_val;
         }
      }

      public static DateTime get_row_datetime_value(IBlueRow r, string col_name, DateTime error_val) {
         try {
            object rv = (r != null && r.Contains(col_name)) ? r[col_name] : error_val;
            return _get_datetime_value(rv, error_val);
         } catch {
            return error_val;
         }
      }

      public static DateTime get_row_datetime_value(IBlueRow r, int col_no, DateTime error_val) {
         try {
            object rv = (r != null && col_no > -1 && col_no < r.schema.columnInfo.Length) ? r[col_no] : error_val;
            return _get_datetime_value(rv, error_val);
         } catch {
            return error_val;
         }
      }

      private static DateTime _get_datetime_value(object v, DateTime error_val) {
         try {
            if (v == null || v == DBNull.Value)
               return error_val;

            if (v is DateTime) {
               return (DateTime)v;
            } else if (v is string) {
               DateTime rv_dt;

               // Try parsing the string as a DateTime
               if (DateTime.TryParse((string)v,
                                     System.Globalization.CultureInfo.InvariantCulture,
                                     System.Globalization.DateTimeStyles.RoundtripKind,
                                     out rv_dt)) {
                  return rv_dt;
               } else {
                  return error_val; // Return the error value if parsing fails
               }
            } else {
               // Assume the input can be converted to a double and use FromOADate
               double oa_date = (double)Convert.ChangeType(v, typeof(Double));
               return DateTime.FromOADate(oa_date);
            }
         } catch {
            return error_val;
         }
      }

      public static TimeSpan? get_timespan(this IBlueRow r, int col_no) =>
         get_timespan(r[col_no]);

      public static TimeSpan? get_timespan(this IBlueRow r, string col_name) =>
         get_timespan(r[col_name]);

      public static TimeSpan? get_timespan(object obj) {
         if (obj is TimeSpan ts) {
            return ts;
         }
         var str = obj as string ?? obj?.ToString();
         if (str is null) {
            return null;
         }
         if (TimeSpan.TryParse(str, out ts)) {
            return ts;
         }
         if (str.StartsWith("PT")) {
            try {
               // Parse ISO 8601 duration
               return System.Xml.XmlConvert.ToTimeSpan(str);
            } catch {
            }
         }
         return null;
      }
      #endregion

      #region CSV methods
      /// <summary>
      /// Reads a schema file name, and return the constructed schema
      /// </summary>
      public static IBlueTableSchema createSchemaFromCSVSchemaFile(string schemaFileName) {
         StreamReader sr = new StreamReader(schemaFileName);
         IBlueTableSchema retVal = createSchemaFromCSVSchemaFile(sr);
         sr.Close();
         return retVal;
      }

      /// <summary>
      /// Builds a schema from a schema file. Note that the stream is not closed, so
      /// the calling code should do so
      /// </summary>
      public static IBlueTableSchema createSchemaFromCSVSchemaFile(TextReader sr) {
         string tableName = sr.ReadLine();
         ArrayList t = new ArrayList();
         string l = "";
         while ((l = sr.ReadLine()) != null)
            t.Add(l.Trim());
         string[] cols = t.ToArray(typeof(string)) as String[];
         ColumnInfo[] cis = new ColumnInfo[cols.Length];
         for (int i = 0; i < cols.Length; i++) {
            string[] colTags = cols[i].Split('/');
            cis[i] = new ColumnInfo(colTags[0], i, TypeInstanceLoader.loadType(colTags[1]));
         }
         IBlueTableSchema retVal = new BlueTableSchema(cis, tableName);
         if (retVal["id"] == null) {
            ArrayList temp = new ArrayList(retVal.columnInfo);
            temp.Add(new ColumnInfo("id", temp.Count, typeof(System.Int32)));
            retVal = new BlueTableSchema(retVal.tableName,
               temp.ToArray(typeof(ColumnInfo)) as ColumnInfo[]);
         }
         return retVal;
      }

      /// <summary>
      /// Creates an IBlueRow from a CSV data file row; if the string already contains an id,
      /// the id argument is ignored
      /// </summary>
      public static IBlueRow createRowFromCSVFileRow(string fileRow, IBlueTableSchema schema, int id) {
         string[] toks = CSVParser.parse(fileRow);
         ArrayList rowList = new ArrayList();
         if (toks.Length == schema.columnNames.Length) {
            for (int i = 0; i < toks.Length; i++)
               rowList.Add(myConvertTo(schema[i].colType, toks[i]));
         } else {
            int idIndex = schema["id"].colNo;
            for (int i = 0; i < toks.Length; i++)
               rowList.Add(myConvertTo(schema[i < idIndex ? i : i + 1].colType, toks[i]));
            rowList.Insert(idIndex, id);
         }
         return new BlueRow(schema, rowList.ToArray());
      }

      /// <summary>
      /// Returns the CSV data string corresponding to the row
      /// </summary>
      public static string toCSVDataString(IBlueRow r) {
         string retVal = "";
         for (int i = 0; i < r.schema.columnNames.Length; i++)
            retVal +=
               r.schema[i].colType.Equals(typeof(string)) ?
               "\"" + r[i].ToString().Replace("\"", "\"\"") + "\"" :
               r[i].ToString();
         return retVal.Remove(retVal.Length - 1, 1);
      }

      public static string[] toCSVSchemaFileStrings(IBlueTableSchema schema) {
         string[] retVal = new string[2];
         retVal[0] = schema.tableName;
         foreach (ColumnInfo ci in schema.columnInfo)
            retVal[1] += ci.colName + "/" + ci.colType.ToString() + " ";
         retVal[1] = retVal[1].Remove(retVal[1].Length - 1, 1);
         return retVal;
      }

      /// <summary>
      /// Converts a CSV data value to its original type.
      /// </summary>
      private static object myConvertTo(Type t, string val) {
         val = val.Trim();
         if (t.Equals(typeof(System.String)))
            return val;
         else if (val != "") {
            if (t.Equals(typeof(System.Char)))
               return Convert.ToChar(val);
            if (t.Equals(typeof(System.Int16)))
               return Convert.ToInt16(val);
            if (t.Equals(typeof(System.Int32)))
               return Convert.ToInt32(val);
            if (t.Equals(typeof(System.Int64)))
               return Convert.ToInt64(val);
            if (t.Equals(typeof(System.Double)))
               return Convert.ToDouble(val);
            if (t.Equals(typeof(System.Single)))
               return Convert.ToSingle(val);
            if (t.Equals(typeof(System.DateTime)))
               return myToDateTime(val);
         }
         return DBNull.Value;
      }

      private static object myToDateTime(string val) {
         try {
            return Convert.ToDateTime(val);
         } catch (FormatException)  //datetime is not stored in one of the string formats
           {
            return val.IndexOf('.') > -1 ?
               DateTime.FromOADate(Convert.ToDouble(val)) : new DateTime(Convert.ToInt64(val));
         }
      }

      public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress) =>
         createRowsFromCSVFileWithoutTypes(fileAddress, out var _);

      public static IBlueRowCollection createRowsFromCSVFileWithoutTypes(string fileAddress, out IBlueTableSchema schema) {
         StreamReader sr = new StreamReader(fileAddress);
         string cols = sr.ReadLine();
         string[] columns = cols.Split(',');
         ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
         ci[0] = new ColumnInfo("id", 0, typeof(int));
         for (int i = 1; i < ci.Length; i++)
            ci[i] = new ColumnInfo(columns[i - 1], i, typeof(string));
         schema = new BlueTableSchema(ci, "csvTable");
         BlueTable result = new BlueTable(schema);
         int id = 0;
         while (!sr.EndOfStream)
            result.addRow(createRowFromCSVFileRow(sr.ReadLine(), schema, id++));
         sr.Close();
         return result;
      }

      #endregion

      #region Magic
      /// <summary>
      /// Uses a cached compiled expression to implement a truly generic add function.  Will throw if given types that cannot be added.
      /// </summary>
      internal class MagicAdder<T> {
         static Func<T, T, T> add;
         static MagicAdder() {
            ParameterExpression left = Expression.Parameter(typeof(T), "a");
            ParameterExpression right = Expression.Parameter(typeof(T), "b");
            add = (Func<T, T, T>)Expression.Lambda(Expression.Add(left, right), left, right).Compile();
         }

         public static T Add(T a, T b) {
            return add(a, b);
         }
      }
      #endregion

      #region Misc

      public static void swap<T>(ref T t1, ref T t2) {
         T temp = t1;
         t1 = t2;
         t2 = temp;
      }

      #endregion
   }
}