KeyValueToRelationalStream added
This commit is contained in:
1 parent
8cbbbc6da3
commit
cc3e289345
16 files changed
+464
-305
No files matched your search
@@ -13,6 +13,7 @@ using System.Linq.Expressions;
|
||||
//using mbts.spark;
|
||||
using mbts;
|
||||
using BCG.Extensions;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace BCG {
|
||||
public static class BlueDataUtility {
|
||||
@@ -35,19 +36,62 @@ namespace BCG {
|
||||
public static T get<T>(this IBlueRow r, int col_no, T err_value = default(T)) {
|
||||
return get_row_value(r, col_no, err_value);
|
||||
}
|
||||
public static IEnumerable<T> get_covalues<T>(this IBlueRowCollection rows, string col) {
|
||||
public static IEnumerable<T> get_column_values<T>(this IBlueRowCollection rows, string col) {
|
||||
List<T> rv = new List<T>();
|
||||
foreach (IBlueRow r in rows)
|
||||
rv.Add(r.get<T>(col));
|
||||
return rv;
|
||||
}
|
||||
public static IEnumerable<T> get_covalues<T>(IEnumerable<IBlueRow> rows, string col) {
|
||||
public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col) {
|
||||
List<T> rv = new List<T>();
|
||||
foreach (IBlueRow r in rows)
|
||||
rv.Add(r.get<T>(col));
|
||||
return rv;
|
||||
}
|
||||
|
||||
public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
|
||||
Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
|
||||
foreach (IBlueRow r in rows) {
|
||||
var gp_val = r.get<T>(gp_col);
|
||||
List<IBlueRow> gp_val_list = null;
|
||||
if (!rv.TryGetValue(gp_val, out gp_val_list)) {
|
||||
gp_val_list = new List<IBlueRow>();
|
||||
rv.Add(gp_val, gp_val_list);
|
||||
}
|
||||
gp_val_list.Add(r);
|
||||
}
|
||||
return rv.ToDictionary(kv => kv.Key, kv => kv.Value as IEnumerable<IBlueRow>);
|
||||
}
|
||||
|
||||
// this function takes in a key value table and converts it into a relational table, converting the key names into columns.
|
||||
// The values are returned as string values
|
||||
// doc_col is the column specifyng the doc name (following mongoDB's nomenclature, one set of key value rows ar a 'doc')
|
||||
public static IBlueRowCollection keyvalue_to_relational<T>(IBlueRowCollection kv_rows, string doc_col, string key_col, string val_col) {
|
||||
HashSet<string> unique_cols = new HashSet<string>();
|
||||
foreach (string col in get_column_values<string>(kv_rows, key_col)) {
|
||||
unique_cols.Add(col);
|
||||
}
|
||||
//unique_cols.Add(doc_col);
|
||||
var schema_cols = unique_cols.Select(c => Tuple.Create(c, TypeCode.String)).ToList();
|
||||
schema_cols.Insert(0, Tuple.Create(doc_col, BlueDataUtility.get_typecode(typeof(T))));
|
||||
schema_cols.Insert(0, Tuple.Create("id", TypeCode.Int32));
|
||||
var schema = BlueDataUtility.form_schema("relational", schema_cols.ToArray());
|
||||
BlueTable rv = new BlueTable(schema);
|
||||
var gp_map = group_by<T>(kv_rows, doc_col);
|
||||
foreach (var kv in gp_map) {
|
||||
MutableBlueRow nr = new MutableBlueRow(schema);
|
||||
nr.setValue("id", rv.Count);
|
||||
nr.setValue(doc_col, kv.Key);
|
||||
foreach (var r in kv.Value) {
|
||||
var col = r.get<string>(key_col);
|
||||
var val = r.get<string>(val_col);
|
||||
nr.setValue(col, val);
|
||||
}
|
||||
rv.addRow(nr);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
public static IBlueRowCollection where(this IBlueRowCollection rows, Func<IBlueRow, bool> where_func) {
|
||||
if (rows == null)
|
||||
return null;
|
||||
@@ -179,6 +223,15 @@ namespace BCG {
|
||||
{ TypeCode.UInt64, typeof(UInt64) }
|
||||
};
|
||||
|
||||
public static TypeCode get_typecode(Type t) {
|
||||
foreach (var kv in _typecode_to_type) {
|
||||
if (kv.Value == t)
|
||||
return kv.Key;
|
||||
}
|
||||
bts.Log.assert_d(false, $"type: {t} not found");
|
||||
return TypeCode.Empty;
|
||||
}
|
||||
|
||||
public static IBlueTableSchema form_schema(string table_name, params Tuple<string, TypeCode>[] cols) {
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||
foreach (Tuple<string, TypeCode> c in cols)
|
||||
|
||||
Reference in new issue
Block a user