KeyValueToRelationalStream added

This commit is contained in:
Taha Afzal committed 2024-01-16 19:08:40 -05:00
1 parent 8cbbbc6da3
commit cc3e289345
16 files changed
+464 -305

No files matched your search

@@ -13,6 +13,7 @@ using System.Linq.Expressions;
//using mbts.spark;
using mbts;
using BCG.Extensions;
using System.Runtime.CompilerServices;
namespace BCG {
public static class BlueDataUtility {
@@ -35,19 +36,62 @@ namespace BCG {
public static T get<T>(this IBlueRow r, int col_no, T err_value = default(T)) {
return get_row_value(r, col_no, err_value);
}
public static IEnumerable<T> get_covalues<T>(this IBlueRowCollection rows, string col) {
public static IEnumerable<T> get_column_values<T>(this IBlueRowCollection rows, string col) {
List<T> rv = new List<T>();
foreach (IBlueRow r in rows)
rv.Add(r.get<T>(col));
return rv;
}
public static IEnumerable<T> get_covalues<T>(IEnumerable<IBlueRow> rows, string col) {
public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col) {
List<T> rv = new List<T>();
foreach (IBlueRow r in rows)
rv.Add(r.get<T>(col));
return rv;
}
public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
foreach (IBlueRow r in rows) {
var gp_val = r.get<T>(gp_col);
List<IBlueRow> gp_val_list = null;
if (!rv.TryGetValue(gp_val, out gp_val_list)) {
gp_val_list = new List<IBlueRow>();
rv.Add(gp_val, gp_val_list);
}
gp_val_list.Add(r);
}
return rv.ToDictionary(kv => kv.Key, kv => kv.Value as IEnumerable<IBlueRow>);
}
// this function takes in a key value table and converts it into a relational table, converting the key names into columns.
// The values are returned as string values
// doc_col is the column specifyng the doc name (following mongoDB's nomenclature, one set of key value rows ar a 'doc')
public static IBlueRowCollection keyvalue_to_relational<T>(IBlueRowCollection kv_rows, string doc_col, string key_col, string val_col) {
HashSet<string> unique_cols = new HashSet<string>();
foreach (string col in get_column_values<string>(kv_rows, key_col)) {
unique_cols.Add(col);
}
//unique_cols.Add(doc_col);
var schema_cols = unique_cols.Select(c => Tuple.Create(c, TypeCode.String)).ToList();
schema_cols.Insert(0, Tuple.Create(doc_col, BlueDataUtility.get_typecode(typeof(T))));
schema_cols.Insert(0, Tuple.Create("id", TypeCode.Int32));
var schema = BlueDataUtility.form_schema("relational", schema_cols.ToArray());
BlueTable rv = new BlueTable(schema);
var gp_map = group_by<T>(kv_rows, doc_col);
foreach (var kv in gp_map) {
MutableBlueRow nr = new MutableBlueRow(schema);
nr.setValue("id", rv.Count);
nr.setValue(doc_col, kv.Key);
foreach (var r in kv.Value) {
var col = r.get<string>(key_col);
var val = r.get<string>(val_col);
nr.setValue(col, val);
}
rv.addRow(nr);
}
return rv;
}
public static IBlueRowCollection where(this IBlueRowCollection rows, Func<IBlueRow, bool> where_func) {
if (rows == null)
return null;
@@ -179,6 +223,15 @@ namespace BCG {
{ TypeCode.UInt64, typeof(UInt64) }
};
public static TypeCode get_typecode(Type t) {
foreach (var kv in _typecode_to_type) {
if (kv.Value == t)
return kv.Key;
}
bts.Log.assert_d(false, $"type: {t} not found");
return TypeCode.Empty;
}
public static IBlueTableSchema form_schema(string table_name, params Tuple<string, TypeCode>[] cols) {
List<ColumnInfo> cis = new List<ColumnInfo>();
foreach (Tuple<string, TypeCode> c in cols)