Files
spark/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByBundle.cs
T

126 lines
4.5 KiB
C#

namespace bts.SilverPine {
using BCG;
using System;
using System.Collections.Generic;
using System.Linq;
//using MathNet.Numerics.Statistics;
public enum MultiRowOp {
Mean,
Median,
Mode,
Range,
StdDev,
Max,
Min,
First,
Last,
Sum,
EdgeDiff,
EdgeDiffPercentage,
//IsEdgeDiff,
}
public class MultiRowOpHelper {
public readonly List<Tuple<string, MultiRowOp>> operations;
public static MultiRowOpHelper parse(string text) {
// text is inp_col: op, inp_col: op, inp_col:op
List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
string[] calcs = text.Split(',');
foreach (string c in calcs) {
string[] toks = c.Split(':');
string inp_col = toks[0].Trim();
string op_txt = toks[1].Trim();
MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
rv.Add(Tuple.Create(inp_col, op));
}
return new MultiRowOpHelper(rv);
}
public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
operations = operations_;
}
public static IBlueRow group_by(List<IBlueRow> rows, MultiRowOpHelper parameters) {
if (rows.Count() == 0) {
return null;
}
IBlueRow ir = rows.ElementAt(0);
IBlueTableSchema in_schema = ir.schema;
IBlueTableSchema out_schema = in_schema;
IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
new Dictionary<string, Tuple<string, MultiRowOp>>();
foreach (var t in parameters.operations) {
string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
out_col_map[out_col_name] = t;
}
MutableBlueRow or = new MutableBlueRow(out_schema, ir);
foreach (var kv in out_col_map) {
string out_col_name = kv.Key;
string in_col_name = kv.Value.Item1;
MultiRowOp op = kv.Value.Item2;
IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
double comp_res = compute_op(col_vals, op);
or.setValue(out_col_name, comp_res);
}
return or;
}
// rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
// calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
if (rolling_rows.Count() == 0) {
return null;
}
IBlueRow ir = rolling_rows.ElementAt(0);
IBlueTableSchema in_schema = ir.schema;
IBlueTableSchema out_schema = in_schema;
IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
new Dictionary<string, Tuple<string, MultiRowOp>>();
foreach (var t in operations) {
string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
out_col_map[out_col_name] = t;
}
MutableBlueRow or = new MutableBlueRow(out_schema, ir);
foreach (var kv in out_col_map) {
string out_col_name = kv.Key;
string in_col_name = kv.Value.Item1;
MultiRowOp op = kv.Value.Item2;
IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
double comp_res = compute_op(col_vals, op);
or.setValue(out_col_name, comp_res);
}
return or;
}
public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
switch (op) {
case MultiRowOp.Max:
return col_vals.Max<double>();
case MultiRowOp.Min:
return col_vals.Min<double>();
case MultiRowOp.Mean:
return col_vals.Sum() / col_vals.Count();
case MultiRowOp.Sum:
return col_vals.Sum();
case MultiRowOp.First:
return col_vals.First();
case MultiRowOp.Last:
return col_vals.Last();
case MultiRowOp.StdDev:
return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
default:
// TODO
Log.assert(false);
return Double.NaN;
}
}
}
}