Files
spark/src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnDifferenceStream.cs
T

124 lines
5.5 KiB
C#

namespace bts.SilverPine {
using BCG;
using bts;
using bts.collections;
using System;
using System.CodeDom;
using System.Collections.Generic;
using System.Linq;
// Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
// so:
// f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
[DataStreamUserRespresentation("ColDiff")]
public class ColumnDifferenceStream : ViewStream {
private readonly ViewStream _base_stream;
private readonly FilterStream.OnFilter _f1, _f2;
private readonly string _c1, _c2;
public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
_base_stream = base_stream;
_f1 = f1;
_f2 = f2;
_c1 = col1;
_c2 = col2;
}
public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
FilterStream.OnFilter f1_del = r => {
BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
return rv.filter(r);
};
FilterStream.OnFilter f2_del = r => {
BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
return rv.filter(r);
};
ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
mySetRealSource(rs);
}
// for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
// we should generlize this for any math operation
public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
// use GlobalMapStream
string diff_col_perc_name = $"{diff_col_name}_percentage";
System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
int minuend_idx = Convert.ToInt32(idx);
var inp_schema = rows[0].schema;
var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
// minuend - subtrahend
IBlueRow minuend_row = rows.ElementAt(minuend_idx);
MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
int subtrahend_index = minuend_idx + row_offset;
double diff_val = Double.NaN, diff_val_perc = Double.NaN;
if (subtrahend_index >= 0) {
try {
IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
} catch (Exception ex) {
Log.assert(false, ex.Message);
}
}
rv.setValue(diff_col_name, diff_val);
rv.setValue(diff_col_perc_name, diff_val_perc);
return rv;
};
ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
if (optional_perc_diff_filter != 0) {
rs = new FilterStream(rs, r => {
double diff_val_perc = r.get<double>(diff_col_perc_name);
double abs_diff_val_perc = Math.Abs(diff_val_perc);
return abs_diff_val_perc >= optional_perc_diff_filter;
});
}
mySetRealSource(rs);
}
protected override void myOnFirstSubscriber() {
_base_stream.onChange += _handle_base_stream_update;
}
protected override void myOnLastSubscriber() {
_base_stream.onChange -= _handle_base_stream_update;
}
private void _handle_base_stream_update(object dp_obj) {
// could have been done with filter and cartesian prod, and then math
DataPacket dp = dp_obj as DataPacket;
List<IBlueRow> rows = dp.newRows.to_list();
var f1_rows = rows.Where(r => _f1(r));
//Log.assert(f1_rows.Count() == 1);
var f2_rows = rows.Where(r => _f2(r));
//Log.assert(f2_rows.Count() == 1);
if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
IBlueRow f1_r = f1_rows.ElementAt(0);
IBlueRow f2_r = f2_rows.ElementAt(0);
double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
BlueTable out_table = new BlueTable(out_schema);
MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
out_row.setValue("diff", diff);
out_table.addRow(out_row);
DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
myFireEvent(dp_out);
} else {
IBlueRow f1_r = rows.ElementAt(0);
IBlueRow f2_r = rows.ElementAt(0);
double diff = Double.NaN;
IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
BlueTable out_table = new BlueTable(out_schema);
MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
out_row.setValue("diff", diff);
out_table.addRow(out_row);
DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
myFireEvent(dp_out);
}
}
}
}