Files
spark/src/DataStreamSuite/DataStream/Primitives/Timeseries/ColumnEvolutionStream.cs
T

51 lines
1.9 KiB
C#

namespace bts.SilverPine {
using BCG;
using bts;
using System.Collections.Generic;
using System.Linq;
// Given a sort order, an optional group by col, it observes the values of a column
// within that group (or the whole table, if grouping not specified) and returns all rows
// where the value changes, ie row[current][col] != row[current-1][col]
// TODO: a more general way to express a filter where row[current][col] != row[previous][col],
// or even expand it to row[current][col_a] - row[previous][col_b] < 0.5
[DataStreamUserRespresentation("ColEvol")]
public class ColumnEvolutionStream : ViewStream {
private readonly ViewStream _base_stream;
private readonly string _sort_col;
private readonly string _evol_col;
public ColumnEvolutionStream(ViewStream base_stream, string sort_col, string evolution_col) {
_base_stream = base_stream;
_sort_col = sort_col;
_evol_col = evolution_col;
}
protected override void myOnFirstSubscriber() {
_base_stream.onChange += _handle_base_stream_update;
}
protected override void myOnLastSubscriber() {
base.myOnLastSubscriber();
}
private void _handle_base_stream_update(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
List<IBlueRow> rows = dp.newRows.to_list();
rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col));
BlueTable rv = new BlueTable(dp.schema);
IBlueRow prev_row = rows.ElementAt(0);
foreach (IBlueRow r in rows) {
var cr = BlueDataUtility.compare_rows(r, prev_row, _evol_col);
if (cr != 0) {
rv.addRow(r);
}
prev_row = r;
}
DataPacket dp_out = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
myFireEvent(dp_out);
}
}
}