* BlueOrbNLStream.cs * ChangeTableNameStream.cs * FilterStream.cs * FilterStream.cs * GroupByOutputTable.cs * GroupByStream.cs * JoinForeignTable.cs * MultiRowRollingMathStream.cs * MutableAppendStream.cs * PersistWindowState.cs * PubsubEchoTestStream.cs * RowDictionaryTable.cs * SortedTable.cs * TimeSeriesStream.cs Also add Metadata prop to IDataPacket
128 lines
5.7 KiB
C#
128 lines
5.7 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Diagnostics;
|
|
using System.Linq;
|
|
using System.Text.Json;
|
|
|
|
using bts;
|
|
//using bts.SilverPine;
|
|
using BCG;
|
|
using mbts.utils;
|
|
|
|
namespace bts.BlueOrb {
|
|
[DataStreamUserRespresentation("GroupByStream")]
|
|
public class GroupByStream : ViewStream {
|
|
|
|
public interface INonIncGroupByEvaluator {
|
|
// grouped_rows
|
|
IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
|
|
string[]? get_group_by_cols();
|
|
}
|
|
|
|
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
|
|
Func<string[]?> _groupby_cols_func;
|
|
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
|
|
public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func, Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
|
|
_groupby_cols_func = groupby_cols_func;
|
|
_evaluate_func = evaluate_func;
|
|
}
|
|
public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
|
|
BlueTable? rv = null;
|
|
foreach (var kv in grouped_rows) {
|
|
List<IBlueRow> g_rows = kv.Value;
|
|
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
|
|
if (out_rows is null || out_rows.Count == 0) {
|
|
continue;
|
|
}
|
|
rv ??= new BlueTable(out_rows.ElementAt(0).schema);
|
|
rv.addRows(out_rows);
|
|
}
|
|
return rv;
|
|
}
|
|
public string[]? get_group_by_cols() {
|
|
return _groupby_cols_func();
|
|
}
|
|
}
|
|
readonly ViewStream? _main_stream;
|
|
readonly INonIncGroupByEvaluator? _groupby_func;
|
|
readonly string[]? _groupby_cols;
|
|
readonly Stopwatch _timer = new();
|
|
|
|
BlueRowHasher? _row_hasher;
|
|
int _n_total_fires = 0;
|
|
int _n_expensive_fires = 0;
|
|
|
|
public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
|
|
_main_stream = main_stream;
|
|
_groupby_func = groupby_func;
|
|
_groupby_cols = _groupby_func.get_group_by_cols();
|
|
_row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
|
|
}
|
|
|
|
public GroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func, Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
|
|
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
|
|
|
|
public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_text)
|
|
: this(parent_str, groupby_cols, standard_group_by_text, String.Empty) { }
|
|
|
|
// this is the main constructor that does everything
|
|
public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
|
|
MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
|
|
Func<string[]?> groupby_cols_func = () => {
|
|
if (String.IsNullOrEmpty(groupby_cols))
|
|
return null;
|
|
return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
|
|
};
|
|
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
|
|
if (!String.IsNullOrEmpty(optional_sort_col))
|
|
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
|
|
// this si where the group by aggregation happens
|
|
IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
|
|
if (rv == null)
|
|
return new List<IBlueRow>();
|
|
return new IBlueRow[] { rv }.ToList();
|
|
};
|
|
ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
|
|
mySetRealSource(rs);
|
|
}
|
|
public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
|
|
this(parent_str, String.Empty, standard_group_by_text) {
|
|
}
|
|
|
|
|
|
protected override void myOnFirstSubscriber() { Log.notnull(_main_stream).onChange += _main_stream_onchange; }
|
|
|
|
protected override void myOnLastSubscriber() { Log.notnull(_main_stream).onChange -= _main_stream_onchange; }
|
|
|
|
private void _main_stream_onchange(object nv) {
|
|
var dp = ExceptionAssert.Argument.Is<IDataPacket>(nv);
|
|
///grouping together
|
|
_timer.Restart();
|
|
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
|
|
new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
|
|
BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
|
|
_timer.Stop();
|
|
var groupElapsedMs = _timer.ElapsedMilliseconds;
|
|
if (groupElapsedMs >= 50) {
|
|
Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
|
|
}
|
|
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
|
/// reducing
|
|
_timer.Restart();
|
|
var total_state = Log.notnull(_groupby_func).evaluate(grouped_rows);
|
|
if (total_state == null)
|
|
// mayeb returna n empty table ?
|
|
return;
|
|
_timer.Stop();
|
|
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
|
_n_total_fires++;
|
|
if (applyElapsedMs >= 50) {
|
|
_n_expensive_fires++;
|
|
Log.DataFlow.info(
|
|
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
|
|
}
|
|
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
|
}
|
|
}
|
|
}
|