Files
spark/src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs
T
John Colvin fb3a6262b3 cleanup: Fix build warnings
* BlueOrbNLStream.cs
* ChangeTableNameStream.cs
* FilterStream.cs
* FilterStream.cs
* GroupByOutputTable.cs
* GroupByStream.cs
* JoinForeignTable.cs
* MultiRowRollingMathStream.cs
* MutableAppendStream.cs
* PersistWindowState.cs
* PubsubEchoTestStream.cs
* RowDictionaryTable.cs
* SortedTable.cs
* TimeSeriesStream.cs

Also add Metadata prop to IDataPacket
2025-11-19 14:03:43 -05:00

128 lines
5.7 KiB
C#

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text.Json;
using bts;
//using bts.SilverPine;
using BCG;
using mbts.utils;
namespace bts.BlueOrb {
[DataStreamUserRespresentation("GroupByStream")]
public class GroupByStream : ViewStream {
public interface INonIncGroupByEvaluator {
// grouped_rows
IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
string[]? get_group_by_cols();
}
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
Func<string[]?> _groupby_cols_func;
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func, Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
_groupby_cols_func = groupby_cols_func;
_evaluate_func = evaluate_func;
}
public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
BlueTable? rv = null;
foreach (var kv in grouped_rows) {
List<IBlueRow> g_rows = kv.Value;
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
if (out_rows is null || out_rows.Count == 0) {
continue;
}
rv ??= new BlueTable(out_rows.ElementAt(0).schema);
rv.addRows(out_rows);
}
return rv;
}
public string[]? get_group_by_cols() {
return _groupby_cols_func();
}
}
readonly ViewStream? _main_stream;
readonly INonIncGroupByEvaluator? _groupby_func;
readonly string[]? _groupby_cols;
readonly Stopwatch _timer = new();
BlueRowHasher? _row_hasher;
int _n_total_fires = 0;
int _n_expensive_fires = 0;
public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
_main_stream = main_stream;
_groupby_func = groupby_func;
_groupby_cols = _groupby_func.get_group_by_cols();
_row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
}
public GroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func, Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_text)
: this(parent_str, groupby_cols, standard_group_by_text, String.Empty) { }
// this is the main constructor that does everything
public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
Func<string[]?> groupby_cols_func = () => {
if (String.IsNullOrEmpty(groupby_cols))
return null;
return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
};
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
if (!String.IsNullOrEmpty(optional_sort_col))
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
// this si where the group by aggregation happens
IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
if (rv == null)
return new List<IBlueRow>();
return new IBlueRow[] { rv }.ToList();
};
ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
mySetRealSource(rs);
}
public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
this(parent_str, String.Empty, standard_group_by_text) {
}
protected override void myOnFirstSubscriber() { Log.notnull(_main_stream).onChange += _main_stream_onchange; }
protected override void myOnLastSubscriber() { Log.notnull(_main_stream).onChange -= _main_stream_onchange; }
private void _main_stream_onchange(object nv) {
var dp = ExceptionAssert.Argument.Is<IDataPacket>(nv);
///grouping together
_timer.Restart();
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
_timer.Stop();
var groupElapsedMs = _timer.ElapsedMilliseconds;
if (groupElapsedMs >= 50) {
Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
}
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
/// reducing
_timer.Restart();
var total_state = Log.notnull(_groupby_func).evaluate(grouped_rows);
if (total_state == null)
// mayeb returna n empty table ?
return;
_timer.Stop();
var applyElapsedMs = _timer.ElapsedMilliseconds;
_n_total_fires++;
if (applyElapsedMs >= 50) {
_n_expensive_fires++;
Log.DataFlow.info(
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
}
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
}
}
}