Files
spark/src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs
T
2025-06-08 21:52:38 -04:00

234 lines
11 KiB
C#

using System;
using System.Collections;
using BCG.Utilities;
using System.Collections.Generic;
using System.Linq;
namespace BCG {
internal class GroupByStream : MethodDrivenStream {
private class MyFuncGroupByEvaluator : IGroupByEvaluator {
private string[] _groupby_cols;
private Func<DataPacket, IBlueRow> _groupby_func;
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
_groupby_cols = groupby_cols.ToArray();
_groupby_func = groupby_func;
}
public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
public string[] groupByColumns { get { return _groupby_cols; } }
public void clear() { }
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
}
/// <summary>
/// Serves as a quick way to access all rows belonging to the same group
/// </summary>
private bool myIdGuard = true;
private RowDictionaryTable myBookKeepingTable;
private IGroupByEvaluator myGroupByEvaluator;
private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
//can map a schema to another
private IBlueTableSchema myOutputSchema;
private int myIdGenerator = 0;
public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
: this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
: this(stream, groupByEvaluator, true) { }
public GroupByStream(ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
: base(stream) {
myGroupByEvaluator = groupbyEvaluator;
myIdGuard = idGuard;
}
protected override void myHandleFirstCall() {
mySchema = myLastArrivedPacket.schema;
myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
myGroupByEvaluator.groupByColumns);
GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
rowGroup, new BlueTable(mySchema)));
if (newRow != null)
temp.addRow(newRow, myGetGroupString(rowGroup));
}
myLastFiredPacket = new DataPacket(temp, new BlueTable(temp), new BlueTable(temp.schema));
}
/// <summary>
/// Adding need not to anything as deletion already takes care
/// of addition
/// </summary>
protected override void myHandleIncrementalAdd() { }
/// <summary>
/// Both deletes and adds are handled here to make the intermediate results not
/// appear in the spitted fired DataPacket (note this is done only in GroupByStream
/// because of the nature of it)
/// </summary>
protected override void myHandleIncrementalDelete() {
myFireSwitch = false;
RowDictionaryTable delGroups, addGroups;
//form groups for deleted and added rows
myInitIncrementalChangeData(out delGroups, out addGroups);
ArrayList handledAddGroups = new ArrayList();
string groupRepString;
foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
// BlueEventLog.logAssert (delGroup.Count > 0);
groupRepString = myGetGroupString(delGroup);
IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
handledAddGroups.Add(addGroup);
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
myFireSwitch |= myMergeRow(newRow, groupRepString);
}
foreach (IBlueRowCollection addGroup in addGroups.getGroups()) {
// BlueEventLog.logAssert (addGroup.Count > 0);
if (!handledAddGroups.Contains(addGroup)) {
//this is the case when there are rows added to a group and none
//have been removed, hence it 'escapes' from being dealt with in
//the loop above
groupRepString = myGetGroupString(addGroup);
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
// BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
new BlueTable(aggGroup.schema)));
myFireSwitch |= myMergeRow(newRow, groupRepString);
}
}
}
private bool myMergeRow(IBlueRow newRow, string groupRepString) {
IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
//BlueEventLog.logAssert (newRow != null || existingRow != null);
if (existingRow == null && newRow == null)
return false;
else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
return false;
else if (existingRow == null) {
//row added for the first time
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
return true;
} else {
((GroupByOutputTable)myLastFiredPacket.newRows).removeRow(groupRepString);
((BlueTable)myLastFiredPacket.deletedRows).addRow(existingRow);
if (newRow != null) {
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
}
return true;
}
}
/// <summary>
/// Divides the added and deletd rows of the input packet into groups, initializes the added and removed rows
/// of the output packet, andupdates the book keeping tables
/// </summary>
/// <param name="delGroups"></param>
/// <param name="addGroups"></param>
private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
delGroups =
new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
addGroups =
new RowDictionaryTable(myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
myBookKeepingTable.removeRow(deletedRow);
foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
myBookKeepingTable.addRow(addedRow);
myLastFiredPacket.addedRows = new BlueTable(myOutputSchema);
myLastFiredPacket.deletedRows = new BlueTable(myOutputSchema);
}
protected override void myOnTableClear() {
if (myBookKeepingTable.Count == 0)
return; //ie there arent any rows, so no need to optimize the 'cleansing' process
myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
(myLastFiredPacket.newRows as GroupByOutputTable).clear();
myBookKeepingTable.clear();
myGroupByEvaluator.clear();
}
private IBlueRow myEvaluate(DataPacket dp) {
IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
return evalRow == null ? null :
myIdGuard ?
BlueDataUtility.getNewReplacedRow(evalRow, myIdGenerator++, "id") : evalRow;
}
private string myGetGroupString(IBlueRowCollection group) {
return BlueDataUtility.rowToString(BlueDataUtility.extractRow(group, 0), myGroupByEvaluator.groupByColumns);
}
protected override void myOnLastSubscriber() {
//the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream
//gets resubscribed, we start the process of building data again, and treat the callback as the first
//callback.
myInit();
myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler);
}
protected override void myInit() {
base.myInit();
if (myBookKeepingTable != null) {
//to avoid calling init before the first callback
myBookKeepingTable.clear();
myGroupByEvaluator.clear();
}
}
public static string getRepString(DataPacket dp, string[] colNames) {
return BlueDataUtility.rowToString(getRepRow(dp), colNames);
}
public static IBlueRow getRepRow(DataPacket dp) {
if (dp.newRows.Count > 0)
return BlueDataUtility.extractRow(dp.newRows, 0);
else if (dp.addedRows.Count > 0)
return BlueDataUtility.extractRow(dp.addedRows, 0);
else if (dp.deletedRows.Count > 0)
return BlueDataUtility.extractRow(dp.deletedRows, 0);
// BlueEventLog.logAssert (false, "Illegal Data Packet");
return null;
}
/// <summary>
/// performs an add with the 2 arguments representing the last and new values and boolean indicating
/// an add or remove. If the
/// </summary>
/// <param name="lastValue"></param>
/// <param name="newValue"></param>
/// <param name="isAdd"></param>
/// <returns></returns>
public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
if (lastValue == null || lastValue.Equals(DBNull.Value))
return newValue;
// BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
// lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
// lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
Object retVal = null;
if (newValue.GetType().Equals(typeof(int)))
retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
else if (newValue.GetType().Equals(typeof(Double)))
retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
else if (lastValue.GetType().Equals(typeof(Decimal)))
retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
else { }
// BlueEventLog.logAssert (false, "Illegal type");
return retVal;
}
}
}