Files
spark/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs
T
2023-09-04 16:25:40 -04:00

244 lines
10 KiB
C#

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using mbts;
namespace BCG
{
public class YXTranspositionStream : ViewStream, INonIncGroupByEvaluator
{
private class OnlyFireOnceStream : ViewStream
{
private ViewStream _stream;
private bool _hooked = false;
public OnlyFireOnceStream(ViewStream stream)
{
_stream = stream;
}
protected override void myOnFirstSubscriber()
{
_stream.onChange += _stream_onChange;
_hooked = true;
}
protected override void myOnLastSubscriber()
{
if (_hooked && _stream != null)
_stream.onChange -= _stream_onChange;
}
private void _stream_onChange(object newValue)
{
DataPacket dp = newValue as DataPacket;
if (dp != null && dp.newRows.Count > 0)
{
_stream.onChange -= _stream_onChange;
myFireEvent(new DataPacket(dp.newRows, dp.newRows, new BlueTable(dp.newRows.schema)));
}
}
}
public delegate void IncrementValues(object[] current_values, object[] values_to_inc);
public delegate string GetColumnPrefix(object[] transposing_col_values);
public delegate string GetColumnSuffix(string col_name);
private GetColumnSuffix _get_column_suffix = new GetColumnSuffix((cn) => { return cn; });
private IncrementValues _increment_values;
private GetColumnPrefix _get_column_prefix;
private IBlueTableSchema _schema;
private string[] _group_by_cols;
private Pair<string, Type>[] _cols_per_transposed;
private string[] _transposing_cols;
private int[] _transposing_col_indices;
private Comparison<Pair<int, object[]>> _transpsed_col_value_comparer_wrapper;
public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer)
: this(stream, transposed_cols_group, cols_per_transposed, group_by_cols, increment_values_del,
get_column_prefix_del, get_column_suffix_del, transposed_col_value_comparer, true)
{ }
public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer, bool only_fire_once)
{
if (get_column_suffix_del != null)
_get_column_suffix = get_column_suffix_del;
if (transposed_col_value_comparer != null)
{
_transpsed_col_value_comparer_wrapper = new Comparison<Pair<int, object[]>>((p1,p2) =>
{
return transposed_col_value_comparer(p1.second, p2.second);
});
}
_group_by_cols = group_by_cols.Split(' ');
_transposing_cols = transposed_cols_group.Split(' ');
string[] cols = cols_per_transposed.Split(' ');
_cols_per_transposed = new Pair<string, Type>[cols.Length];
for (int i = 0; i < _cols_per_transposed.Length; i++)
{
string[] name_type = cols[i].Split('\\');
// mbts.Logger.assert(name_type.Length == 2, "Expected cols_per_transposed to be in format name\\type");
_cols_per_transposed[i] = new Pair<string, Type>(name_type[0], Type.GetType(name_type[1]));
}
_increment_values = increment_values_del;
_get_column_prefix = get_column_prefix_del;
if(only_fire_once)
mySetRealSource(new NonIncGroupByStream(new OnlyFireOnceStream(stream), this));
else
mySetRealSource(new NonIncGroupByStream(stream, this));
}
private void _set_schema(List<string> tc_prefixes, Pair<string, Type>[] group_by_col_info, string table_name)
{
int col_index = 0;
List<ColumnInfo> col_info = new List<ColumnInfo>();
col_info.Add(new ColumnInfo("id", col_index++, typeof(int)));
foreach (Pair<string, Type> p in group_by_col_info)
col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
foreach (string tc_prefix in tc_prefixes)
foreach (Pair<string, Type> p in _cols_per_transposed)
col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first), col_index++, p.second));
IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
if (_schema == null)
_schema = schema;
}
private string _get_full_column_name(string prefix, string col_name)
{
string suffix = _get_column_suffix(col_name);
if (suffix == string.Empty)
return prefix;
else
return prefix + "_" + suffix;
}
#region INonIncGroupByEvaluator Members
public string[] get_group_by_cols()
{
return _group_by_cols;
}
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
{
Dictionary<int, Dictionary<int, object[]>> all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object[]>>();
Dictionary<int, object[]> group_by_cols_value_dict = new Dictionary<int, object[]>();
BlueRowHasher hasher = new BlueRowHasher(_transposing_cols.Length);
Pair<string, Type>[] group_by_col_info = new Pair<string, Type>[_group_by_cols.Length];
string table_name = string.Empty;
if (grouped_rows.Count > 0)
{
IBlueTableSchema schema = null;
foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
{
schema = kvp.Value[0].schema;
break;
}
table_name = schema.tableName;
_transposing_col_indices = BlueDataUtility.get_column_indices(schema, _transposing_cols);
int[] group_by_indices = BlueDataUtility.get_column_indices(schema, _group_by_cols);
for (int i = 0; i < group_by_indices.Length; i++)
{
ColumnInfo ci = schema.columnInfo[group_by_indices[i]];
group_by_col_info[i] = new Pair<string, Type>(ci.colName, ci.colType);
}
}
else
return new BlueTable(new BlueTableSchema(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }, table_name));
Dictionary<int, string> tc_prefixes = new Dictionary<int, string>();
List<Pair<int, object[]>> tc_values = new List<Pair<int, object[]>>();
foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
{
//Dictionary of trans_col_group_hash -> pair: first=trans_col_group_values second=per_transposing_col_values
Dictionary<int, object[]> incrementing_values = new Dictionary<int, object[]>();
foreach (IBlueRow r in kvp.Value)
{
int hash = hasher.get_hash(r, _transposing_col_indices);
object[] values;
if (!incrementing_values.TryGetValue(hash, out values))
{
incrementing_values.Add(hash, _get_per_transposing_col_values(r));
string prefix = _get_column_prefix(_get_transposing_col_values(r));
if (!tc_prefixes.ContainsKey(hash))
{
tc_prefixes.Add(hash, prefix);
tc_values.Add(new Pair<int,object[]>(hash, _get_transposing_col_values(r)));
}
}
else
_increment_values(values, _get_per_transposing_col_values(r));
}
all_tc_infos_and_values.Add(kvp.Key, incrementing_values);
//An ugly way to extract group information needed to insert values later
foreach (IBlueRow r in kvp.Value)
{
group_by_cols_value_dict.Add(kvp.Key, _get_group_by_cols_values(r));
break;
}
}
//Sort the transposed schema
if (_transpsed_col_value_comparer_wrapper != null)
tc_values.Sort(_transpsed_col_value_comparer_wrapper);
List<string> prefixes = new List<string>();
foreach (Pair<int, object[]> p in tc_values)
prefixes.Add(tc_prefixes[p.first]);
_set_schema(prefixes, group_by_col_info, table_name);
BlueTable result = new BlueTable(_schema);
int id = 1;
foreach (KeyValuePair<int, Dictionary<int, object[]>> kvp in all_tc_infos_and_values)
{
MutableBlueRow mbr = new MutableBlueRow(_schema);
mbr.setValue("id", id++);
for (int i = 0; i < _group_by_cols.Length; i++)
mbr.setValue(_group_by_cols[i], group_by_cols_value_dict[kvp.Key][i]);
foreach (KeyValuePair<int, object[]> tc_cols in kvp.Value)
{
string prefix = tc_prefixes[tc_cols.Key];
for (int i = 0; i < _cols_per_transposed.Length; i++)
{
string column = _get_full_column_name(prefix, _cols_per_transposed[i].first);
if(mbr.schema.columnNames.Contains(column))
mbr.setValue(column, tc_cols.Value[i]);
}
}
result.addRow(mbr);
}
return result;
}
#endregion
private object[] _get_transposing_col_values(IBlueRow row)
{
object[] values = new object[_transposing_cols.Length];
for (int i = 0; i < _transposing_cols.Length; i++)
values[i] = row[_transposing_cols[i]];
return values;
}
private object[] _get_per_transposing_col_values(IBlueRow row)
{
object[] values = new object[_cols_per_transposed.Length];
for (int i = 0; i < _cols_per_transposed.Length; i++)
values[i] = row[_cols_per_transposed[i].first];
return values;
}
private object[] _get_group_by_cols_values(IBlueRow row)
{
object[] values = new object[_group_by_cols.Length];
for (int i = 0; i < values.Length; i++)
values[i] = row[_group_by_cols[i]];
return values;
}
}
}