Files
spark/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs
T

235 lines
9.9 KiB
C#

using mbts;
using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic;
using System.Linq;
namespace BCG
{
public class YXTranspositionStream : ViewStream, INonIncGroupByEvaluator
{
private class OnlyFireOnceStream : ViewStream
{
private ViewStream _stream;
private bool _hooked = false;
public OnlyFireOnceStream(ViewStream stream)
{
_stream = stream;
}
protected override void myOnFirstSubscriber()
{
_stream.onChange += _stream_onChange;
_hooked = true;
}
protected override void myOnLastSubscriber()
{
if (_hooked && _stream != null)
_stream.onChange -= _stream_onChange;
}
private void _stream_onChange(object newValue)
{
var dp = (IDataPacket)newValue;
if (dp != null && dp.newRows.Count > 0)
{
_stream.onChange -= _stream_onChange;
myFireEvent(new DataPacket(dp.newRows, dp.newRows, new BlueTable(dp.newRows.schema)));
}
}
}
public delegate void IncrementValues(object?[] current_values, object?[] values_to_inc);
public delegate string GetColumnPrefix(object?[] transposing_col_values);
public delegate string GetColumnSuffix(string col_name);
private GetColumnSuffix _get_column_suffix = new GetColumnSuffix((cn) => { return cn; });
private IncrementValues _increment_values;
private GetColumnPrefix _get_column_prefix;
IBlueTableSchema? _schema;
private string[] _group_by_cols;
private Pair<string, Type>[] _cols_per_transposed;
private string[] _transposing_cols;
int[]? _transposing_col_indices;
Comparison<Pair<int, object?[]>>? _transpsed_col_value_comparer_wrapper;
public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer)
: this(stream, transposed_cols_group, cols_per_transposed, group_by_cols, increment_values_del,
get_column_prefix_del, get_column_suffix_del, transposed_col_value_comparer, true)
{ }
public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer, bool only_fire_once)
{
if (get_column_suffix_del != null)
_get_column_suffix = get_column_suffix_del;
if (transposed_col_value_comparer != null)
{
_transpsed_col_value_comparer_wrapper = new((p1, p2) =>
transposed_col_value_comparer(ExceptionAssert.NotNull(p1.second), ExceptionAssert.NotNull(p2.second)));
}
_group_by_cols = group_by_cols.Split(' ');
_transposing_cols = transposed_cols_group.Split(' ');
string[] cols = cols_per_transposed.Split(' ');
_cols_per_transposed = new Pair<string, Type>[cols.Length];
for (int i = 0; i < _cols_per_transposed.Length; i++)
{
string[] name_type = cols[i].Split('\\');
// mbts.Logger.assert(name_type.Length == 2, "Expected cols_per_transposed to be in format name\\type");
_cols_per_transposed[i] = new Pair<string, Type>(name_type[0], Type.GetType(name_type[1]));
}
_increment_values = increment_values_del;
_get_column_prefix = get_column_prefix_del;
if(only_fire_once)
mySetRealSource(new NonIncGroupByStream(new OnlyFireOnceStream(stream), this));
else
mySetRealSource(new NonIncGroupByStream(stream, this));
}
private void _set_schema(List<string> tc_prefixes, Pair<string, Type>[] group_by_col_info, string table_name)
{
int col_index = 0;
List<ColumnInfo> col_info = new List<ColumnInfo>();
col_info.Add(new ColumnInfo("id", col_index++, typeof(int)));
foreach (Pair<string, Type> p in group_by_col_info)
col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
foreach (string tc_prefix in tc_prefixes)
foreach (Pair<string, Type> p in _cols_per_transposed)
col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, p.second));
IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
if (_schema == null)
_schema = schema;
}
private string _get_full_column_name(string prefix, string col_name)
{
string suffix = _get_column_suffix(col_name);
if (suffix == string.Empty)
return prefix;
else
return prefix + "_" + suffix;
}
#region INonIncGroupByEvaluator Members
public string[] get_group_by_cols()
{
return _group_by_cols;
}
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
{
var all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object?[]>>();
var group_by_cols_value_dict = new Dictionary<int, object?[]>();
BlueRowHasher hasher = new BlueRowHasher(_transposing_cols.Length);
Pair<string, Type>[] group_by_col_info = new Pair<string, Type>[_group_by_cols.Length];
string table_name = string.Empty;
if (grouped_rows.Count > 0)
{
var schema = grouped_rows.Values.First()[0].schema;
table_name = schema.tableName;
_transposing_col_indices = BlueDataUtility.get_column_indices(schema, _transposing_cols);
int[] group_by_indices = BlueDataUtility.get_column_indices(schema, _group_by_cols);
for (int i = 0; i < group_by_indices.Length; i++)
{
ColumnInfo ci = schema.columnInfo[group_by_indices[i]];
group_by_col_info[i] = new Pair<string, Type>(ci.colName, ci.colType);
}
}
else
return new BlueTable(new BlueTableSchema(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }, table_name));
Dictionary<int, string> tc_prefixes = new Dictionary<int, string>();
var tc_values = new List<Pair<int, object?[]>>();
foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
{
//Dictionary of trans_col_group_hash -> pair: first=trans_col_group_values second=per_transposing_col_values
var incrementing_values = new Dictionary<int, object?[]>();
foreach (IBlueRow r in kvp.Value)
{
int hash = hasher.get_hash(r, _transposing_col_indices);
if (!incrementing_values.TryGetValue(hash, out var values))
{
incrementing_values.Add(hash, _get_per_transposing_col_values(r));
string prefix = _get_column_prefix(_get_transposing_col_values(r));
if (!tc_prefixes.ContainsKey(hash))
{
tc_prefixes.Add(hash, prefix);
tc_values.Add(new(hash, _get_transposing_col_values(r)));
}
}
else
_increment_values(values, _get_per_transposing_col_values(r));
}
all_tc_infos_and_values.Add(kvp.Key, incrementing_values);
//An ugly way to extract group information needed to insert values later
foreach (IBlueRow r in kvp.Value)
{
group_by_cols_value_dict.Add(kvp.Key, _get_group_by_cols_values(r));
break;
}
}
//Sort the transposed schema
if (_transpsed_col_value_comparer_wrapper != null)
tc_values.Sort(_transpsed_col_value_comparer_wrapper);
List<string> prefixes = new List<string>();
foreach (Pair<int, object[]> p in tc_values)
prefixes.Add(tc_prefixes[p.first]);
_set_schema(prefixes, group_by_col_info, table_name);
BlueTable result = new BlueTable(_schema);
int id = 1;
foreach (var kvp in all_tc_infos_and_values)
{
MutableBlueRow mbr = new MutableBlueRow(_schema);
mbr.setValue("id", id++);
for (int i = 0; i < _group_by_cols.Length; i++)
mbr.setValue(_group_by_cols[i], group_by_cols_value_dict[kvp.Key][i]);
foreach (var tc_cols in kvp.Value)
{
string prefix = tc_prefixes[tc_cols.Key];
for (int i = 0; i < _cols_per_transposed.Length; i++)
{
string column = _get_full_column_name(prefix, _cols_per_transposed[i].first ?? string.Empty);
if(mbr.schema.columnNames.Contains(column))
mbr.setValue(column, tc_cols.Value[i]);
}
}
result.addRow(mbr);
}
return result;
}
#endregion
object?[] _get_transposing_col_values(IBlueRow row) {
var values = new object?[_transposing_cols.Length];
for (int i = 0; i < _transposing_cols.Length; i++)
values[i] = row[_transposing_cols[i]];
return values;
}
object?[] _get_per_transposing_col_values(IBlueRow row) {
var values = new object?[_cols_per_transposed.Length];
for (int i = 0; i < _cols_per_transposed.Length; i++)
values[i] = row[ExceptionAssert.NotNull(_cols_per_transposed[i].first)];
return values;
}
object?[] _get_group_by_cols_values(IBlueRow row) {
var values = new object?[_group_by_cols.Length];
for (int i = 0; i < values.Length; i++)
values[i] = row[_group_by_cols[i]];
return values;
}
}
}