Files
spark/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs
T

152 lines
5.9 KiB
C#

using bts;
using bts.collections;
using mbts;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Security.Cryptography;
namespace BCG {
public class NonIncJoinBookkeeper : IDisposable {
public Action on_new_data;
private readonly NonIncJoinSpec _join_spec;
private readonly int _join_position;
private readonly int _start_index_in_output;
private readonly BlueRowHasher _row_hasher;
private readonly bool _is_left_most;
private readonly bool _is_tag_along;
private int _n_total_fires = 0, _n_expensive_fires = 0;
private IBlueTableSchema _schema = null;
private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
private IBlueRow _solitary_row = null;
private int[] _join_col_indices = null,
_data_col_indices = null;
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
_join_spec = the_join_spec;
_join_position = join_position;
_start_index_in_output = start_index_in_output;
_row_hasher = row_hasher;
_is_left_most = _join_position == 0;
_is_tag_along = _join_spec.join_col_names.Length == 0;
}
private bool _filled_missing_data = false;
private void _fill_missing_data(DataPacket dp) {
if (_filled_missing_data)
return;
Func<string[], bool> null_or_empty = (str_arr) => {
if (str_arr == null || str_arr.Length == 0)
return true;
foreach (string s in str_arr) {
if (!String.IsNullOrEmpty(s)) {
return false;
}
}
return true;
};
Func<string[], string[]> remove_bad_cols = (col_names) => {
HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
List<string> cols_list = new List<string>(col_names);
cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
return cols_list.ToArray();
};
if (null_or_empty(_join_spec.data_col_names)) {
_join_spec.data_col_names = dp.schema.columnNames;
}
_join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
if (null_or_empty(_join_spec.data_col_names)) {
_join_spec.data_col_names = dp.schema.columnNames;
}
if (null_or_empty(_join_spec.projected_col_names)) {
_join_spec.projected_col_names = _join_spec.data_col_names;
}
}
public void init() {
_join_spec.stream.onChange += _on_stream_cb;
}
public void _on_stream_cb(object dp_obj) {
Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
var timer = Stopwatch.StartNew();
DataPacket dp = (DataPacket)dp_obj;
_fill_missing_data(dp);
if (_schema == null) {
_schema = dp.schema;
_join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
_data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
}
if (_is_left_most) {
_row_list.Clear();
foreach (IBlueRow r in dp.newRows)
_row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
} else {
if (_is_tag_along)
_solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
else {
_row_map.Clear();
foreach (IBlueRow r in dp.newRows)
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
}
}
timer.Stop();
_n_total_fires++;
if (timer.ElapsedMilliseconds >= 50) {
_n_expensive_fires++;
Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
+ $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
}
if (on_new_data != null)
on_new_data();
}
public bool is_ready { get { return _schema != null; } }
public void constribute_cols_to_schema(List<ColumnInfo> cis) {
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
}
public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
public void copy_master_row_to_output_row(int row_index, object[] output_row) {
IBlueRow this_row = _row_list[row_index].second;
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
}
public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
if (_is_tag_along) {
if (_solitary_row != null)
BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
else
copy_dbnull_to_output_row(output_row);
} else {
IBlueRow this_row = null;
if (_row_map.TryGetValue(row_hash, out this_row))
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
else
copy_dbnull_to_output_row(output_row);
}
}
private void copy_dbnull_to_output_row(object[] output_row) {
for (int i = 0; i < _data_col_indices.Length; i++)
output_row[i + _start_index_in_output] = DBNull.Value;
}
#region IDisposable
public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
#endregion
}
}