152 lines
5.9 KiB
C#
152 lines
5.9 KiB
C#
using bts;
|
|
using bts.collections;
|
|
|
|
using mbts;
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Diagnostics;
|
|
using System.Linq;
|
|
using System.Security.Cryptography;
|
|
|
|
namespace BCG {
|
|
public class NonIncJoinBookkeeper : IDisposable {
|
|
public Action on_new_data;
|
|
private readonly NonIncJoinSpec _join_spec;
|
|
private readonly int _join_position;
|
|
private readonly int _start_index_in_output;
|
|
private readonly BlueRowHasher _row_hasher;
|
|
private readonly bool _is_left_most;
|
|
private readonly bool _is_tag_along;
|
|
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
|
|
|
private IBlueTableSchema _schema = null;
|
|
private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
|
|
private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
|
|
private IBlueRow _solitary_row = null;
|
|
|
|
private int[] _join_col_indices = null,
|
|
_data_col_indices = null;
|
|
|
|
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
|
|
_join_spec = the_join_spec;
|
|
_join_position = join_position;
|
|
_start_index_in_output = start_index_in_output;
|
|
_row_hasher = row_hasher;
|
|
_is_left_most = _join_position == 0;
|
|
_is_tag_along = _join_spec.join_col_names.Length == 0;
|
|
}
|
|
|
|
private bool _filled_missing_data = false;
|
|
private void _fill_missing_data(DataPacket dp) {
|
|
if (_filled_missing_data)
|
|
return;
|
|
Func<string[], bool> null_or_empty = (str_arr) => {
|
|
if (str_arr == null || str_arr.Length == 0)
|
|
return true;
|
|
foreach (string s in str_arr) {
|
|
if (!String.IsNullOrEmpty(s)) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
};
|
|
Func<string[], string[]> remove_bad_cols = (col_names) => {
|
|
HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
|
|
List<string> cols_list = new List<string>(col_names);
|
|
cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
|
|
return cols_list.ToArray();
|
|
};
|
|
if (null_or_empty(_join_spec.data_col_names)) {
|
|
_join_spec.data_col_names = dp.schema.columnNames;
|
|
}
|
|
_join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
|
|
if (null_or_empty(_join_spec.data_col_names)) {
|
|
_join_spec.data_col_names = dp.schema.columnNames;
|
|
}
|
|
if (null_or_empty(_join_spec.projected_col_names)) {
|
|
_join_spec.projected_col_names = _join_spec.data_col_names;
|
|
}
|
|
}
|
|
|
|
public void init() {
|
|
_join_spec.stream.onChange += _on_stream_cb;
|
|
}
|
|
|
|
public void _on_stream_cb(object dp_obj) {
|
|
Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
|
|
var timer = Stopwatch.StartNew();
|
|
DataPacket dp = (DataPacket)dp_obj;
|
|
_fill_missing_data(dp);
|
|
if (_schema == null) {
|
|
_schema = dp.schema;
|
|
_join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
|
|
_data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
|
|
}
|
|
|
|
if (_is_left_most) {
|
|
_row_list.Clear();
|
|
foreach (IBlueRow r in dp.newRows)
|
|
_row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
|
|
} else {
|
|
if (_is_tag_along)
|
|
_solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
|
|
else {
|
|
_row_map.Clear();
|
|
foreach (IBlueRow r in dp.newRows)
|
|
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
|
|
}
|
|
}
|
|
timer.Stop();
|
|
_n_total_fires++;
|
|
if (timer.ElapsedMilliseconds >= 50) {
|
|
_n_expensive_fires++;
|
|
Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
|
|
+ $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
|
|
}
|
|
if (on_new_data != null)
|
|
on_new_data();
|
|
}
|
|
|
|
public bool is_ready { get { return _schema != null; } }
|
|
|
|
public void constribute_cols_to_schema(List<ColumnInfo> cis) {
|
|
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
|
|
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
|
|
}
|
|
|
|
public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
|
|
|
|
public void copy_master_row_to_output_row(int row_index, object[] output_row) {
|
|
IBlueRow this_row = _row_list[row_index].second;
|
|
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
|
}
|
|
|
|
public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
|
|
if (_is_tag_along) {
|
|
if (_solitary_row != null)
|
|
BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
|
|
else
|
|
copy_dbnull_to_output_row(output_row);
|
|
} else {
|
|
IBlueRow this_row = null;
|
|
if (_row_map.TryGetValue(row_hash, out this_row))
|
|
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
|
else
|
|
copy_dbnull_to_output_row(output_row);
|
|
}
|
|
}
|
|
|
|
private void copy_dbnull_to_output_row(object[] output_row) {
|
|
for (int i = 0; i < _data_col_indices.Length; i++)
|
|
output_row[i + _start_index_in_output] = DBNull.Value;
|
|
}
|
|
|
|
#region IDisposable
|
|
|
|
public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
|
|
|
|
#endregion
|
|
}
|
|
}
|