Files
spark/src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs
T
2025-07-13 21:21:10 -04:00

188 lines
6.9 KiB
C#

namespace bts.BlueOrb {
using BCG;
using RowHash = System.Int32;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Collections;
using System.Data.Entity.Core.Mapping;
[DataStreamUserRespresentation("JoinStream")]
public class SimpleJoinStream : ViewStream {
public class JoinBookkeepingData {
// initialized in cons
private readonly List<string> _join_col_names;
private readonly bool _is_left;
private readonly BlueRowHasher _hasher;
// initialized on first udpate
private IBlueTableSchema _schema;
private int[] _join_cols;
private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
_join_col_names = join_cols.Where (c => !String.IsNullOrEmpty(c)).ToList ();
_is_left = is_left;
_hasher = row_hasher;
}
public IBlueTableSchema schema => _schema;
public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
public List<IBlueRow> get_matching_rows (int hash) {
List<IBlueRow> rv = null;
_hash_row_map.TryGetValue(hash, out rv);
return rv;
}
private RowHash _get_hash(IBlueRow r) {
if (_join_cols == null || _join_cols.Length == 0) {
return 0;
}
return _hasher.get_hash(r, _join_cols);
}
public void update_data(DataPacket dp) {
if (!ready) {
_schema = dp.newRows.schema;
_join_cols = _join_col_names.Count == 0 ? new int[] { }: _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
}
_hash_row_map.Clear();
foreach (IBlueRow r in dp.newRows) {
int hash = _get_hash(r);
List<IBlueRow> hash_rows = null;
if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
hash_rows = new List<IBlueRow>();
_hash_row_map.Add(hash, hash_rows);
}
hash_rows.Add(r);
}
}
public bool ready {
get {
return _schema != null;
}
}
}
private ViewStream _ls, _rs;
private JoinBookkeepingData _ldata, _rdata;
private ViewStreamDatabase _vsd;
private IBlueTableSchema _schema = null;
public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
_ls = ls;
_rs = rs;
List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
Log.assert(lj_col_list.Count == rj_col_list.Count);
int n_cols = lj_col_list.Count;
BlueRowHasher hasher = new BlueRowHasher(n_cols);
_ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
_rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
}
protected override void myOnFirstSubscriber() {
_ls.onChange += _ls_onchange;
_rs.onChange += _rs_onchange;
}
private void _ls_onchange(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
_ldata.update_data(dp);
_form_and_fire();
}
private void _rs_onchange(object dp_obj) {
DataPacket dp = dp_obj as DataPacket ;
_rdata.update_data(dp);
_form_and_fire();
}
protected override void myOnLastSubscriber() {
_ls.onChange -= _ls_onchange;
_rs.onChange -= _rs_onchange;
}
private void _build_schema() {
if (_schema != null)
return;
List<ColumnInfo> cis = new List<ColumnInfo>();
cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
}
foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
}
_schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
}
private void _form_and_fire() {
if (!_ldata.ready||!_rdata.ready)
return;
_build_schema();
List<object[]> res_rows = new List<object[]>();
Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
int offset = 1;
JoinBookkeepingData sdata = _ldata;
if (is_left) {
offset = 1;
sdata = _ldata;
} else {
sdata = _rdata;
offset = 1 + _ldata.schema.columnInfo.Length;
}
for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
rval[offset + i] = r[i];
}
};
Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
object[] rv = new object[_schema.columnNames.Length];
for (int i = 0; i < _schema.columnNames.Length; i++) {
rv[i] = DBNull.Value;
}
rv[0] = id;
copy_row(rv, l_row, true);
return rv;
};
// this is where the join happens
foreach (var kv in _ldata.rows) {
RowHash hash = kv.Key;
List<IBlueRow> l_rows = kv.Value;
List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
foreach (IBlueRow lr in l_rows) {
if (r_rows != null) {
foreach (IBlueRow r in r_rows) {
object[] rowvals = build_row_array(res_rows.Count, lr);
copy_row(rowvals, r, false);
res_rows.Add(rowvals);
}
} else {
object[] rowvals = build_row_array(res_rows.Count, lr);
res_rows.Add(rowvals);
}
}
}
BlueTable rv = new BlueTable(_schema);
foreach (object[] rval in res_rows) {
rv.addRow(new BlueRow(_schema, rval));
}
var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
myFireEvent(dp);
}
}
}