Files
spark/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs
T

164 lines
6.9 KiB
C#

using bts;
using mbts;
using System;
using System.Collections.Generic;
using System.Diagnostics;
namespace BCG {
[DataStreamUserRespresentation("fastJoin")]
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
private readonly NonIncJoinSpec[] _join_specs;
private readonly INonIncJoinInlineMapper _output_mapper;
private readonly BlueRowHasher _row_hasher;
private readonly TimeSpan _throttle_period;
private readonly ViewStreamDatabase _vsd;
private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
private IBlueTableSchema _output_schema = null;
private DateTime _last_fire_time = DateTime.MinValue;
private bool _is_dirty = false;
public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
ViewStream right, string right_join_cols, string right_proj)
: this(new Triplet<ViewStream, string, string>[]
{
new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
}) { }
public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
: this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
params Triplet<ViewStream, string, string>[] join_specs)
: this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
: this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
_join_specs = join_specs;
_output_mapper = output_mapper;
_row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
_throttle_period = throttles_ms == 0 ? TimeSpan.Zero : new TimeSpan(0, 0, 0, 0, throttles_ms);
_vsd = vsd;
}
protected override void myOnFirstSubscriber() {
int data_col_no = 1;
for (int jp = 0; jp < _join_specs.Length; jp++) {
NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
data_col_no += _join_specs[jp].data_col_names.Length;
s.on_new_data += _stream_callback;
_book_keeping_data.Add(s);
}
if (_output_mapper != null)
_output_mapper.onMapperChange += _on_mapper_change;
_vsd.addWorker(this);
foreach (var bkd in _book_keeping_data)
bkd.init();
}
protected override void myOnLastSubscriber() {
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
d.Dispose();
if (_output_mapper != null)
_output_mapper.onMapperChange -= _on_mapper_change;
_vsd.removeWorker(this);
}
private void _stream_callback() {
Log.Gen.info("NonIncJoinStream._stream_callback received callback");
_is_dirty = true;
doWork();
}
private void _on_mapper_change() {
_is_dirty = true;
doWork();
}
private bool _is_ready() {
bool rv = true;
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
rv &= d.is_ready;
return rv;
}
private void _construct_schema() {
if (_output_schema == null) {
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
d.constribute_cols_to_schema(cis);
_output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
}
}
private int _n_total_fires = 0, _n_expensive_fires = 0;
private void _form_and_fire() {
Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
if (_is_ready()) {
_construct_schema();
var timer = Stopwatch.StartNew();
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
List<object[]> output_rows = new List<object[]>();
for (int i = 0; i < left_row_list.Count; i++) {
object[] or = new object[_output_schema.columnInfo.Length];
or[0] = i;
output_rows.Add(or);
}
for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
for (int i = 0; i < left_row_list.Count; i++) {
if (ji == 0)
bk_data.copy_master_row_to_output_row(i, output_rows[i]);
else
bk_data.copy_matching_row_to_output_row(left_row_list[i].first, output_rows[i]);
}
}
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
timer.Stop();
_n_total_fires++;
//if (timer.ElapsedMilliseconds >= 250) {
_n_expensive_fires++;
string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+ $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
Log.Gen.info(log_msg);
//}
var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
myFireEvent(dp_out);
//myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
}
}
private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
if (_output_mapper != null) {
IBlueRowCollection output_rows;
var timer = Stopwatch.StartNew();
output_rows = _output_mapper.map_rows(unmapped_rows);
timer.Stop();
if (timer.ElapsedMilliseconds >= 50) {
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
}
return output_rows;
} else {
return unmapped_rows;
}
}
public void doWork() {
Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
_is_dirty = false;
_last_fire_time = DateTime.Now;
_form_and_fire();
}
}
}
}