164 lines
6.9 KiB
C#
164 lines
6.9 KiB
C#
using bts;
|
|
|
|
using mbts;
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Diagnostics;
|
|
|
|
namespace BCG {
|
|
[DataStreamUserRespresentation("fastJoin")]
|
|
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
|
|
private readonly NonIncJoinSpec[] _join_specs;
|
|
private readonly INonIncJoinInlineMapper _output_mapper;
|
|
private readonly BlueRowHasher _row_hasher;
|
|
private readonly TimeSpan _throttle_period;
|
|
private readonly ViewStreamDatabase _vsd;
|
|
|
|
private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
|
|
private IBlueTableSchema _output_schema = null;
|
|
private DateTime _last_fire_time = DateTime.MinValue;
|
|
private bool _is_dirty = false;
|
|
|
|
public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj,
|
|
ViewStream right, string right_join_cols, string right_proj)
|
|
: this(new Triplet<ViewStream, string, string>[]
|
|
{
|
|
new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
|
|
new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
|
|
}) { }
|
|
|
|
public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
|
|
: this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
|
|
|
|
public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
|
|
params Triplet<ViewStream, string, string>[] join_specs)
|
|
: this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
|
|
|
|
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
|
|
: this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
|
|
|
|
public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
|
|
_join_specs = join_specs;
|
|
_output_mapper = output_mapper;
|
|
_row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
|
|
_throttle_period = throttles_ms == 0 ? TimeSpan.Zero : new TimeSpan(0, 0, 0, 0, throttles_ms);
|
|
_vsd = vsd;
|
|
}
|
|
|
|
protected override void myOnFirstSubscriber() {
|
|
int data_col_no = 1;
|
|
for (int jp = 0; jp < _join_specs.Length; jp++) {
|
|
NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
|
|
Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
|
|
data_col_no += _join_specs[jp].data_col_names.Length;
|
|
s.on_new_data += _stream_callback;
|
|
_book_keeping_data.Add(s);
|
|
}
|
|
if (_output_mapper != null)
|
|
_output_mapper.onMapperChange += _on_mapper_change;
|
|
_vsd.addWorker(this);
|
|
foreach (var bkd in _book_keeping_data)
|
|
bkd.init();
|
|
}
|
|
|
|
protected override void myOnLastSubscriber() {
|
|
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
|
d.Dispose();
|
|
if (_output_mapper != null)
|
|
_output_mapper.onMapperChange -= _on_mapper_change;
|
|
_vsd.removeWorker(this);
|
|
}
|
|
|
|
private void _stream_callback() {
|
|
Log.Gen.info("NonIncJoinStream._stream_callback received callback");
|
|
_is_dirty = true;
|
|
doWork();
|
|
}
|
|
|
|
private void _on_mapper_change() {
|
|
_is_dirty = true;
|
|
doWork();
|
|
}
|
|
|
|
private bool _is_ready() {
|
|
bool rv = true;
|
|
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
|
rv &= d.is_ready;
|
|
return rv;
|
|
}
|
|
|
|
private void _construct_schema() {
|
|
if (_output_schema == null) {
|
|
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
|
|
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
|
d.constribute_cols_to_schema(cis);
|
|
_output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray());
|
|
}
|
|
}
|
|
|
|
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
|
private void _form_and_fire() {
|
|
Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
|
|
if (_is_ready()) {
|
|
_construct_schema();
|
|
var timer = Stopwatch.StartNew();
|
|
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
|
|
List<object[]> output_rows = new List<object[]>();
|
|
|
|
for (int i = 0; i < left_row_list.Count; i++) {
|
|
object[] or = new object[_output_schema.columnInfo.Length];
|
|
or[0] = i;
|
|
output_rows.Add(or);
|
|
}
|
|
|
|
for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
|
|
NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
|
|
for (int i = 0; i < left_row_list.Count; i++) {
|
|
if (ji == 0)
|
|
bk_data.copy_master_row_to_output_row(i, output_rows[i]);
|
|
else
|
|
bk_data.copy_matching_row_to_output_row(left_row_list[i].first, output_rows[i]);
|
|
}
|
|
}
|
|
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
|
|
timer.Stop();
|
|
_n_total_fires++;
|
|
//if (timer.ElapsedMilliseconds >= 250) {
|
|
_n_expensive_fires++;
|
|
string log_msg = $"Join and map time for {_book_keeping_data.Count} "
|
|
+ $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
|
|
Log.Gen.info(log_msg);
|
|
//}
|
|
var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
|
|
myFireEvent(dp_out);
|
|
//myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
|
|
}
|
|
}
|
|
|
|
private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
|
|
if (_output_mapper != null) {
|
|
IBlueRowCollection output_rows;
|
|
var timer = Stopwatch.StartNew();
|
|
output_rows = _output_mapper.map_rows(unmapped_rows);
|
|
timer.Stop();
|
|
if (timer.ElapsedMilliseconds >= 50) {
|
|
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
|
|
}
|
|
return output_rows;
|
|
} else {
|
|
return unmapped_rows;
|
|
}
|
|
}
|
|
|
|
public void doWork() {
|
|
Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
|
|
if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
|
|
_is_dirty = false;
|
|
_last_fire_time = DateTime.Now;
|
|
_form_and_fire();
|
|
}
|
|
}
|
|
}
|
|
}
|