using bts; using mbts; using System; using System.Collections.Generic; using System.Diagnostics; namespace BCG { [DataStreamUserRespresentation("fastJoin")] public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker { private readonly NonIncJoinSpec[] _join_specs; private readonly INonIncJoinInlineMapper _output_mapper; private readonly BlueRowHasher _row_hasher; private readonly TimeSpan _throttle_period; private readonly ViewStreamDatabase _vsd; private List _book_keeping_data = new List(); private IBlueTableSchema _output_schema = null; private DateTime _last_fire_time = DateTime.MinValue; private bool _is_dirty = false; public NonIncJoinStream(ViewStream left, string left_join_cols, string left_proj, ViewStream right, string right_join_cols, string right_proj) : this(new Triplet[] { new Triplet (left, left_join_cols, left_proj), new Triplet (right, right_join_cols, right_proj) }) { } public NonIncJoinStream(params Triplet[] join_specs) : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { } public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms, params Triplet[] join_specs) : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { } public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { } public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) { _join_specs = join_specs; _output_mapper = output_mapper; _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length); _throttle_period = throttles_ms == 0 ? TimeSpan.Zero : new TimeSpan(0, 0, 0, 0, throttles_ms); _vsd = vsd; } protected override void myOnFirstSubscriber() { int data_col_no = 1; for (int jp = 0; jp < _join_specs.Length; jp++) { NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher); Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}"); data_col_no += _join_specs[jp].data_col_names.Length; s.on_new_data += _stream_callback; _book_keeping_data.Add(s); } if (_output_mapper != null) _output_mapper.onMapperChange += _on_mapper_change; _vsd.addWorker(this); foreach (var bkd in _book_keeping_data) bkd.init(); } protected override void myOnLastSubscriber() { foreach (NonIncJoinBookkeeper d in _book_keeping_data) d.Dispose(); if (_output_mapper != null) _output_mapper.onMapperChange -= _on_mapper_change; _vsd.removeWorker(this); } private void _stream_callback() { Log.Gen.info("NonIncJoinStream._stream_callback received callback"); _is_dirty = true; doWork(); } private void _on_mapper_change() { _is_dirty = true; doWork(); } private bool _is_ready() { bool rv = true; foreach (NonIncJoinBookkeeper d in _book_keeping_data) rv &= d.is_ready; return rv; } private void _construct_schema() { if (_output_schema == null) { List cis = new List(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }); foreach (NonIncJoinBookkeeper d in _book_keeping_data) d.constribute_cols_to_schema(cis); _output_schema = new BlueTableSchema("NonIncJoinStream", cis.ToArray()); } } private int _n_total_fires = 0, _n_expensive_fires = 0; private void _form_and_fire() { Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}"); if (_is_ready()) { _construct_schema(); var timer = Stopwatch.StartNew(); IList> left_row_list = _book_keeping_data[0].get_list(); List output_rows = new List(); for (int i = 0; i < left_row_list.Count; i++) { object[] or = new object[_output_schema.columnInfo.Length]; or[0] = i; output_rows.Add(or); } for (int ji = 0; ji < _book_keeping_data.Count; ji++) { NonIncJoinBookkeeper bk_data = _book_keeping_data[ji]; for (int i = 0; i < left_row_list.Count; i++) { if (ji == 0) bk_data.copy_master_row_to_output_row(i, output_rows[i]); else bk_data.copy_matching_row_to_output_row(left_row_list[i].first, output_rows[i]); } } IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows)); timer.Stop(); _n_total_fires++; //if (timer.ElapsedMilliseconds >= 250) { _n_expensive_fires++; string log_msg = $"Join and map time for {_book_keeping_data.Count} " + $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}"; Log.Gen.info(log_msg); //} var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket); myFireEvent(dp_out); //myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows)); } } private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) { if (_output_mapper != null) { IBlueRowCollection output_rows; var timer = Stopwatch.StartNew(); output_rows = _output_mapper.map_rows(unmapped_rows); timer.Stop(); if (timer.ElapsedMilliseconds >= 50) { Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}"); } return output_rows; } else { return unmapped_rows; } } public void doWork() { Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}"); if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) { _is_dirty = false; _last_fire_time = DateTime.Now; _form_and_fire(); } } } }