diff --git a/.vscode/launch.json b/.vscode/launch.json
index 224b1ff56..800b9d764 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -1,6 +1,11 @@
{
"version": "0.2.0",
"configurations": [
+ {
+ "name": ".NET Core Attach",
+ "type": "coreclr",
+ "request": "attach"
+ },
{
"name": "DataStreamPublisher",
"type": "coreclr",
@@ -138,4 +143,4 @@
"default": "http://localhost:8000/config",
},
]
-}
+}
\ No newline at end of file
diff --git a/src/DataStreamSuite/DataStream/DataStream.csproj b/src/DataStreamSuite/DataStream/DataStream.csproj
index d101ba8cc..8c8ed8d6b 100644
--- a/src/DataStreamSuite/DataStream/DataStream.csproj
+++ b/src/DataStreamSuite/DataStream/DataStream.csproj
@@ -20,6 +20,7 @@
+
diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageGroupsStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageGroupsStream.cs
index c9ccf45a5..8ce1a3159 100644
--- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageGroupsStream.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageGroupsStream.cs
@@ -1,102 +1,102 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
-namespace BCG
-{
+// namespace BCG
+// {
- public class AverageGroupsStream : ViewStream
- {
- public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
- {
+// public class AverageGroupsStream : ViewStream
+// {
+// public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
+// {
- myColToAverage = colToAverage;
- myInput = input;
- myKeyColumn = keyColumn;
- }
+// myColToAverage = colToAverage;
+// myInput = input;
+// myKeyColumn = keyColumn;
+// }
- private ViewStream myInput;
- private string myColToAverage;
- private BlueTableSchema mySchema;
- private string myKeyColumn;
+// private ViewStream myInput;
+// private string myColToAverage;
+// private BlueTableSchema mySchema;
+// private string myKeyColumn;
- protected override void myOnFirstSubscriber()
- {
- myInput.onChange += new ChangeDelegate(myInput_onChange);
- }
- protected override void myOnLastSubscriber()
- {
- myInput.onChange -= myInput_onChange;
- }
- private void myInput_onChange(object newValue)
- {
- if (mySchema == null)
- myBuildSchema(((DataPacket)newValue).schema);
+// protected override void myOnFirstSubscriber()
+// {
+// myInput.onChange += new ChangeDelegate(myInput_onChange);
+// }
+// protected override void myOnLastSubscriber()
+// {
+// myInput.onChange -= myInput_onChange;
+// }
+// private void myInput_onChange(object newValue)
+// {
+// if (mySchema == null)
+// myBuildSchema(((DataPacket)newValue).schema);
- IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
+// IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
- if (myLastValue == null)
- myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
- else
- {
- IBlueRowCollection added, deleted;
- BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
- myFireEvent(new DataPacket(newRows, added, deleted));
- }
- }
+// if (myLastValue == null)
+// myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
+// else
+// {
+// IBlueRowCollection added, deleted;
+// BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
+// myFireEvent(new DataPacket(newRows, added, deleted));
+// }
+// }
- private void myBuildSchema(IBlueTableSchema schema)
- {
- List columns = new List();
+// private void myBuildSchema(IBlueTableSchema schema)
+// {
+// List columns = new List();
- foreach (ColumnInfo ci in schema.columnInfo)
- columns.Add(ci);
+// foreach (ColumnInfo ci in schema.columnInfo)
+// columns.Add(ci);
- columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
- mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
- }
+// columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
+// mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
+// }
- private IBlueRowCollection myEvaluate(DataPacket packet)
- {
+// private IBlueRowCollection myEvaluate(DataPacket packet)
+// {
- IDictionary colToRunningTotal = new Dictionary();
- IDictionary colToCount = new Dictionary();
- IDictionary colToLastRow = new Dictionary();
+// IDictionary colToRunningTotal = new Dictionary();
+// IDictionary colToCount = new Dictionary();
+// IDictionary colToLastRow = new Dictionary();
- foreach (IBlueRow r in packet.newRows)
- {
- double thisVal = Convert.ToDouble(r[myColToAverage]);
+// foreach (IBlueRow r in packet.newRows)
+// {
+// double thisVal = Convert.ToDouble(r[myColToAverage]);
- string keyVal = r[myKeyColumn].ToString();
- if (!colToCount.ContainsKey(keyVal))
- {
- colToRunningTotal[keyVal] = thisVal;
- colToCount[keyVal] = 1;
- }
- else
- {
- colToRunningTotal[keyVal] += thisVal;
- colToCount[keyVal] += 1;
- }
- colToLastRow[keyVal] = r;
- }
+// string keyVal = r[myKeyColumn].ToString();
+// if (!colToCount.ContainsKey(keyVal))
+// {
+// colToRunningTotal[keyVal] = thisVal;
+// colToCount[keyVal] = 1;
+// }
+// else
+// {
+// colToRunningTotal[keyVal] += thisVal;
+// colToCount[keyVal] += 1;
+// }
+// colToLastRow[keyVal] = r;
+// }
- BlueTable result = new BlueTable(mySchema);
- foreach (string key in colToCount.Keys)
- {
- object[] values = new object[mySchema.columnInfo.Length];
+// BlueTable result = new BlueTable(mySchema);
+// foreach (string key in colToCount.Keys)
+// {
+// object[] values = new object[mySchema.columnInfo.Length];
- colToLastRow[key].CopyTo(values, 0);
- values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
- result.addRow(new BlueRow(mySchema, values));
- }
+// colToLastRow[key].CopyTo(values, 0);
+// values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
+// result.addRow(new BlueRow(mySchema, values));
+// }
- return result;
- }
+// return result;
+// }
- }
-}
+// }
+// }
diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageNonIncGroupsStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageNonIncGroupsStream.cs
new file mode 100644
index 000000000..a5daa62ce
--- /dev/null
+++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/AverageNonIncGroupsStream.cs
@@ -0,0 +1,31 @@
+namespace bts.SilverPine {
+ using BCG;
+
+ using System;
+ using System.Collections.Generic;
+ using System.Linq;
+
+ [DataStreamUserRespresentation("Average")]
+ public class AverageNonIncGroupsStream : ViewStream {
+ public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
+ NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
+ (g_rows) => _find_mean(inp_col_name, g_rows));
+ mySetRealSource(rs);
+ }
+
+ private List _find_mean(string inp_col_name, List g_rows) {
+ IBlueRow inp_row = g_rows.ElementAt(0);
+ IEnumerable inp_vals = BlueDataUtility.get_colvalues(g_rows, inp_col_name);
+ double sum = 0.0;
+ inp_vals.ToList().ForEach(d => sum += d);
+ string out_col_name = $"average_{inp_col_name}";
+ double average = sum / inp_vals.Count();
+ IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
+ MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
+ out_row.setValue(out_col_name, average);
+ List rv = new List();
+ rv.Add(out_row);
+ return rv;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs
index 8f55cab92..f87da6ebc 100644
--- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs
@@ -10,6 +10,6 @@ namespace BCG {
}
public MaxStream(ViewStream stream, string outputCol)
- : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+ : this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { }
}
}
diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs
index 2e0b28992..c8b60b508 100644
--- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs
@@ -1,13 +1,37 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
+using System.Linq;
using bts;
-namespace BCG
-{
- public class NonIncGroupByStream : ViewStream
- {
+namespace BCG {
+ public class NonIncGroupByStream : ViewStream {
+ private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
+ Func _groupby_cols_func;
+ Func, List> _evaluate_func;
+ public DelNonIncGroupByEvaluator(Func groupby_cols_func,
+ Func, List> evaluate_func) {
+ _groupby_cols_func = groupby_cols_func;
+ _evaluate_func = evaluate_func;
+ }
+ public IBlueRowCollection evaluate(IDictionary> grouped_rows) {
+ BlueTable rv = null;
+ foreach (var kv in grouped_rows) {
+ List g_rows = kv.Value;
+ List out_rows = _evaluate_func.Invoke(g_rows);
+ if (out_rows != null && out_rows.Count > 0) {
+ rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
+ }
+ rv.addRows(out_rows);
+ }
+ return rv;
+ }
+ public string[] get_group_by_cols() {
+ return _groupby_cols_func();
+ }
+
+ }
private ViewStream _main_stream;
private INonIncGroupByEvaluator _groupby_func;
private BlueRowHasher _row_hasher;
@@ -15,24 +39,29 @@ namespace BCG
private Stopwatch _timer = new();
private int _n_total_fires = 0, _n_expensive_fires = 0;
- public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
- {
+ public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
_main_stream = main_stream;
_groupby_func = groupby_func;
_groupby_cols = _groupby_func.get_group_by_cols();
_row_hasher = new BlueRowHasher(_groupby_cols.Length);
}
+ public NonIncGroupByStream(ViewStream main_stream, Func groupby_cols_func,
+ Func, List> evaluate_func)
+ : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
+
+
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
- private void _main_stream_onchange(object nv)
- {
+ private void _main_stream_onchange(object nv) {
DataPacket dp = nv as DataPacket;
///grouping together
_timer.Restart();
- IDictionary> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
+ IDictionary> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
+ new Dictionary>() { { 0, dp.newRows.to_list() } } :
+ BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
_timer.Stop();
var groupElapsedMs = _timer.ElapsedMilliseconds;
if (groupElapsedMs >= 50) {
@@ -44,14 +73,13 @@ namespace BCG
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
_timer.Stop();
var applyElapsedMs = _timer.ElapsedMilliseconds;
- _n_total_fires++;
- if (applyElapsedMs >= 50)
- {
+ _n_total_fires++;
+ if (applyElapsedMs >= 50) {
_n_expensive_fires++;
Log.DataFlow.info(
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
- }
- myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
+ }
+ myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
}
}
}
diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs
index 4ce9eaa4e..ef5ad6180 100644
--- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs
@@ -5,11 +5,10 @@ using mbts;
using System;
using System.Collections.Generic;
using System.Diagnostics;
+using System.Security.Cryptography;
-namespace BCG
-{
- public class NonIncJoinBookkeeper : IDisposable
- {
+namespace BCG {
+ public class NonIncJoinBookkeeper : IDisposable {
public Action on_new_data;
private readonly NonIncJoinSpec _join_spec;
private readonly int _join_position;
@@ -26,41 +25,38 @@ namespace BCG
private int[] _join_col_indices = null,
_data_col_indices = null;
-
- public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
- {
+
+ public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
_join_spec = the_join_spec;
_join_position = join_position;
_start_index_in_output = start_index_in_output;
_row_hasher = row_hasher;
_is_left_most = _join_position == 0;
_is_tag_along = _join_spec.join_col_names.Length == 0;
+ }
+
+ public void init() {
_join_spec.stream.onChange += _on_stream_cb;
}
- public void _on_stream_cb(object dp_obj)
- {
+ public void _on_stream_cb(object dp_obj) {
+ Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
var timer = Stopwatch.StartNew();
DataPacket dp = (DataPacket)dp_obj;
- if (_schema == null)
- {
+ if (_schema == null) {
_schema = dp.schema;
_join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
_data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
}
- if (_is_left_most)
- {
+ if (_is_left_most) {
_row_list.Clear();
foreach (IBlueRow r in dp.newRows)
- _row_list.Add(new Pair(_row_hasher.get_hash(r, _join_col_indices), r));
- }
- else
- {
+ _row_list.Add(new Pair(_row_hasher.get_hash(r, _join_col_indices), r));
+ } else {
if (_is_tag_along)
_solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
- else
- {
+ else {
_row_map.Clear();
foreach (IBlueRow r in dp.newRows)
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
@@ -79,31 +75,25 @@ namespace BCG
public bool is_ready { get { return _schema != null; } }
- public void constribute_cols_to_schema(List cis)
- {
+ public void constribute_cols_to_schema(List cis) {
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
}
public List> get_list() { return _row_list; }
- public void copy_master_row_to_output_row(int row_index, object[] output_row)
- {
+ public void copy_master_row_to_output_row(int row_index, object[] output_row) {
IBlueRow this_row = _row_list[row_index].second;
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
}
- public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
- {
- if (_is_tag_along)
- {
+ public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
+ if (_is_tag_along) {
if (_solitary_row != null)
BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
else
copy_dbnull_to_output_row(output_row);
- }
- else
- {
+ } else {
IBlueRow this_row = null;
if (_row_map.TryGetValue(row_hash, out this_row))
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
@@ -112,8 +102,7 @@ namespace BCG
}
}
- private void copy_dbnull_to_output_row(object[] output_row)
- {
+ private void copy_dbnull_to_output_row(object[] output_row) {
for (int i = 0; i < _data_col_indices.Length; i++)
output_row[i + _start_index_in_output] = DBNull.Value;
}
diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs
index c6cb15fb6..adea44bac 100644
--- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs
@@ -3,63 +3,58 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
+
using mbts;
-namespace BCG
-{
- public class NonIncJoinSpec
- {
+namespace BCG {
+ public class NonIncJoinSpec {
public ViewStream stream;
public string[] join_col_names, data_col_names, projected_col_names;
public NonIncJoinSpec() { }
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
- : this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
- { }
+ : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
- public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
- {
+ public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
stream = the_stream;
join_col_names = the_join_col_names;
data_col_names = the_data_col_names;
projected_col_names = the_projected_col_names;
}
- public static NonIncJoinSpec[] from_triplet(params Triplet[] join_specs)
- {
+ public static NonIncJoinSpec[] from_triplet(params Triplet[] join_specs) {
List rv = new List();
- foreach (Triplet r in join_specs)
- {
- string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
+ foreach (Triplet r in join_specs) {
+ string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
string[] visible_cols = r.third.Split(' ');
List data_col_names = new List(), mapped_col_names = new List();
- foreach (string s in visible_cols)
- {
+ foreach (string s in visible_cols) {
string[] toks = s.Split(':');
- if (toks.Length == 1)
- {
+ if (toks.Length == 1) {
data_col_names.Add(toks[0]);
mapped_col_names.Add(toks[0]);
- }
- else if (toks.Length == 2)
- {
+ } else if (toks.Length == 2) {
data_col_names.Add(toks[0]);
mapped_col_names.Add(toks[1]);
}
}
- rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
+ rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
}
- return rv.ToArray ();
+ return rv.ToArray();
+ }
+
+ public override string ToString() {
+ Func arr_txt = (a) => {
+ string rv = "";
+ foreach (string s in a) {
+ rv += "{s} ";
+ }
+ rv = rv.Trim();
+ return rv;
+ };
+ string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
+ return rv;
}
}
-
- //[DataStreamUserRespresentation("ninctest")]
- //public class NonIncJoinTestStream : ViewStream
- //{
- // public NonIncJoinTestStream (string key_cols, string data_cols)
- // {
- // NonIncJoinSpec.from_triplet(new Triplet(null, key_cols, data_cols));
- // }
- //}
}
diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs
index 0a341413e..5c6be1a14 100644
--- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs
+++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs
@@ -6,11 +6,9 @@ using System;
using System.Collections.Generic;
using System.Diagnostics;
-namespace BCG
-{
+namespace BCG {
[DataStreamUserRespresentation("fastJoin")]
- public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
- {
+ public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
private readonly NonIncJoinSpec[] _join_specs;
private readonly INonIncJoinInlineMapper _output_mapper;
private readonly BlueRowHasher _row_hasher;
@@ -28,24 +26,19 @@ namespace BCG
{
new Triplet (left, left_join_cols, left_proj),
new Triplet (right, right_join_cols, right_proj)
- })
- { }
+ }) { }
public NonIncJoinStream(params Triplet[] join_specs)
- : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
- { }
+ : this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
params Triplet[] join_specs)
- : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
- { }
+ : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
- : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
- { }
+ : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
- public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
- {
+ public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
_join_specs = join_specs;
_output_mapper = output_mapper;
_row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -53,12 +46,11 @@ namespace BCG
_vsd = vsd;
}
- protected override void myOnFirstSubscriber()
- {
+ protected override void myOnFirstSubscriber() {
int data_col_no = 1;
- for (int jp = 0; jp < _join_specs.Length; jp++)
- {
+ for (int jp = 0; jp < _join_specs.Length; jp++) {
NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
+ Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
data_col_no += _join_specs[jp].data_col_names.Length;
s.on_new_data += _stream_callback;
_book_keeping_data.Add(s);
@@ -66,10 +58,11 @@ namespace BCG
if (_output_mapper != null)
_output_mapper.onMapperChange += _on_mapper_change;
_vsd.addWorker(this);
+ foreach (var bkd in _book_keeping_data)
+ bkd.init();
}
- protected override void myOnLastSubscriber()
- {
+ protected override void myOnLastSubscriber() {
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
d.Dispose();
if (_output_mapper != null)
@@ -77,30 +70,26 @@ namespace BCG
_vsd.removeWorker(this);
}
- private void _stream_callback()
- {
+ private void _stream_callback() {
+ Log.Gen.info("NonIncJoinStream._stream_callback received callback");
_is_dirty = true;
doWork();
}
- private void _on_mapper_change()
- {
+ private void _on_mapper_change() {
_is_dirty = true;
doWork();
}
- private bool _is_ready()
- {
+ private bool _is_ready() {
bool rv = true;
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
rv &= d.is_ready;
return rv;
}
- private void _construct_schema()
- {
- if (_output_schema == null)
- {
+ private void _construct_schema() {
+ if (_output_schema == null) {
List cis = new List(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
d.constribute_cols_to_schema(cis);
@@ -109,27 +98,23 @@ namespace BCG
}
private int _n_total_fires = 0, _n_expensive_fires = 0;
- private void _form_and_fire()
- {
- if (_is_ready())
- {
+ private void _form_and_fire() {
+ Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
+ if (_is_ready()) {
_construct_schema();
var timer = Stopwatch.StartNew();
IList> left_row_list = _book_keeping_data[0].get_list();
List