cleanup: Fix more build warnings in DataStreamSuite
Alot more
This commit is contained in:
1 parent
989d103eef
commit
bcd207f89e
118 files changed
+1812
-1597
No files matched your search
@@ -1,5 +1,8 @@
|
||||
namespace bts.BlueOrb {
|
||||
using BCG;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
@@ -21,22 +24,24 @@ namespace bts.BlueOrb {
|
||||
|
||||
#region IStreamFilter Members
|
||||
public bool filter(IBlueRow dr) {
|
||||
bool rv = true;
|
||||
for (int i = 0; i < myCols.Length; i++) {
|
||||
rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
|
||||
if (!rv)
|
||||
return rv;
|
||||
var val = dr[myCols[i]];
|
||||
var str = val is DBNull
|
||||
? string.Empty
|
||||
: (val?.ToString() ?? string.Empty);
|
||||
if (!myRegExps[i].IsMatch(str)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return rv;
|
||||
return true;
|
||||
}
|
||||
#endregion
|
||||
}
|
||||
private BlueTable myLastFiredRows;
|
||||
private IBlueRowCollection myLastReceivedRows;
|
||||
BlueTable? myLastFiredRows;
|
||||
IBlueRowCollection? myLastReceivedRows;
|
||||
private IStreamFilter myFilter;
|
||||
private ViewStream myParentStream;
|
||||
private string myFilterString;
|
||||
private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
|
||||
|
||||
#region constructors
|
||||
public delegate bool OnFilter(IBlueRow r);
|
||||
@@ -52,12 +57,11 @@ namespace bts.BlueOrb {
|
||||
|
||||
|
||||
public FilterStream(ViewStream vs, IStreamFilter filter) {
|
||||
{
|
||||
myParentStream = vs;
|
||||
myFilter = filter;
|
||||
myFilterString = filter.ToString();
|
||||
if (filter is IDynamicStreamFilter)
|
||||
(filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
|
||||
myParentStream = vs;
|
||||
myFilter = filter;
|
||||
myFilterString = filter.ToString() ?? string.Empty;
|
||||
if (filter is IDynamicStreamFilter dynFilter) {
|
||||
dynFilter.onFilterChange += myHandleFilterChange;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,16 +103,20 @@ namespace bts.BlueOrb {
|
||||
if (myLastFiredRows != null) {
|
||||
retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
|
||||
new BlueTable(myLastFiredRows.schema));
|
||||
foreach (IBlueRow r in arrivedPacket.deletedRows)
|
||||
if (myLastFiredRows.containsID((int)r["id"])) {
|
||||
var deletedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.deletedRows);
|
||||
foreach (IBlueRow r in arrivedPacket.deletedRows) {
|
||||
if (myLastFiredRows.containsID(r.Id())) {
|
||||
myLastFiredRows.deleteRow(r);
|
||||
(retVal.deletedRows as BlueTable).addRow(r);
|
||||
deletedRows.Value.addRow(r);
|
||||
}
|
||||
foreach (IBlueRow r in arrivedPacket.addedRows)
|
||||
}
|
||||
var addedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.addedRows);
|
||||
foreach (IBlueRow r in arrivedPacket.addedRows) {
|
||||
if (myFilter.filter(r)) {
|
||||
myLastFiredRows.addRow(r);
|
||||
(retVal.addedRows as BlueTable).addRow(r);
|
||||
addedRows.Value.addRow(r);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
myLastFiredRows = new BlueTable(arrivedPacket.Schema);
|
||||
retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
using System;
|
||||
using BCG;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
|
||||
using bts;
|
||||
//using bts.SilverPine;
|
||||
using BCG;
|
||||
|
||||
namespace bts.BlueOrb {
|
||||
[DataStreamUserRespresentation("GroupByStream")]
|
||||
@@ -14,20 +15,21 @@ namespace bts.BlueOrb {
|
||||
|
||||
public interface INonIncGroupByEvaluator {
|
||||
// grouped_rows
|
||||
IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
|
||||
string[] get_group_by_cols();
|
||||
IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
|
||||
string[]? get_group_by_cols();
|
||||
}
|
||||
|
||||
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
|
||||
Func<string[]> _groupby_cols_func;
|
||||
Func<string[]?> _groupby_cols_func;
|
||||
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
|
||||
public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
|
||||
public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func,
|
||||
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
|
||||
_groupby_cols_func = groupby_cols_func;
|
||||
_evaluate_func = evaluate_func;
|
||||
}
|
||||
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
|
||||
BlueTable rv = null;
|
||||
|
||||
public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
|
||||
BlueTable? rv = null;
|
||||
foreach (var kv in grouped_rows) {
|
||||
List<IBlueRow> g_rows = kv.Value;
|
||||
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
|
||||
@@ -39,17 +41,19 @@ namespace bts.BlueOrb {
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
public string[] get_group_by_cols() {
|
||||
|
||||
public string[]? get_group_by_cols() {
|
||||
return _groupby_cols_func();
|
||||
}
|
||||
|
||||
}
|
||||
private ViewStream _main_stream;
|
||||
private INonIncGroupByEvaluator _groupby_func;
|
||||
private BlueRowHasher _row_hasher;
|
||||
private string[] _groupby_cols;
|
||||
ViewStream? _main_stream;
|
||||
INonIncGroupByEvaluator? _groupby_func;
|
||||
BlueRowHasher? _row_hasher;
|
||||
string[]? _groupby_cols;
|
||||
private Stopwatch _timer = new();
|
||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||
IDisposable? subscription;
|
||||
|
||||
public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
|
||||
_main_stream = main_stream;
|
||||
@@ -58,7 +62,7 @@ namespace bts.BlueOrb {
|
||||
_row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
|
||||
}
|
||||
|
||||
public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
|
||||
public GroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func,
|
||||
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
|
||||
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
|
||||
|
||||
@@ -68,33 +72,41 @@ namespace bts.BlueOrb {
|
||||
// this is the main constructor that does everything
|
||||
public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
|
||||
MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
|
||||
Func<string[]> groupby_cols_func = () => {
|
||||
if (String.IsNullOrEmpty(groupby_cols))
|
||||
return null;
|
||||
return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
|
||||
};
|
||||
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
|
||||
if (!String.IsNullOrEmpty(optional_sort_col))
|
||||
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
|
||||
// this si where the group by aggregation happens
|
||||
IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
|
||||
IBlueRow? rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
|
||||
if (rv == null)
|
||||
return new List<IBlueRow>();
|
||||
return new IBlueRow[] { rv }.ToList();
|
||||
};
|
||||
ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
|
||||
mySetRealSource(rs);
|
||||
|
||||
string[]? groupby_cols_func() {
|
||||
if (string.IsNullOrEmpty(groupby_cols)) {
|
||||
return null;
|
||||
}
|
||||
return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
|
||||
}
|
||||
}
|
||||
public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
|
||||
this(parent_str, String.Empty, standard_group_by_text) {
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
Log.isnull(subscription);
|
||||
subscription = ExceptionAssert.MemberNotNull(_main_stream).Subscribe(_main_stream_onchange);
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
|
||||
|
||||
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
|
||||
protected override void myOnLastSubscriber() {
|
||||
subscription?.Dispose();
|
||||
subscription = null;
|
||||
}
|
||||
|
||||
private void _main_stream_onchange(object nv) {
|
||||
Log.notnull(_groupby_func);
|
||||
var dp = (IDataPacket)nv;
|
||||
///grouping together
|
||||
_timer.Restart();
|
||||
@@ -109,7 +121,7 @@ namespace bts.BlueOrb {
|
||||
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
||||
/// reducing
|
||||
_timer.Restart();
|
||||
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
||||
IBlueRowCollection? total_state = _groupby_func.evaluate(grouped_rows);
|
||||
if (total_state == null)
|
||||
// mayeb returna n empty table ?
|
||||
return;
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
namespace bts.BlueOrb {
|
||||
using BCG;
|
||||
|
||||
using Bts.DataStream;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Linq;
|
||||
|
||||
using RowHash = System.Int32;
|
||||
@@ -11,29 +16,29 @@
|
||||
public class SimpleJoinStream : ViewStream {
|
||||
public class JoinBookkeepingData {
|
||||
// initialized in cons
|
||||
private readonly List<string> _join_col_names;
|
||||
private readonly bool _is_left;
|
||||
readonly string[] _join_col_names;
|
||||
private readonly BlueRowHasher _hasher;
|
||||
// initialized on first udpate
|
||||
private IBlueTableSchema _schema;
|
||||
private int[] _join_cols;
|
||||
IBlueTableSchema? _schema;
|
||||
int[]? _join_cols;
|
||||
private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
|
||||
|
||||
public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
|
||||
_join_col_names = join_cols.Where(c => !String.IsNullOrEmpty(c)).ToList();
|
||||
_is_left = is_left;
|
||||
public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher) {
|
||||
_join_col_names = join_cols.Where(c => !string.IsNullOrEmpty(c)).ToArray();
|
||||
_hasher = row_hasher;
|
||||
}
|
||||
|
||||
public IBlueTableSchema schema => _schema;
|
||||
[NotNullIfNotNull(nameof(_schema))]
|
||||
public IBlueTableSchema? schema => _schema;
|
||||
|
||||
public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
|
||||
|
||||
public List<IBlueRow> get_matching_rows(int hash) {
|
||||
List<IBlueRow> rv = null;
|
||||
_hash_row_map.TryGetValue(hash, out rv);
|
||||
return rv;
|
||||
}
|
||||
[MemberNotNull(nameof(_schema))]
|
||||
public IBlueTableSchema RequireSchema() =>
|
||||
ExceptionAssert.MemberNotNull(_schema);
|
||||
|
||||
public List<IBlueRow>? get_matching_rows(int hash) =>
|
||||
_hash_row_map.GetValueOrDefault(hash);
|
||||
|
||||
private RowHash _get_hash(IBlueRow r) {
|
||||
if (_join_cols == null || _join_cols.Length == 0) {
|
||||
@@ -45,13 +50,12 @@
|
||||
public void update_data(IDataPacket dp) {
|
||||
if (!ready) {
|
||||
_schema = dp.newRows.schema;
|
||||
_join_cols = _join_col_names.Count == 0 ? new int[] { } : _join_col_names.Select(n => _schema[n].colNo).ToArray();
|
||||
_join_cols = _schema.ResolveIndices(_join_col_names);
|
||||
}
|
||||
_hash_row_map.Clear();
|
||||
foreach (IBlueRow r in dp.newRows) {
|
||||
int hash = _get_hash(r);
|
||||
List<IBlueRow> hash_rows = null;
|
||||
if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
|
||||
if (!_hash_row_map.TryGetValue(hash, out var hash_rows)) {
|
||||
hash_rows = new List<IBlueRow>();
|
||||
_hash_row_map.Add(hash, hash_rows);
|
||||
}
|
||||
@@ -59,6 +63,7 @@
|
||||
}
|
||||
}
|
||||
|
||||
[MemberNotNullWhen(true, nameof(_schema))]
|
||||
public bool ready {
|
||||
get {
|
||||
return _schema != null;
|
||||
@@ -69,8 +74,7 @@
|
||||
|
||||
private ViewStream _ls, _rs;
|
||||
private JoinBookkeepingData _ldata, _rdata;
|
||||
private ViewStreamDatabase _vsd;
|
||||
private IBlueTableSchema _schema = null;
|
||||
IBlueTableSchema? _schema;
|
||||
|
||||
public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
|
||||
this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
|
||||
@@ -83,8 +87,8 @@
|
||||
Log.assert(lj_col_list.Count == rj_col_list.Count);
|
||||
int n_cols = lj_col_list.Count;
|
||||
BlueRowHasher hasher = new BlueRowHasher(n_cols);
|
||||
_ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
|
||||
_rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
|
||||
_ldata = new JoinBookkeepingData(ls, lj_col_list, hasher);
|
||||
_rdata = new JoinBookkeepingData(rs, rj_col_list, hasher);
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
@@ -109,20 +113,24 @@
|
||||
_rs.onChange -= _rs_onchange;
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(_schema))]
|
||||
private void _build_schema() {
|
||||
if (_schema != null)
|
||||
return;
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||
cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
|
||||
foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
|
||||
string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
|
||||
var lschema = Log.notnull(_ldata.schema);
|
||||
var rschema = Log.notnull(_rdata.schema);
|
||||
var cis = new List<ColumnInfo> {
|
||||
new("id", 0, TypeCode.Int32)
|
||||
};
|
||||
foreach (ColumnInfo ci in lschema.columnInfo) {
|
||||
string col_name = rschema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
|
||||
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
|
||||
}
|
||||
foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
|
||||
string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
|
||||
foreach (ColumnInfo ci in rschema.columnInfo) {
|
||||
string col_name = lschema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
|
||||
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
|
||||
}
|
||||
_schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
|
||||
_schema = new BlueTableSchema($"{lschema.tableName}_join_{rschema.tableName}", cis.ToArray());
|
||||
}
|
||||
|
||||
private void _form_and_fire() {
|
||||
@@ -131,34 +139,11 @@
|
||||
_build_schema();
|
||||
List<object[]> res_rows = new List<object[]>();
|
||||
|
||||
Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
|
||||
int offset = 1;
|
||||
JoinBookkeepingData sdata = _ldata;
|
||||
if (is_left) {
|
||||
offset = 1;
|
||||
sdata = _ldata;
|
||||
} else {
|
||||
sdata = _rdata;
|
||||
offset = 1 + _ldata.schema.columnInfo.Length;
|
||||
}
|
||||
for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
|
||||
rval[offset + i] = r[i];
|
||||
}
|
||||
};
|
||||
Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
|
||||
object[] rv = new object[_schema.columnNames.Length];
|
||||
for (int i = 0; i < _schema.columnNames.Length; i++) {
|
||||
rv[i] = DBNull.Value;
|
||||
}
|
||||
rv[0] = id;
|
||||
copy_row(rv, l_row, true);
|
||||
return rv;
|
||||
};
|
||||
// this is where the join happens
|
||||
foreach (var kv in _ldata.rows) {
|
||||
RowHash hash = kv.Key;
|
||||
List<IBlueRow> l_rows = kv.Value;
|
||||
List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
|
||||
List<IBlueRow>? r_rows = _rdata.get_matching_rows(hash);
|
||||
|
||||
foreach (IBlueRow lr in l_rows) {
|
||||
if (r_rows != null) {
|
||||
@@ -179,6 +164,31 @@
|
||||
}
|
||||
var dp = BlueDataUtility.build_dp(rv, myLastValue as IDataPacket);
|
||||
myFireEvent(dp);
|
||||
|
||||
object[] build_row_array(int id, IBlueRow l_row) {
|
||||
object[] rv = new object[_schema.columnNames.Length];
|
||||
for (int i = 0; i < _schema.columnNames.Length; i++) {
|
||||
rv[i] = DBNull.Value;
|
||||
}
|
||||
rv[0] = id;
|
||||
copy_row(rv, l_row, true);
|
||||
return rv;
|
||||
}
|
||||
|
||||
void copy_row(object?[] rval, IBlueRow r, bool is_left) {
|
||||
int offset = 1;
|
||||
JoinBookkeepingData sdata = _ldata;
|
||||
if (is_left) {
|
||||
offset = 1;
|
||||
sdata = _ldata;
|
||||
} else {
|
||||
sdata = _rdata;
|
||||
offset = 1 + _ldata.RequireSchema().columnInfo.Length;
|
||||
}
|
||||
for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++) {
|
||||
rval[offset + i] = r[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -45,8 +45,8 @@ namespace bts.BlueOrb {
|
||||
}
|
||||
|
||||
// aggregates multiple rows into one - ie does group_by compute
|
||||
public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
|
||||
if (rows.Count() == 0) {
|
||||
public static IBlueRow? group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
|
||||
if (rows.Count == 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -73,8 +73,8 @@ namespace bts.BlueOrb {
|
||||
}
|
||||
|
||||
// aggregates multiple rows into one, used by MultiRowRollingMathStream
|
||||
public static IBlueRow group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
|
||||
if (rows.Count() == 0) {
|
||||
public static IBlueRow? group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
|
||||
if (rows.Count == 0) {
|
||||
return null;
|
||||
}
|
||||
IBlueRow ir = rows.ElementAt(start_idx);
|
||||
@@ -104,8 +104,8 @@ namespace bts.BlueOrb {
|
||||
// rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
|
||||
// calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
|
||||
|
||||
public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
|
||||
if (rolling_rows.Count() == 0) {
|
||||
public IBlueRow? rolling_calc(List<IBlueRow> rolling_rows) {
|
||||
if (rolling_rows.Count == 0) {
|
||||
return null;
|
||||
}
|
||||
IBlueRow ir = rolling_rows.ElementAt(0);
|
||||
@@ -156,7 +156,6 @@ namespace bts.BlueOrb {
|
||||
double first = col_vals.First();
|
||||
double last = col_vals.Last();
|
||||
return ((first - last) / last) * 100.0;
|
||||
return col_vals.First() - col_vals.Last();
|
||||
}
|
||||
default:
|
||||
// TODO
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
namespace bts.BlueOrb {
|
||||
using BCG;
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System;
|
||||
|
||||
[DataStreamUserRespresentation("MultiRowRollingMathStream")]
|
||||
public class MultiRowRollingMathStream : ViewStream {
|
||||
@@ -26,9 +25,10 @@
|
||||
private void _parent_str_onchange(object dp_obj) {
|
||||
var dp = (IDataPacket)dp_obj;
|
||||
var rows = dp.newRows.to_list();
|
||||
BlueTable out_table = null;
|
||||
BlueTable? out_table = null;
|
||||
for (int i = 0; i < rows.Count; i++) {
|
||||
var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op);
|
||||
var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op)
|
||||
?? throw new InvalidOperationException("group_by_compute returned null row");
|
||||
if (out_table == null)
|
||||
out_table = new BlueTable(out_r.schema);
|
||||
out_table.addRow(out_r);
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
namespace bts.BlueOrb {
|
||||
using BCG;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
@@ -54,7 +57,7 @@ namespace bts.BlueOrb {
|
||||
|
||||
public bool IsReadOnly { get { return true; } }
|
||||
|
||||
public object this[object key] {
|
||||
public object? this[object key] {
|
||||
get {
|
||||
if (key is string)
|
||||
return this[(string)key];
|
||||
@@ -63,16 +66,15 @@ namespace bts.BlueOrb {
|
||||
set { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||
}
|
||||
|
||||
public object this[int index] {
|
||||
public object? this[int index] {
|
||||
get {
|
||||
return this[schema.columnInfo[index].colName];
|
||||
}
|
||||
}
|
||||
|
||||
public object this[string col_name] {
|
||||
public object? this[string col_name] {
|
||||
get {
|
||||
string mapped_to_col = null;
|
||||
if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
|
||||
if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out var mapped_to_col)) {
|
||||
return _row[mapped_to_col];
|
||||
}
|
||||
return _row[col_name];
|
||||
@@ -89,7 +91,8 @@ namespace bts.BlueOrb {
|
||||
}
|
||||
}
|
||||
|
||||
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||
public void Add(object key, object? value) =>
|
||||
throw new NotSupportedException();
|
||||
|
||||
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||
|
||||
@@ -114,7 +117,7 @@ namespace bts.BlueOrb {
|
||||
|
||||
private ViewStream _stream;
|
||||
private string _project_expression;
|
||||
private ProjectData _project_data = null;
|
||||
ProjectData? _project_data;
|
||||
//public ProjectData
|
||||
public NewProjectStream(ViewStream vs, string project_expression) {
|
||||
_stream = vs;
|
||||
@@ -128,9 +131,8 @@ namespace bts.BlueOrb {
|
||||
return _project_data.out_schema;
|
||||
}
|
||||
|
||||
public IBlueRow map(IBlueRow row) {
|
||||
return new NewProjectRow(_project_data, row);
|
||||
}
|
||||
public IBlueRow map(IBlueRow row) =>
|
||||
new NewProjectRow(ExceptionAssert.MemberNotNull(_project_data), row);
|
||||
|
||||
// this is where most of the logic is, in terms of translating cols
|
||||
private static ProjectData _create_project_data(IBlueTableSchema input_schema,
|
||||
|
||||
@@ -2,9 +2,6 @@ namespace bts {
|
||||
using System.IO;
|
||||
using BCG;
|
||||
|
||||
using System.IO;
|
||||
using System.Threading.Tasks.Dataflow;
|
||||
|
||||
public class RemoteRequest {
|
||||
public readonly string stream_expression;
|
||||
public readonly string published_table_name;
|
||||
@@ -28,4 +25,4 @@ namespace bts {
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,33 +1,28 @@
|
||||
using Spark.Core.Collections;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace BCG {
|
||||
public class AddColumnsStream : ViewStream {
|
||||
public static Tuple<string, Type, object> parse_single_tok(string exp) {
|
||||
public static (string Name, Type Type, object Value) parse_single_tok(string exp) {
|
||||
string[] toks = exp.Split(':');
|
||||
string col_name = toks[0].Trim();
|
||||
string col_value = toks[1].Trim();
|
||||
Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
|
||||
return Tuple.Create(col_name, tup.Item1, tup.Item2);
|
||||
return (col_name, tup.Item1, tup.Item2);
|
||||
}
|
||||
|
||||
public static List<Tuple<string, Type, object>> parse_exp(string exp) {
|
||||
string[] toks = exp.Split(",");
|
||||
List<Tuple<string, Type, object>> rv = new List<Tuple<string, Type, object>>();
|
||||
foreach (string s in toks) {
|
||||
var tup = parse_single_tok(s.Trim());
|
||||
rv.Add(tup);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
public static List<(string Name, Type Type, object Value)> parse_exp(string exp) =>
|
||||
exp.Split(",").ToList(t => parse_single_tok(t.Trim()));
|
||||
|
||||
private Tuple<string, Type, object> _new_col;
|
||||
private List<Tuple<string, Type, object>> _new_col_map;
|
||||
private IBlueTableSchema _out_schema;
|
||||
readonly List<(string Name, Type Type, object Value)> _new_col_map;
|
||||
IBlueTableSchema? _out_schema;
|
||||
|
||||
public AddColumnsStream(ViewStream vs, string col_def) {
|
||||
_new_col_map = parse_exp(col_def);
|
||||
ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
|
||||
ViewStream rs = new MapStream(vs, _get_schema, _map);
|
||||
mySetRealSource(rs);
|
||||
}
|
||||
|
||||
@@ -44,7 +39,7 @@ namespace BCG {
|
||||
private IBlueRow _map(IBlueRow r) {
|
||||
MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
|
||||
foreach (var m in _new_col_map) {
|
||||
rv.setValue(m.Item1, m.Item3);
|
||||
rv.setValue(m.Name, m.Value);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
+33
-43
@@ -1,9 +1,10 @@
|
||||
using System;
|
||||
using mbts;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using mbts;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -42,8 +43,8 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public delegate void IncrementValues(object[] current_values, object[] values_to_inc);
|
||||
public delegate string GetColumnPrefix(object[] transposing_col_values);
|
||||
public delegate void IncrementValues(object?[] current_values, object?[] values_to_inc);
|
||||
public delegate string GetColumnPrefix(object?[] transposing_col_values);
|
||||
|
||||
public delegate string GetColumnSuffix(string col_name);
|
||||
private GetColumnSuffix _get_column_suffix = new GetColumnSuffix((cn) => { return cn; });
|
||||
@@ -51,33 +52,31 @@ namespace BCG
|
||||
private IncrementValues _increment_values;
|
||||
private GetColumnPrefix _get_column_prefix;
|
||||
|
||||
private IBlueTableSchema _schema;
|
||||
IBlueTableSchema? _schema;
|
||||
|
||||
private string[] _group_by_cols;
|
||||
private Pair<string, Type>[] _cols_per_transposed;
|
||||
private string[] _transposing_cols;
|
||||
|
||||
private int[] _transposing_col_indices;
|
||||
int[]? _transposing_col_indices;
|
||||
|
||||
private Comparison<Pair<int, object[]>> _transpsed_col_value_comparer_wrapper;
|
||||
Comparison<Pair<int, object?[]>>? _transpsed_col_value_comparer_wrapper;
|
||||
|
||||
public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
|
||||
IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer)
|
||||
IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer)
|
||||
: this(stream, transposed_cols_group, cols_per_transposed, group_by_cols, increment_values_del,
|
||||
get_column_prefix_del, get_column_suffix_del, transposed_col_value_comparer, true)
|
||||
{ }
|
||||
|
||||
public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
|
||||
IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer, bool only_fire_once)
|
||||
IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer, bool only_fire_once)
|
||||
{
|
||||
if (get_column_suffix_del != null)
|
||||
_get_column_suffix = get_column_suffix_del;
|
||||
if (transposed_col_value_comparer != null)
|
||||
{
|
||||
_transpsed_col_value_comparer_wrapper = new Comparison<Pair<int, object[]>>((p1,p2) =>
|
||||
{
|
||||
return transposed_col_value_comparer(p1.second, p2.second);
|
||||
});
|
||||
_transpsed_col_value_comparer_wrapper = new((p1, p2) =>
|
||||
transposed_col_value_comparer(ExceptionAssert.NotNull(p1.second), ExceptionAssert.NotNull(p2.second)));
|
||||
}
|
||||
_group_by_cols = group_by_cols.Split(' ');
|
||||
_transposing_cols = transposed_cols_group.Split(' ');
|
||||
@@ -106,7 +105,7 @@ namespace BCG
|
||||
col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
|
||||
foreach (string tc_prefix in tc_prefixes)
|
||||
foreach (Pair<string, Type> p in _cols_per_transposed)
|
||||
col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first), col_index++, p.second));
|
||||
col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, p.second));
|
||||
IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
|
||||
if (_schema == null)
|
||||
_schema = schema;
|
||||
@@ -130,19 +129,14 @@ namespace BCG
|
||||
|
||||
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
|
||||
{
|
||||
Dictionary<int, Dictionary<int, object[]>> all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object[]>>();
|
||||
Dictionary<int, object[]> group_by_cols_value_dict = new Dictionary<int, object[]>();
|
||||
var all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object?[]>>();
|
||||
var group_by_cols_value_dict = new Dictionary<int, object?[]>();
|
||||
BlueRowHasher hasher = new BlueRowHasher(_transposing_cols.Length);
|
||||
Pair<string, Type>[] group_by_col_info = new Pair<string, Type>[_group_by_cols.Length];
|
||||
string table_name = string.Empty;
|
||||
if (grouped_rows.Count > 0)
|
||||
{
|
||||
IBlueTableSchema schema = null;
|
||||
foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
|
||||
{
|
||||
schema = kvp.Value[0].schema;
|
||||
break;
|
||||
}
|
||||
var schema = grouped_rows.Values.First()[0].schema;
|
||||
table_name = schema.tableName;
|
||||
_transposing_col_indices = BlueDataUtility.get_column_indices(schema, _transposing_cols);
|
||||
int[] group_by_indices = BlueDataUtility.get_column_indices(schema, _group_by_cols);
|
||||
@@ -155,23 +149,22 @@ namespace BCG
|
||||
else
|
||||
return new BlueTable(new BlueTableSchema(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }, table_name));
|
||||
Dictionary<int, string> tc_prefixes = new Dictionary<int, string>();
|
||||
List<Pair<int, object[]>> tc_values = new List<Pair<int, object[]>>();
|
||||
var tc_values = new List<Pair<int, object?[]>>();
|
||||
foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
|
||||
{
|
||||
//Dictionary of trans_col_group_hash -> pair: first=trans_col_group_values second=per_transposing_col_values
|
||||
Dictionary<int, object[]> incrementing_values = new Dictionary<int, object[]>();
|
||||
var incrementing_values = new Dictionary<int, object?[]>();
|
||||
foreach (IBlueRow r in kvp.Value)
|
||||
{
|
||||
int hash = hasher.get_hash(r, _transposing_col_indices);
|
||||
object[] values;
|
||||
if (!incrementing_values.TryGetValue(hash, out values))
|
||||
if (!incrementing_values.TryGetValue(hash, out var values))
|
||||
{
|
||||
incrementing_values.Add(hash, _get_per_transposing_col_values(r));
|
||||
string prefix = _get_column_prefix(_get_transposing_col_values(r));
|
||||
if (!tc_prefixes.ContainsKey(hash))
|
||||
{
|
||||
tc_prefixes.Add(hash, prefix);
|
||||
tc_values.Add(new Pair<int,object[]>(hash, _get_transposing_col_values(r)));
|
||||
tc_values.Add(new(hash, _get_transposing_col_values(r)));
|
||||
}
|
||||
}
|
||||
else
|
||||
@@ -194,18 +187,18 @@ namespace BCG
|
||||
_set_schema(prefixes, group_by_col_info, table_name);
|
||||
BlueTable result = new BlueTable(_schema);
|
||||
int id = 1;
|
||||
foreach (KeyValuePair<int, Dictionary<int, object[]>> kvp in all_tc_infos_and_values)
|
||||
foreach (var kvp in all_tc_infos_and_values)
|
||||
{
|
||||
MutableBlueRow mbr = new MutableBlueRow(_schema);
|
||||
mbr.setValue("id", id++);
|
||||
for (int i = 0; i < _group_by_cols.Length; i++)
|
||||
mbr.setValue(_group_by_cols[i], group_by_cols_value_dict[kvp.Key][i]);
|
||||
foreach (KeyValuePair<int, object[]> tc_cols in kvp.Value)
|
||||
foreach (var tc_cols in kvp.Value)
|
||||
{
|
||||
string prefix = tc_prefixes[tc_cols.Key];
|
||||
for (int i = 0; i < _cols_per_transposed.Length; i++)
|
||||
{
|
||||
string column = _get_full_column_name(prefix, _cols_per_transposed[i].first);
|
||||
string column = _get_full_column_name(prefix, _cols_per_transposed[i].first ?? string.Empty);
|
||||
if(mbr.schema.columnNames.Contains(column))
|
||||
mbr.setValue(column, tc_cols.Value[i]);
|
||||
}
|
||||
@@ -217,28 +210,25 @@ namespace BCG
|
||||
|
||||
#endregion
|
||||
|
||||
private object[] _get_transposing_col_values(IBlueRow row)
|
||||
{
|
||||
object[] values = new object[_transposing_cols.Length];
|
||||
object?[] _get_transposing_col_values(IBlueRow row) {
|
||||
var values = new object?[_transposing_cols.Length];
|
||||
for (int i = 0; i < _transposing_cols.Length; i++)
|
||||
values[i] = row[_transposing_cols[i]];
|
||||
return values;
|
||||
}
|
||||
|
||||
private object[] _get_per_transposing_col_values(IBlueRow row)
|
||||
{
|
||||
object[] values = new object[_cols_per_transposed.Length];
|
||||
object?[] _get_per_transposing_col_values(IBlueRow row) {
|
||||
var values = new object?[_cols_per_transposed.Length];
|
||||
for (int i = 0; i < _cols_per_transposed.Length; i++)
|
||||
values[i] = row[_cols_per_transposed[i].first];
|
||||
values[i] = row[ExceptionAssert.NotNull(_cols_per_transposed[i].first)];
|
||||
return values;
|
||||
}
|
||||
|
||||
private object[] _get_group_by_cols_values(IBlueRow row)
|
||||
{
|
||||
object[] values = new object[_group_by_cols.Length];
|
||||
object?[] _get_group_by_cols_values(IBlueRow row) {
|
||||
var values = new object?[_group_by_cols.Length];
|
||||
for (int i = 0; i < values.Length; i++)
|
||||
values[i] = row[_group_by_cols[i]];
|
||||
return values;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,22 +1,19 @@
|
||||
using System;
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class DerivativeStream : ViewStream
|
||||
{
|
||||
|
||||
private IBlueTableSchema mySchema;
|
||||
IBlueTableSchema? mySchema;
|
||||
private string myDerivandColumn;
|
||||
private string myOutputColumn;
|
||||
private string myKeyColumn;
|
||||
private DateTime myLastPacketArrival = DateTime.MinValue;
|
||||
private ViewStream myDataSource;
|
||||
|
||||
private IDictionary<string, double> myPreviousValues;
|
||||
|
||||
Dictionary<string, double>? myPreviousValues;
|
||||
|
||||
public DerivativeStream(ViewStream input, string keyColumn, string columnToDifferentiate, string outputColName)
|
||||
{
|
||||
@@ -64,12 +61,13 @@ namespace BCG
|
||||
|
||||
protected IBlueRow myMakeRow(IBlueRow r, double timeScale)
|
||||
{
|
||||
ExceptionAssert.NotNull(mySchema, myPreviousValues);
|
||||
object[] outValues = new object[mySchema.columnInfo.Length];
|
||||
r.Values.CopyTo(outValues, 0);
|
||||
|
||||
object derivand = r[myDerivandColumn];
|
||||
double lastValue = default(double);
|
||||
if (myPreviousValues.TryGetValue(r[myKeyColumn].ToString(), out lastValue) && derivand != null && derivand != DBNull.Value)
|
||||
var derivand = r[myDerivandColumn];
|
||||
var key = getRowKey(r);
|
||||
if (myPreviousValues.TryGetValue(key, out double lastValue) && derivand != null && derivand != DBNull.Value)
|
||||
outValues[mySchema.columnInfo.Length - 1] = (Convert.ToDouble(derivand) - lastValue) * timeScale;
|
||||
else
|
||||
outValues[mySchema.columnInfo.Length - 1] = 0.0;
|
||||
@@ -90,10 +88,13 @@ namespace BCG
|
||||
myLastPacketArrival = DateTime.Now;
|
||||
foreach (IBlueRow row in dp.newRows)
|
||||
{
|
||||
object derivand = row[myDerivandColumn];
|
||||
var derivand = row[myDerivandColumn];
|
||||
if (derivand != null && derivand != DBNull.Value)
|
||||
myPreviousValues[row[myKeyColumn].ToString()] = Convert.ToDouble(derivand);
|
||||
myPreviousValues[getRowKey(row)] = Convert.ToDouble(derivand);
|
||||
}
|
||||
}
|
||||
|
||||
string getRowKey(IBlueRow row) =>
|
||||
row[myKeyColumn]?.ToString() ?? throw new ArgumentException("Row key is null", nameof(row));
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using mbts;
|
||||
|
||||
using Spark.Core.Collections;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using mbts;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -12,9 +14,9 @@ namespace BCG
|
||||
{
|
||||
ViewStream myLStream, myRStream;
|
||||
|
||||
DataPacket myLastR = null, myLastL = null;
|
||||
|
||||
BlueTable myLastFired = null;
|
||||
DataPacket? myLastR;
|
||||
DataPacket? myLastL;
|
||||
BlueTable? myLastFired;
|
||||
|
||||
double myThreshold = .00005;
|
||||
|
||||
@@ -24,14 +26,14 @@ namespace BCG
|
||||
public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) :
|
||||
this(null, v1, v2, thresh, ColumnsToMatch) { }
|
||||
|
||||
public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
|
||||
public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
|
||||
{
|
||||
myThreshold = thresh;
|
||||
myLStream = v1;
|
||||
myRStream = v2;
|
||||
myColumnsToMatch = columnsToMatch;
|
||||
}
|
||||
public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
|
||||
public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
|
||||
: this(db,v1,v2,.00005,columnsToMatch)
|
||||
{
|
||||
}
|
||||
@@ -50,32 +52,9 @@ namespace BCG
|
||||
myRStream.onChange -= myRChange;
|
||||
}
|
||||
|
||||
struct ListComparer : IEqualityComparer<List<object>>
|
||||
{
|
||||
public bool Equals(List<object> x, List<object> y)
|
||||
{
|
||||
if (x.Count != y.Count)
|
||||
return false;
|
||||
|
||||
for (int i = 0; i < x.Count; ++i)
|
||||
if (!x[i].Equals(y[i]))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
public int GetHashCode(List<object> obj)
|
||||
{
|
||||
int ret = 0;
|
||||
for(int i = 0; i < obj.Count; ++i)
|
||||
ret ^= obj[i].GetHashCode();
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
void myHandleCallbacks()
|
||||
{
|
||||
DataPacket packet = null;
|
||||
|
||||
DataPacket? packet;
|
||||
if (myLastL == null || myLastR == null)
|
||||
{
|
||||
packet = myLastL == null ? myLastR : myLastL;
|
||||
@@ -88,7 +67,7 @@ namespace BCG
|
||||
|
||||
BlueTable table = new BlueTable(myLastL.newRows.schema);
|
||||
|
||||
Dictionary<List<object>, Pair<BlueRow, BlueRow>> rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(new ListComparer());
|
||||
var rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(EnumerableComparer.Create<object>());
|
||||
|
||||
foreach(BlueRow row in myLastL.newRows)
|
||||
{
|
||||
@@ -118,11 +97,7 @@ namespace BCG
|
||||
}
|
||||
catch (Exception) { }
|
||||
}
|
||||
|
||||
Pair<BlueRow, BlueRow> pair;
|
||||
rowPairs.TryGetValue(hash, out pair);
|
||||
|
||||
|
||||
rowPairs.TryGetValue(hash, out var pair);
|
||||
if (pair == null)
|
||||
pair = new Pair<BlueRow, BlueRow>(null, row);
|
||||
else
|
||||
@@ -138,8 +113,7 @@ namespace BCG
|
||||
|
||||
if (pair.first == null || pair.second == null)
|
||||
{
|
||||
BlueRow row = pair.first == null ? pair.second : pair.first;
|
||||
|
||||
var row = pair.first ?? pair.second ?? throw new InvalidOperationException("Both rows in rowPair are null");
|
||||
Dictionary<string, object> newRow = new Dictionary<string, object>();
|
||||
foreach (ColumnInfo info in row.schema.columnInfo)
|
||||
{
|
||||
@@ -211,4 +185,4 @@ namespace BCG
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,7 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using BCG.Utilities.General;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using System;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class DateBasedFilter: IDynamicStreamFilter, IWorker
|
||||
@@ -26,20 +25,14 @@ namespace BCG
|
||||
myReference = myTodaysDate.AddDays(-1 * myNDays);
|
||||
}
|
||||
|
||||
public event FilterChangeDelegate onFilterChange;
|
||||
public event FilterChangeDelegate? onFilterChange;
|
||||
|
||||
public bool filter(IBlueRow r)
|
||||
{
|
||||
if (DBNull.Value.Equals(r[myDateColumn]))
|
||||
{
|
||||
//return false;
|
||||
return true; //it can be considered as our assumption whenever datecolumn is db null we return true
|
||||
}
|
||||
DateTime t = (DateTime)r[myDateColumn];
|
||||
bool returnVallue = t < myReference ? false : true;
|
||||
return returnVallue;
|
||||
}
|
||||
|
||||
public bool filter(IBlueRow r) =>
|
||||
r[myDateColumn] switch {
|
||||
DBNull or null => true,
|
||||
DateTime t => t >= myReference,
|
||||
_ => throw new ArgumentException("Row has invalid date column value", nameof(r)),
|
||||
};
|
||||
|
||||
public void doWork()
|
||||
{
|
||||
@@ -47,7 +40,7 @@ namespace BCG
|
||||
{
|
||||
myTodaysDate = DateTime.Today;
|
||||
myReference = myTodaysDate.AddDays(-1 * myNDays);
|
||||
onFilterChange();
|
||||
onFilterChange?.Invoke();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using BCG.Utilities.General;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using System;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -26,7 +24,7 @@ namespace BCG
|
||||
|
||||
#region IDynamicStreamFilter Members
|
||||
|
||||
public event FilterChangeDelegate onFilterChange;
|
||||
public event FilterChangeDelegate? onFilterChange;
|
||||
|
||||
#endregion
|
||||
|
||||
@@ -34,7 +32,10 @@ namespace BCG
|
||||
|
||||
public bool filter(IBlueRow r)
|
||||
{
|
||||
DateTime rowDate = DBNull.Value.Equals (r[myDateColName]) ? DateTime.Now.Date : ((DateTime)r[myDateColName]).Date;
|
||||
DateTime rowDate = r[myDateColName] switch {
|
||||
DBNull or null => DateTime.Now.Date,
|
||||
object obj => (DateTime)obj,
|
||||
};
|
||||
DateTime today = DateTime.Now.Date;
|
||||
return rowDate > today ? true : !(today.Subtract(rowDate).Days >= myNPastTodayDays);
|
||||
}
|
||||
@@ -48,7 +49,7 @@ namespace BCG
|
||||
if (DateTime.Now.Date != myLastCheckedTime.Date)
|
||||
{
|
||||
myLastCheckedTime = DateTime.Now;
|
||||
onFilterChange();
|
||||
onFilterChange?.Invoke();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,18 +1,23 @@
|
||||
using bts;
|
||||
|
||||
using Bts.DataStream;
|
||||
using Bts.DataStream.Primitives.Filter;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using mbts.config;
|
||||
using Spark.Core.Diagnostics;
|
||||
using Spark.Core.Disposables;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace BCG.StreamWorld.Addition {
|
||||
[DataStreamUserRespresentation(false)]
|
||||
public class FilterStream : ViewStream {
|
||||
private ViewStream myBaseStream;
|
||||
private BlueTable myBaseTable, myStateTable;
|
||||
BlueTable? myBaseTable;
|
||||
BlueTable? myStateTable;
|
||||
private IStreamFilter myFilter;
|
||||
|
||||
public FilterStream(ViewStream vs)
|
||||
@@ -25,8 +30,8 @@ namespace BCG.StreamWorld.Addition {
|
||||
myBaseStream = vs;
|
||||
myFilter = filter;
|
||||
vs.ErrorChanged += OnError;
|
||||
if (filter is IDynamicStreamFilter)
|
||||
(filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
|
||||
if (filter is IDynamicStreamFilter dfilter)
|
||||
dfilter.onFilterChange += myHandleFilterChange;
|
||||
}
|
||||
|
||||
|
||||
@@ -36,8 +41,8 @@ namespace BCG.StreamWorld.Addition {
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
|
||||
myBaseTable.clear();
|
||||
myStateTable.clear();
|
||||
myBaseTable?.clear();
|
||||
myStateTable?.clear();
|
||||
}
|
||||
|
||||
bool filterRow(IBlueRow r) {
|
||||
@@ -54,16 +59,12 @@ namespace BCG.StreamWorld.Addition {
|
||||
try {
|
||||
var dp = (IDataPacket)newValue;
|
||||
var schema = dp.Schema;
|
||||
if (myStateTable == null) {
|
||||
bts.Log.assert(myBaseTable == null);
|
||||
myStateTable = new BlueTable(schema);
|
||||
myBaseTable = new BlueTable(schema);
|
||||
}
|
||||
init(schema);
|
||||
var dels = new BlueTable(schema);
|
||||
var adds = new BlueTable(schema);
|
||||
foreach (IBlueRow r in dp.deletedRows) {
|
||||
myBaseTable.deleteRow(r);
|
||||
if (myStateTable.containsID((int)r["id"])) {
|
||||
if (myStateTable.containsID(r.Id())) {
|
||||
dels.addRow(r);
|
||||
myStateTable.deleteRow(r);
|
||||
}
|
||||
@@ -86,6 +87,7 @@ namespace BCG.StreamWorld.Addition {
|
||||
set {
|
||||
myFilter = value;
|
||||
if (myBaseTable != null) {
|
||||
ExceptionAssert.NotNull(myStateTable);
|
||||
IBlueRowCollection oldState = new BlueTable(myStateTable);
|
||||
myStateTable.clear();
|
||||
// If the filter throws an exception, we can rethrow it as a StreamFilterException
|
||||
@@ -113,6 +115,17 @@ namespace BCG.StreamWorld.Addition {
|
||||
//public string filterString { set { filter = new BooleanExpressionFilter(value); } }
|
||||
|
||||
private void myHandleFilterChange() { filter = myFilter; }
|
||||
|
||||
[MemberNotNull(nameof(myStateTable), nameof(myBaseTable))]
|
||||
void init(IBlueTableSchema schema) {
|
||||
if (myStateTable is null) {
|
||||
Log.isnull(myBaseTable);
|
||||
myStateTable = new BlueTable(schema);
|
||||
myBaseTable = new BlueTable(schema);
|
||||
} else {
|
||||
Log.notnull(myBaseTable);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -136,29 +149,24 @@ namespace BCG
|
||||
}
|
||||
|
||||
#region IStreamFilter Members
|
||||
public bool filter(IBlueRow dr)
|
||||
{
|
||||
bool rv = true;
|
||||
for (int i = 0; i < myCols.Length; i++)
|
||||
{
|
||||
rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
|
||||
if (!rv)
|
||||
return rv;
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
public bool filter(IBlueRow dr) =>
|
||||
myCols.Zip(myRegExps).All(p => p.Second.IsMatch(dr[p.First] switch {
|
||||
DBNull or null => string.Empty,
|
||||
object obj => obj.ToString() ?? string.Empty
|
||||
}));
|
||||
#endregion
|
||||
}
|
||||
|
||||
public class FilterStream : ViewStream
|
||||
{
|
||||
private BlueTable myLastFiredRows;
|
||||
private IBlueRowCollection myLastReceivedRows;
|
||||
private IStreamFilter myFilter;
|
||||
private ViewStream myParentStream;
|
||||
private string myFilterString;
|
||||
private BCG.StreamWorld.Addition.FilterStream myFilterStream;
|
||||
private bool myUseNewFilter = false;
|
||||
BlueTable? myLastFiredRows;
|
||||
IBlueRowCollection? myLastReceivedRows;
|
||||
IStreamFilter? myFilter;
|
||||
readonly ViewStream? myParentStream;
|
||||
string? myFilterString;
|
||||
readonly StreamWorld.Addition.FilterStream? myFilterStream;
|
||||
readonly bool myUseNewFilter = false;
|
||||
IDisposable? subscription;
|
||||
|
||||
#region constructors
|
||||
public delegate bool OnFilter(IBlueRow r);
|
||||
@@ -188,9 +196,10 @@ namespace BCG
|
||||
{
|
||||
myParentStream = vs;
|
||||
myFilter = filter;
|
||||
myFilterString = filter.ToString();
|
||||
if (filter is IDynamicStreamFilter)
|
||||
(filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
|
||||
myFilterString = filter.ToString() ?? string.Empty;
|
||||
if (filter is IDynamicStreamFilter dfilter) {
|
||||
dfilter.onFilterChange += myHandleFilterChange;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,13 +223,15 @@ namespace BCG
|
||||
#endregion
|
||||
|
||||
#region overridden methods
|
||||
protected override void myOnFirstSubscriber()
|
||||
{ myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
|
||||
protected override void myOnFirstSubscriber() {
|
||||
ExceptionAssert.IsNull(subscription);
|
||||
subscription = ExceptionAssert.NotNull(myParentStream).Subscribe(myHandleCallbacks);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
myLastFiredRows = null;
|
||||
myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
|
||||
Disposable.Dispose(ref subscription);
|
||||
}
|
||||
|
||||
protected override void myFireEvent(object o)
|
||||
@@ -239,19 +250,20 @@ namespace BCG
|
||||
try {
|
||||
if (myLastFiredRows != null)
|
||||
{
|
||||
retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
|
||||
new BlueTable(myLastFiredRows.schema));
|
||||
var addedRows = new BlueTable(myLastFiredRows.schema);
|
||||
var deletedRows = new BlueTable(myLastFiredRows.schema);
|
||||
retVal = new DataPacket(myLastFiredRows, addedRows, deletedRows);
|
||||
foreach (IBlueRow r in arrivedPacket.deletedRows)
|
||||
if (myLastFiredRows.containsID((int)r["id"]))
|
||||
if (myLastFiredRows.containsID(r.Id()))
|
||||
{
|
||||
myLastFiredRows.deleteRow(r);
|
||||
(retVal.deletedRows as BlueTable).addRow(r);
|
||||
deletedRows.addRow(r);
|
||||
}
|
||||
foreach (IBlueRow r in arrivedPacket.addedRows)
|
||||
if (filterRow(r))
|
||||
{
|
||||
myLastFiredRows.addRow(r);
|
||||
(retVal.addedRows as BlueTable).addRow(r);
|
||||
addedRows.addRow(r);
|
||||
}
|
||||
}
|
||||
else
|
||||
@@ -274,7 +286,7 @@ namespace BCG
|
||||
{
|
||||
if (myUseNewFilter)
|
||||
{
|
||||
myFilterStream.filter = filter;
|
||||
requireNewFilter().filter = filter;
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -303,9 +315,9 @@ namespace BCG
|
||||
get
|
||||
{
|
||||
if (myUseNewFilter)
|
||||
return myFilterStream.filter;
|
||||
return requireNewFilter().filter;
|
||||
else
|
||||
return myFilter;
|
||||
return requireOldFilter();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -318,17 +330,25 @@ namespace BCG
|
||||
|
||||
private void myHandleFilterChange()
|
||||
{
|
||||
filter = myFilter;
|
||||
filter = requireOldFilter();
|
||||
}
|
||||
|
||||
bool filterRow(IBlueRow r) {
|
||||
try {
|
||||
return myFilter.filter(r);
|
||||
return requireOldFilter().filter(r);
|
||||
} catch (Exception ex) {
|
||||
Log.Gen.error($"FilterStream delegate threw exception: {ex}");
|
||||
OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(myFilterStream))]
|
||||
StreamWorld.Addition.FilterStream requireNewFilter() =>
|
||||
ExceptionAssert.MemberNotNull(myFilterStream);
|
||||
|
||||
[MemberNotNull(nameof(myFilter))]
|
||||
IStreamFilter requireOldFilter() =>
|
||||
ExceptionAssert.MemberNotNull(myFilter);
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using BCG.Utilities.General;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using System;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -29,7 +27,7 @@ namespace BCG
|
||||
myReference = myCurrentTime.AddSeconds(-1 * myNSecs);
|
||||
}
|
||||
|
||||
public event FilterChangeDelegate onFilterChange;
|
||||
public event FilterChangeDelegate? onFilterChange;
|
||||
|
||||
public bool filter(IBlueRow r)
|
||||
{
|
||||
@@ -38,9 +36,8 @@ namespace BCG
|
||||
//it can be considered as our assumption whenever datecolumn is db null we return true
|
||||
return true; //dont use this for volparamsched use true
|
||||
}
|
||||
DateTime t = (DateTime)r[myDateColumn];
|
||||
bool returnVallue = t < myReference ? false : true;
|
||||
return returnVallue;
|
||||
var t = (DateTime)r[myDateColumn]!;
|
||||
return t >= myReference;
|
||||
}
|
||||
|
||||
|
||||
@@ -51,8 +48,7 @@ namespace BCG
|
||||
myCurrentTime = DateTime.Now;
|
||||
myReference = myCurrentTime.AddSeconds(-1 * myNSecs);
|
||||
myLastTimeFilteredApplied = DateTime.Now;
|
||||
onFilterChange();
|
||||
|
||||
onFilterChange?.Invoke();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
using System;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using FuzzyRowComparer = System.Func<BCG.IBlueRow, BCG.IBlueRow, double>;
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -40,12 +42,12 @@ namespace BCG
|
||||
private IComparer<IBlueRow> myRightCompare;
|
||||
protected FuzzyRowComparer myFuzzyCompare;
|
||||
protected TimeSpan myTimeWindow;
|
||||
protected IBlueTableSchema myLeftSchema;
|
||||
protected IBlueTableSchema myRightSchema;
|
||||
protected IBlueTableSchema myOutputSchema;
|
||||
protected IBlueTableSchema? myLeftSchema;
|
||||
protected IBlueTableSchema? myRightSchema;
|
||||
protected IBlueTableSchema? myOutputSchema;
|
||||
private int myID = 0;
|
||||
|
||||
protected TimeWindowTable myStateRows;
|
||||
protected TimeWindowTable? myStateRows;
|
||||
|
||||
private DateTime myNextCull;
|
||||
const long MinCullIntervalMS = 1000;
|
||||
@@ -65,7 +67,7 @@ namespace BCG
|
||||
this.myFuzzyCompare = fuzzyCompare;
|
||||
}
|
||||
|
||||
public virtual IBlueTableSchema OutputSchema
|
||||
public virtual IBlueTableSchema? OutputSchema
|
||||
{
|
||||
get
|
||||
{
|
||||
@@ -150,7 +152,7 @@ namespace BCG
|
||||
int count;
|
||||
for (count = 0; count < list.Count; count++)
|
||||
{
|
||||
if ((DateTime)list[count][timeColumn] > expiry)
|
||||
if ((DateTime)list[count][timeColumn]! > expiry)
|
||||
{
|
||||
// Stop when we find the first row that we can keep. Since this is sorted all others are also kept.
|
||||
break;
|
||||
@@ -172,16 +174,16 @@ namespace BCG
|
||||
this.myFitness = fitness;
|
||||
}
|
||||
public IBlueRow myRightRow { get; private set; }
|
||||
public IBlueRow myLeftRow { get; set; }
|
||||
public IBlueRow? myLeftRow { get; set; }
|
||||
public double myFitness { get; private set; }
|
||||
public IBlueRow myMergedRow { get; set; }
|
||||
public IBlueRow? myMergedRow { get; set; }
|
||||
|
||||
#region IComparable<RowMatch> Members
|
||||
|
||||
public int CompareTo(RowMatch other)
|
||||
{
|
||||
return myFitness.CompareTo(other.myFitness);
|
||||
}
|
||||
public int CompareTo(RowMatch? other) =>
|
||||
other is null
|
||||
? 1
|
||||
: myFitness.CompareTo(other.myFitness);
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -262,12 +264,12 @@ namespace BCG
|
||||
|
||||
private void HandleCorrection(List<RowMatch> corrections, BlueTable adds, BlueTable dels)
|
||||
{
|
||||
ExceptionAssert.NotNull(myStateRows);
|
||||
// We already emitted correction.MergedRow so we need to redact that and
|
||||
// then emit correction.left as an add since it is no longer matched
|
||||
foreach (RowMatch correction in corrections.Distinct())
|
||||
{
|
||||
Debug.Assert(correction.myMergedRow != null);
|
||||
this.myStateRows.RemoveRow(correction.myMergedRow);
|
||||
myStateRows.RemoveRow(ExceptionAssert.NotNull(correction.myMergedRow));
|
||||
adds.addRow(this.MergeRows(correction.myLeftRow, null));
|
||||
dels.addRow(correction.myMergedRow);
|
||||
}
|
||||
@@ -285,7 +287,7 @@ namespace BCG
|
||||
int r = 0;
|
||||
for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; )
|
||||
{
|
||||
if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn])
|
||||
if ((DateTime)leftExpired[l][this.myLeftTimeColumn]! <= (DateTime)rightExpired[r][this.myRightTimeColumn]!)
|
||||
{
|
||||
// Advance Left
|
||||
IEnumerable<RowMatch> rm = rightMatches[l];
|
||||
@@ -332,7 +334,7 @@ namespace BCG
|
||||
|
||||
private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable<RowMatch> rightMatches)
|
||||
{
|
||||
IBlueRow merged = null;
|
||||
IBlueRow? merged = null;
|
||||
if (rightMatches != null)
|
||||
{
|
||||
foreach (RowMatch match in rightMatches)
|
||||
@@ -366,9 +368,9 @@ namespace BCG
|
||||
/// Override this function to provide a custom row merge.
|
||||
/// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting.
|
||||
/// </summary>
|
||||
protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right)
|
||||
protected virtual IBlueRow MergeRows(IBlueRow? left, IBlueRow? right)
|
||||
{
|
||||
Dictionary<string, object> newRow = new Dictionary<string, object>();
|
||||
Dictionary<string, object?> newRow = new();
|
||||
|
||||
if (left == null && right == null) throw new ArgumentException("oh come on.");
|
||||
|
||||
@@ -393,14 +395,15 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
newRow["id"] = this.myID++;
|
||||
newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn];
|
||||
|
||||
newRow[NEW_TIME_COLUMN] = left is not null
|
||||
? left[myLeftTimeColumn]
|
||||
: right![myRightTimeColumn];
|
||||
return new BlueRow(this.OutputSchema, newRow);
|
||||
}
|
||||
|
||||
private IEnumerable<RowMatch> FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
|
||||
{
|
||||
List<RowMatch> matches = null;
|
||||
List<RowMatch>? matches = null;
|
||||
|
||||
// Search through all rows in list and compare with the fuzzy comparer
|
||||
for (int x = 0; x < list.Count; x++)
|
||||
@@ -421,8 +424,7 @@ namespace BCG
|
||||
matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch);
|
||||
|
||||
// Mark the best match for this particular right hand match
|
||||
RowMatch oldMatch;
|
||||
if (this.myBestMatch.TryGetValue(match, out oldMatch))
|
||||
if (this.myBestMatch.TryGetValue(match, out var oldMatch))
|
||||
{
|
||||
if (currentMatch.myFitness < oldMatch.myFitness)
|
||||
{
|
||||
@@ -441,7 +443,7 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
return matches;
|
||||
return matches ?? Enumerable.Empty<RowMatch>();
|
||||
}
|
||||
}
|
||||
}
|
||||
+3
-7
@@ -1,14 +1,10 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public interface INonIncGroupByEvaluator
|
||||
{
|
||||
IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
|
||||
string[] get_group_by_cols();
|
||||
IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
|
||||
string[]? get_group_by_cols();
|
||||
}
|
||||
}
|
||||
+32
-22
@@ -1,46 +1,51 @@
|
||||
using System;
|
||||
using bts;
|
||||
using bts.SilverPine;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
using Spark.Core.Disposables;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
|
||||
using bts;
|
||||
using bts.SilverPine;
|
||||
|
||||
namespace BCG {
|
||||
//[DataStreamUserRespresentation("AggregateStream")]
|
||||
public class NonIncGroupByStream : ViewStream {
|
||||
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
|
||||
Func<string[]> _groupby_cols_func;
|
||||
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
|
||||
public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
|
||||
Func<string[]?> _groupby_cols_func;
|
||||
Func<List<IBlueRow>, List<IBlueRow>?> _evaluate_func;
|
||||
public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func,
|
||||
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
|
||||
_groupby_cols_func = groupby_cols_func;
|
||||
_evaluate_func = evaluate_func;
|
||||
}
|
||||
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
|
||||
BlueTable rv = null;
|
||||
public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
|
||||
BlueTable? rv = null;
|
||||
foreach (var kv in grouped_rows) {
|
||||
List<IBlueRow> g_rows = kv.Value;
|
||||
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
|
||||
var out_rows = _evaluate_func(g_rows);
|
||||
if (out_rows != null && out_rows.Count > 0) {
|
||||
rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
|
||||
rv.addRows(out_rows);
|
||||
}
|
||||
rv.addRows(out_rows);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
public string[] get_group_by_cols() {
|
||||
public string[]? get_group_by_cols() {
|
||||
return _groupby_cols_func();
|
||||
}
|
||||
|
||||
}
|
||||
private ViewStream _main_stream;
|
||||
private INonIncGroupByEvaluator _groupby_func;
|
||||
private BlueRowHasher _row_hasher;
|
||||
private string[] _groupby_cols;
|
||||
readonly ViewStream? _main_stream;
|
||||
readonly INonIncGroupByEvaluator? _groupby_func;
|
||||
BlueRowHasher? _row_hasher;
|
||||
readonly string[]? _groupby_cols;
|
||||
private Stopwatch _timer = new();
|
||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||
IDisposable? subscription;
|
||||
|
||||
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
|
||||
_main_stream = main_stream;
|
||||
@@ -49,7 +54,7 @@ namespace BCG {
|
||||
_row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
|
||||
}
|
||||
|
||||
public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
|
||||
public NonIncGroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func,
|
||||
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
|
||||
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
|
||||
|
||||
@@ -58,7 +63,7 @@ namespace BCG {
|
||||
|
||||
public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
|
||||
MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
|
||||
Func<string[]> groupby_cols_func = () => {
|
||||
Func<string[]?> groupby_cols_func = () => {
|
||||
return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
|
||||
};
|
||||
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
|
||||
@@ -75,11 +80,16 @@ namespace BCG {
|
||||
}
|
||||
|
||||
|
||||
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
|
||||
protected override void myOnFirstSubscriber() {
|
||||
ExceptionAssert.IsNull(subscription);
|
||||
ExceptionAssert.NotNull(_main_stream).Subscribe(_main_stream_onchange);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
|
||||
protected override void myOnLastSubscriber() =>
|
||||
Disposable.Dispose(ref subscription);
|
||||
|
||||
private void _main_stream_onchange(object nv) {
|
||||
ExceptionAssert.NotNull(_groupby_func);
|
||||
var dp = (IDataPacket)nv;
|
||||
///grouping together
|
||||
_timer.Restart();
|
||||
@@ -94,7 +104,7 @@ namespace BCG {
|
||||
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
||||
/// reducing
|
||||
_timer.Restart();
|
||||
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
||||
var total_state = _groupby_func.evaluate(grouped_rows);
|
||||
_timer.Stop();
|
||||
if (total_state is null) {
|
||||
Log.Gen.warn("NonIncGroupByStream _groupby_func.evaluate returned null (ignoring last update).");
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
using System;
|
||||
using BCG.Utilities;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -11,12 +11,10 @@ namespace BCG
|
||||
/// </summary>
|
||||
public class RowDictionaryTable : IBlueRowCollection
|
||||
{
|
||||
protected IDictionary myRowCollections = new Hashtable ();
|
||||
protected BlueTable myLinearCollection;
|
||||
protected string[] myCols;
|
||||
protected readonly Dictionary<string, BlueTable> myRowCollections = new();
|
||||
protected readonly BlueTable myLinearCollection;
|
||||
protected readonly string[] myCols;
|
||||
protected IBlueTableSchema mySchema;
|
||||
|
||||
protected RowDictionaryTable () {}
|
||||
|
||||
public RowDictionaryTable (IBlueRowCollection rows, params string[] cols)
|
||||
: this (rows.schema, cols)
|
||||
@@ -40,7 +38,7 @@ namespace BCG
|
||||
|
||||
public virtual void addRow (IBlueRow row)
|
||||
{
|
||||
BlueTable rows = (BlueTable)getGroup (row);
|
||||
var rows = getMutableGroup(row);
|
||||
if (rows == null)
|
||||
{
|
||||
//i.e we dont have any group for this row => we need to create a new group
|
||||
@@ -53,7 +51,7 @@ namespace BCG
|
||||
|
||||
public virtual void removeRow (IBlueRow row)
|
||||
{
|
||||
BlueTable rowGroup = (BlueTable)getGroup (row);
|
||||
var rowGroup = requireMutableGroup(row);
|
||||
// BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0);
|
||||
rowGroup.deleteRow (row);
|
||||
myLinearCollection.deleteRow (row);
|
||||
@@ -65,7 +63,7 @@ namespace BCG
|
||||
|
||||
public void removeGroup (IBlueRow representativeRow)
|
||||
{
|
||||
IBlueRowCollection rowGroup = getGroup (representativeRow);
|
||||
var rowGroup = requireGroup(representativeRow);
|
||||
// BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0);
|
||||
foreach (IBlueRow row in rowGroup)
|
||||
myLinearCollection.deleteRow (row);
|
||||
@@ -85,24 +83,31 @@ namespace BCG
|
||||
/// <returns></returns>
|
||||
//public IBlueRowCollection getGroup (IBlueRow row) {return getGroup (
|
||||
|
||||
public IBlueRowCollection getGroup (IBlueRow r) {return getGroup (myRowToString (r));}
|
||||
public IBlueRowCollection? getGroup (IBlueRow r) =>
|
||||
getGroup(myRowToString(r));
|
||||
|
||||
public ICollection getGroups () {return myRowCollections.Values;}
|
||||
|
||||
private bool myAreRowsEqual (IBlueRow r1, IBlueRow r2)
|
||||
{
|
||||
bool retVal = true;
|
||||
foreach (string col in myCols)
|
||||
retVal &= r1[col].Equals(r2[col]);
|
||||
return retVal;
|
||||
}
|
||||
public IBlueRowCollection? getGroup(string repString) =>
|
||||
getMutableGroup(repString);
|
||||
|
||||
public IBlueRowCollection getGroup (string repString)
|
||||
{
|
||||
IBlueRowCollection retVal = myRowCollections[repString] != null ?
|
||||
(IBlueRowCollection) myRowCollections[repString] : null;
|
||||
return retVal;
|
||||
}
|
||||
BlueTable? getMutableGroup(IBlueRow row) =>
|
||||
getMutableGroup(myRowToString(row));
|
||||
|
||||
BlueTable? getMutableGroup(string group) =>
|
||||
myRowCollections.GetValueOrDefault(group);
|
||||
|
||||
IBlueRowCollection requireGroup(IBlueRow row) =>
|
||||
requireGroup(myRowToString(row));
|
||||
|
||||
IBlueRowCollection requireGroup(string group) =>
|
||||
getGroup(group) ?? throw new ArgumentException("No table for group", nameof(group));
|
||||
|
||||
BlueTable requireMutableGroup(IBlueRow row) =>
|
||||
requireMutableGroup(myRowToString(row));
|
||||
|
||||
BlueTable requireMutableGroup(string group) =>
|
||||
getMutableGroup(group) ?? throw new ArgumentException("No table for group", nameof(group));
|
||||
|
||||
private string myRowToString (IBlueRow r) {return BlueDataUtility.rowToString (r, myCols);}
|
||||
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
using System;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
[DataStreamUserRespresentation (false)]
|
||||
@@ -17,6 +15,7 @@ namespace BCG
|
||||
|
||||
protected override void myInitFiredPacket ()
|
||||
{
|
||||
requireSchema();
|
||||
myLastFiredPacket = new DataPacket ();
|
||||
myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
|
||||
myLastFiredPacket.addedRows = new BlueTable (mySchema);
|
||||
@@ -25,6 +24,10 @@ namespace BCG
|
||||
|
||||
public override void onTableClear (bool isLeft)
|
||||
{
|
||||
if (myLastFiredPacket is null) {
|
||||
return;
|
||||
}
|
||||
requireSchema();
|
||||
//incase of natural join, left or right wont matter, the end result zeroes out
|
||||
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
||||
myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -13,12 +15,10 @@ namespace BCG
|
||||
protected JoinOperand myLeftOperand, myRightOperand;
|
||||
protected int myLeftJoinColOrdinal, myRightJoinColOrdinal;
|
||||
protected ViewStream myLeftStream, myRightStream;
|
||||
protected JoinTableSchema mySchema;
|
||||
protected DataPacket myLastFiredPacket;
|
||||
protected JoinTableSchema? mySchema;
|
||||
protected DataPacket? myLastFiredPacket;
|
||||
protected bool myOnceFired = false;
|
||||
|
||||
public IBlueTableSchema schema {get{return mySchema;}}
|
||||
|
||||
public InternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight)
|
||||
{
|
||||
myLeftStream = vsLeft;
|
||||
@@ -27,6 +27,9 @@ namespace BCG
|
||||
myRightOperand = new JoinOperand (jcRight, false);
|
||||
}
|
||||
|
||||
public IBlueTableSchema schema =>
|
||||
requireSchema();
|
||||
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
myLeftOperand.master = this;
|
||||
@@ -62,29 +65,32 @@ namespace BCG
|
||||
{
|
||||
myLeftJoinColOrdinal = myLeftOperand.joinColOridinal;
|
||||
myRightJoinColOrdinal = myRightOperand.joinColOridinal;
|
||||
myLeftOperand.RequireInitialized();
|
||||
myRightOperand.RequireInitialized();
|
||||
mySchema = new JoinTableSchema (myLeftOperand.operandSchema, myLeftOperand.joinCol,
|
||||
myRightOperand.operandSchema, myRightOperand.joinCol);
|
||||
myInitFiredPacket ();
|
||||
}
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(mySchema), nameof(myLastFiredPacket))]
|
||||
protected virtual void myInitFiredPacket ()
|
||||
{
|
||||
var schema = requireSchema();
|
||||
myLastFiredPacket = new DataPacket ();
|
||||
myLastFiredPacket.newRows = new JoinTable (mySchema);
|
||||
myLastFiredPacket.addedRows = new BlueTable (mySchema);
|
||||
myLastFiredPacket.deletedRows = new BlueTable (mySchema);
|
||||
myLastFiredPacket.newRows = new JoinTable (schema);
|
||||
myLastFiredPacket.addedRows = new BlueTable (schema);
|
||||
myLastFiredPacket.deletedRows = new BlueTable (schema);
|
||||
}
|
||||
|
||||
public IBlueTableSchema getSchema (bool isLeft)
|
||||
{
|
||||
IBlueTableSchema retVal;
|
||||
retVal = isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema;
|
||||
return retVal;
|
||||
}
|
||||
public IBlueTableSchema? getSchema (bool isLeft) =>
|
||||
isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema;
|
||||
|
||||
public void endSession ()
|
||||
{
|
||||
if (myLastFiredPacket is null) {
|
||||
return;
|
||||
}
|
||||
if (!myOnceFired)
|
||||
{
|
||||
myOnceFired = true;
|
||||
@@ -97,13 +103,17 @@ namespace BCG
|
||||
|
||||
protected virtual void myInit ()
|
||||
{
|
||||
myLeftOperand.init ();
|
||||
myRightOperand.init ();
|
||||
myLeftOperand.Reset ();
|
||||
myRightOperand.Reset ();
|
||||
myLastFiredPacket = null;
|
||||
}
|
||||
|
||||
public virtual void onTableClear (bool isLeft)
|
||||
{
|
||||
if (myLastFiredPacket is null) {
|
||||
return;
|
||||
}
|
||||
requireSchema();
|
||||
//incase of natural join, left or right wont matter, the end result zeroes out
|
||||
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
||||
myLastFiredPacket.newRows = new JoinTable (mySchema);
|
||||
@@ -112,15 +122,19 @@ namespace BCG
|
||||
|
||||
public virtual void deleteRows (IBlueRowCollection rows, bool isLeft)
|
||||
{
|
||||
var last = myLastFiredPacket;
|
||||
if (last is null) {
|
||||
return;
|
||||
}
|
||||
JoinOperand thisOp, otherOp;
|
||||
int joinColOrdinal;
|
||||
myAssignAccordingly (out thisOp, out otherOp, out joinColOrdinal, isLeft);
|
||||
foreach (IBlueRow row in rows)
|
||||
{
|
||||
thisOp.deleteRow (row);
|
||||
if (otherOp.getMatchingRows (row[joinColOrdinal]).Count == 0)
|
||||
if (otherOp.getMatchingRows(getJoinKey(row, joinColOrdinal)).Count == 0)
|
||||
continue; //row does not participate in join
|
||||
((JoinTable)(myLastFiredPacket.newRows)).deleteRow (isLeft, row, myLastFiredPacket);
|
||||
((JoinTable)last.newRows).deleteRow(isLeft, row, last);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,10 +149,12 @@ namespace BCG
|
||||
otherOp = myLeftOperand;
|
||||
joinColOrdinal = myRightJoinColOrdinal;
|
||||
}
|
||||
var schema = requireSchema();
|
||||
myLastFiredPacket ??= new DataPacket(new BlueTable(schema), new BlueTable(schema), new BlueTable(schema));
|
||||
foreach (IBlueRow row in rows)
|
||||
{
|
||||
thisOp.addRow (row);
|
||||
RowList matchedNodes = otherOp.getMatchingRows (row[joinColOrdinal]);
|
||||
RowList matchedNodes = otherOp.getMatchingRows(getJoinKey(row, joinColOrdinal));
|
||||
if (matchedNodes.Count == 0)
|
||||
continue; //this row did not participate in the join
|
||||
if (isLeft)
|
||||
@@ -146,10 +162,22 @@ namespace BCG
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow (row, matchedNode.row, myLastFiredPacket);
|
||||
else
|
||||
foreach (RowNode matchedNode in matchedNodes)
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow (matchedNode.row, row, myLastFiredPacket);
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow(ExceptionAssert.NotNull(matchedNode.row), row, myLastFiredPacket);
|
||||
}
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(mySchema))]
|
||||
protected JoinTableSchema requireSchema() =>
|
||||
ExceptionAssert.NotNull(mySchema);
|
||||
|
||||
[MemberNotNull(nameof(myLastFiredPacket))]
|
||||
protected DataPacket requireLastFiredPacket() {
|
||||
if (myLastFiredPacket is null) {
|
||||
myInitFiredPacket();
|
||||
}
|
||||
return myLastFiredPacket;
|
||||
}
|
||||
|
||||
private void myAssignAccordingly (out JoinOperand thisOp, out JoinOperand otherOp, out int joinColOrdinal, bool isLeft)
|
||||
{
|
||||
if (isLeft)
|
||||
@@ -166,6 +194,9 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
object getJoinKey(IBlueRow row, int joinColOrdinal) =>
|
||||
row[joinColOrdinal] ?? throw new ArgumentException("Row has null join key", nameof(row));
|
||||
|
||||
/**************************************TEST METHOD(S)*********************************************/
|
||||
/// <summary>
|
||||
/// Used for testing purposes only. This enables the test to work without a DataSource (PubServer,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -8,7 +9,7 @@ namespace BCG
|
||||
/// </summary>
|
||||
internal class JoinDeletionIndex
|
||||
{
|
||||
IDictionary myIndex = new Hashtable ();
|
||||
Dictionary<object, RowList> myIndex = new();
|
||||
private string myOpIdName ;
|
||||
private bool myIsLeft;
|
||||
|
||||
@@ -28,10 +29,8 @@ namespace BCG
|
||||
/// <param name="row">JoinRow to add</param>
|
||||
public void add (JoinRow row)
|
||||
{
|
||||
object o = row[myOpIdName];
|
||||
if (myIndex[o] == null)
|
||||
myIndex[o] = new RowList ();
|
||||
RowNode nodeAdded = ((RowList)myIndex[o]).add (row);
|
||||
object o = ExceptionAssert.NotNull(row[myOpIdName]);
|
||||
RowNode nodeAdded = myIndex.GetOrAdd(o, _ => new()).add(row);
|
||||
if (myIsLeft)
|
||||
row.leftNode = nodeAdded;
|
||||
else
|
||||
@@ -43,7 +42,10 @@ namespace BCG
|
||||
/// To remove a node, if node does not exist exception is thrown
|
||||
/// </summary>
|
||||
/// <param name="node"></param>
|
||||
public void remove (RowNode node) {((RowList)myIndex[node.row[myOpIdName]]).remove (node);}
|
||||
public void remove (RowNode node) {
|
||||
var nodeId = ExceptionAssert.NotNull(node.row?[myOpIdName]);
|
||||
myIndex[nodeId].remove(node);
|
||||
}
|
||||
|
||||
public RowList getMatchingRows (object key) {
|
||||
RowList retVal = (RowList)myIndex[key];
|
||||
|
||||
@@ -1,19 +1,18 @@
|
||||
using System;
|
||||
using BCG.Utilities;
|
||||
|
||||
using bts;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
internal class JoinForeignTable : JoinTable
|
||||
{
|
||||
public JoinForeignTable (JoinTableSchema schema) : base (schema) {}
|
||||
|
||||
public override void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp)
|
||||
{
|
||||
JoinRow newRow = new JoinRow (leftRow, rightRow, mySchema);
|
||||
Log.assert(myIdIndex[leftRow["id"]] == null);
|
||||
myIdIndex[leftRow["id"]] = newRow;
|
||||
public override void addRow(IBlueRow leftRow, IBlueRow? rightRow, IDataPacket dp) {
|
||||
var newRow = new JoinRow(leftRow, rightRow, mySchema);
|
||||
int leftId = leftRow.Id();
|
||||
Log.assert(myIdIndex[leftId] == null);
|
||||
myIdIndex[leftId] = newRow;
|
||||
((BlueTable)(dp.addedRows)).addRow (newRow);
|
||||
myLeftIndex.add (newRow);
|
||||
myRightIndex.add (newRow);
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
|
||||
using bts;
|
||||
|
||||
using Bts.ColumnMapExpressions;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
/// <summary>
|
||||
@@ -37,7 +38,8 @@ namespace BCG
|
||||
private JoinTableSchema myJoinSchema;
|
||||
private IBlueTableSchema myLeftSchema, myRightSchema;
|
||||
private string myLeftColExpression, myRightColExpression;
|
||||
private string[] myUserRules, myResolvedNames;
|
||||
string[] myUserRules;
|
||||
string[]? myResolvedNames;
|
||||
private IDictionary myFinalMap = new Hashtable ();
|
||||
|
||||
|
||||
@@ -87,16 +89,18 @@ namespace BCG
|
||||
/// </summary>
|
||||
private void myFormUserRules ()
|
||||
{
|
||||
var leftExp = ColumnMapExpression.Parse(myLeftColExpression) ?? throw new InvalidOperationException("Failed to parse left column expression");
|
||||
var rightExp = ColumnMapExpression.Parse(myRightColExpression) ?? throw new InvalidOperationException("Failed to parse right column expression");
|
||||
if (DataStreamLoadFileConfig.instance.useStaticParser)
|
||||
{
|
||||
myFormUserRules(true, ColumnMapExpression.Parse(myLeftColExpression));
|
||||
myFormUserRules(false, ColumnMapExpression.Parse(myRightColExpression));
|
||||
myFormUserRules(true, leftExp);
|
||||
myFormUserRules(false, rightExp);
|
||||
}
|
||||
else
|
||||
{
|
||||
// TODO (maybe): Actually use non-static parser (unclear what advantages it will have)
|
||||
myFormUserRules(true, ColumnMapExpression.Parse(myLeftColExpression));
|
||||
myFormUserRules(false, ColumnMapExpression.Parse(myRightColExpression));
|
||||
myFormUserRules(true, leftExp);
|
||||
myFormUserRules(false, rightExp);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -115,13 +119,7 @@ namespace BCG
|
||||
OriginalNameInfo oni = myOriginalColInfo[i];
|
||||
if (oni.isLeft == isLeft)
|
||||
{
|
||||
if (userMap[oni.colName] != null)
|
||||
{
|
||||
if ((string)userMap[oni.colName] != oni.colName)
|
||||
myUserRules[i] = (string)userMap[oni.colName];
|
||||
else
|
||||
myUserRules[i] = oni.colName;
|
||||
}
|
||||
myUserRules[i] = userMap[oni.colName] as string ?? oni.colName;
|
||||
}
|
||||
}
|
||||
else
|
||||
@@ -131,10 +129,13 @@ namespace BCG
|
||||
OriginalNameInfo oni = myOriginalColInfo[i];
|
||||
if (oni.isLeft == isLeft)
|
||||
{
|
||||
if (userMap[oni.colName] == null)
|
||||
var colName = userMap[oni.colName] as string;
|
||||
// myUserRules[i] = userMap[oni.colName] as string
|
||||
if (colName is null) {
|
||||
myUserRules[i] = oni.colName;
|
||||
else if ((string)userMap[oni.colName] != oni.colName)
|
||||
myUserRules[i] = (string)userMap[oni.colName];
|
||||
} else if (colName != oni.colName) {
|
||||
myUserRules[i] = colName;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -165,26 +166,12 @@ namespace BCG
|
||||
|
||||
private void myFormFinalMap ()
|
||||
{
|
||||
ExceptionAssert.NotNull(myResolvedNames);
|
||||
foreach (string resolvedName in myResolvedNames)
|
||||
if (resolvedName != null)
|
||||
myFinalMap [resolvedName] = myJoinNames[Array.IndexOf (myResolvedNames, resolvedName)];
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Utility method to invert a hashtable
|
||||
/// </summary>
|
||||
/// <param name="arg"></param>
|
||||
/// <returns></returns>
|
||||
|
||||
private IDictionary reciprocateDictionary (IDictionary arg)
|
||||
{
|
||||
IDictionary retVal = new Hashtable ();
|
||||
foreach (DictionaryEntry de in arg)
|
||||
retVal[de.Value] = de.Key;
|
||||
return retVal;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Utility method to find all occurances of an obkect in an IList
|
||||
/// </summary>
|
||||
@@ -194,9 +181,11 @@ namespace BCG
|
||||
private ArrayList myFindAllInCollection (IList list, object obj)
|
||||
{
|
||||
ArrayList retVal = new ArrayList ();
|
||||
for (int i = 0; i < list.Count; i++)
|
||||
if (list[i] != null && list[i].Equals (obj))
|
||||
for (int i = 0; i < list.Count; i++) {
|
||||
if (Equals(list[i], obj)) {
|
||||
retVal.Add (i);
|
||||
}
|
||||
}
|
||||
return retVal;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -12,32 +13,45 @@ namespace BCG
|
||||
{
|
||||
private string myJoinCol;
|
||||
IDataPacket? myLastArrivedPacket;
|
||||
private JoinOperand myOtherOperand;
|
||||
JoinOperand? myOtherOperand;
|
||||
private bool myIsLeft;
|
||||
private JoinOperandRowIndex myRowIndex;
|
||||
private InternalJoinStream myMaster;
|
||||
private IBlueTableSchema myOpSchema;
|
||||
|
||||
JoinOperandRowIndex? myRowIndex;
|
||||
InternalJoinStream? myMaster;
|
||||
|
||||
public JoinOperand (string joinCol, bool isLeft)
|
||||
{
|
||||
myJoinCol = joinCol;
|
||||
myIsLeft = isLeft;
|
||||
}
|
||||
|
||||
[MemberNotNullWhen(true, nameof(myMaster), nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
|
||||
bool isInitialized {
|
||||
get {
|
||||
if (myLastArrivedPacket is null) {
|
||||
return false;
|
||||
}
|
||||
ExceptionAssert.NotNull(myMaster, myLastArrivedPacket, operandSchema, myOtherOperand, myRowIndex);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
internal void changeHandler (object o)
|
||||
{
|
||||
var dp = (IDataPacket)o;
|
||||
if (myLastArrivedPacket != null)
|
||||
if (isInitialized) {
|
||||
mySubsequentChangeHandler (dp);
|
||||
else
|
||||
} else {
|
||||
myFirstChangeHandler(dp);
|
||||
}
|
||||
}
|
||||
|
||||
void myFirstChangeHandler(IDataPacket arrivedDp) {
|
||||
[MemberNotNull(nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
|
||||
bool myFirstChangeHandler(IDataPacket arrivedDp) {
|
||||
ExceptionAssert.NotNull(myMaster);
|
||||
myLastArrivedPacket = arrivedDp;
|
||||
myOpSchema = myLastArrivedPacket.newRows.schema;
|
||||
operandSchema = myLastArrivedPacket.newRows.schema;
|
||||
myOtherOperand = myMaster.getOtherOperand (this);
|
||||
myRowIndex = new JoinOperandRowIndex (myOpSchema[myJoinCol].colNo, myOpSchema["id"].colNo);
|
||||
myRowIndex = new JoinOperandRowIndex (operandSchema[myJoinCol].colNo, operandSchema["id"].colNo);
|
||||
if (myOtherOperand.hasReceivedData)
|
||||
{
|
||||
//the other operand has already received data, so the join would
|
||||
@@ -48,13 +62,16 @@ namespace BCG
|
||||
myMaster.addRows (myLastArrivedPacket.newRows, myIsLeft);
|
||||
myMaster.endSession ();
|
||||
}
|
||||
else
|
||||
else {
|
||||
myRowIndex = new JoinOperandRowIndex (myLastArrivedPacket.newRows,
|
||||
myOpSchema[myJoinCol].colNo,
|
||||
myOpSchema["id"].colNo);
|
||||
operandSchema[myJoinCol].colNo,
|
||||
operandSchema["id"].colNo);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void mySubsequentChangeHandler (IDataPacket arrivedDp) {
|
||||
RequireInitialized();
|
||||
myLastArrivedPacket = arrivedDp;
|
||||
if (!myOtherOperand.hasReceivedData)
|
||||
{
|
||||
@@ -83,29 +100,46 @@ namespace BCG
|
||||
myMaster.endSession ();
|
||||
}
|
||||
|
||||
internal void addRow (IBlueRow row) {myRowIndex.add (row);}
|
||||
internal void addRow (IBlueRow row) =>
|
||||
ExceptionAssert.NotNull(myRowIndex).add(row);
|
||||
|
||||
internal void deleteRow (IBlueRow row) {myRowIndex.remove (row);}
|
||||
internal void deleteRow (IBlueRow row) {
|
||||
RequireInitialized();
|
||||
myRowIndex.remove (row);
|
||||
}
|
||||
|
||||
internal IBlueTableSchema operandSchema {get{return myOpSchema;}}
|
||||
internal IBlueTableSchema? operandSchema { get; private set; }
|
||||
|
||||
internal InternalJoinStream master {set{myMaster = value;}}
|
||||
|
||||
internal string joinCol {get{return myJoinCol;}}
|
||||
|
||||
internal int joinColOridinal {get{return myOpSchema[myJoinCol].colNo;}}
|
||||
internal int joinColOridinal {
|
||||
get {
|
||||
RequireInitialized();
|
||||
return operandSchema[myJoinCol].colNo;
|
||||
}
|
||||
}
|
||||
|
||||
internal RowList getMatchingRows (object key) {return (RowList) myRowIndex.getMatchingRows (key);}
|
||||
internal RowList getMatchingRows (object key) {
|
||||
RequireInitialized();
|
||||
return myRowIndex.getMatchingRows(key);
|
||||
}
|
||||
|
||||
internal bool hasReceivedData {get{return myLastArrivedPacket != null;}}
|
||||
|
||||
internal void init ()
|
||||
{
|
||||
internal void Reset() {
|
||||
///should be brought back to the the state before the first callback
|
||||
///(ie the state after construction)
|
||||
myRowIndex = null;
|
||||
myLastArrivedPacket = null;
|
||||
}
|
||||
|
||||
|
||||
[MemberNotNull(nameof(myMaster), nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
|
||||
internal void RequireInitialized() {
|
||||
if (!isInitialized) {
|
||||
throw new InvalidOperationException("Internal state not properly initialized");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -11,8 +11,10 @@ namespace BCG
|
||||
/// </summary>
|
||||
internal class JoinOperandRowIndex
|
||||
{
|
||||
private IDictionary myJoinColDict = new Hashtable (), myIdDict = new Hashtable ();
|
||||
private int myJoinColOrdinal, myIdColOrdinal;
|
||||
readonly Dictionary<object, RowList> myJoinColDict = new();
|
||||
readonly Dictionary<int, RowNode> myIdDict = new();
|
||||
readonly int myJoinColOrdinal;
|
||||
readonly int myIdColOrdinal;
|
||||
|
||||
/// <summary>
|
||||
///
|
||||
@@ -34,19 +36,14 @@ namespace BCG
|
||||
|
||||
public void add (IBlueRow row)
|
||||
{
|
||||
RowList list;
|
||||
if (myJoinColDict[row[myJoinColOrdinal]] == null)
|
||||
myJoinColDict[row[myJoinColOrdinal]] = list = new RowList ();
|
||||
else
|
||||
list = (RowList)myJoinColDict[row[myJoinColOrdinal]];
|
||||
myIdDict[row[myIdColOrdinal]] = list.add (row);
|
||||
myIdDict[getId(row)] = myJoinColDict.GetOrAdd(getJoinKey(row), _ => new()).add(row);
|
||||
}
|
||||
|
||||
public void remove (IBlueRow row)
|
||||
{
|
||||
RowNode node = (RowNode)myIdDict[row[myIdColOrdinal]];
|
||||
((RowList)myJoinColDict[row[myJoinColOrdinal]]).remove (node);
|
||||
myIdDict.Remove (row[myIdColOrdinal]);
|
||||
var id = getId(row);
|
||||
myJoinColDict[getJoinKey(row)].remove(myIdDict[id]);
|
||||
myIdDict.Remove(id);
|
||||
}
|
||||
|
||||
public RowList getMatchingRows (object key)
|
||||
@@ -62,5 +59,11 @@ namespace BCG
|
||||
myJoinColDict.Clear ();
|
||||
myIdDict.Clear ();
|
||||
}
|
||||
|
||||
object getJoinKey(IBlueRow row) =>
|
||||
ExceptionAssert.NotNull(row[myJoinColOrdinal]);
|
||||
|
||||
int getId(IBlueRow row) =>
|
||||
(int)row[myIdColOrdinal]!;
|
||||
}
|
||||
}
|
||||
@@ -1,17 +1,21 @@
|
||||
using System;
|
||||
using BCG.Utilities;
|
||||
using System.Collections;
|
||||
using bts;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
internal class JoinRow : IBlueRow
|
||||
{
|
||||
private RowNode myLeftNode, myRightNode;
|
||||
protected IBlueRow myLeftRow, myRightRow;
|
||||
RowNode? myLeftNode;
|
||||
RowNode? myRightNode;
|
||||
protected IBlueRow myLeftRow;
|
||||
protected IBlueRow? myRightRow;
|
||||
protected int myId;
|
||||
protected JoinTableSchema mySchema;
|
||||
private object[] myValues;
|
||||
private object?[] myValues;
|
||||
|
||||
#region Constructor(s)
|
||||
|
||||
@@ -21,8 +25,8 @@ namespace BCG
|
||||
/// <param name="leftRow">left row of the join</param>
|
||||
/// <param name="rightRow">right row of the join</param>
|
||||
/// <param name="schema">schema of the joined row</param>
|
||||
public JoinRow (IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
|
||||
: this ((int)leftRow["id"], leftRow, rightRow, schema) {}
|
||||
public JoinRow (IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema)
|
||||
: this(leftRow.Id(), leftRow, rightRow, schema) { }
|
||||
|
||||
/// <summary>
|
||||
/// Constructs a row assigning it the ID provioded as the first argument
|
||||
@@ -31,8 +35,7 @@ namespace BCG
|
||||
/// <param name="leftRow">left row of the join</param>
|
||||
/// <param name="rightRow">right row of the join</param>
|
||||
/// <param name="schema">schema of the joined row</param>
|
||||
public JoinRow (int id, IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
|
||||
{
|
||||
public JoinRow (int id, IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema) {
|
||||
myId = id;
|
||||
myLeftRow = leftRow;
|
||||
myRightRow = rightRow;
|
||||
@@ -48,13 +51,13 @@ namespace BCG
|
||||
|
||||
public bool isPartialJoin{get{return myRightRow == null;}}
|
||||
|
||||
public RowNode leftNode
|
||||
public RowNode? leftNode
|
||||
{
|
||||
get{return myLeftNode;}
|
||||
set{myLeftNode = value;}
|
||||
}
|
||||
|
||||
public RowNode rightNode
|
||||
public RowNode? rightNode
|
||||
{
|
||||
get{return myRightNode;}
|
||||
set{myRightNode = value;}
|
||||
@@ -85,7 +88,7 @@ namespace BCG
|
||||
/// The key can either be the string corrsponding to the column name or
|
||||
/// the order of the column
|
||||
/// </summary>
|
||||
public virtual object this [object key]
|
||||
public virtual object? this[object key]
|
||||
{
|
||||
get
|
||||
{
|
||||
@@ -96,7 +99,7 @@ namespace BCG
|
||||
set {throw new NotSupportedException ("The IBlueRow is read only");}
|
||||
}
|
||||
|
||||
public object this [int index]
|
||||
public object? this[int index]
|
||||
{
|
||||
get
|
||||
{
|
||||
@@ -119,7 +122,8 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public object this [string colName] {get{return this[mySchema[colName].colNo];}}
|
||||
public object? this[string colName] =>
|
||||
this[mySchema[colName].colNo];
|
||||
|
||||
public ICollection Keys{get{return mySchema.columnNames;}}
|
||||
|
||||
@@ -133,7 +137,8 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||
public void Add (object key, object? value) =>
|
||||
throw new NotSupportedException ("The IBlueRow is readonly");
|
||||
|
||||
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||
|
||||
|
||||
@@ -1,20 +1,26 @@
|
||||
using System;
|
||||
using System.Data;
|
||||
using System.Collections;
|
||||
using System.Text.RegularExpressions;
|
||||
using BCG.Utilities;
|
||||
using BCG.Utilities.General;
|
||||
using bts;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
using Spark.Core.Disposables;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class JoinStream : ViewStream
|
||||
{
|
||||
protected InternalJoinStream? myIjs;
|
||||
protected readonly InternalJoinStream? myIjs;
|
||||
protected readonly ColumnMap? myLeftCm;
|
||||
protected readonly ColumnMap? myRightCm;
|
||||
protected readonly bool myIsProject = true;
|
||||
protected ProjectStream? myPs;
|
||||
protected ColumnMap? myLeftCm, myRightCm;
|
||||
protected bool myIsProject = true;
|
||||
private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
|
||||
|
||||
readonly bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
|
||||
IDisposable? internalSubscription;
|
||||
IDisposable? projectSubscription;
|
||||
|
||||
protected JoinStream () {}
|
||||
public JoinStream (ViewStream v1, string joinCol1, ColumnMap cm1,
|
||||
@@ -111,47 +117,61 @@ namespace BCG
|
||||
{
|
||||
if (myPs == null)
|
||||
{
|
||||
JoinNameResolver resolver = new JoinNameResolver ((JoinTableSchema)myIjs.schema, myIjs.getSchema (true), myLeftCm.columnExpression,
|
||||
myIjs.getSchema (false), myRightCm.columnExpression);
|
||||
ExceptionAssert.NotNull(myIjs, myLeftCm, myRightCm);
|
||||
var leftSchema = myIjs.getSchema(true) ?? throw new InvalidOperationException("Left schema is null");
|
||||
var rightSchema = myIjs.getSchema(true) ?? throw new InvalidOperationException("Right schema is null");
|
||||
var joinSchema = (JoinTableSchema)myIjs.schema;
|
||||
JoinNameResolver resolver = new JoinNameResolver(
|
||||
joinSchema, leftSchema, myLeftCm.columnExpression, rightSchema, myRightCm.columnExpression);
|
||||
IDictionary map = resolver.getMap ();
|
||||
map["id"] = "id";
|
||||
Disposable.Dispose(ref projectSubscription);
|
||||
myPs = new ProjectStream (myIjs, map);
|
||||
myPs.onChange += new DataStream.ChangeDelegate (myProjectChangeHandler);
|
||||
projectSubscription = myPs.Subscribe(myProjectChangeHandler);
|
||||
return;
|
||||
}
|
||||
else
|
||||
Log.assert(false, "This call back should not have been called");
|
||||
}
|
||||
else
|
||||
myFireEvent ((o as ICloneable).Clone());
|
||||
else {
|
||||
myFireEvent(cloneDataPacket(o));
|
||||
}
|
||||
}
|
||||
|
||||
private void myProjectChangeHandler (object o)
|
||||
{
|
||||
if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1)
|
||||
myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
|
||||
myFireEvent ((o as ICloneable).Clone());
|
||||
|
||||
ExceptionAssert.NotNull(myIjs);
|
||||
if (myIjs.numberOfSubscribers > 1) {
|
||||
Disposable.Dispose(ref internalSubscription);
|
||||
}
|
||||
myFireEvent(cloneDataPacket(o));
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber ()
|
||||
{
|
||||
myIjs.onChange += new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
|
||||
Log.isnull(internalSubscription);
|
||||
internalSubscription = ExceptionAssert.NotNull(myIjs).Subscribe(myInternalJoinChangeHandler);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber ()
|
||||
{
|
||||
// how does this relate to the -= in myProjectChangeHandler?
|
||||
myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
|
||||
|
||||
Disposable.Dispose(ref internalSubscription);
|
||||
// if there is a project after the ijs we need to
|
||||
// unsubscribe to it otherwise references will be kept and
|
||||
// a memory leak will occur
|
||||
if (myPs != null)
|
||||
myPs.onChange -= new DataStream.ChangeDelegate (myProjectChangeHandler);
|
||||
Disposable.Dispose(ref projectSubscription);
|
||||
myPs = null;
|
||||
}
|
||||
|
||||
object cloneDataPacket(object dpObj) {
|
||||
var rv = (dpObj as ICloneable)?.Clone();
|
||||
if (rv is null) {
|
||||
var dp = (IDataPacket)dpObj;
|
||||
rv = new DataPacket(new BlueTable(dp.newRows), new BlueTable(dp.addedRows), new BlueTable(dp.deletedRows));
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
/**************************************TEST METHOD(S)*********************************************/
|
||||
/// <summary>
|
||||
/// Used for testing purposes only. This enables the test to work without a DataSource (PubServer,
|
||||
@@ -159,7 +179,9 @@ namespace BCG
|
||||
/// </summary>
|
||||
/// <param name="leftPacket"></param>
|
||||
/// <param name="rightPacket"></param>
|
||||
public void test (DataPacket dp, bool isLeft) {myIjs.test (dp, isLeft);}
|
||||
public void test (DataPacket dp, bool isLeft) {
|
||||
ExceptionAssert.NotNull(myIjs).test(dp, isLeft);
|
||||
}
|
||||
/***********************************************************************************************/
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.IO;
|
||||
using BCG.Utilities.General;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
internal class JoinTable : IBlueRowCollection
|
||||
{
|
||||
protected JoinTableSchema mySchema;
|
||||
protected IDictionary myIdIndex = new Hashtable ();
|
||||
protected readonly Dictionary<int, IBlueRow> myIdIndex = new();
|
||||
protected JoinDeletionIndex myLeftIndex, myRightIndex;
|
||||
private int myNewRowId;
|
||||
|
||||
@@ -19,19 +22,16 @@ namespace BCG
|
||||
myRightIndex = new JoinDeletionIndex (mySchema.getRenamedName ("id", false), false);
|
||||
}
|
||||
|
||||
public IBlueRow this [int id]
|
||||
{
|
||||
get
|
||||
{
|
||||
if (myIdIndex[id] == null)
|
||||
throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
|
||||
return (IBlueRow)myIdIndex[id];
|
||||
public IBlueRow this [int id] {
|
||||
get {
|
||||
var rowObj = myIdIndex[id]
|
||||
?? throw new BlueDataException($"ID {id} does not exist in table {mySchema.tableName}");
|
||||
return (IBlueRow)rowObj;
|
||||
}
|
||||
}
|
||||
|
||||
#region JoinTable members
|
||||
public virtual void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp)
|
||||
{
|
||||
public virtual void addRow(IBlueRow leftRow, IBlueRow? rightRow, IDataPacket dp) {
|
||||
JoinRow newRow = new JoinRow (myNewRowId, leftRow, rightRow, mySchema);
|
||||
myIdIndex[myNewRowId ++] = newRow;
|
||||
((BlueTable)(dp.addedRows)).addRow (newRow);
|
||||
@@ -40,7 +40,7 @@ namespace BCG
|
||||
myRightIndex.add (newRow);
|
||||
}
|
||||
|
||||
public virtual void deleteRow (bool isLeft, IBlueRow deletedRow, DataPacket dp)
|
||||
public virtual void deleteRow (bool isLeft, IBlueRow deletedRow, IDataPacket dp)
|
||||
{
|
||||
JoinDeletionIndex thisIndex = myRightIndex, otherIndex = myLeftIndex;
|
||||
if (isLeft)
|
||||
@@ -48,17 +48,17 @@ namespace BCG
|
||||
thisIndex = myLeftIndex;
|
||||
otherIndex = myRightIndex;
|
||||
}
|
||||
RowList matchingNodes = thisIndex.getMatchingRows (deletedRow["id"]);
|
||||
RowList matchingNodes = thisIndex.getMatchingRows(deletedRow.Id());
|
||||
foreach (RowNode node in matchingNodes)
|
||||
{
|
||||
JoinRow row = (JoinRow)node.row;
|
||||
RowNode correspondingNode = isLeft ? row.rightNode : row.leftNode;
|
||||
var row = ExceptionAssert.Argument.Is<JoinRow>(node.row);
|
||||
RowNode? correspondingNode = isLeft ? row.rightNode : row.leftNode;
|
||||
if (correspondingNode != null)
|
||||
otherIndex.remove (correspondingNode); //in case of partial left joinss
|
||||
myIdIndex.Remove (row["id"]);
|
||||
myIdIndex.Remove(row.Id());
|
||||
((BlueTable)dp.deletedRows).addRow (row);
|
||||
}
|
||||
thisIndex.removeAll (deletedRow["id"]);
|
||||
thisIndex.removeAll(deletedRow.Id());
|
||||
}
|
||||
#endregion
|
||||
#region ICollection members
|
||||
@@ -68,7 +68,11 @@ namespace BCG
|
||||
|
||||
public object SyncRoot{get{return this;}}
|
||||
|
||||
public void CopyTo (Array array, int index) {myIdIndex.Values.CopyTo (array, index);}
|
||||
public void CopyTo (Array array, int index) {
|
||||
foreach (var row in myIdIndex.Values) {
|
||||
array.SetValue(row, index++);
|
||||
}
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region IEnumerable members
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
|
||||
using bts;
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
/// <summary>
|
||||
@@ -13,10 +12,11 @@ namespace BCG
|
||||
public class JoinTableSchema : BlueTableSchema
|
||||
{
|
||||
private string myLeftJc = "", myRightJc = "", myLeftId = "", myRightId = "";
|
||||
private IDictionary myLeftNameToAlias = new Hashtable ();
|
||||
private IDictionary myRightNameToAlias = new Hashtable ();
|
||||
private IDictionary myAliasToLeftName = new Hashtable(),
|
||||
myAliasToRightName = new Hashtable();
|
||||
readonly Dictionary<string, string> myLeftNameToAlias = new();
|
||||
readonly Dictionary<string, string> myRightNameToAlias = new();
|
||||
readonly Dictionary<string, string> myAliasToLeftName = new();
|
||||
readonly Dictionary<string, string> myAliasToRightName = new();
|
||||
|
||||
/// <summary>
|
||||
/// Constructs the schema of the Jointable with the schema of the left and right tables. Incase of
|
||||
/// a name being there in both the shemas it prefixes the name with "L." or "R." accordingly. Column "id"
|
||||
@@ -85,9 +85,9 @@ namespace BCG
|
||||
renamed = false;
|
||||
}
|
||||
myIndexByColName["id"] = new ColumnInfo("id", myIndexByColName.Count, typeof(int));
|
||||
myIndexByColOrder[myIndexByColOrder.Length - 1] = (ColumnInfo)myIndexByColName["id"];
|
||||
myAliasToLeftName = ProjectHelper.invertDic(myLeftNameToAlias);
|
||||
myAliasToRightName = ProjectHelper.invertDic(myRightNameToAlias);
|
||||
myIndexByColOrder[myIndexByColOrder.Length - 1] = myIndexByColName["id"];
|
||||
myAliasToLeftName = myLeftNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key);
|
||||
myAliasToRightName = myRightNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key);
|
||||
}
|
||||
|
||||
public string leftJoinCol {get{return myLeftJc;}}
|
||||
|
||||
+50
-25
@@ -1,17 +1,21 @@
|
||||
using bts;
|
||||
using bts.collections;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using mbts;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
using Spark.Core.Disposables;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace BCG {
|
||||
public class NonIncJoinBookkeeper : IDisposable {
|
||||
public Action on_new_data;
|
||||
public event Action? on_new_data;
|
||||
private readonly NonIncJoinSpec _join_spec;
|
||||
private readonly int _join_position;
|
||||
private readonly int _start_index_in_output;
|
||||
@@ -20,13 +24,14 @@ namespace BCG {
|
||||
private readonly bool _is_tag_along;
|
||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||
|
||||
private IBlueTableSchema _schema = null;
|
||||
IBlueTableSchema? _schema;
|
||||
private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
|
||||
private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
|
||||
private IBlueRow _solitary_row = null;
|
||||
IBlueRow? _solitary_row;
|
||||
|
||||
private int[] _join_col_indices = null,
|
||||
_data_col_indices = null;
|
||||
int[]? _join_col_indices;
|
||||
int[]? _data_col_indices;
|
||||
IDisposable? subscription;
|
||||
|
||||
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
|
||||
_join_spec = the_join_spec;
|
||||
@@ -70,7 +75,8 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public void init() {
|
||||
_join_spec.stream.onChange += _on_stream_cb;
|
||||
ExceptionAssert.MemberIsNull(subscription);
|
||||
subscription = ExceptionAssert.MemberNotNull(_join_spec.stream).Subscribe(_on_stream_cb);
|
||||
}
|
||||
|
||||
public void _on_stream_cb(object dp_obj) {
|
||||
@@ -78,12 +84,7 @@ namespace BCG {
|
||||
var timer = Stopwatch.StartNew();
|
||||
var dp = (IDataPacket)dp_obj;
|
||||
_fill_missing_data(dp);
|
||||
if (_schema == null) {
|
||||
_schema = dp.Schema;
|
||||
_join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
|
||||
_data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
|
||||
}
|
||||
|
||||
init(dp);
|
||||
if (_is_left_most) {
|
||||
_row_list.Clear();
|
||||
foreach (IBlueRow r in dp.newRows)
|
||||
@@ -108,44 +109,68 @@ namespace BCG {
|
||||
on_new_data();
|
||||
}
|
||||
|
||||
[MemberNotNullWhen(true, nameof(_schema))]
|
||||
public bool is_ready { get { return _schema != null; } }
|
||||
|
||||
public void constribute_cols_to_schema(List<ColumnInfo> cis) {
|
||||
requireReady();
|
||||
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
|
||||
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
|
||||
}
|
||||
|
||||
public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
|
||||
|
||||
public void copy_master_row_to_output_row(int row_index, object[] output_row) {
|
||||
IBlueRow this_row = _row_list[row_index].second;
|
||||
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
||||
}
|
||||
public void copy_master_row_to_output_row(int row_index, object?[] output_row) =>
|
||||
copyRow(ExceptionAssert.NotNull(_row_list[row_index].second), output_row);
|
||||
|
||||
public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
|
||||
public void copy_matching_row_to_output_row(int row_hash, object?[] output_row) {
|
||||
if (_is_tag_along) {
|
||||
if (_solitary_row != null)
|
||||
BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
|
||||
copyRow(_solitary_row, output_row);
|
||||
else
|
||||
copy_dbnull_to_output_row(output_row);
|
||||
} else {
|
||||
IBlueRow this_row = null;
|
||||
if (_row_map.TryGetValue(row_hash, out this_row))
|
||||
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
||||
if (_row_map.TryGetValue(row_hash, out var this_row))
|
||||
copyRow(this_row, output_row);
|
||||
else
|
||||
copy_dbnull_to_output_row(output_row);
|
||||
}
|
||||
}
|
||||
|
||||
private void copy_dbnull_to_output_row(object[] output_row) {
|
||||
void copy_dbnull_to_output_row(object?[] output_row) {
|
||||
requireDataColIndices();
|
||||
for (int i = 0; i < _data_col_indices.Length; i++)
|
||||
output_row[i + _start_index_in_output] = DBNull.Value;
|
||||
}
|
||||
|
||||
#region IDisposable
|
||||
|
||||
public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
|
||||
public void Dispose() =>
|
||||
Disposable.Dispose(ref subscription);
|
||||
|
||||
#endregion
|
||||
|
||||
void copyRow(IBlueRow src, object?[] dst) {
|
||||
requireDataColIndices();
|
||||
BlueDataUtility.copy_to_row(src, _data_col_indices, dst, _start_index_in_output);
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(_schema), nameof(_join_col_indices), nameof(_data_col_indices))]
|
||||
void init(IDataPacket dp) {
|
||||
_schema ??= dp.Schema;
|
||||
_join_col_indices ??= BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
|
||||
_data_col_indices ??= BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(_data_col_indices))]
|
||||
int[] requireDataColIndices() =>
|
||||
ExceptionAssert.NotNull(_data_col_indices);
|
||||
|
||||
[MemberNotNull(nameof(_schema))]
|
||||
void requireReady() {
|
||||
if (!is_ready) {
|
||||
throw new InvalidOperationException("Not ready");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,22 +1,25 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using mbts;
|
||||
|
||||
using mbts;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG {
|
||||
// this a collection of data, theres no joining logic in here
|
||||
public class NonIncJoinSpec {
|
||||
public ViewStream stream;
|
||||
public ViewStream? stream;
|
||||
// join col names are the names on which the streams are joined (for equality),
|
||||
// data col names are the names in the original stream
|
||||
// projected col names are their new names, ie if its x:l_x, then x is in the
|
||||
// data_col_names and l_x in projected cols
|
||||
public string[] join_col_names, data_col_names, projected_col_names;
|
||||
public string[] join_col_names;
|
||||
public string[] data_col_names;
|
||||
public string[] projected_col_names;
|
||||
|
||||
public NonIncJoinSpec() { }
|
||||
public NonIncJoinSpec() {
|
||||
join_col_names = Array.Empty<string>();
|
||||
data_col_names = Array.Empty<string>();
|
||||
projected_col_names = Array.Empty<string>();
|
||||
}
|
||||
|
||||
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
|
||||
: this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
|
||||
|
||||
@@ -5,18 +5,19 @@ using mbts;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace BCG {
|
||||
[DataStreamUserRespresentation("fastJoin")]
|
||||
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
|
||||
private readonly NonIncJoinSpec[] _join_specs;
|
||||
private readonly INonIncJoinInlineMapper _output_mapper;
|
||||
readonly INonIncJoinInlineMapper? _output_mapper;
|
||||
private readonly BlueRowHasher _row_hasher;
|
||||
private readonly TimeSpan _throttle_period;
|
||||
private readonly ViewStreamDatabase _vsd;
|
||||
|
||||
private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
|
||||
private IBlueTableSchema _output_schema = null;
|
||||
IBlueTableSchema? _output_schema;
|
||||
private DateTime _last_fire_time = DateTime.MinValue;
|
||||
private bool _is_dirty = false;
|
||||
|
||||
@@ -35,10 +36,10 @@ namespace BCG {
|
||||
params Triplet<ViewStream, string, string>[] join_specs)
|
||||
: this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
|
||||
|
||||
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
|
||||
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper? output_mapper, int throttles_ms = 0)
|
||||
: this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
|
||||
|
||||
public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
|
||||
public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper? output_mapper, int throttles_ms = 0) {
|
||||
_join_specs = join_specs;
|
||||
_output_mapper = output_mapper;
|
||||
_row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
|
||||
@@ -88,6 +89,7 @@ namespace BCG {
|
||||
return rv;
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(_output_schema))]
|
||||
private void _construct_schema() {
|
||||
if (_output_schema == null) {
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
@@ -9,7 +11,7 @@ namespace BCG
|
||||
/// </summary>
|
||||
internal class RowList : ICollection
|
||||
{
|
||||
private RowNode myFirstPtr;
|
||||
RowNode? myFirstPtr;
|
||||
private int myCount = 0;
|
||||
|
||||
#region ICollection methods
|
||||
@@ -67,19 +69,19 @@ namespace BCG
|
||||
}
|
||||
else if (node.isFirst)
|
||||
{
|
||||
myFirstPtr = myFirstPtr.next;
|
||||
myFirstPtr.previous = null;
|
||||
myFirstPtr = ExceptionAssert.NotNull(myFirstPtr).next;
|
||||
ExceptionAssert.NotNull(myFirstPtr).previous = null;
|
||||
return;
|
||||
}
|
||||
else if (node.isLast)
|
||||
{
|
||||
node.previous.next = null;
|
||||
ExceptionAssert.NotNull(node.previous).next = null;
|
||||
return;
|
||||
}
|
||||
else if (!node.isFirst && !node.isLast)
|
||||
{
|
||||
node.previous.next = node.next;
|
||||
node.next.previous = node.previous;
|
||||
ExceptionAssert.NotNull(node.previous).next = node.next;
|
||||
ExceptionAssert.NotNull(node.next).previous = node.previous;
|
||||
return;
|
||||
}
|
||||
else
|
||||
@@ -91,14 +93,16 @@ namespace BCG
|
||||
|
||||
internal class RowListEnumerator : IEnumerator
|
||||
{
|
||||
private RowNode myPointer;
|
||||
RowNode? myPointer;
|
||||
private RowList myList;
|
||||
private int myPos = -1;
|
||||
public RowListEnumerator (RowList list) {myList = list;}
|
||||
public object Current {get{return myPointer;}}
|
||||
public object? Current => myPointer;
|
||||
public bool MoveNext()
|
||||
{
|
||||
myPointer = myPos == -1 ? myList.myFirstPtr : myPointer.next;
|
||||
myPointer = myPos == -1
|
||||
? myList.myFirstPtr
|
||||
: ExceptionAssert.NotNull(myPointer).next;
|
||||
myPos ++;
|
||||
return myPointer != null;
|
||||
}
|
||||
@@ -108,28 +112,24 @@ namespace BCG
|
||||
|
||||
internal class RowNode
|
||||
{
|
||||
private IBlueRow myRow;
|
||||
private RowNode myNext;
|
||||
private RowNode myPrevious;
|
||||
private RowList myList;
|
||||
|
||||
public RowNode (IBlueRow row){myRow = row;}
|
||||
public RowNode (IBlueRow row, RowList list) : this (row) {myList = list;}
|
||||
public bool isLast {get{return myNext == null;}}
|
||||
public bool isFirst {get{return myPrevious == null;}}
|
||||
public RowList list {get{return myList;}}
|
||||
public IBlueRow row
|
||||
{
|
||||
get{return myRow;}
|
||||
set{myRow = value;}
|
||||
}
|
||||
public RowNode next {
|
||||
get{return myNext;}
|
||||
set{myNext = value;}
|
||||
}
|
||||
public RowNode previous {
|
||||
get{return myPrevious;}
|
||||
set{myPrevious = value;}
|
||||
}
|
||||
public RowNode (IBlueRow row) =>
|
||||
this.row = row;
|
||||
|
||||
public RowNode (IBlueRow row, RowList list)
|
||||
: this(row) =>
|
||||
this.list = list;
|
||||
|
||||
public bool isLast => next is null;
|
||||
public bool isFirst => previous is null;
|
||||
public RowList? list { get; }
|
||||
public IBlueRow? row { get; set; }
|
||||
public RowNode? next { get; set; }
|
||||
public RowNode? previous { get; set; }
|
||||
|
||||
public RowList RequireList() =>
|
||||
ExceptionAssert.MemberNotNull(list);
|
||||
|
||||
public IBlueRow RequireRow() =>
|
||||
ExceptionAssert.MemberNotNull(row);
|
||||
}
|
||||
}
|
||||
@@ -1,125 +1,17 @@
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using mbts;
|
||||
|
||||
using Spark.Core.Collections;
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using mbts;
|
||||
using Spark.Core.Collections;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
internal class SingleRowJoinDicionary<TKey, TValue> : IDictionary<TKey, TValue>
|
||||
{
|
||||
private TKey myKey;
|
||||
private TValue myValue;
|
||||
|
||||
#region IDictionary<TKey,TValue> Members
|
||||
|
||||
public void Add(TKey key, TValue value)
|
||||
{
|
||||
myKey = key;
|
||||
myValue = value;
|
||||
}
|
||||
|
||||
public bool ContainsKey(TKey key)
|
||||
{
|
||||
return myKey.Equals(key);
|
||||
}
|
||||
|
||||
public ICollection<TKey> Keys
|
||||
{
|
||||
get { return new TKey[] { myKey }; }
|
||||
}
|
||||
|
||||
public bool Remove(TKey key)
|
||||
{
|
||||
bool rv = myKey.Equals(key);
|
||||
myKey = default(TKey);
|
||||
return rv;
|
||||
}
|
||||
|
||||
public bool TryGetValue(TKey key, out TValue value)
|
||||
{
|
||||
bool rv = false;
|
||||
value = (rv = myKey.Equals(key)) ? myValue : default(TValue);
|
||||
return rv;
|
||||
}
|
||||
|
||||
public ICollection<TValue> Values
|
||||
{
|
||||
get { throw new Exception("The method or operation is not implemented."); }
|
||||
}
|
||||
|
||||
public TValue this[TKey key]
|
||||
{
|
||||
get
|
||||
{
|
||||
throw new Exception("The method or operation is not implemented.");
|
||||
}
|
||||
set
|
||||
{
|
||||
throw new Exception("The method or operation is not implemented.");
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region ICollection<KeyValuePair<TKey,TValue>> Members
|
||||
|
||||
public void Add(KeyValuePair<TKey, TValue> item)
|
||||
{
|
||||
throw new Exception("The method or operation is not implemented.");
|
||||
}
|
||||
|
||||
public void Clear()
|
||||
{
|
||||
throw new Exception("The method or operation is not implemented.");
|
||||
}
|
||||
|
||||
public bool Contains(KeyValuePair<TKey, TValue> item)
|
||||
{
|
||||
throw new Exception("The method or operation is not implemented.");
|
||||
}
|
||||
|
||||
public void CopyTo(KeyValuePair<TKey, TValue>[] array, int arrayIndex)
|
||||
{
|
||||
throw new Exception("The method or operation is not implemented.");
|
||||
}
|
||||
|
||||
public int Count
|
||||
{
|
||||
get { throw new Exception("The method or operation is not implemented."); }
|
||||
}
|
||||
|
||||
public bool IsReadOnly
|
||||
{
|
||||
get { throw new Exception("The method or operation is not implemented."); }
|
||||
}
|
||||
|
||||
public bool Remove(KeyValuePair<TKey, TValue> item)
|
||||
{
|
||||
throw new Exception("The method or operation is not implemented.");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region IEnumerable<KeyValuePair<TKey,TValue>> Members
|
||||
|
||||
public IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator()
|
||||
{
|
||||
throw new Exception("The method or operation is not implemented.");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region IEnumerable Members
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator()
|
||||
{
|
||||
throw new Exception("The method or operation is not implemented.");
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
#region Helper class to deal with stream projections
|
||||
|
||||
public class JoinProjectHelper
|
||||
@@ -167,7 +59,7 @@ namespace BCG
|
||||
{
|
||||
Pair<string, bool> p = myAliasToOriginal[col];
|
||||
isLeft = p.second;
|
||||
return p.first;
|
||||
return ExceptionAssert.NotNull(p.first);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -178,10 +70,11 @@ namespace BCG
|
||||
public class SuperJoinRow : IBlueRow
|
||||
{
|
||||
private JoinProjectHelper myHelper;
|
||||
private IBlueRow myLeftRow, myRightRow;
|
||||
IBlueRow myLeftRow;
|
||||
IBlueRow? myRightRow;
|
||||
int myID;
|
||||
|
||||
public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow rightRow, JoinProjectHelper joinProjectHelper)
|
||||
public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow? rightRow, JoinProjectHelper joinProjectHelper)
|
||||
{
|
||||
myID = id;
|
||||
myLeftRow = leftRow;
|
||||
@@ -189,21 +82,22 @@ namespace BCG
|
||||
myHelper = joinProjectHelper;
|
||||
}
|
||||
|
||||
public int leftID { get { return (int)myLeftRow["id"]; } }
|
||||
public int leftID => myLeftRow.Id();
|
||||
|
||||
public int rightID { get { return (int)myRightRow["id"]; } }
|
||||
public int rightID => myRightRow.Id();
|
||||
|
||||
public IBlueRow leftRow { get { return myLeftRow; } }
|
||||
|
||||
public IBlueRow rightRow { get { return myRightRow; } }
|
||||
public IBlueRow? rightRow => myRightRow;
|
||||
|
||||
#region IBlueRow Members
|
||||
|
||||
public IBlueTableSchema schema { get { return myHelper.schema; } }
|
||||
|
||||
public object this[int index] { get { return this[schema[index].colName]; } }
|
||||
public object? this[int index] =>
|
||||
this[schema[index].colName];
|
||||
|
||||
public object this[string colName]
|
||||
public object? this[string colName]
|
||||
{
|
||||
get
|
||||
{
|
||||
@@ -229,7 +123,7 @@ namespace BCG
|
||||
|
||||
public bool Contains(object key)
|
||||
{
|
||||
return myLeftRow.Contains(key) || myRightRow.Contains(key);
|
||||
return myLeftRow.Contains(key) || (myRightRow?.Contains(key) ?? false);
|
||||
}
|
||||
|
||||
public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
|
||||
@@ -242,11 +136,11 @@ namespace BCG
|
||||
|
||||
public void Remove(object key) { throw new NotSupportedException("IBlueRow is readonly"); }
|
||||
|
||||
public System.Collections.ICollection Values
|
||||
public ICollection Values
|
||||
{
|
||||
get
|
||||
{
|
||||
List<object> rv = new List<object>();
|
||||
var rv = new List<object?>(Keys.Count);
|
||||
foreach (string col in Keys)
|
||||
rv.Add(this[col]);
|
||||
return rv;
|
||||
@@ -291,9 +185,10 @@ namespace BCG
|
||||
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
|
||||
private bool myManageUnmappedLeftRows = false;
|
||||
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
|
||||
protected IBlueTableSchema myLeftSchema, myRightSchema;
|
||||
protected JoinProjectHelper myJoinProjectHelper;
|
||||
protected BlueTable myState;
|
||||
protected IBlueTableSchema? myLeftSchema;
|
||||
protected IBlueTableSchema? myRightSchema;
|
||||
protected JoinProjectHelper? myJoinProjectHelper;
|
||||
protected BlueTable? myState;
|
||||
protected int myID;
|
||||
protected ViewStream myLeftStream, myRightStream;
|
||||
protected string myLeftKeyCol, myRightKeyCol;
|
||||
@@ -357,12 +252,8 @@ namespace BCG
|
||||
private void myHandleStreamChanges(IDataPacket dp, bool isLeft)
|
||||
{
|
||||
myInitSchema(dp.Schema, isLeft);
|
||||
BlueTable dels = null, adds = null;
|
||||
if (myState != null)
|
||||
{
|
||||
dels = new BlueTable(myState.schema);
|
||||
adds = new BlueTable(myState.schema);
|
||||
}
|
||||
var dels = new BlueTable(myState.schema);
|
||||
var adds = new BlueTable(myState.schema);
|
||||
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
|
||||
myAddRows(dp.addedRows, isLeft, myState, adds);
|
||||
if (myState != null)
|
||||
@@ -373,6 +264,7 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(myState))]
|
||||
private void myInitSchema(IBlueTableSchema schema, bool isLeft)
|
||||
{
|
||||
if (myState == null)
|
||||
@@ -402,6 +294,7 @@ namespace BCG
|
||||
|
||||
private void myAddRows(IBlueRowCollection addedRows, bool isLeft, BlueTable state, BlueTable addedRowsOut)
|
||||
{
|
||||
ExceptionAssert.NotNull(myJoinProjectHelper);
|
||||
object thisKeyCol = isLeft ? myLeftKeyCol : myRightKeyCol,
|
||||
otherSideKeyCol = isLeft ? myRightKeyCol : myLeftKeyCol;
|
||||
Dictionary<object, IDictionary<int, IBlueRow>> thisKeyToRows = isLeft ? myLeftKeyToLeftRows : myRightKeyToRightRows,
|
||||
@@ -410,8 +303,8 @@ namespace BCG
|
||||
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
|
||||
foreach (IBlueRow r in addedRows)
|
||||
{
|
||||
object currentKey = r[thisKeyCol];
|
||||
int currentID = (int)r["id"];
|
||||
object currentKey = ExceptionAssert.NotNull(r[thisKeyCol]);
|
||||
int currentID = r.Id();
|
||||
///add it to 1.
|
||||
if (!thisKeyToRows.ContainsKey(currentKey))
|
||||
thisKeyToRows.Add(currentKey, new Dictionary<int, IBlueRow>());
|
||||
@@ -422,14 +315,14 @@ namespace BCG
|
||||
ICollection<IBlueRow> matchingRows = otherSideKeyToRows[currentKey].Values;
|
||||
foreach (IBlueRow mr in matchingRows)
|
||||
{
|
||||
int mrID = (int)mr["id"];
|
||||
int mrID = mr.Id();
|
||||
SuperJoinRow rr = new SuperJoinRow(myID++, isLeft ? r : mr, isLeft ? mr : r, myJoinProjectHelper);
|
||||
state.addRow(rr);
|
||||
addedRowsOut.addRow(rr);
|
||||
///add to 3.
|
||||
if (!thisIDToResultRows.ContainsKey(currentID))
|
||||
thisIDToResultRows.Add(currentID, new Dictionary<int, SuperJoinRow>());
|
||||
thisIDToResultRows[currentID].Add((int)rr["id"], rr);
|
||||
thisIDToResultRows[currentID].Add(rr.Id(), rr);
|
||||
///add to 4.
|
||||
if (!otherIDToResultRows.ContainsKey(mrID))
|
||||
{
|
||||
@@ -437,7 +330,7 @@ namespace BCG
|
||||
if (myManageUnmappedLeftRows && !isLeft)
|
||||
myTakeCareOfUnmatachedLeftRows(mr, false);
|
||||
}
|
||||
otherIDToResultRows[mrID].Add((int)rr["id"], rr);
|
||||
otherIDToResultRows[mrID].Add(rr.Id(), rr);
|
||||
}
|
||||
}
|
||||
else
|
||||
@@ -458,22 +351,22 @@ namespace BCG
|
||||
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
|
||||
foreach (IBlueRow r in deletedRows)
|
||||
{
|
||||
int id = (int)r["id"];
|
||||
object key = r[thisKeyCol];
|
||||
int id = r.Id();
|
||||
object key = ExceptionAssert.NotNull(r[thisKeyCol]);
|
||||
///delete it from 1.
|
||||
thisKeyToRows[key].Remove(id);
|
||||
if (thisKeyToRows[key].Count == 0)
|
||||
thisKeyToRows.Remove(key);
|
||||
///form return rows and delete from 3.
|
||||
if (thisIDToResultRows.ContainsKey(id))
|
||||
if (thisIDToResultRows.TryGetValue(id, out IDictionary<int, SuperJoinRow>? removedResultRowMap))
|
||||
{
|
||||
ICollection removedResultRows = thisIDToResultRows[id].Values as ICollection;
|
||||
var removedResultRows = (ICollection)removedResultRowMap.Values;
|
||||
thisIDToResultRows.Remove(id);
|
||||
foreach (SuperJoinRow rr in removedResultRows)
|
||||
{
|
||||
///delete from 4.
|
||||
int otherId = isLeft ? rr.rightID : rr.leftID;
|
||||
otherIDToResultRows[otherId].Remove((int)rr["id"]);
|
||||
otherIDToResultRows[otherId].Remove(rr.Id());
|
||||
if (otherIDToResultRows[otherId].Count == 0)
|
||||
otherIDToResultRows.Remove(otherId);
|
||||
delRowsOut.addRow(rr);
|
||||
@@ -493,7 +386,7 @@ namespace BCG
|
||||
|
||||
private void myTakeCareOfUnmatachedLeftRows(IBlueRow leftRow, bool add)
|
||||
{
|
||||
int lid = (int)leftRow["id"];
|
||||
int lid = leftRow.Id();
|
||||
if (add)
|
||||
myLeftUnmatchedRowsAdds.Add(lid, leftRow);
|
||||
else
|
||||
@@ -501,7 +394,7 @@ namespace BCG
|
||||
if (myLeftUnmatchedRowsAdds.ContainsKey(lid))
|
||||
myLeftUnmatchedRowsAdds.Remove(lid);
|
||||
else
|
||||
myLeftUnmatchedRowsDels.Add((int)leftRow["id"], leftRow);
|
||||
myLeftUnmatchedRowsDels.Add(lid, leftRow);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -526,38 +419,45 @@ namespace BCG
|
||||
}
|
||||
|
||||
void myHandleJoinResults(Dictionary<int, IBlueRow> unmatchedLeftRowsDels, Dictionary<int, IBlueRow> unmatchedLeftRowAdds, IDataPacket resultPacket) {
|
||||
BlueTable dels = resultPacket.deletedRows as BlueTable,
|
||||
adds = resultPacket.addedRows as BlueTable;
|
||||
ExceptionAssert.NotNull(myJoinProjectHelper);
|
||||
var dels = resultPacket.deletedRows;
|
||||
var adds = resultPacket.addedRows;
|
||||
|
||||
foreach (IBlueRow lr in unmatchedLeftRowsDels.Values)
|
||||
{
|
||||
///an lr will be here if its actually deleted or
|
||||
///a right row is added, so now it has a partner
|
||||
///and is no longer unmatched.
|
||||
///in both cases the half row that this lr
|
||||
///created needs to be removed from this
|
||||
///class's state and added to dels
|
||||
///it means a right is added
|
||||
int lid = (int)lr["id"];
|
||||
SuperJoinRow rr = myLeftIDToHalfRows[lid];
|
||||
myLeftIDToHalfRows.Remove(lid);
|
||||
myState.deleteRow(rr);
|
||||
dels.addRow(rr);
|
||||
if (unmatchedLeftRowsDels.Count > 0) {
|
||||
ExceptionAssert.NotNull(myState);
|
||||
var mutableDels = (BlueTable)dels;
|
||||
foreach (IBlueRow lr in unmatchedLeftRowsDels.Values) {
|
||||
///an lr will be here if its actually deleted or
|
||||
///a right row is added, so now it has a partner
|
||||
///and is no longer unmatched.
|
||||
///in both cases the half row that this lr
|
||||
///created needs to be removed from this
|
||||
///class's state and added to dels
|
||||
///it means a right is added
|
||||
int lid = lr.Id();
|
||||
SuperJoinRow rr = myLeftIDToHalfRows[lid];
|
||||
myLeftIDToHalfRows.Remove(lid);
|
||||
myState.deleteRow(rr);
|
||||
mutableDels.addRow(rr);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (IBlueRow lr in unmatchedLeftRowAdds.Values)
|
||||
{
|
||||
///an lr will be here if its added with no
|
||||
///matching right, or a right matching row
|
||||
///is gone so it is unmatched now.
|
||||
///in both cases a half row needs to be
|
||||
///added to this class's state and added
|
||||
///to adds
|
||||
int lid = (int)lr["id"];
|
||||
SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
|
||||
myLeftIDToHalfRows.Add(lid, rr);
|
||||
myState.addRow(rr);
|
||||
adds.addRow(rr);
|
||||
if (unmatchedLeftRowAdds.Count > 0) {
|
||||
ExceptionAssert.NotNull(myState);
|
||||
var mutableAdds = (BlueTable)adds;
|
||||
foreach (IBlueRow lr in unmatchedLeftRowAdds.Values) {
|
||||
///an lr will be here if its added with no
|
||||
///matching right, or a right matching row
|
||||
///is gone so it is unmatched now.
|
||||
///in both cases a half row needs to be
|
||||
///added to this class's state and added
|
||||
///to adds
|
||||
int lid = lr.Id();
|
||||
SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
|
||||
myLeftIDToHalfRows.Add(lid, rr);
|
||||
myState.addRow(rr);
|
||||
mutableAdds.addRow(rr);
|
||||
}
|
||||
}
|
||||
base.myFireEvent(new DataPacket(myState, adds, dels));
|
||||
}
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
using System.Collections;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -12,7 +17,8 @@ namespace BCG
|
||||
protected IDataPacket? myLeftDataPacket;
|
||||
protected IDataPacket? myRightDataPacket;
|
||||
//Container to contain all the left rows that are orphaned during a deletion operation
|
||||
private IDictionary myLeftOrphanedRows;
|
||||
Dictionary<int, RowNode>? myLeftOrphanedRows;
|
||||
|
||||
public LeftInternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight) :
|
||||
base (vsLeft, jcLeft, vsRight, jcRight) {}
|
||||
|
||||
@@ -61,7 +67,9 @@ namespace BCG
|
||||
private void myTestAndFire (bool isLeft)
|
||||
{
|
||||
JoinOperand jo = isLeft ? myLeftOperand : myRightOperand;
|
||||
var dp = isLeft ? myLeftDataPacket : myRightDataPacket;
|
||||
var dp = isLeft
|
||||
? ExceptionAssert.NotNull(myLeftDataPacket)
|
||||
: ExceptionAssert.NotNull(myRightDataPacket);
|
||||
//to test whether both have received call backs
|
||||
if (myLeftOperand.hasReceivedData)
|
||||
jo.changeHandler (dp);
|
||||
@@ -79,11 +87,13 @@ namespace BCG
|
||||
/// <param name="isLeft"></param>
|
||||
public override void addRows (IBlueRowCollection rows, bool isLeft)
|
||||
{
|
||||
ExceptionAssert.NotNull(myLastFiredPacket);
|
||||
foreach (IBlueRow row in rows)
|
||||
myAddRow (row, isLeft);
|
||||
if (!isLeft)
|
||||
foreach (RowNode node in myLeftOrphanedRows.Values)
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow (node.row, null, myLastFiredPacket);
|
||||
foreach (RowNode node in ExceptionAssert.NotNull(myLeftOrphanedRows).Values) {
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow(node.RequireRow(), null, myLastFiredPacket);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -93,17 +103,19 @@ namespace BCG
|
||||
/// <param name="isLeft"></param>
|
||||
public override void deleteRows (IBlueRowCollection rows, bool isLeft)
|
||||
{
|
||||
myLeftOrphanedRows = new Hashtable ();
|
||||
myLeftOrphanedRows = new();
|
||||
foreach (IBlueRow row in rows)
|
||||
myDeleteRow (row, isLeft);
|
||||
}
|
||||
|
||||
private void myAddRow (IBlueRow row, bool isLeft)
|
||||
{
|
||||
ExceptionAssert.NotNull(myLastFiredPacket);
|
||||
if (isLeft)
|
||||
{
|
||||
myLeftOperand.addRow (row);
|
||||
RowList matchedRows = myRightOperand.getMatchingRows (row[myLeftJoinColOrdinal]);
|
||||
var joinKey = ExceptionAssert.NotNull(row[myLeftJoinColOrdinal]);
|
||||
RowList matchedRows = myRightOperand.getMatchingRows(joinKey);
|
||||
if (matchedRows.Count == 0)
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
|
||||
else
|
||||
@@ -112,22 +124,25 @@ namespace BCG
|
||||
}
|
||||
else
|
||||
{
|
||||
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]);
|
||||
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
|
||||
var joinKey = ExceptionAssert.NotNull(row[myRightJoinColOrdinal]);
|
||||
RowList matchedRows = myLeftOperand.getMatchingRows(joinKey);
|
||||
int rightMatchingRowCount = myRightOperand.getMatchingRows(joinKey).Count;
|
||||
myRightOperand.addRow (row);
|
||||
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
|
||||
int newRightMatchingRowCount = myRightOperand.getMatchingRows(joinKey).Count;
|
||||
if (matchedRows.Count != 0){
|
||||
foreach (RowNode matchedNode in matchedRows)
|
||||
{
|
||||
ExceptionAssert.NotNull(matchedNode.row);
|
||||
//if the left rows were orphaned in this transaction (ie in this callback), they should
|
||||
//just be removed from the orphanedlist and we dont need to worry about the consistency of
|
||||
//the jointable as a new row will replace it
|
||||
if (rightMatchingRowCount == 0 && newRightMatchingRowCount > 0)
|
||||
{
|
||||
if (myLeftOrphanedRows[matchedNode.row["id"]] == null)
|
||||
((JoinTable)myLastFiredPacket.newRows).deleteRow (true, matchedNode.row, myLastFiredPacket);
|
||||
else
|
||||
myLeftOrphanedRows.Remove (matchedNode.row["id"]);
|
||||
ExceptionAssert.NotNull(myLeftOrphanedRows);
|
||||
int matchedId = matchedNode.row.Id();
|
||||
if (!myLeftOrphanedRows.Remove(matchedId)) {
|
||||
((JoinTable)myLastFiredPacket.newRows).deleteRow(true, matchedNode.row, myLastFiredPacket);
|
||||
}
|
||||
}
|
||||
//they were orphaned by a previous transaction and hence the orphans need to be
|
||||
//removed from the Jointable and the complete row added
|
||||
@@ -140,41 +155,52 @@ namespace BCG
|
||||
|
||||
private void myDeleteRow (IBlueRow row, bool isLeft)
|
||||
{
|
||||
var last = requireLastFiredPacket();
|
||||
if (isLeft)
|
||||
{
|
||||
myLeftOperand.deleteRow (row);
|
||||
((JoinTable)myLastFiredPacket.newRows).deleteRow (true, row, myLastFiredPacket);
|
||||
((JoinTable)last.newRows).deleteRow (true, row, myLastFiredPacket);
|
||||
}
|
||||
else
|
||||
{
|
||||
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]);
|
||||
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
|
||||
var rightJoinKey = row[myRightJoinColOrdinal] ?? throw new InvalidOperationException("RightJoinKey is null");
|
||||
RowList matchedRows = myLeftOperand.getMatchingRows(rightJoinKey);
|
||||
int rightMatchingRowCount = myRightOperand.getMatchingRows(rightJoinKey).Count;
|
||||
myRightOperand.deleteRow (row);
|
||||
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
|
||||
int newRightMatchingRowCount = myRightOperand.getMatchingRows(rightJoinKey).Count;
|
||||
if (matchedRows.Count != 0)
|
||||
{
|
||||
if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0)
|
||||
if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0) {
|
||||
ExceptionAssert.MemberNotNull(myLeftOrphanedRows);
|
||||
//ie all the matchedRows were joined with 'this' right row (method arg), and hence they need to
|
||||
//be handled specially, ie joined with nulls, if there are no right rows with this
|
||||
//joincolumn value being inserted.
|
||||
ExceptionAssert.NotNull(myLeftOrphanedRows);
|
||||
foreach (RowNode matchedNode in matchedRows)
|
||||
{
|
||||
// BlueEventLog.logAssert (myLeftOrphanedRows[matchedNode.row["id"]] == null);
|
||||
myLeftOrphanedRows[matchedNode.row["id"]] = matchedNode;
|
||||
myLeftOrphanedRows[matchedNode.row.Id()] = matchedNode;
|
||||
}
|
||||
((JoinTable)myLastFiredPacket.newRows).deleteRow (false, row, myLastFiredPacket);
|
||||
}
|
||||
((JoinTable)last.newRows).deleteRow (false, row, myLastFiredPacket);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public override void onTableClear (bool isLeft)
|
||||
{
|
||||
if (myLastFiredPacket is null) {
|
||||
return;
|
||||
}
|
||||
requireSchema();
|
||||
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
||||
myLastFiredPacket.newRows = new JoinTable (mySchema);
|
||||
myLastFiredPacket.addedRows = new BlueTable (mySchema);
|
||||
if (!isLeft)
|
||||
if (!isLeft) {
|
||||
ExceptionAssert.MemberNotNull(myLeftDataPacket);
|
||||
foreach (IBlueRow row in myLeftDataPacket.newRows)
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ namespace BCG
|
||||
/// <summary>
|
||||
/// Node containing the referencedRow
|
||||
/// </summary>
|
||||
IBlueRow referencedRow {get;}
|
||||
IBlueRow? referencedRow { get; }
|
||||
|
||||
/// <summary>
|
||||
/// removes all referenced or referencing nodes (indicated by the bool), p.s. theres only one referenced row
|
||||
|
||||
@@ -15,12 +15,14 @@ namespace BCG
|
||||
/// </summary>
|
||||
/// <param name="id"></param>
|
||||
/// <returns></returns>
|
||||
IBlueRow getRow (int id);
|
||||
IBlueRow? getRow(int id);
|
||||
|
||||
/// <summary>
|
||||
/// Gets all the rows in the node
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
IBlueRow[] getRows ();
|
||||
IBlueRow?[] getRows();
|
||||
|
||||
/// <summary>
|
||||
/// Replaces the row with the given id, with the row passed as the second arg
|
||||
/// </summary>
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
using BCG;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -56,13 +54,18 @@ namespace BCG
|
||||
/// The key can either be the string corrsponding to the column name or
|
||||
/// the order of the column
|
||||
/// </summary>
|
||||
public virtual object this [object key]
|
||||
public virtual object? this [object key]
|
||||
{
|
||||
get {return key is string ? this [key as string] : this [(int)key];}
|
||||
// get {return key is string ? this [key as string] : this [(int)key];}
|
||||
get => key switch {
|
||||
string colName => this[colName],
|
||||
int colIndex => this[colIndex],
|
||||
_ => throw new ArgumentException("Must be string or int", nameof(key)),
|
||||
};
|
||||
set {throw new NotSupportedException ("The IBlueRow is read only");}
|
||||
}
|
||||
|
||||
public virtual object this [int index]
|
||||
public virtual object? this [int index]
|
||||
{
|
||||
get
|
||||
{
|
||||
@@ -77,7 +80,8 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public object this [string colName] {get{return this[mySchema[colName].colNo];}}
|
||||
public object? this [string colName] =>
|
||||
this[mySchema[colName].colNo];
|
||||
|
||||
public ICollection Keys{get{return mySchema.columnNames;}}
|
||||
|
||||
|
||||
+8
-6
@@ -1,6 +1,6 @@
|
||||
using System;
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -70,7 +70,8 @@ namespace BCG
|
||||
}
|
||||
else
|
||||
{
|
||||
SimpleNode sn = n as SimpleNode;
|
||||
var sn = (SimpleNode)n;
|
||||
ExceptionAssert.NotNull(sn.prev);
|
||||
sn.prev.next = sn.next;
|
||||
if (sn.next != null)
|
||||
sn.next.prev = sn.prev;
|
||||
@@ -81,7 +82,7 @@ namespace BCG
|
||||
|
||||
public int count {get {return myFirst.listCount;}}
|
||||
|
||||
public IBlueRow referencedRow {get {return myFirst.row;}}
|
||||
public IBlueRow? referencedRow => myFirst.row;
|
||||
|
||||
public void removeAll (bool isReD)
|
||||
{
|
||||
@@ -110,10 +111,11 @@ namespace BCG
|
||||
public class LightForeignJoinRowListEnumerator : IEnumerator
|
||||
{
|
||||
private ILightForeignJoinRowList myList;
|
||||
private SimpleNode myPtr, myFirst;
|
||||
private SimpleNode? myPtr;
|
||||
private SimpleNode myFirst;
|
||||
public LightForeignJoinRowListEnumerator (ILightForeignJoinRowList list, SimpleNode first)
|
||||
{myList = list; myFirst = first;}
|
||||
public object Current {get{return myPtr.row;}}
|
||||
public object? Current => myPtr?.row;
|
||||
public bool MoveNext() {return (myPtr = myPtr == null ? myFirst : myPtr.next) != null;}
|
||||
public void Reset() {myPtr = null;}
|
||||
public ILightForeignJoinRowList list {get {return myList;}}
|
||||
|
||||
+1
-1
@@ -51,7 +51,7 @@ namespace BCG
|
||||
|
||||
public void CopyTo (Array array, int index) {throw new NotSupportedException ("");}
|
||||
|
||||
public object SyncRoot {get{return null;}}
|
||||
public object SyncRoot => this;
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
+1
-3
@@ -1,5 +1,4 @@
|
||||
using System;
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -13,8 +12,7 @@ namespace BCG
|
||||
: base (reDRow, rinGRow, schema)
|
||||
{}
|
||||
|
||||
public override object this [int index]
|
||||
{
|
||||
public override object? this [int index] {
|
||||
get
|
||||
{
|
||||
if (index == mySchema.columnNames.Length - 1)
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
using System;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
/// <summary>
|
||||
@@ -9,15 +7,15 @@ namespace BCG
|
||||
/// </summary>
|
||||
public class SimpleNode : INode
|
||||
{
|
||||
public IBlueRow row;
|
||||
public SimpleNode next, prev;
|
||||
public IBlueRow? row;
|
||||
public SimpleNode? next;
|
||||
public SimpleNode? prev;
|
||||
|
||||
public SimpleNode (IBlueRow r)
|
||||
: this (r, null, null)
|
||||
{}
|
||||
|
||||
public SimpleNode (IBlueRow r, SimpleNode nx, SimpleNode pv)
|
||||
{
|
||||
public SimpleNode(IBlueRow r, SimpleNode? nx, SimpleNode? pv) {
|
||||
row = r;
|
||||
next = nx;
|
||||
prev = pv;
|
||||
@@ -25,13 +23,15 @@ namespace BCG
|
||||
|
||||
#region INode Members
|
||||
|
||||
public IBlueRow getRow (int id) {return row;}
|
||||
public IBlueRow? getRow(int id) =>
|
||||
row;
|
||||
|
||||
public IBlueRow[] getRows() {return new IBlueRow[] {row};}
|
||||
public IBlueRow?[] getRows() =>
|
||||
new[] {row};
|
||||
|
||||
public void replace (int id, IBlueRow nr) {row = nr;}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,8 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
using Amazon.Runtime.Internal.Util;
|
||||
|
||||
using BCG;
|
||||
using BCG.Utilities;
|
||||
|
||||
using bts;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG {
|
||||
public class LiteralStream : ViewStream {
|
||||
|
||||
@@ -42,14 +36,14 @@ namespace BCG {
|
||||
return new BlueRow(map);
|
||||
};
|
||||
string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
|
||||
BlueTable table = null;
|
||||
BlueTable? table = null;
|
||||
foreach (string json_string in row_arr) {
|
||||
IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
|
||||
get_row(json_string, table == null ? 0 : table.Count);
|
||||
table = table ?? new BlueTable(r.schema);
|
||||
table ??= new BlueTable(r.schema);
|
||||
table.addRow(r);
|
||||
}
|
||||
ViewStream str = new BlueTableStream(table);
|
||||
ViewStream str = new BlueTableStream(table ?? throw new ArgumentException("No rows specified, unable to determine schema"));
|
||||
str = new SchemaGuesserStream(str);
|
||||
mySetRealSource(str);
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
using BCG;
|
||||
using System;
|
||||
using System.IO;
|
||||
using MongoDB.Bson;
|
||||
using BCG.Utilities.General;
|
||||
using System.Linq;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Intrinsics.X86;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
|
||||
namespace bts {
|
||||
// listens to the underlying stream, and writes each seriialized packet in a
|
||||
@@ -32,15 +32,9 @@ namespace bts {
|
||||
|
||||
private void _handle_update(object dp_obj) {
|
||||
var dp = (IDataPacket)dp_obj;
|
||||
byte[] ser_data = null;
|
||||
if (_n_updates == 0) {
|
||||
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
|
||||
ser_data = _serializer.serialize(new StreamIOMessage(cr_msg));
|
||||
} else {
|
||||
StreamIOMessage.MultipleRemoveAddMessage mra = new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows);
|
||||
ser_data = _serializer.serialize(new StreamIOMessage(mra));
|
||||
}
|
||||
Log.assert(ser_data != null);
|
||||
byte[] ser_data = n_updates == 0
|
||||
? _serializer.serialize(new(new StreamIOMessage.CreateMessage(dp.newRows)))
|
||||
: _serializer.serialize(new(new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
|
||||
_n_updates++;
|
||||
string file_name = $"{_folder}\\{_n_updates}.bin";
|
||||
_write_to_file(ser_data, file_name);
|
||||
@@ -62,7 +56,7 @@ namespace bts {
|
||||
IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
|
||||
private int _file_ptr = 0;
|
||||
private List<Tuple<string, byte[]>> _file_data = new List<Tuple<string, byte[]>>();
|
||||
private BlueTable _state = null;
|
||||
BlueTable? _state;
|
||||
|
||||
public DataPacketReplayStream(string folder_path, string fire_period)
|
||||
:
|
||||
@@ -98,6 +92,7 @@ namespace bts {
|
||||
break;
|
||||
}
|
||||
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: {
|
||||
Log.notnull(_state);
|
||||
var mra = msg.multiple_remove_add_message;
|
||||
foreach (IBlueRow r in mra.removed_rows)
|
||||
_state.deleteRow(r);
|
||||
@@ -139,4 +134,4 @@ namespace bts {
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,12 @@
|
||||
using BCG;
|
||||
using BCG.Utilities.General;
|
||||
using System.Collections.Generic;
|
||||
|
||||
using Bts.PubSub.Routing;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using bts.api;
|
||||
using Bts.PubSub.Routing;
|
||||
|
||||
namespace bts
|
||||
{
|
||||
@@ -17,9 +15,9 @@ namespace bts
|
||||
{
|
||||
public class PayloadMessage
|
||||
{
|
||||
public readonly string msg_id;
|
||||
public readonly string? msg_id;
|
||||
public readonly Int64 timestamp = 0;
|
||||
public readonly byte[] payload = null;
|
||||
public readonly byte[]? payload = null;
|
||||
|
||||
public static byte[] create_data(int payload_size, byte payload_byte)
|
||||
{
|
||||
@@ -53,10 +51,10 @@ namespace bts
|
||||
{
|
||||
var str = new MemoryStream();
|
||||
BinaryWriter bw = new BinaryWriter(str);
|
||||
bw.Write(msg_id);
|
||||
bw.Write(msg_id ?? string.Empty);
|
||||
bw.Write(timestamp);
|
||||
bw.Write(payload.Length);
|
||||
bw.Write(payload);
|
||||
bw.Write(payload?.Length ?? 0);
|
||||
bw.Write(payload ?? Array.Empty<byte>());
|
||||
str.Position = 0;
|
||||
var bytes = str.ToArray();
|
||||
return bytes;
|
||||
@@ -73,18 +71,15 @@ namespace bts
|
||||
return new PayloadMessage(guid, timestamp_, payload);
|
||||
}
|
||||
}
|
||||
//public void
|
||||
private readonly int _msg_size;
|
||||
|
||||
private readonly TimeSpan _publish_period;
|
||||
private DateTime _last_publish_time = DateTime.MinValue;
|
||||
private PublisherCore _pub_core;
|
||||
private ISubscriberCore _sub_core;
|
||||
PublisherCore? _pub_core;
|
||||
ISubscriberCore? _sub_core;
|
||||
private BlueTable _table;
|
||||
private int _n_pubs = 0;
|
||||
private readonly int _max_pubs;
|
||||
private int _n_updates = 0;
|
||||
private ViewStreamDatabase _vsd;
|
||||
private PayloadMessage _last_sent_msg, _last_recvd_msg;
|
||||
PayloadMessage? _last_sent_msg;
|
||||
PayloadMessage? _last_recvd_msg;
|
||||
private const byte _payload_byte = (byte)'A';
|
||||
private readonly string _tag_name;
|
||||
private readonly int _payload_sz;
|
||||
@@ -131,13 +126,13 @@ namespace bts
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
_vsd.removeWorker(this);
|
||||
_sub_core.unsubscribe(_tag_name);
|
||||
Log.notnull(_sub_core).unsubscribe(_tag_name);
|
||||
}
|
||||
|
||||
private void _fire_dp_after_send()
|
||||
{
|
||||
Log.notnull(_last_sent_msg);
|
||||
// as soon as the data is published, its reflected in the stream state
|
||||
Log.assert(_last_sent_msg != null);
|
||||
BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
|
||||
DBNull.Value, DBNull.Value);
|
||||
_table.addRow(r);
|
||||
@@ -147,7 +142,8 @@ namespace bts
|
||||
|
||||
private void _fire_dp_after_recv()
|
||||
{
|
||||
Log.assert(_last_recvd_msg != null);
|
||||
Log.notnull(_last_recvd_msg);
|
||||
Log.notnull(_last_sent_msg);
|
||||
// update the last publish row
|
||||
int id = _table.Count - 1;
|
||||
IBlueRow last_row = _table[id];
|
||||
@@ -167,28 +163,16 @@ namespace bts
|
||||
|
||||
private void _publish_next_packet()
|
||||
{
|
||||
Log.notnull(_pub_core);
|
||||
var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
|
||||
_last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
|
||||
var ser_bytes = _last_sent_msg.serialize();
|
||||
_n_pubs++;
|
||||
_pub_core.publish("Blob", _tag_name, ser_bytes);
|
||||
_last_publish_time = DateTime.Now;
|
||||
}
|
||||
|
||||
public void doWork()
|
||||
{
|
||||
Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
|
||||
{
|
||||
foreach (var u in updates)
|
||||
{
|
||||
var recv_msg = PayloadMessage.deserialize(u.data);
|
||||
if (recv_msg.msg_id == _last_sent_msg.msg_id)
|
||||
{
|
||||
return recv_msg;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
switch (_state)
|
||||
{
|
||||
case IOState.SEND:
|
||||
@@ -202,7 +186,7 @@ namespace bts
|
||||
}
|
||||
break;
|
||||
case IOState.RECV:
|
||||
var updates = _sub_core.get_updates(TimeSpan.Zero);
|
||||
var updates = Log.notnull(_sub_core).get_updates(TimeSpan.Zero);
|
||||
updates = updates.Where(u => u.tag == _tag_name);
|
||||
var recv_msg = find_packet(updates);
|
||||
if (recv_msg != null)
|
||||
@@ -214,6 +198,19 @@ namespace bts
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
PayloadMessage? find_packet(IEnumerable<PubSubServerMessage> updates) {
|
||||
if (_last_sent_msg is null) {
|
||||
return null;
|
||||
}
|
||||
foreach (var u in updates) {
|
||||
var recv_msg = PayloadMessage.deserialize(Log.notnull(u.data));
|
||||
if (recv_msg.msg_id == _last_sent_msg.msg_id) {
|
||||
return recv_msg;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static long epoch_now()
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
using bts;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
@@ -197,7 +199,6 @@ namespace BCG {
|
||||
|
||||
|
||||
private TestResult _run_schema_serialization_test() {
|
||||
var table_name = "test_table";
|
||||
var excluded_types = new HashSet<Type>() { typeof(DateTime), typeof(Decimal) };
|
||||
IBlueTableSchema schema = create_schema_with_all_types(excluded_types);
|
||||
MemoryStream str = new MemoryStream();
|
||||
@@ -375,7 +376,7 @@ namespace BCG {
|
||||
StreamIOSerializationController ser_controller = new StreamIOSerializationController();
|
||||
StreamIODeserializationController deser_controller = new StreamIODeserializationController();
|
||||
|
||||
IBlueTableSchema schema_p = null;
|
||||
IBlueTableSchema? schema_p = null;
|
||||
var schema_file = @"c:\Temp\tests\schema.bin";
|
||||
var row_file = @"C:\Temp\tests\row.bin";
|
||||
var create_op_file = @"C:\Temp\tests\create_op.bin";
|
||||
@@ -619,7 +620,7 @@ namespace BCG {
|
||||
private static TestResult _run_single_tag_data_test_clean() {
|
||||
// var tagHandlerFactory
|
||||
// create message
|
||||
var test_name = MethodBase.GetCurrentMethod()?.Name;
|
||||
var test_name = MethodBase.GetCurrentMethod()?.Name ?? string.Empty;
|
||||
var schema = create_schema_with_all_types();
|
||||
var tag = "DS_" + schema.tableName;
|
||||
BlueTable table = new BlueTable(schema);
|
||||
@@ -695,14 +696,14 @@ namespace BCG {
|
||||
|
||||
}
|
||||
|
||||
public static PublishMsg build_pub_msg(IBlueRow added_row, IBlueRow removed_row, bool is_create,
|
||||
public static PublishMsg build_pub_msg(IBlueRow? added_row, IBlueRow? removed_row, bool is_create,
|
||||
ref IStreamIOMsgSerializer serializer) {
|
||||
Func<IBlueTableSchema> get_schema = () => {
|
||||
return (added_row != null ? added_row : removed_row).schema;
|
||||
return (added_row != null ? added_row : ExceptionAssert.NotNull(removed_row)).schema;
|
||||
};
|
||||
bts.StreamIOMessage str_io_msg = null;
|
||||
StreamIOMessage? str_io_msg;
|
||||
if (is_create) {
|
||||
Log.assert(added_row != null);
|
||||
ExceptionAssert.NotNull(added_row);
|
||||
var aux_new = new BlueTable(added_row.schema);
|
||||
aux_new.addRow(added_row);
|
||||
var cr_msg = new bts.StreamIOMessage.CreateMessage(aux_new);
|
||||
@@ -729,7 +730,7 @@ namespace BCG {
|
||||
return pub_msg;
|
||||
}
|
||||
|
||||
public static IBlueTableSchema create_schema_with_all_types(HashSet<Type> excluded_col_map = null) {
|
||||
public static IBlueTableSchema create_schema_with_all_types(HashSet<Type>? excluded_col_map = null) {
|
||||
excluded_col_map = excluded_col_map == null ? new HashSet<Type>() : excluded_col_map;
|
||||
var table_name = "test_table";
|
||||
BlueTableSchema schema = new BlueTableSchema();
|
||||
@@ -800,7 +801,6 @@ namespace BCG {
|
||||
return BitConverter.ToUInt64(ulongBuffer, 0);
|
||||
default:
|
||||
throw new ArgumentException($"Unsupported TypeCode: {type_code}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
#endregion
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -16,7 +13,7 @@ namespace BCG
|
||||
private string myInColumn;
|
||||
private string myOutValue;
|
||||
private object[] myMuxedValues;
|
||||
private BlueTableSchema mySchema;
|
||||
BlueTableSchema? mySchema;
|
||||
|
||||
public MuxStream(ViewStream inStream, string outColumn, string outValue, string inColumn, params object[] muxedValues)
|
||||
{
|
||||
@@ -49,7 +46,7 @@ namespace BCG
|
||||
if (Array.IndexOf(myMuxedValues, r[myInColumn]) != -1) // muxable value?
|
||||
r.setValue(myOutColumn, myOutValue);
|
||||
else
|
||||
r.setValue(myOutColumn, row[myInColumn].ToString());
|
||||
r.setValue(myOutColumn, row[myInColumn]?.ToString());
|
||||
return r;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
using mbts;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
using BCG;
|
||||
using BCG.Utilities;
|
||||
using System.Collections.Generic;
|
||||
using mbts;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class NullToConstantMapper : IStreamMapper
|
||||
@@ -27,17 +24,11 @@ namespace BCG
|
||||
|
||||
#region IDynamicStreamMapper Members
|
||||
|
||||
public IBlueRow map(IBlueRow row)
|
||||
{
|
||||
if(row[myField].Equals(DBNull.Value))
|
||||
{
|
||||
Hashtable newRow = new Hashtable(row);
|
||||
newRow[myField]= myConstant;
|
||||
return new BlueRow(row.schema, newRow);
|
||||
}
|
||||
else
|
||||
return row;
|
||||
}
|
||||
public IBlueRow map(IBlueRow row) =>
|
||||
row[myField] switch {
|
||||
null or DBNull => new BlueRow(row.schema, new Hashtable(row) { { myField, myConstant } }),
|
||||
_ => row,
|
||||
};
|
||||
|
||||
#endregion
|
||||
|
||||
@@ -69,10 +60,11 @@ namespace BCG
|
||||
Hashtable newRow = new Hashtable(row);
|
||||
foreach(ColumnInfo info in row.schema.columnInfo)
|
||||
{
|
||||
if(row[info.colName].Equals(DBNull.Value))
|
||||
{
|
||||
change = true;
|
||||
newRow[info.colName] = Convert.ChangeType(myConstant, info.colType);
|
||||
switch (row[info.colName]) {
|
||||
case null or DBNull:
|
||||
change = true;
|
||||
newRow[info.colName] = Convert.ChangeType(myConstant, info.colType);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(change)
|
||||
|
||||
@@ -19,15 +19,15 @@ namespace BCG
|
||||
|
||||
public IBlueTableSchema schema { get { return mySchema; } }
|
||||
|
||||
public object this[int index] { get { return myRow[index]; } }
|
||||
public object? this[int index] { get { return myRow[index]; } }
|
||||
|
||||
public object this[string colName] { get { return myRow[colName]; } }
|
||||
public object? this[string colName] { get { return myRow[colName]; } }
|
||||
|
||||
#endregion
|
||||
|
||||
#region IDictionary Members
|
||||
|
||||
public void Add(object key, object value)
|
||||
public void Add(object key, object? value)
|
||||
{
|
||||
throw new Exception("The method or operation is not implemented.");
|
||||
}
|
||||
@@ -54,7 +54,7 @@ namespace BCG
|
||||
|
||||
public ICollection Values { get { return myRow.Values; } }
|
||||
|
||||
public object this[object key]
|
||||
public object? this[object key]
|
||||
{
|
||||
get { return myRow[key]; }
|
||||
set { throw new Exception("The method or operation is not implemented."); }
|
||||
@@ -82,7 +82,7 @@ namespace BCG
|
||||
}
|
||||
|
||||
private string myNewName;
|
||||
private IBlueTableSchema mySchema;
|
||||
IBlueTableSchema? mySchema;
|
||||
|
||||
public ChangeTableNameStream (ViewStream vs, string name)
|
||||
{
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
using Bts.ColumnMapExpressions;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Data;
|
||||
using System.Text.RegularExpressions;
|
||||
using BCG.Utilities;
|
||||
using Bts.ColumnMapExpressions;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -56,7 +56,7 @@ namespace BCG
|
||||
{
|
||||
IDictionary retVal = new Hashtable ();
|
||||
foreach (DictionaryEntry de in dic)
|
||||
retVal[de.Value] = de.Key;
|
||||
retVal[ExceptionAssert.NotNull(de.Value)] = de.Key;
|
||||
return retVal;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
using Bts.ColumnMapExpressions;
|
||||
using Bts.DataStream;
|
||||
|
||||
using mbts.config;
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
// using BCG.Evaluator.ColumnMapExpression;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -26,7 +24,7 @@ namespace BCG
|
||||
/// <param name="isch">original(unprojected) schema</param>
|
||||
/// <param name="exp">expression defining the mapping</param>
|
||||
/// <param name="tableName">table name of the new schema</param>
|
||||
internal ProjectHelper (IBlueTableSchema isch, string exp, string tableName)
|
||||
internal ProjectHelper (IBlueTableSchema isch, string exp, string? tableName)
|
||||
: this (isch, myGetDictionary (isch, exp), tableName)
|
||||
{}
|
||||
|
||||
@@ -36,7 +34,7 @@ namespace BCG
|
||||
/// <param name="origToAlias">dictionary mapping unprojected names to their projected names, dropped cols should not be there</param>
|
||||
/// <param name="tableName">table name of the new schema</param>
|
||||
internal ProjectHelper (
|
||||
IBlueTableSchema isch, IDictionary<string, string> origToAlias, string tableName)
|
||||
IBlueTableSchema isch, IDictionary<string, string> origToAlias, string? tableName)
|
||||
{
|
||||
ArrayList colMapAl = new ArrayList (),
|
||||
colInfoAl = new ArrayList ();
|
||||
@@ -61,7 +59,7 @@ namespace BCG
|
||||
|
||||
schema = new BlueTableSchema (tableName == null ? isch.tableName : tableName,
|
||||
colInfoAl.ToArray (typeof(ColumnInfo)) as ColumnInfo[]);
|
||||
colMap = colMapAl.ToArray (typeof(int)) as int[];
|
||||
colMap = (int[])colMapAl.ToArray(typeof(int));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -74,8 +72,8 @@ namespace BCG
|
||||
private static IDictionary<string, string> myGetDictionary (IBlueTableSchema isch, string exp)
|
||||
{
|
||||
IDictionary<string, string> origToAlias = new Dictionary<string, string>();
|
||||
|
||||
var e = ColumnMapExpression.Parse(exp);
|
||||
var e = ColumnMapExpression.Parse(exp)
|
||||
?? throw new ArgumentException("Failed to parse ColumnMapExpression", nameof(exp));
|
||||
if (e.IsInclusive) {
|
||||
origToAlias = e.Columns.ToDictionary();
|
||||
origToAlias["id"] = "id";
|
||||
@@ -95,7 +93,7 @@ namespace BCG
|
||||
{
|
||||
IDictionary retVal = new Hashtable (dic.Count);
|
||||
foreach (DictionaryEntry de in dic)
|
||||
retVal.Add (de.Value, de.Key);
|
||||
retVal.Add(ExceptionAssert.NotNull(de.Value), de.Key);
|
||||
return retVal;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
namespace BCG {
|
||||
using Spark.Core.Collections;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using Spark.Core.Collections;
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
@@ -8,10 +12,13 @@ namespace BCG {
|
||||
[DataStreamUserRespresentation("oldproject")]
|
||||
public class ProjectStream : ViewStream
|
||||
{
|
||||
private ProjectHelper myHelper;
|
||||
private string myColExp, myTableName;
|
||||
private ViewStream myVs;
|
||||
private IDictionary<string, string> myDic;
|
||||
private ProjectHelper? myHelper;
|
||||
readonly string? myColExp;
|
||||
readonly string? myTableName;
|
||||
readonly ViewStream? myVs;
|
||||
private IDictionary<string, string>? myDic;
|
||||
IDisposable? subscription;
|
||||
|
||||
/// <summary>
|
||||
/// </summary>
|
||||
/// <param name="vs">stream to susbsribe to as the input stream</param>
|
||||
@@ -52,14 +59,13 @@ namespace BCG {
|
||||
myDic = dic.ToDictionary(kv => kv.Value, kv => kv.Key);
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
myVs.onChange += new BCG.DataStream.ChangeDelegate (myHandleChange);
|
||||
}
|
||||
protected override void myOnFirstSubscriber() =>
|
||||
subscription = ExceptionAssert.MemberNotNull(myVs).Subscribe(myHandleChange);
|
||||
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
myVs.onChange -= new BCG.DataStream.ChangeDelegate (myHandleChange);;
|
||||
subscription?.Dispose();
|
||||
subscription = null;
|
||||
}
|
||||
|
||||
internal void myHandleChange (object newValue)
|
||||
@@ -79,7 +85,7 @@ namespace BCG {
|
||||
else
|
||||
{ //first time a callback is received
|
||||
myHelper = myDic == null ?
|
||||
new ProjectHelper (dp.Schema, myColExp, myTableName) :
|
||||
new ProjectHelper(dp.Schema, ExceptionAssert.MemberNotNull(myColExp), myTableName) :
|
||||
new ProjectHelper (dp.Schema, myDic, myTableName);
|
||||
|
||||
IBlueRowCollection temp;
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -22,9 +21,11 @@ namespace BCG
|
||||
|
||||
#region IBlueRow Members
|
||||
|
||||
public object this [string colName] {get {return this[myHelper.schema[colName].colNo];}}
|
||||
public object? this [string colName] =>
|
||||
this[myHelper.schema[colName].colNo];
|
||||
|
||||
public object this [int index] {get {return myRow[myHelper.colMap[index]];}}
|
||||
public object? this [int index] =>
|
||||
myRow[myHelper.colMap[index]];
|
||||
|
||||
public IBlueTableSchema schema {get {return myHelper.schema;}}
|
||||
|
||||
@@ -36,9 +37,13 @@ namespace BCG
|
||||
|
||||
public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator (this);}
|
||||
|
||||
object System.Collections.IDictionary.this[object key]
|
||||
object? IDictionary.this[object key]
|
||||
{
|
||||
get {return key is int ? this[(int)key] : this[key as string];}
|
||||
get => key switch {
|
||||
int i => this[i],
|
||||
string s => this[s],
|
||||
_ => throw new ArgumentException("Must be int or string", nameof(key)),
|
||||
};
|
||||
set {throw new ApplicationException ("Is read only");}
|
||||
}
|
||||
|
||||
@@ -58,7 +63,8 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public void Add (object key, object value) {throw new ApplicationException ("Is read only");}
|
||||
public void Add(object key, object? value) =>
|
||||
throw new NotSupportedException();
|
||||
|
||||
public ICollection Keys {get {return myHelper.schema.columnNames;}}
|
||||
|
||||
@@ -78,7 +84,7 @@ namespace BCG
|
||||
array.SetValue (this[ci.colNo], index ++);
|
||||
}
|
||||
|
||||
public object SyncRoot {get {return null;}}
|
||||
public object SyncRoot => this;
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
@@ -42,7 +44,7 @@ namespace BCG
|
||||
public void CopyTo (Array array, int index)
|
||||
{
|
||||
foreach (IBlueRow r in myRows)
|
||||
array.SetValue (this[(int)r["id"]], index++);
|
||||
array.SetValue(this[r.Id()], index++);
|
||||
}
|
||||
|
||||
|
||||
@@ -72,14 +74,10 @@ namespace BCG
|
||||
|
||||
public void Reset() {myMaster.Reset();}
|
||||
|
||||
public object Current
|
||||
{
|
||||
get
|
||||
{
|
||||
return myMaster.Current == null ? null :
|
||||
new ProjectedRow (myMaster.Current as IBlueRow, myHelper);
|
||||
}
|
||||
}
|
||||
public object? Current =>
|
||||
myMaster.Current is null
|
||||
? null
|
||||
: new ProjectedRow((IBlueRow)myMaster.Current, myHelper);
|
||||
|
||||
public bool MoveNext() {return myMaster.MoveNext ();}
|
||||
|
||||
|
||||
@@ -1,25 +1,16 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using System.Data.SQLite;
|
||||
using System.Diagnostics;
|
||||
using System.Collections;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
|
||||
|
||||
using BCG;
|
||||
using System.Linq;
|
||||
using System.Data.Entity;
|
||||
using System.Collections.Generic;
|
||||
using System.Data.SQLite;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
|
||||
|
||||
namespace bts.BlueOrb{
|
||||
|
||||
public static class SQLiteHelper {
|
||||
public static string location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:\\BlueOrb_Data\\" : Path.Combine(Environment.GetEnvironmentVariable("HOME"), "BlueOrb_Data");
|
||||
public static string location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
|
||||
? "C:\\BlueOrb_Data\\"
|
||||
: Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? string.Empty, "BlueOrb_Data");
|
||||
public static string db_name = "stream_data.db";
|
||||
|
||||
public static DataTable execute_select_query(string connection_string, string query) {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.IO;
|
||||
|
||||
namespace BCG {
|
||||
[DataStreamUserRespresentation("CSVSaver")]
|
||||
@@ -77,7 +80,7 @@ namespace BCG {
|
||||
if (col != "id") {
|
||||
string colText = String.Empty;
|
||||
if (r[col] != null && r[col] != DBNull.Value)
|
||||
colText = r[col].ToString();
|
||||
colText = r[col]?.ToString() ?? string.Empty;
|
||||
if (r.schema[col].colType == typeof(string))
|
||||
colText = "\"" + colText + "\"";
|
||||
rv += (i++ == 0 ? colText : "," + colText);
|
||||
@@ -135,13 +138,14 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public class CSVReaderStreamMapper : IStreamMapper {
|
||||
private IBlueTableSchema mySchema;
|
||||
IBlueTableSchema? mySchema;
|
||||
private readonly string[] myOutputSchemaStringArray;
|
||||
|
||||
public CSVReaderStreamMapper(string outputSchema) {
|
||||
myOutputSchemaStringArray = outputSchema.Split(' ');
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(mySchema))]
|
||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
|
||||
if (mySchema == null) {
|
||||
ColumnInfo[] ci = inputSchema.columnInfo;
|
||||
@@ -161,6 +165,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public IBlueRow map(IBlueRow row) {
|
||||
ExceptionAssert.NotNull(mySchema);
|
||||
object[] vals = new object[row.schema.columnInfo.Length];
|
||||
row.Values.CopyTo(vals, 0);
|
||||
for (int i = 0; i < mySchema.columnInfo.Length; i++) {
|
||||
|
||||
@@ -4,6 +4,7 @@ using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
using Bts.ColumnSortExpressions;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
namespace BCG;
|
||||
|
||||
@@ -54,7 +55,7 @@ public class SortedTable : IBlueRowCollection {
|
||||
for (int i = 0; i < this.rows.Count; i++) {
|
||||
if (newRow.CompareTo(this.rows[i]) <= 0) {
|
||||
this.rows.Insert(i, newRow);
|
||||
rowIndex[(int)newRow["id"]] = newRow;
|
||||
rowIndex[newRow.Id()] = newRow;
|
||||
if (++addedRowsIndex == addedRows.Count) {
|
||||
return;
|
||||
}
|
||||
@@ -65,17 +66,17 @@ public class SortedTable : IBlueRowCollection {
|
||||
for (int i = addedRowsIndex; i < addedRows.Count; i++) {
|
||||
newRow = addedRows.getRowByIndex(i);
|
||||
this.rows.Add(newRow);
|
||||
rowIndex[(int)newRow["id"]] = newRow;
|
||||
rowIndex[newRow.Id()] = newRow;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteRows(IBlueRowCollection deletedRows) {
|
||||
foreach (IBlueRow deletedRow in deletedRows) {
|
||||
var row = rowIndex[(int)deletedRow["id"]!] as SortRow
|
||||
var row = rowIndex[deletedRow.Id()] as SortRow
|
||||
?? throw new InvalidOperationException("Index did not return IBlueRow");
|
||||
rows.Remove(row);
|
||||
rowIndex.Remove((int)row["id"]!);
|
||||
rowIndex.Remove(row.Id());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,7 +102,7 @@ public class SortedTable : IBlueRowCollection {
|
||||
SortRow newRow = comparer is null ?
|
||||
new(row, sortCols) : new(row, comparer);
|
||||
this.rows.Add(newRow);
|
||||
rowIndex[(int)row["id"]!] = newRow;
|
||||
rowIndex[row.Id()] = newRow;
|
||||
}
|
||||
this.rows.Sort();
|
||||
}
|
||||
|
||||
@@ -19,7 +19,8 @@ namespace BCG
|
||||
StreamReader reader = new StreamReader(myFile);
|
||||
|
||||
#region get schema
|
||||
IBlueTableSchema schema = null; string schemaStr;
|
||||
IBlueTableSchema? schema;
|
||||
string? schemaStr;
|
||||
if ((schemaStr = reader.ReadLine()) == null)
|
||||
{
|
||||
reader.Close();
|
||||
@@ -33,7 +34,7 @@ namespace BCG
|
||||
BlueTable adds = new BlueTable(schema);
|
||||
BlueTable dels = new BlueTable(schema);
|
||||
|
||||
string rowStr;
|
||||
string? rowStr;
|
||||
while ((rowStr = reader.ReadLine()) != null)
|
||||
{
|
||||
adds.addRow(myGetRow(schema, rowStr));
|
||||
|
||||
@@ -101,7 +101,7 @@ namespace BCG
|
||||
rowStr += DELIM;
|
||||
}
|
||||
|
||||
rowStr += row[columnName].ToString();
|
||||
rowStr += row[columnName]?.ToString();
|
||||
}
|
||||
|
||||
writer.WriteLine(rowStr);
|
||||
@@ -144,7 +144,7 @@ namespace BCG
|
||||
private int myWriteFreqSecs;
|
||||
private DateTime myNextWrite;
|
||||
private bool myGotUpdate;
|
||||
private string mySchemaStr;
|
||||
private string? mySchemaStr;
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
@@ -20,7 +20,7 @@ namespace BCG {
|
||||
|
||||
[DataStreamUserRespresentation("dateStandardize")]
|
||||
public class DateStandardizationStream : ViewStream, IStreamMapper {
|
||||
private IBlueTableSchema mySchema;
|
||||
private IBlueTableSchema? mySchema;
|
||||
private DateConversion myConversion;
|
||||
private string myConversionColName;
|
||||
private readonly Dictionary<DateConversion, Type> _date_conversion_to_type_map = new Dictionary<DateConversion, Type>()
|
||||
@@ -60,8 +60,7 @@ namespace BCG {
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||
foreach (ColumnInfo c in inputSchema.columnInfo) {
|
||||
if (c.colName == myConversionColName) {
|
||||
Type convType;
|
||||
if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType)) {
|
||||
if (!_date_conversion_to_type_map.TryGetValue(myConversion, out var convType)) {
|
||||
//BCG.Utilities.BlueEventLog.logEvent("Please update the _date_conversion_to_type_map in DateStandardizationStream.cs.", BCG.Utilities.BlueEventLevel.Error);
|
||||
convType = typeof(DateTime);
|
||||
}
|
||||
@@ -87,7 +86,7 @@ namespace BCG {
|
||||
|
||||
#endregion
|
||||
|
||||
public static object mapDateTime(object obj_in, DateConversion conv) {
|
||||
public static object mapDateTime(object? obj_in, DateConversion conv) {
|
||||
if (obj_in == DBNull.Value || obj_in == null) {
|
||||
return DBNull.Value;
|
||||
} else {
|
||||
@@ -118,7 +117,7 @@ namespace BCG {
|
||||
return day_period.TotalSeconds;
|
||||
}
|
||||
case DateConversion.ISO8601: {
|
||||
string iso8601Date = obj_in.ToString();
|
||||
var iso8601Date = obj_in.ToString() ?? string.Empty;
|
||||
return DateTime.Parse(iso8601Date, null, System.Globalization.DateTimeStyles.RoundtripKind);
|
||||
}
|
||||
case DateConversion.EpochToDatetime: {
|
||||
@@ -135,4 +134,4 @@ namespace BCG {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -11,7 +8,7 @@ namespace BCG
|
||||
{
|
||||
private ViewStream _src;
|
||||
private string _time_stamp_field;
|
||||
private IBlueTableSchema _schema;
|
||||
IBlueTableSchema? _schema;
|
||||
private string _latency_field;
|
||||
|
||||
public LatencyStream(ViewStream input, string time_stamp_field, string latency_field)
|
||||
@@ -38,10 +35,11 @@ namespace BCG
|
||||
public IBlueRow map(IBlueRow row)
|
||||
{
|
||||
var ht = new Hashtable(row);
|
||||
if (row[_time_stamp_field].Equals(DBNull.Value))
|
||||
ht[_latency_field] = DBNull.Value;
|
||||
else
|
||||
ht[_latency_field] = (DateTime.Now - (DateTime)row[_time_stamp_field]).TotalMilliseconds;
|
||||
ht[_latency_field] = row[_time_stamp_field] switch {
|
||||
null or DBNull => DBNull.Value,
|
||||
DateTime ts => (DateTime.Now - ts).TotalMilliseconds,
|
||||
_ => throw new ArgumentException("Row timestamp must be DateTime", nameof(row))
|
||||
};
|
||||
return new BlueRow(_schema, ht);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
using System;
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class TimeAverageStream : ViewStream
|
||||
{
|
||||
|
||||
private IDictionary<string, SortedList<DateTime, double>> myKeyToTimeToValue;
|
||||
Dictionary<string, SortedList<DateTime, double>>? myKeyToTimeToValue;
|
||||
private ViewStream myToAverage;
|
||||
private string myCol_Key, myCol_Time, myCol_Value;
|
||||
private double myAveragePeriodSeconds;
|
||||
@@ -42,8 +43,8 @@ namespace BCG
|
||||
BlueTable result = new BlueTable(dp.Schema);
|
||||
foreach (IBlueRow r in dp.newRows)
|
||||
{
|
||||
string key = r[myCol_Key].ToString();
|
||||
DateTime dt = (DateTime) r[myCol_Time];
|
||||
string key = r[myCol_Key]?.ToString() ?? string.Empty;
|
||||
DateTime dt = (DateTime)r[myCol_Time]!;
|
||||
double value = Convert.ToDouble(r[myCol_Value]);
|
||||
|
||||
object[] rowValues = new object[r.Count];
|
||||
@@ -67,9 +68,8 @@ namespace BCG
|
||||
|
||||
private double myUpdateValue(string key, DateTime time, double value)
|
||||
{
|
||||
|
||||
SortedList<DateTime, double> seriesValues;
|
||||
if (!myKeyToTimeToValue.TryGetValue(key, out seriesValues))
|
||||
ExceptionAssert.MemberNotNull(myKeyToTimeToValue);
|
||||
if (!myKeyToTimeToValue.TryGetValue(key, out var seriesValues))
|
||||
{
|
||||
seriesValues = new SortedList<DateTime, double>();
|
||||
myKeyToTimeToValue[key] = seriesValues;
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -10,9 +13,9 @@ namespace BCG
|
||||
internal class AppendCollection : IBlueRowCollection
|
||||
{
|
||||
private int myId = 0;
|
||||
private IDictionary myLeftIndex = new Hashtable (),
|
||||
myRightIndex = new Hashtable ();
|
||||
private IBlueTableSchema mySchema;
|
||||
readonly Dictionary<int, IBlueRow> myLeftIndex = new();
|
||||
readonly Dictionary<int, IBlueRow> myRightIndex = new();
|
||||
IBlueTableSchema? mySchema;
|
||||
|
||||
public AppendCollection () {}
|
||||
|
||||
@@ -24,15 +27,15 @@ namespace BCG
|
||||
IDictionary index = isLeft ? myLeftIndex : myRightIndex;
|
||||
foreach (IBlueRow r in dp.deletedRows)
|
||||
{
|
||||
IBlueRow dr = index[r["id"]] as IBlueRow;
|
||||
var dr = index[r.Id()] as IBlueRow;
|
||||
dRows.addRow (dr);
|
||||
index.Remove (r["id"]);
|
||||
index.Remove(r.Id());
|
||||
}
|
||||
foreach (IBlueRow r in dp.addedRows)
|
||||
{
|
||||
IBlueRow ar = new AppendRow (r, myId++, mySchema);
|
||||
aRows.addRow (ar);
|
||||
index.Add (r["id"], ar);
|
||||
index.Add(r.Id(), ar);
|
||||
}
|
||||
return new DataPacket (this, aRows, dRows);
|
||||
}
|
||||
@@ -43,7 +46,8 @@ namespace BCG
|
||||
|
||||
public bool containsID (int id) {throw new ApplicationException ("IBlueRow is readonly");}
|
||||
|
||||
public IBlueTableSchema schema {get {return mySchema;}}
|
||||
public IBlueTableSchema schema =>
|
||||
mySchema ?? throw new InvalidOperationException("Schema is unknown (no rows added)");
|
||||
|
||||
#endregion
|
||||
|
||||
@@ -60,7 +64,7 @@ namespace BCG
|
||||
array.SetValue (r, i++);
|
||||
}
|
||||
|
||||
public object SyncRoot {get {return null;}}
|
||||
public object SyncRoot => this;
|
||||
|
||||
#endregion
|
||||
|
||||
@@ -87,7 +91,7 @@ namespace BCG
|
||||
|
||||
public void Reset() {myEnum = myLeft.GetEnumerator ();}
|
||||
|
||||
public object Current {get {return myEnum.Current;}}
|
||||
public object? Current {get {return myEnum.Current;}}
|
||||
|
||||
public bool MoveNext()
|
||||
{
|
||||
|
||||
@@ -23,12 +23,12 @@ namespace BCG
|
||||
|
||||
#region IBlueRow Members
|
||||
|
||||
public object this [string colName]
|
||||
public object? this [string colName]
|
||||
{
|
||||
get {return colName.Equals("id") ? myId : myRow[colName];}
|
||||
}
|
||||
|
||||
public object this [int index]
|
||||
public object? this [int index]
|
||||
{
|
||||
get {return this[schema[index].colName];}
|
||||
}
|
||||
@@ -43,10 +43,13 @@ namespace BCG
|
||||
|
||||
public System.Collections.IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator (this);}
|
||||
|
||||
object System.Collections.IDictionary.this[object key]
|
||||
{
|
||||
get {return key is string ? this[key as string] : this [(int)key];}
|
||||
set {throw new ApplicationException ("IBlueRow is readonly");}
|
||||
object? IDictionary.this[object key] {
|
||||
get => key switch {
|
||||
string colName => this[colName],
|
||||
int colIndex => this[colIndex],
|
||||
_ => throw new ArgumentException("Must be string or int", nameof(key)),
|
||||
};
|
||||
set => throw new NotSupportedException();
|
||||
}
|
||||
|
||||
public void Remove (object key) {throw new ApplicationException ("IBlueRow is readonly");}
|
||||
@@ -65,7 +68,8 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public void Add (object key, object value) {throw new ApplicationException ("IBlueRow is readonly");}
|
||||
public void Add(object key, object? value) =>
|
||||
throw new NotSupportedException();
|
||||
|
||||
public System.Collections.ICollection Keys {get {return schema.columnNames;}}
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ namespace BCG {
|
||||
|
||||
public IBlueTableSchema schema { get { return myRow.schema; } }
|
||||
|
||||
public object this[int index] {
|
||||
public object? this[int index] {
|
||||
get {
|
||||
if (myRow.schema[index].colName == "id")
|
||||
return myID;
|
||||
@@ -38,15 +38,18 @@ namespace BCG {
|
||||
}
|
||||
}
|
||||
|
||||
public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
|
||||
public object? this[string colName] =>
|
||||
colName == "id" ? myID : myRow[colName];
|
||||
|
||||
#endregion
|
||||
|
||||
#region IDictionary Members
|
||||
|
||||
public void Add(object key, object value) { throw new Exception("Row is read only."); }
|
||||
public void Add(object key, object? value) =>
|
||||
throw new NotSupportedException();
|
||||
|
||||
public void Clear() { throw new Exception("Row is read only."); }
|
||||
public void Clear() =>
|
||||
throw new NotSupportedException();
|
||||
|
||||
public bool Contains(object key) { return myRow.Contains(key); }
|
||||
|
||||
@@ -68,9 +71,13 @@ namespace BCG {
|
||||
}
|
||||
}
|
||||
|
||||
public object this[object key] {
|
||||
get { return key is string ? this[key as string] : this[(int)key]; }
|
||||
set { throw new Exception("The method or operation is not implemented."); }
|
||||
public object? this[object key] {
|
||||
get => key switch {
|
||||
string colName => this[colName],
|
||||
int colIndex => this[colIndex],
|
||||
_ => throw new ArgumentException("Must be string or int", nameof(key)),
|
||||
};
|
||||
set => throw new NotSupportedException();
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -112,18 +119,18 @@ namespace BCG {
|
||||
myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
|
||||
IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
|
||||
myRows.addRow(nr);
|
||||
myIDMap[tableNo].Add((int)r["id"], nr);
|
||||
myIDMap[tableNo].Add(r.Id(), nr);
|
||||
return nr;
|
||||
}
|
||||
|
||||
public IBlueRow removeRow(IBlueRow r, int tableNo) {
|
||||
IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
|
||||
IBlueRow dr = myIDMap[tableNo][r.Id()];
|
||||
myRows.deleteRow(dr);
|
||||
myIDMap[tableNo].Remove((int)r["id"]);
|
||||
myIDMap[tableNo].Remove(r.Id());
|
||||
return dr;
|
||||
}
|
||||
|
||||
public IBlueRowCollection removeTable(int tableNo) {
|
||||
public IBlueRowCollection? removeTable(int tableNo) {
|
||||
if (!myIDMap.ContainsKey(tableNo))
|
||||
return null;
|
||||
else {
|
||||
@@ -170,7 +177,7 @@ namespace BCG {
|
||||
#endregion
|
||||
}
|
||||
private bool mySubscriptionActive = false;
|
||||
private MutableAppendTable myTable;
|
||||
MutableAppendTable? myTable;
|
||||
private int myMovingIndex;
|
||||
bool isFirstUpdate = true;
|
||||
ImmutableDictionary<int, (ViewStream Input, ViewStream Output, IDisposable? Subscription)> myStreamIndex =
|
||||
@@ -222,7 +229,7 @@ namespace BCG {
|
||||
if (mySubscriptionActive) {
|
||||
sub?.Dispose();
|
||||
if (myTable != null) {
|
||||
IBlueRowCollection delRows = myTable.removeTable(kvp.Key);
|
||||
IBlueRowCollection? delRows = myTable.removeTable(kvp.Key);
|
||||
if (delRows != null)
|
||||
myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
|
||||
}
|
||||
|
||||
@@ -99,7 +99,8 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||
public void Add(object key, object? value) =>
|
||||
throw new NotSupportedException();
|
||||
|
||||
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
using System;
|
||||
#nullable enable
|
||||
using System.Collections;
|
||||
|
||||
namespace BCG
|
||||
@@ -10,14 +10,7 @@ namespace BCG
|
||||
public class BlueRowDictionaryEnumerator : BlueRowEnumerator, IDictionaryEnumerator
|
||||
{
|
||||
public BlueRowDictionaryEnumerator (IBlueRow row) : base (row) {}
|
||||
public override object Current {get{return Entry;}}
|
||||
|
||||
#region IDictionaryEnumerator members
|
||||
public object Key {get{return myRow.schema[myPos].colName;}}
|
||||
|
||||
public object Value {get{return myRow[myPos];}}
|
||||
|
||||
public DictionaryEntry Entry {get{return new DictionaryEntry (Key, Value);}}
|
||||
#endregion
|
||||
public override object? Current => Entry;
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,14 @@
|
||||
using System;
|
||||
using System.Data;
|
||||
using System.Collections;
|
||||
using System.IO;
|
||||
using System.Reflection;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using Spark.Core.Collections;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// The BlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
|
||||
@@ -117,7 +118,7 @@ namespace BCG {
|
||||
// }
|
||||
}
|
||||
|
||||
protected Hashtable myIndexByColName = new Hashtable();
|
||||
protected Dictionary<string, ColumnInfo> myIndexByColName = new();
|
||||
protected ColumnInfo[] myIndexByColOrder;
|
||||
private string[] myTableColNames;
|
||||
protected string myTableName;
|
||||
@@ -187,18 +188,8 @@ namespace BCG {
|
||||
/// <summary>
|
||||
/// The returned Array is sorted according to the Column No
|
||||
/// </summary>
|
||||
public string[] columnNames {
|
||||
get {
|
||||
if (myTableColNames == null) {
|
||||
myTableColNames = new string[myIndexByColName.Count];
|
||||
foreach (DictionaryEntry de in myIndexByColName) {
|
||||
ColumnInfo temp = (ColumnInfo)de.Value;
|
||||
myTableColNames[temp.colNo] = temp.colName;
|
||||
}
|
||||
}
|
||||
return myTableColNames;
|
||||
}
|
||||
}
|
||||
public string[] columnNames =>
|
||||
myTableColNames ??= myIndexByColName.Values.ToIndexedArray(c => (c.colNo, c.colName));
|
||||
|
||||
/// <summary>
|
||||
/// The returned Array is sorted according to the Column No
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
@@ -6,10 +7,10 @@ namespace BCG {
|
||||
/// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
|
||||
/// </summary>
|
||||
[Serializable]
|
||||
public class ColumnInfo : IComparable {
|
||||
public class ColumnInfo : IComparable<ColumnInfo>, IComparable, IEquatable<ColumnInfo> {
|
||||
|
||||
//the default column info does not cach values in a lookup table, therefore it is in group 0
|
||||
public ColumnInfo(string colName, int colNo, Type colType, object colValue = null) {
|
||||
public ColumnInfo(string colName, int colNo, Type colType, object? colValue = null) {
|
||||
myColName = colName;
|
||||
myColNo = colNo;
|
||||
myColType = colType;
|
||||
@@ -27,8 +28,14 @@ namespace BCG {
|
||||
type == typeof(string) ||
|
||||
type == typeof(DateTime);
|
||||
|
||||
public int CompareTo(object obj) {
|
||||
int otherColNo = ((ColumnInfo)(obj)).colNo;
|
||||
public int CompareTo(object? obj) =>
|
||||
CompareTo(obj as ColumnInfo);
|
||||
|
||||
public int CompareTo(ColumnInfo? other) {
|
||||
if (other is null) {
|
||||
return 1;
|
||||
}
|
||||
int otherColNo = other.colNo;
|
||||
if (myColNo < otherColNo)
|
||||
return -1;
|
||||
else if (myColNo > otherColNo)
|
||||
@@ -36,18 +43,21 @@ namespace BCG {
|
||||
return 0;
|
||||
}
|
||||
|
||||
public override bool Equals(object obj) {
|
||||
if (obj is ColumnInfo) {
|
||||
ColumnInfo other = obj as ColumnInfo;
|
||||
return (colNo == other.colNo) && (colName == other.colName) && (colType == other.colType);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
public override bool Equals(object? obj) =>
|
||||
Equals(obj as ColumnInfo);
|
||||
|
||||
public bool Equals(ColumnInfo? other) =>
|
||||
other is not null
|
||||
&& colNo == other.colNo
|
||||
&& colName == other.colName
|
||||
&& colType == other.colType;
|
||||
|
||||
public override string ToString() {
|
||||
return colName + " " + colNo + " " + colType;
|
||||
}
|
||||
|
||||
public override int GetHashCode() =>
|
||||
HashCode.Combine(colNo, colName, colType);
|
||||
|
||||
// for backwards compatibility to allow build
|
||||
public bool isConstant {get{return myColValue != null;}}
|
||||
@@ -55,7 +65,7 @@ namespace BCG {
|
||||
private int myColNo;
|
||||
private string myColName;
|
||||
private Type myColType;
|
||||
private object myColValue;
|
||||
private object? myColValue;
|
||||
|
||||
[JsonPropertyName("colNo")]
|
||||
public int colNo { get { return myColNo; } }
|
||||
@@ -67,6 +77,6 @@ namespace BCG {
|
||||
public Type colType { get { return myColType; } }
|
||||
|
||||
[JsonIgnore]
|
||||
public object colValue {get{return myColValue;}}
|
||||
public object? colValue {get{return myColValue;}}
|
||||
}
|
||||
}
|
||||
+53
-42
@@ -1,24 +1,39 @@
|
||||
using System;
|
||||
#nullable enable
|
||||
using BCG;
|
||||
using System.Collections;
|
||||
|
||||
using Spark.Core.Collections;
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
// common place for all serialization interfaces
|
||||
|
||||
namespace bts {
|
||||
// this is the data that is sent across the write for stream IO
|
||||
public class StreamIOMessage {
|
||||
public class StreamIOMessage : IEquatable<StreamIOMessage> {
|
||||
public enum StreamOp {
|
||||
CREATE = 0,
|
||||
// removes and adds multiple rows
|
||||
// other operations can be expressed with this
|
||||
MULTIPLE_REMOVE_ADD = 1,
|
||||
}
|
||||
public class CreateMessage {
|
||||
|
||||
public class CreateMessage : IEquatable<CreateMessage> {
|
||||
public IBlueRowCollection rows;
|
||||
public CreateMessage(IBlueRowCollection rows_) {
|
||||
rows = rows_;
|
||||
}
|
||||
|
||||
public bool Equals(CreateMessage? other) =>
|
||||
other is not null
|
||||
&& BlueDataUtility.compute_equality(rows, other.rows);
|
||||
|
||||
public override bool Equals(object? obj) =>
|
||||
Equals(obj as CreateMessage);
|
||||
|
||||
public override int GetHashCode() =>
|
||||
rows.Count;
|
||||
|
||||
public override string ToString() {
|
||||
string rv = "Create message ";
|
||||
if (rows == null) {
|
||||
@@ -30,21 +45,27 @@ namespace bts {
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
public override bool Equals(object obj) {
|
||||
if (obj != null && obj is CreateMessage) {
|
||||
var other = obj as CreateMessage;
|
||||
return BlueDataUtility.compute_equality(rows, other.rows);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
public class MultipleRemoveAddMessage {
|
||||
|
||||
public class MultipleRemoveAddMessage : IEquatable<MultipleRemoveAddMessage> {
|
||||
public IBlueRowCollection removed_rows;
|
||||
public IBlueRowCollection added_rows;
|
||||
public MultipleRemoveAddMessage(IBlueRowCollection removed_rows_, IBlueRowCollection added_rows_) {
|
||||
removed_rows = removed_rows_;
|
||||
added_rows = added_rows_;
|
||||
}
|
||||
|
||||
public bool Equals(MultipleRemoveAddMessage? other) =>
|
||||
other is not null
|
||||
&& BlueDataUtility.compute_equality(removed_rows, other.removed_rows)
|
||||
&& BlueDataUtility.compute_equality(added_rows, other.added_rows);
|
||||
|
||||
public override bool Equals(object? obj) =>
|
||||
Equals(obj as MultipleRemoveAddMessage);
|
||||
|
||||
public override int GetHashCode() =>
|
||||
HashCode.Combine(removed_rows.Count, added_rows.Count);
|
||||
|
||||
public override string ToString() {
|
||||
string rv = "MultipleRemoveAdd removed: ";
|
||||
foreach (IBlueRow r in removed_rows) {
|
||||
@@ -56,56 +77,46 @@ namespace bts {
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
public override bool Equals(object obj) {
|
||||
if (obj != null && obj is MultipleRemoveAddMessage) {
|
||||
MultipleRemoveAddMessage other = obj as MultipleRemoveAddMessage;
|
||||
return BlueDataUtility.compute_equality(removed_rows, other.removed_rows) &&
|
||||
BlueDataUtility.compute_equality(added_rows, other.added_rows);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
public StreamOp op;
|
||||
public readonly CreateMessage create_message = null;
|
||||
public readonly MultipleRemoveAddMessage multiple_remove_add_message = null;
|
||||
public readonly CreateMessage? create_message;
|
||||
public readonly MultipleRemoveAddMessage? multiple_remove_add_message;
|
||||
|
||||
public StreamIOMessage(CreateMessage create_message_) {
|
||||
op = StreamOp.CREATE;
|
||||
create_message = create_message_;
|
||||
multiple_remove_add_message = null;
|
||||
}
|
||||
|
||||
public StreamIOMessage(MultipleRemoveAddMessage multiple_remove_add_message_) {
|
||||
op = StreamOp.MULTIPLE_REMOVE_ADD;
|
||||
create_message = null;
|
||||
multiple_remove_add_message = multiple_remove_add_message_;
|
||||
}
|
||||
|
||||
public bool Equals(StreamIOMessage? other) =>
|
||||
other is not null
|
||||
&& op == other.op
|
||||
&& op switch {
|
||||
StreamOp.CREATE => NullCheckEqualityComparer.Equals(create_message, other.create_message),
|
||||
StreamOp.MULTIPLE_REMOVE_ADD => NullCheckEqualityComparer.Equals(multiple_remove_add_message, other.multiple_remove_add_message),
|
||||
_ => false,
|
||||
};
|
||||
|
||||
public override bool Equals(object? obj) =>
|
||||
Equals(obj as StreamIOMessage);
|
||||
|
||||
public override int GetHashCode() =>
|
||||
HashCode.Combine(op, create_message, multiple_remove_add_message);
|
||||
|
||||
public override string ToString() {
|
||||
switch (op) {
|
||||
case StreamIOMessage.StreamOp.CREATE:
|
||||
return create_message.ToString();
|
||||
return create_message?.ToString() ?? string.Empty;
|
||||
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD:
|
||||
return multiple_remove_add_message.ToString();
|
||||
return multiple_remove_add_message?.ToString() ?? string.Empty;
|
||||
}
|
||||
bts.Log.assert(false);
|
||||
return String.Empty;
|
||||
}
|
||||
public override bool Equals(object obj) {
|
||||
if (obj != null && obj is StreamIOMessage) {
|
||||
StreamIOMessage other = obj as StreamIOMessage;
|
||||
if (op == other.op) {
|
||||
switch (op) {
|
||||
case StreamOp.CREATE:
|
||||
return create_message.Equals(other.create_message);
|
||||
case StreamOp.MULTIPLE_REMOVE_ADD:
|
||||
return multiple_remove_add_message.Equals(other.multiple_remove_add_message);
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
#region serialization
|
||||
// This helps putting stream callbacks (op + data) on the wire, to send it to the pubsubserver.
|
||||
@@ -134,4 +145,4 @@ namespace bts {
|
||||
}
|
||||
#endregion
|
||||
|
||||
}
|
||||
}
|
||||
+15
-10
@@ -1,12 +1,10 @@
|
||||
#nullable enable
|
||||
namespace bts {
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Reactive.Subjects;
|
||||
using System.Reflection.Metadata;
|
||||
using System.Runtime.Serialization.Formatters.Binary;
|
||||
using System.Text;
|
||||
|
||||
public static class StreamSerializationUtils {
|
||||
@@ -122,17 +120,24 @@ namespace bts {
|
||||
}
|
||||
|
||||
public static object deserialize_column(ref BinaryReader br, Type t) {
|
||||
Func<BinaryReader, object> reader_func;
|
||||
Log.assert(_col_deser_map.TryGetValue(t, out reader_func));
|
||||
return reader_func(br);
|
||||
if (_col_deser_map.TryGetValue(t, out var deserialize)) {
|
||||
return deserialize(br);
|
||||
}
|
||||
var msg = $"Deserialization function not defined for type '{t}'";
|
||||
Log.assert(false, msg);
|
||||
throw new ArgumentException(msg, nameof(t));
|
||||
}
|
||||
|
||||
public static void serialize_row_column(BinaryWriter bw, IBlueRow r, int col_no) {
|
||||
var ci = r.schema.columnInfo[col_no];
|
||||
var type = ci.colType;
|
||||
Action<BinaryWriter, IBlueRow, int> ser_action;
|
||||
Log.assert(_col_ser_map.TryGetValue(type, out ser_action));
|
||||
ser_action(bw, r, col_no);
|
||||
if (_col_ser_map.TryGetValue(type, out var serialize)) {
|
||||
serialize(bw, r, col_no);
|
||||
return;
|
||||
}
|
||||
var msg = $"Serialization function not defined for type '{type}' (failed on column '{ci.colName}')";
|
||||
Log.assert(false, msg);
|
||||
throw new ArgumentException(msg, nameof(r));
|
||||
}
|
||||
|
||||
public static ColumnInfo deserialize_column_info(ref BinaryReader br) {
|
||||
@@ -226,4 +231,4 @@ namespace bts {
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,4 @@
|
||||
using NUnit.Framework;
|
||||
|
||||
using System.Reactive.Disposables;
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
namespace bts {
|
||||
[TestFixture]
|
||||
@@ -41,7 +39,7 @@ namespace bts {
|
||||
sub_msg_desered.construct_deserialize(str);
|
||||
Assert.That(sub_msg_desered.connection, Is.EqualTo(sub_msg.connection));
|
||||
Assert.That(sub_msg_desered.subscribing_app, Is.EqualTo(sub_msg.subscribing_app));
|
||||
compare_arrays(sub_msg.tags, sub_msg_desered.tags);
|
||||
compare_arrays(sub_msg.tags, ExceptionAssert.NotNull(sub_msg_desered.tags));
|
||||
Assert.That(sub_msg_desered, Is.EqualTo(sub_msg));
|
||||
}
|
||||
|
||||
@@ -60,7 +58,7 @@ namespace bts {
|
||||
sub_msg_desered.construct_deserialize(str);
|
||||
Assert.That(sub_msg_desered.connection, Is.EqualTo(unsub_msg.connection));
|
||||
Assert.That(sub_msg_desered.subscribing_app, Is.EqualTo(unsub_msg.subscribing_app));
|
||||
compare_arrays<string>(unsub_msg.tags, sub_msg_desered.tags);
|
||||
compare_arrays<string>(unsub_msg.tags, ExceptionAssert.NotNull(sub_msg_desered.tags));
|
||||
}
|
||||
|
||||
[Test]
|
||||
@@ -87,7 +85,7 @@ namespace bts {
|
||||
Assert.That(pub_msg_desered.tag_type, Is.EqualTo(pub_msg.tag_type));
|
||||
Assert.That(pub_msg_desered.is_compressed, Is.EqualTo(pub_msg.is_compressed));
|
||||
Assert.That(pub_msg_desered.publish_time_epoch_ms, Is.EqualTo(pub_msg.publish_time_epoch_ms));
|
||||
Assert.That(pub_msg_desered.data_to_publish.Length, Is.EqualTo(pub_msg.data_to_publish.Length));
|
||||
Assert.That(ExceptionAssert.NotNull(pub_msg_desered.data_to_publish).Length, Is.EqualTo(pub_msg.data_to_publish.Length));
|
||||
compare_arrays(pub_msg.data_to_publish, pub_msg_desered.data_to_publish);
|
||||
}
|
||||
|
||||
@@ -105,7 +103,7 @@ namespace bts {
|
||||
ps_server_msg_desered.construct_deserialize(str);
|
||||
Assert.That(ps_server_msg_desered.tag, Is.EqualTo(ps_server_msg.tag));
|
||||
Assert.That(ps_server_msg_desered.tag_type, Is.EqualTo(ps_server_msg.tag_type));
|
||||
compare_arrays(ps_server_msg.data, ps_server_msg_desered.data);
|
||||
compare_arrays(ps_server_msg.data, ExceptionAssert.NotNull(ps_server_msg_desered.data));
|
||||
}
|
||||
|
||||
[Test]
|
||||
|
||||
@@ -20,8 +20,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public IPubSubServerConnection get_connection(RemoteConnectionInfo server_rci) {
|
||||
IPubSubServerConnection rv = null;
|
||||
if (!_conn_map.TryGetValue(server_rci, out rv)) {
|
||||
if (!_conn_map.TryGetValue(server_rci, out var rv)) {
|
||||
rv = _build_connection(server_rci);
|
||||
_conn_map.Add(server_rci, rv);
|
||||
}
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
using bts;
|
||||
|
||||
using mbts;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.IO;
|
||||
//using BCG.Utilities;
|
||||
using mbts;
|
||||
using System.Linq;
|
||||
|
||||
namespace BCG {
|
||||
@@ -14,8 +16,8 @@ namespace BCG {
|
||||
/// </summary>
|
||||
public class TagPubSubInfo {
|
||||
public string tag;
|
||||
public IDictionary<string, string> subscribers,
|
||||
publishers;
|
||||
public IDictionary<string, string> subscribers;
|
||||
public IDictionary<string, string> publishers;
|
||||
public ICollection<Pair<string, string>> dead_publishers;
|
||||
public string pubsub_server;
|
||||
|
||||
@@ -58,6 +60,8 @@ namespace BCG {
|
||||
}
|
||||
bw.Write(dead_publishers.Count);
|
||||
foreach (Pair<string, string> p in dead_publishers) {
|
||||
Log.notnull(p.first);
|
||||
Log.notnull(p.second);
|
||||
bw.Write(p.first);
|
||||
bw.Write(p.second);
|
||||
}
|
||||
@@ -91,6 +95,7 @@ namespace BCG {
|
||||
/// constructs itself from the stream
|
||||
/// </summary>
|
||||
/// <param name="s"></param>
|
||||
[MemberNotNull(nameof(tag), nameof(subscribers), nameof(publishers), nameof(dead_publishers), nameof(pubsub_server))]
|
||||
private void _read_from(Stream s) {
|
||||
BinaryReader br = new BinaryReader(s);
|
||||
tag = br.ReadString();
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
// using BCG.Utilities;
|
||||
using BCG.Utilities.General;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using mbts;
|
||||
|
||||
using System;
|
||||
using System.Threading;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
/// <summary>
|
||||
@@ -62,7 +61,7 @@ namespace BCG
|
||||
// w.ToString(), myWorkers.Count), PS3LogLevels.connection);
|
||||
myWorkers.Remove(w);
|
||||
}
|
||||
foreach (KeyValuePair<IWorker, Pair<TimeSpan, DateTime>> p in myJustAdded)
|
||||
foreach (var p in myJustAdded)
|
||||
{
|
||||
// BlueEventLog.logEvent(String.Format("ThreadDrivenMasterWorker : adding worker {0} , worker count before adding this = {1}",
|
||||
// p.Key.ToString(), myWorkers.Count), PS3LogLevels.connection);
|
||||
@@ -78,7 +77,7 @@ namespace BCG
|
||||
{
|
||||
// BlueEventLog.logEvent(ex);
|
||||
if (ex is OutOfMemoryException)
|
||||
throw ex;
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public interface IGenericFormatter
|
||||
{
|
||||
T Deserialize<T> (Stream serializationStream);
|
||||
void Serialize<T> (Stream serializationStream, T graph);
|
||||
T Deserialize<T> (Stream serializationStream) where T : notnull;
|
||||
void Serialize<T> (Stream serializationStream, T graph) where T : notnull;
|
||||
}
|
||||
}
|
||||
@@ -2,17 +2,20 @@ namespace bts;
|
||||
|
||||
using BCG;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
|
||||
// opens a TCP port and starts listening to connections
|
||||
public class AsyncTCPConnectionAcceptor_APM {
|
||||
private readonly int _port;
|
||||
private Socket _accept_socket;
|
||||
private ITCPConnectionHandler _connection_handler;
|
||||
private HashSet<string> _key_map = new HashSet<string>();
|
||||
Socket? _accept_socket;
|
||||
readonly ITCPConnectionHandler _connection_handler;
|
||||
readonly HashSet<string> _key_map = new();
|
||||
|
||||
public AsyncTCPConnectionAcceptor_APM(int port, ITCPConnectionHandler connection_handler) {
|
||||
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM. cons started on port: {port}");
|
||||
@@ -25,6 +28,7 @@ public class AsyncTCPConnectionAcceptor_APM {
|
||||
_begin_accepting();
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(_accept_socket))]
|
||||
void _init() {
|
||||
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._init started on port: {_port}");
|
||||
_accept_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
@@ -34,7 +38,9 @@ public class AsyncTCPConnectionAcceptor_APM {
|
||||
_accept_socket.Listen(250);
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(_accept_socket))]
|
||||
void _begin_accepting() {
|
||||
ArgumentNullException.ThrowIfNull(_accept_socket);
|
||||
var accept_key = Guid.NewGuid().ToString();
|
||||
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._begin_accepting port: {_port} accept_key: {accept_key}");
|
||||
var res = _accept_socket.BeginAccept(new AsyncCallback(_on_accept), accept_key);
|
||||
@@ -46,14 +52,14 @@ public class AsyncTCPConnectionAcceptor_APM {
|
||||
var log_msg = $"AsyncTCPConnectionAcceptor_APM._on_accept state: {res.AsyncState}";
|
||||
Log.Gen.info(log_msg);
|
||||
Console.WriteLine(log_msg);
|
||||
string key = res.AsyncState.ToString();
|
||||
var key = ExceptionAssert.NotNull(res.AsyncState).ToString() ?? string.Empty;
|
||||
if (_key_map.Contains(key)) {
|
||||
return;
|
||||
}
|
||||
_key_map.Add(key);
|
||||
Socket new_sock;
|
||||
try {
|
||||
new_sock = _accept_socket.EndAccept(res);
|
||||
new_sock = ExceptionAssert.NotNull(_accept_socket).EndAccept(res);
|
||||
} catch (Exception ex) {
|
||||
bts.Log.Gen.info("AsyncTCPConnectionAcceptor_APM._on_accept exception thrown by EndAccept ex: {ex}");
|
||||
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(250));
|
||||
|
||||
@@ -5,6 +5,8 @@ using BCG;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
|
||||
namespace bts {
|
||||
@@ -13,10 +15,10 @@ namespace bts {
|
||||
// it uses TcpListener, instead of raw sockets.
|
||||
public class AsyncTCPConnectionAcceptor_AsyncWrapper {
|
||||
private readonly int _accept_port;
|
||||
private Socket _accept_socket;
|
||||
// private Socket _accept_socket;
|
||||
private ITCPConnectionHandler _connection_handler;
|
||||
private TcpListener _server;
|
||||
private Action<ITCPConnection> _cbs = (c) => { };
|
||||
TcpListener? _server;
|
||||
// private Action<ITCPConnection> _cbs = (c) => { };
|
||||
|
||||
public AsyncTCPConnectionAcceptor_AsyncWrapper(int accept_port, ITCPConnectionHandler connection_handler) {
|
||||
Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper.cons started on port: {accept_port}");
|
||||
@@ -25,9 +27,10 @@ namespace bts {
|
||||
}
|
||||
|
||||
public void resgiter_callback(Action<ITCPConnection> cb) {
|
||||
_cbs += cb;
|
||||
// _cbs += cb;
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(_server))]
|
||||
public void start() {
|
||||
Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._init started on port: {_accept_port}");
|
||||
_server = new TcpListener(IPAddress.Any, _accept_port);
|
||||
@@ -36,6 +39,7 @@ namespace bts {
|
||||
}
|
||||
|
||||
private void _accept_connections_forever() {
|
||||
ExceptionAssert.MemberNotNull(_server);
|
||||
Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port}");
|
||||
while (true) {
|
||||
// this is a blocking call
|
||||
@@ -55,4 +59,4 @@ namespace bts {
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -2,13 +2,13 @@ namespace bts
|
||||
{
|
||||
using BCG;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Net.Sockets;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
|
||||
// this ITCPConnection uses the TCPClient class instead of a raw socket (in the hope, that it would configure the socket correctly)
|
||||
|
||||
@@ -34,7 +34,7 @@ namespace bts
|
||||
private const int HB_MSG_SIGNAL = 1;
|
||||
private TcpClient _tcp_client;
|
||||
private NetworkStream _nw_stream;
|
||||
private ITCPMessageHandler _msg_handler;
|
||||
ITCPMessageHandler? _msg_handler;
|
||||
private bool _is_alive = true;
|
||||
ConnectionID _connection_id;
|
||||
|
||||
@@ -191,7 +191,7 @@ namespace bts
|
||||
private class OneRecvMessage
|
||||
{
|
||||
public int msg_type;
|
||||
public byte[] data;
|
||||
public byte[]? data;
|
||||
public OneRecvMessage(int msg_type_, byte[] data_)
|
||||
{
|
||||
msg_type = msg_type_;
|
||||
@@ -211,7 +211,7 @@ namespace bts
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
string rv = $"OneMessage msg_type: {msg_type} data_len: {data.Length}";
|
||||
string rv = $"OneMessage msg_type: {msg_type} data_len: {data?.Length ?? 0}";
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
@@ -237,6 +237,7 @@ namespace bts
|
||||
|
||||
private void _read_msgs_forever()
|
||||
{
|
||||
ExceptionAssert.MemberNotNull(_msg_handler);
|
||||
while (_is_alive)
|
||||
{
|
||||
try
|
||||
@@ -270,7 +271,7 @@ namespace bts
|
||||
public class ErrorMessage
|
||||
{
|
||||
public readonly SocketError? socket_error = null;
|
||||
public readonly Exception exception = null;
|
||||
public readonly Exception? exception = null;
|
||||
public readonly bool is_exception = false;
|
||||
public readonly string context = String.Empty;
|
||||
public ErrorMessage(SocketError socket_error_, string context_)
|
||||
@@ -313,9 +314,9 @@ namespace bts
|
||||
{
|
||||
String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket");
|
||||
if (ex is OutOfMemoryException)
|
||||
throw ex;
|
||||
throw;
|
||||
}
|
||||
_msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _connection_id.peer));
|
||||
_msg_handler?.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _connection_id.peer));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -325,8 +326,10 @@ namespace bts
|
||||
lock (this)
|
||||
{
|
||||
Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
|
||||
if (err_msg.oom)
|
||||
if (err_msg.oom) {
|
||||
Log.notnull(err_msg.exception);
|
||||
throw err_msg.exception;
|
||||
}
|
||||
Dispose();
|
||||
}
|
||||
}
|
||||
@@ -336,4 +339,4 @@ namespace bts
|
||||
return _connection_id.ToString();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,10 @@
|
||||
using System.Net.Sockets;
|
||||
using System.Net;
|
||||
|
||||
using BCG;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
|
||||
namespace bts {
|
||||
// Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests
|
||||
@@ -42,7 +42,7 @@ namespace bts {
|
||||
|
||||
private static void _on_connect(IAsyncResult res) {
|
||||
Log.Gen.info($"AsyncTCPConnector_APM._on_connect res: {res.AsyncState}");
|
||||
SocketConnectionArgs sc_args = res.AsyncState as SocketConnectionArgs;
|
||||
var sc_args = ExceptionAssert.Cast(res.AsyncState).To<SocketConnectionArgs>();
|
||||
try {
|
||||
sc_args.sock.EndConnect(res);
|
||||
} catch (SocketException socket_ex) {
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
using bts;
|
||||
|
||||
using Bts.Net;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.Net;
|
||||
using bts.utils;
|
||||
using System.Net.NetworkInformation;
|
||||
using bts;
|
||||
using Bts.Net;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
|
||||
namespace BCG {
|
||||
public class RemoteConnectionInfo : IEquatable<RemoteConnectionInfo> {
|
||||
@@ -106,13 +105,7 @@ namespace BCG {
|
||||
return ip.GetHashCode() ^ port.GetHashCode();
|
||||
}
|
||||
|
||||
public override bool Equals(object? obj) {
|
||||
if (obj == null) {
|
||||
return false;
|
||||
}
|
||||
RemoteConnectionInfo other = obj as RemoteConnectionInfo;
|
||||
return other != null && other.ip == ip && other.port == port;
|
||||
}
|
||||
|
||||
public override bool Equals(object? obj) =>
|
||||
obj is RemoteConnectionInfo other && other.ip == ip && other.port == port;
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,14 @@
|
||||
using bts;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using bts;
|
||||
using System.Threading;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace BCG {
|
||||
// utility functions for TCP socket async communications. Does the following
|
||||
@@ -50,7 +50,9 @@ namespace BCG {
|
||||
Marshal.Copy(ptr, inOptionValues, 0, size);
|
||||
Marshal.FreeHGlobal(ptr);
|
||||
|
||||
server_socket.IOControl(IOControlCode.KeepAliveValues, inOptionValues, null);
|
||||
if (OperatingSystem.IsWindows()) {
|
||||
server_socket.IOControl(IOControlCode.KeepAliveValues, inOptionValues, null);
|
||||
}
|
||||
}
|
||||
|
||||
// Set other socket options as needed
|
||||
@@ -67,14 +69,14 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public static RemoteConnectionInfo get_peer_address(Socket sock) {
|
||||
IPEndPoint ip_ep = sock.RemoteEndPoint as IPEndPoint;
|
||||
var ip_ep = ExceptionAssert.Cast(sock.RemoteEndPoint).To<IPEndPoint>();
|
||||
int port = ip_ep.Port;
|
||||
string ip = ip_ep.Address.ToString();
|
||||
return new RemoteConnectionInfo(ip, port);
|
||||
}
|
||||
|
||||
public static RemoteConnectionInfo get_self_address(Socket socket) {
|
||||
IPEndPoint ip_ep = socket.LocalEndPoint as IPEndPoint;
|
||||
var ip_ep = ExceptionAssert.Cast(socket.LocalEndPoint).To<IPEndPoint>();
|
||||
int port = ip_ep.Port;
|
||||
string ip = ip_ep.Address.ToString();
|
||||
return new RemoteConnectionInfo(ip, port);
|
||||
@@ -101,7 +103,9 @@ namespace BCG {
|
||||
Marshal.Copy(ptr, inOptionValues, 0, size);
|
||||
Marshal.FreeHGlobal(ptr);
|
||||
|
||||
client_socket.IOControl(IOControlCode.KeepAliveValues, inOptionValues, null);
|
||||
if (OperatingSystem.IsWindows()) {
|
||||
client_socket.IOControl(IOControlCode.KeepAliveValues, inOptionValues, null);
|
||||
}
|
||||
}
|
||||
|
||||
// Set other socket options as needed
|
||||
@@ -126,7 +130,7 @@ namespace BCG {
|
||||
Socket listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
listener.Bind(ip_end_point);
|
||||
listener.Listen(10);
|
||||
Socket client_socket = null;
|
||||
Socket? client_socket;
|
||||
while (true) {
|
||||
try {
|
||||
// keeps on accepting connections
|
||||
@@ -135,16 +139,16 @@ namespace BCG {
|
||||
handler.onConnection(new_conn);
|
||||
} catch (Exception ex) {
|
||||
bts.Log.Gen.error($"TCPConnection.accept ex: {ex} trying again");
|
||||
Task.Delay(wait_after_error);
|
||||
await Task.Delay(wait_after_error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// connects to the server and ona successful connection, calls con__handler.on_connection_async with an ITCPConnection.
|
||||
// In case of failures to connect it waits for a given amount of time before retrying
|
||||
public static async Task<Socket> connect_to_server_async(string ip, int server_port, ITCPConnectionHandler conn_handler,
|
||||
public static async Task<Socket?> connect_to_server_async(string ip, int server_port, ITCPConnectionHandler conn_handler,
|
||||
TimeSpan wait_after_failed_attempt, int n_max_attempts = 1024) {
|
||||
Socket client_socket = null;
|
||||
Socket? client_socket = null;
|
||||
for (int i = 0; i < n_max_attempts; i++) {
|
||||
try {
|
||||
string log_conn = $"{ip}:{server_port}";
|
||||
@@ -158,7 +162,7 @@ namespace BCG {
|
||||
break;
|
||||
} catch (Exception ex) {
|
||||
bts.Log.Gen.error($"AsyncTCPConnection2.connect_to_server threw ex: {ex}, will reattempt now");
|
||||
Task.Delay(wait_after_failed_attempt);
|
||||
await Task.Delay(wait_after_failed_attempt);
|
||||
}
|
||||
}
|
||||
return client_socket;
|
||||
@@ -281,4 +285,4 @@ namespace BCG {
|
||||
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
@@ -2,9 +2,6 @@ using System;
|
||||
using System.Collections;
|
||||
using System.Threading;
|
||||
|
||||
using BCG;
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
[Serializable]
|
||||
@@ -36,13 +33,10 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public object dequeue()
|
||||
{
|
||||
return dequeue(Timeout.Infinite);
|
||||
}
|
||||
public object? dequeue() =>
|
||||
dequeue(Timeout.Infinite);
|
||||
|
||||
public object dequeue (int millisecondsToWait)
|
||||
{
|
||||
public object? dequeue (int millisecondsToWait) {
|
||||
for (;;)
|
||||
lock (this)
|
||||
{
|
||||
@@ -60,13 +54,13 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public object[] dequeueMultipleElements(int millisecondsToWait, int maxElements)
|
||||
public object?[] dequeueMultipleElements(int millisecondsToWait, int maxElements)
|
||||
{
|
||||
ArrayList rv = new ArrayList();
|
||||
int n = getCount();
|
||||
for (int i = 0; i < n; i++)
|
||||
{
|
||||
object o = dequeue(0);
|
||||
var o = dequeue(0);
|
||||
if (o != null)
|
||||
{
|
||||
rv.Add(o);
|
||||
@@ -78,7 +72,7 @@ namespace BCG
|
||||
int timeRemaining = millisecondsToWait;
|
||||
while (true)
|
||||
{
|
||||
object o = dequeue(timeRemaining);
|
||||
var o = dequeue(timeRemaining);
|
||||
if (o != null)
|
||||
rv.Add(o);
|
||||
timeRemaining = millisecondsToWait - Convert.ToInt32 (((TimeSpan)(DateTime.Now - startTime)).TotalMilliseconds);
|
||||
@@ -87,10 +81,8 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public object[] dequeueMultipleElements(TimeSpan ts, int maxElements)
|
||||
{
|
||||
return dequeueMultipleElements(Convert.ToInt32 (ts.TotalMilliseconds), maxElements);
|
||||
}
|
||||
public object?[] dequeueMultipleElements(TimeSpan ts, int maxElements) =>
|
||||
dequeueMultipleElements(Convert.ToInt32 (ts.TotalMilliseconds), maxElements);
|
||||
|
||||
public int getCount()
|
||||
{
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
using System;
|
||||
using System.Text;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -43,7 +40,7 @@ namespace BCG
|
||||
set { myStartTime = value; }
|
||||
|
||||
}
|
||||
private string myProcessString;
|
||||
string? myProcessString;
|
||||
public string processString
|
||||
{
|
||||
get
|
||||
@@ -84,7 +81,7 @@ namespace BCG
|
||||
myStartTime = System.Diagnostics.Process.GetCurrentProcess().StartTime;
|
||||
}
|
||||
|
||||
private static ClientInfo myLocalClient;
|
||||
static ClientInfo? myLocalClient;
|
||||
private static object myLock = new object();
|
||||
|
||||
public static ClientInfo thisClient
|
||||
@@ -135,4 +132,4 @@ namespace BCG
|
||||
return builder.ToString();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -33,8 +33,8 @@
|
||||
// exception statement from your version.
|
||||
|
||||
using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
|
||||
namespace BCG.Utilities.SharpZipLib.Tar {
|
||||
|
||||
@@ -55,10 +55,10 @@ namespace BCG.Utilities.SharpZipLib.Tar {
|
||||
public static readonly int DEFAULT_RCDSIZE = 512;
|
||||
public static readonly int DEFAULT_BLKSIZE = DEFAULT_RCDSIZE * 20;
|
||||
|
||||
Stream inputStream;
|
||||
Stream outputStream;
|
||||
Stream? inputStream;
|
||||
Stream? outputStream;
|
||||
|
||||
byte[] blockBuffer;
|
||||
byte[]? blockBuffer;
|
||||
int currBlkIdx;
|
||||
int currRecIdx;
|
||||
int blockSize;
|
||||
@@ -110,6 +110,7 @@ namespace BCG.Utilities.SharpZipLib.Tar {
|
||||
/// <summary>
|
||||
/// Initialization common to all constructors.
|
||||
/// </summary>
|
||||
[MemberNotNull(nameof(blockBuffer))]
|
||||
void Initialize(int blockSize, int recordSize)
|
||||
{
|
||||
this.debug = false;
|
||||
@@ -197,7 +198,7 @@ namespace BCG.Utilities.SharpZipLib.Tar {
|
||||
/// <returns>
|
||||
/// The record data.
|
||||
/// </returns>
|
||||
public byte[] ReadRecord()
|
||||
public byte[]? ReadRecord()
|
||||
{
|
||||
if (this.debug) {
|
||||
Console.Error.WriteLine( "ReadRecord: recIdx = " + this.currRecIdx + " blkIdx = " + this.currBlkIdx );
|
||||
@@ -214,7 +215,7 @@ namespace BCG.Utilities.SharpZipLib.Tar {
|
||||
}
|
||||
|
||||
byte[] result = new byte[this.recordSize];
|
||||
|
||||
requireBlockBuffer();
|
||||
Array.Copy(this.blockBuffer, (this.currRecIdx * this.recordSize), result, 0, this.recordSize );
|
||||
this.currRecIdx++;
|
||||
return result;
|
||||
@@ -238,34 +239,37 @@ namespace BCG.Utilities.SharpZipLib.Tar {
|
||||
|
||||
int offset = 0;
|
||||
int bytesNeeded = this.blockSize;
|
||||
for (; bytesNeeded > 0 ;) {
|
||||
long numBytes = this.inputStream.Read(this.blockBuffer, offset, bytesNeeded);
|
||||
|
||||
//
|
||||
// NOTE
|
||||
// We have fit EOF, and the block is not full!
|
||||
//
|
||||
// This is a broken archive. It does not follow the standard
|
||||
// blocking algorithm. However, because we are generous, and
|
||||
// it requires little effort, we will simply ignore the error
|
||||
// and continue as if the entire block were read. This does
|
||||
// not appear to break anything upstream. We used to return
|
||||
// false in this case.
|
||||
//
|
||||
// Thanks to 'Yohann.Roussel@alcatel.fr' for this fix.
|
||||
//
|
||||
if (numBytes <= 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
offset += (int)numBytes;
|
||||
bytesNeeded -= (int)numBytes;
|
||||
if (numBytes != this.blockSize) {
|
||||
if (this.debug) {
|
||||
Console.Error.WriteLine("ReadBlock: INCOMPLETE READ " + numBytes + " of " + this.blockSize + " bytes read.");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (bytesNeeded > 0) {
|
||||
requireBlockBuffer();
|
||||
do {
|
||||
long numBytes = this.inputStream.Read(this.blockBuffer, offset, bytesNeeded);
|
||||
|
||||
//
|
||||
// NOTE
|
||||
// We have fit EOF, and the block is not full!
|
||||
//
|
||||
// This is a broken archive. It does not follow the standard
|
||||
// blocking algorithm. However, because we are generous, and
|
||||
// it requires little effort, we will simply ignore the error
|
||||
// and continue as if the entire block were read. This does
|
||||
// not appear to break anything upstream. We used to return
|
||||
// false in this case.
|
||||
//
|
||||
// Thanks to 'Yohann.Roussel@alcatel.fr' for this fix.
|
||||
//
|
||||
if (numBytes <= 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
offset += (int)numBytes;
|
||||
bytesNeeded -= (int)numBytes;
|
||||
if (numBytes != this.blockSize) {
|
||||
if (this.debug) {
|
||||
Console.Error.WriteLine("ReadBlock: INCOMPLETE READ " + numBytes + " of " + this.blockSize + " bytes read.");
|
||||
}
|
||||
}
|
||||
} while(bytesNeeded > 0);
|
||||
}
|
||||
|
||||
this.currBlkIdx++;
|
||||
return true;
|
||||
@@ -318,6 +322,7 @@ namespace BCG.Utilities.SharpZipLib.Tar {
|
||||
if (this.currRecIdx >= this.recsPerBlock) {
|
||||
this.WriteBlock();
|
||||
}
|
||||
requireBlockBuffer();
|
||||
Array.Copy(record, 0, this.blockBuffer, (this.currRecIdx * this.recordSize), this.recordSize );
|
||||
this.currRecIdx++;
|
||||
}
|
||||
@@ -350,7 +355,7 @@ namespace BCG.Utilities.SharpZipLib.Tar {
|
||||
if (this.currRecIdx >= this.recsPerBlock) {
|
||||
this.WriteBlock();
|
||||
}
|
||||
|
||||
requireBlockBuffer();
|
||||
Array.Copy(buf, offset, this.blockBuffer, (this.currRecIdx * this.recordSize), this.recordSize );
|
||||
|
||||
this.currRecIdx++;
|
||||
@@ -368,7 +373,7 @@ namespace BCG.Utilities.SharpZipLib.Tar {
|
||||
if (this.outputStream == null) {
|
||||
throw new System.IO.IOException("no output stream defined");
|
||||
}
|
||||
|
||||
requireBlockBuffer();
|
||||
this.outputStream.Write(this.blockBuffer, 0, this.blockSize);
|
||||
this.outputStream.Flush();
|
||||
|
||||
@@ -418,6 +423,13 @@ namespace BCG.Utilities.SharpZipLib.Tar {
|
||||
inputStream = null;
|
||||
}
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(blockBuffer))]
|
||||
void requireBlockBuffer() {
|
||||
if (blockBuffer is null) {
|
||||
throw new InvalidOperationException("blockBuffer is null");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/* The original Java file had this header:
|
||||
|
||||
+10
-12
@@ -32,7 +32,6 @@
|
||||
// obligated to do so. If you do not wish to do so, delete this
|
||||
// exception statement from your version.
|
||||
|
||||
#nullable disable
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
@@ -53,11 +52,11 @@ namespace BCG.Utilities.SharpZipLib.Tar {
|
||||
protected int entrySize;
|
||||
protected int entryOffset;
|
||||
|
||||
protected byte[] readBuf;
|
||||
protected byte[]? readBuf;
|
||||
|
||||
protected TarBuffer buffer;
|
||||
protected TarEntry currEntry;
|
||||
protected IEntryFactory eFactory;
|
||||
protected TarEntry? currEntry;
|
||||
protected IEntryFactory? eFactory;
|
||||
|
||||
Stream inputStream;
|
||||
/// <summary>
|
||||
@@ -308,7 +307,7 @@ namespace BCG.Utilities.SharpZipLib.Tar {
|
||||
this.readBuf = null;
|
||||
}
|
||||
|
||||
byte[] headerBuf = this.buffer.ReadRecord();
|
||||
byte[]? headerBuf = this.buffer.ReadRecord();
|
||||
|
||||
if (headerBuf == null) {
|
||||
if (this.debug) {
|
||||
@@ -327,6 +326,9 @@ namespace BCG.Utilities.SharpZipLib.Tar {
|
||||
if (this.hasHitEOF) {
|
||||
this.currEntry = null;
|
||||
} else {
|
||||
if (headerBuf is null) {
|
||||
throw new InvalidOperationException("headerBuf is null");
|
||||
}
|
||||
try {
|
||||
if (this.eFactory == null) {
|
||||
this.currEntry = new TarEntry(headerBuf);
|
||||
@@ -422,13 +424,9 @@ namespace BCG.Utilities.SharpZipLib.Tar {
|
||||
}
|
||||
|
||||
while (numToRead > 0) {
|
||||
byte[] rec = this.buffer.ReadRecord();
|
||||
if (rec == null) {
|
||||
// Unexpected EOF!
|
||||
throw new IOException("unexpected EOF with " + numToRead + " bytes unread");
|
||||
}
|
||||
|
||||
int sz = numToRead;
|
||||
byte[]? rec = buffer.ReadRecord()
|
||||
?? throw new IOException("unexpected EOF with " + numToRead + " bytes unread");
|
||||
int sz = numToRead;
|
||||
int recLen = rec.Length;
|
||||
|
||||
if (recLen > sz) {
|
||||
|
||||
+8
-12
@@ -44,6 +44,8 @@ using BCG.Utilities.SharpZipLib.Checksums;
|
||||
using BCG.Utilities.SharpZipLib.Zip.Compression;
|
||||
using BCG.Utilities.SharpZipLib.Zip.Compression.Streams;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
namespace BCG.Utilities.SharpZipLib.Zip {
|
||||
|
||||
/// <summary>
|
||||
@@ -96,9 +98,9 @@ namespace BCG.Utilities.SharpZipLib.Zip {
|
||||
/// </example>
|
||||
public class ZipOutputStream : DeflaterOutputStream
|
||||
{
|
||||
private ArrayList entries = new ArrayList();
|
||||
private Crc32 crc = new Crc32();
|
||||
private ZipEntry curEntry = null;
|
||||
private ArrayList? entries = new();
|
||||
private Crc32 crc = new();
|
||||
private ZipEntry? curEntry = null;
|
||||
|
||||
private CompressionMethod curMethod;
|
||||
private int size;
|
||||
@@ -308,10 +310,7 @@ namespace BCG.Utilities.SharpZipLib.Zip {
|
||||
if (name.Length > 0xFFFF) {
|
||||
throw new ZipException("Name too long.");
|
||||
}
|
||||
byte[] extra = entry.ExtraData;
|
||||
if (extra == null) {
|
||||
extra = new byte[0];
|
||||
}
|
||||
byte[] extra = entry.ExtraData ?? Array.Empty<byte>();
|
||||
if (extra.Length > 0xFFFF) {
|
||||
throw new ZipException("Extra data too long.");
|
||||
}
|
||||
@@ -397,7 +396,7 @@ namespace BCG.Utilities.SharpZipLib.Zip {
|
||||
}
|
||||
}
|
||||
|
||||
entries.Add(curEntry);
|
||||
ExceptionAssert.MemberNotNull(entries).Add(curEntry);
|
||||
curEntry = null;
|
||||
}
|
||||
|
||||
@@ -468,10 +467,7 @@ namespace BCG.Utilities.SharpZipLib.Zip {
|
||||
if (name.Length > 0xffff) {
|
||||
throw new ZipException("Name too long.");
|
||||
}
|
||||
byte[] extra = entry.ExtraData;
|
||||
if (extra == null) {
|
||||
extra = new byte[0];
|
||||
}
|
||||
byte[] extra = entry.ExtraData ?? Array.Empty<byte>();
|
||||
string strComment = entry.Comment;
|
||||
byte[] comment = strComment != null ? ZipConstants.ConvertToArray(strComment) : new byte[0];
|
||||
if (comment.Length > 0xffff) {
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Runtime.Serialization.Formatters.Binary;
|
||||
@@ -10,14 +9,11 @@ namespace BCG
|
||||
{
|
||||
IFormatter myFormatter = new F ();
|
||||
|
||||
public T Deserialize<T> (Stream serializationStream)
|
||||
{
|
||||
return (T) myFormatter.Deserialize (serializationStream);
|
||||
}
|
||||
public void Serialize<T> (Stream serializationStream, T graph)
|
||||
{
|
||||
public T Deserialize<T> (Stream serializationStream) where T : notnull =>
|
||||
(T)myFormatter.Deserialize (serializationStream);
|
||||
|
||||
public void Serialize<T> (Stream serializationStream, T graph) where T : notnull =>
|
||||
myFormatter.Serialize (serializationStream, graph);
|
||||
}
|
||||
}
|
||||
|
||||
public class GenericBinaryFormatter : GenericFormatter<BinaryFormatter> { }
|
||||
|
||||
@@ -17,12 +17,13 @@ namespace BCG {
|
||||
myLastLogTime = DateTime.MinValue;
|
||||
protected TimeSpan myPeriod = new TimeSpan(0, 0, 0, 0, 100),
|
||||
myLogPeriod = new TimeSpan(0, 1, 0);
|
||||
protected Dictionary<IWorker, Pair<TimeSpan, DateTime>> myWorkers = new Dictionary<IWorker, Pair<TimeSpan, DateTime>>(),
|
||||
myJustAdded = new Dictionary<IWorker, Pair<TimeSpan, DateTime>>(),
|
||||
myJustRemoved = new Dictionary<IWorker, Pair<TimeSpan, DateTime>>();
|
||||
protected readonly Dictionary<IWorker, Pair<TimeSpan, DateTime>?> myWorkers = new();
|
||||
protected readonly Dictionary<IWorker, Pair<TimeSpan, DateTime>?> myJustAdded = new();
|
||||
protected readonly Dictionary<IWorker, Pair<TimeSpan, DateTime>?> myJustRemoved = new();
|
||||
readonly Stopwatch timer = new();
|
||||
private static MasterWorker myThreadDriven = null,
|
||||
myDoWorkDriven = null;
|
||||
static MasterWorker? myThreadDriven;
|
||||
static MasterWorker? myDoWorkDriven;
|
||||
|
||||
internal MasterWorker() {
|
||||
myInit();
|
||||
}
|
||||
@@ -35,7 +36,6 @@ namespace BCG {
|
||||
// TODO
|
||||
myThreadDriven = myThreadDriven ?? new ThreadDrivenMasterWorker();
|
||||
return myThreadDriven;
|
||||
return null;
|
||||
} else {
|
||||
myDoWorkDriven = myDoWorkDriven ?? new MasterWorker();
|
||||
return myDoWorkDriven;
|
||||
@@ -61,10 +61,10 @@ namespace BCG {
|
||||
|
||||
public virtual void doWork() {
|
||||
|
||||
foreach (KeyValuePair<IWorker, Pair<TimeSpan, DateTime>> p in myJustAdded) {
|
||||
foreach (var p in myJustAdded) {
|
||||
myWorkers.Add(p.Key, p.Value);
|
||||
}
|
||||
foreach (KeyValuePair<IWorker, Pair<TimeSpan, DateTime>> p in myJustRemoved) {
|
||||
foreach (var p in myJustRemoved) {
|
||||
myWorkers.Remove(p.Key);
|
||||
}
|
||||
myJustAdded.Clear();
|
||||
|
||||
@@ -1,12 +1,5 @@
|
||||
//using ProtoBuf;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
//using System.Windows;
|
||||
//using System.Windows.Media;
|
||||
|
||||
namespace mbts
|
||||
{
|
||||
@@ -18,7 +11,7 @@ namespace mbts
|
||||
public S? second;
|
||||
|
||||
public Pair() { }
|
||||
public Pair(F f, S s)
|
||||
public Pair(F? f, S? s)
|
||||
{
|
||||
first = f;
|
||||
second = s;
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
using BCG.Utilities;
|
||||
|
||||
using bts;
|
||||
|
||||
using mbts;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// SingleTagPubSubMap consists of all the state w.r.t to
|
||||
@@ -79,6 +77,7 @@ namespace BCG {
|
||||
|
||||
public bool handle_sub(SubscribeMsg sub_msg) {
|
||||
Log.Gen.info($"SingleTagConnectionData.handle_sub sub_msg: {sub_msg}");
|
||||
ExceptionAssert.NotNull(sub_msg.connection);
|
||||
if (_sub_conn_to_app_map.ContainsKey(sub_msg.connection)) {
|
||||
bts.Log.Gen.info($"tag: {_tag} has already been subscribed by connection: {sub_msg.connection}");
|
||||
return false;
|
||||
@@ -89,6 +88,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public bool handle_unsub(UnsubscribeMsg unsub_msg) {
|
||||
ExceptionAssert.NotNull(unsub_msg.connection);
|
||||
if (_sub_conn_to_app_map.ContainsKey(unsub_msg.connection)) {
|
||||
_sub_conn_to_app_map.Remove(unsub_msg.connection);
|
||||
return true;
|
||||
@@ -123,6 +123,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public void handle_pub(PublishMsg pub_msg, out bool new_publisher) {
|
||||
ExceptionAssert.NotNull(pub_msg.connection);
|
||||
if (_active_pub_conn_to_app_map.ContainsKey(pub_msg.connection)) {
|
||||
new_publisher = false;
|
||||
} else {
|
||||
@@ -131,4 +132,4 @@ namespace BCG {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
using bts;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using bts;
|
||||
using System.Threading;
|
||||
|
||||
namespace BCG {
|
||||
@@ -40,15 +43,13 @@ namespace BCG {
|
||||
return;
|
||||
}
|
||||
RemoteConnectionInfo rci = msg.remoteConnectionInfo;
|
||||
ITCPConnection this_conn;
|
||||
bool found = _conn_map.TryGetValue(rci, out this_conn);
|
||||
if (!found) {
|
||||
if (!_conn_map.TryGetValue(rci, out var this_conn)) {
|
||||
Log.Gen.error($"Could not find connection: {rci}");
|
||||
return;
|
||||
}
|
||||
Log.Gen.info($"\n\n\nSimpleTCPServerAsync.onData msgs_received: {_n_msgs_received} msg: {msg} ci: {rci} NOW echoing it back\n\n\n");
|
||||
if (msg.messageType == RemoteMessageType.Data) {
|
||||
this_conn.send(msg.message);
|
||||
this_conn.send(ExceptionAssert.NotNull(msg.message));
|
||||
} else {
|
||||
if (msg.messageType == RemoteMessageType.Disconnection) {
|
||||
var ci = msg.remoteConnectionInfo;
|
||||
|
||||
@@ -1,103 +1,102 @@
|
||||
using bts.SilverPine;
|
||||
|
||||
namespace bts.BlueOrb {
|
||||
using bts;
|
||||
//using bts.api;
|
||||
//using bts.spark;
|
||||
|
||||
using System;
|
||||
|
||||
using ZstdSharp.Unsafe;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
public static class ContractUtils {
|
||||
public class FutureTradableContract {
|
||||
public string symbol;
|
||||
public DateTime expiration;
|
||||
public Exchange exchange;
|
||||
public string contract_text;
|
||||
}
|
||||
public record FutureTradableContract(
|
||||
string Symbol,
|
||||
DateTime Expiration,
|
||||
Exchange Exchange,
|
||||
string? ContractText);
|
||||
|
||||
public class FutureSpreadContract {
|
||||
public List<Tuple<double, FutureTradableContract>> legs;
|
||||
public string symbol {
|
||||
public List<Tuple<double, FutureTradableContract>> legs = new(2);
|
||||
public string Symbol {
|
||||
get {
|
||||
return legs == null || legs.Count == 0 ? string.Empty : legs.ElementAt(0).Item2.symbol;
|
||||
return legs == null || legs.Count == 0 ? string.Empty : legs.ElementAt(0).Item2.Symbol;
|
||||
}
|
||||
}
|
||||
public string contract_text;
|
||||
}
|
||||
public class FutureContinousContract {
|
||||
public string symbol;
|
||||
public Exchange exchange;
|
||||
public string? ContractText;
|
||||
}
|
||||
|
||||
public record FutureContinousContract(string Symbol, Exchange Exchange);
|
||||
|
||||
// this is for all future conytracts belonging to a symbol
|
||||
public class FutureWildcardContract {
|
||||
public string symbol;
|
||||
public Exchange exchange;
|
||||
}
|
||||
public record FutureWildcardContract(string Symbol, Exchange Exchange);
|
||||
|
||||
public class FutureContract {
|
||||
public FutureWildcardContract wildcard_contract = null;
|
||||
public FutureTradableContract future_tradable_contract = null;
|
||||
public FutureSpreadContract spread_tradable_contract = null;
|
||||
public FutureContinousContract continous_constract = null;
|
||||
public FutureWildcardContract? wildcard_contract;
|
||||
public FutureTradableContract? future_tradable_contract;
|
||||
public FutureSpreadContract? spread_tradable_contract;
|
||||
public FutureContinousContract? continous_constract;
|
||||
|
||||
public FutureContract(string txt) {
|
||||
wildcard_contract = parse_wildcard_contract(txt);
|
||||
if (wildcard_contract != null)
|
||||
return;
|
||||
FutureTradableContract fc = parse_future_contract(txt);
|
||||
var fc = parse_future_contract(txt);
|
||||
if (fc != null) {
|
||||
future_tradable_contract = fc;
|
||||
return;
|
||||
}
|
||||
FutureSpreadContract sc = parse_future_spread_contract(txt);
|
||||
var sc = parse_future_spread_contract(txt);
|
||||
if (sc != null) {
|
||||
spread_tradable_contract = sc;
|
||||
return;
|
||||
}
|
||||
// this is not totally true, since ParseFutureSpreadContract does not
|
||||
// parse ICS spreads
|
||||
continous_constract = new FutureContinousContract() { symbol = txt };
|
||||
continous_constract = new(txt, Exchange.CME);
|
||||
}
|
||||
|
||||
[MemberNotNullWhen(true, nameof(continous_constract))]
|
||||
public bool is_continous_contract => continous_constract != null;
|
||||
|
||||
[MemberNotNullWhen(true, nameof(wildcard_contract))]
|
||||
public bool is_wildcard_contract => wildcard_contract != null;
|
||||
|
||||
[MemberNotNullWhen(true, nameof(future_tradable_contract))]
|
||||
public bool is_future_tradable_contract => future_tradable_contract != null;
|
||||
|
||||
[MemberNotNullWhen(true, nameof(spread_tradable_contract))]
|
||||
public bool is_spread_tradable_contract => spread_tradable_contract != null;
|
||||
|
||||
public string symbol() {
|
||||
if (wildcard_contract != null)
|
||||
return wildcard_contract.symbol;
|
||||
return wildcard_contract.Symbol;
|
||||
if (continous_constract != null) {
|
||||
return continous_constract.symbol;
|
||||
return continous_constract.Symbol;
|
||||
} else if (future_tradable_contract != null) {
|
||||
return future_tradable_contract.symbol;
|
||||
return future_tradable_contract.Symbol;
|
||||
} else if (spread_tradable_contract != null) {
|
||||
return spread_tradable_contract.symbol;
|
||||
return spread_tradable_contract.Symbol;
|
||||
}
|
||||
Log.assert(false);
|
||||
return string.Empty;
|
||||
}
|
||||
public string contract_text {
|
||||
public string? contract_text {
|
||||
get {
|
||||
if (future_tradable_contract != null)
|
||||
return future_tradable_contract.contract_text;
|
||||
return future_tradable_contract.ContractText;
|
||||
if (spread_tradable_contract != null)
|
||||
return spread_tradable_contract.contract_text;
|
||||
return spread_tradable_contract.ContractText;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
#region Parsing
|
||||
public static FutureWildcardContract parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) {
|
||||
public static FutureWildcardContract? parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) {
|
||||
if (contract_txt.EndsWith("*")) {
|
||||
contract_txt = contract_txt.Replace("*", string.Empty);
|
||||
return new FutureWildcardContract() {
|
||||
symbol = contract_txt,
|
||||
exchange = exchange
|
||||
};
|
||||
return new FutureWildcardContract(contract_txt, exchange);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
public static FutureTradableContract parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) {
|
||||
public static FutureTradableContract? parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) {
|
||||
// Validate the future string length
|
||||
if (!contract_txt.All(c => char.IsLetterOrDigit(c)))
|
||||
return null;
|
||||
@@ -143,17 +142,10 @@ namespace bts.BlueOrb {
|
||||
|
||||
// Create the FutureContract instance and set its properties
|
||||
DateTime expiration = new DateTime(year, month, 1);
|
||||
var futureContract = new FutureTradableContract {
|
||||
symbol = symbol,
|
||||
expiration = expiration,
|
||||
exchange = exchange,
|
||||
contract_text = contract_txt
|
||||
};
|
||||
|
||||
return futureContract;
|
||||
return new(symbol, expiration, exchange, contract_txt);
|
||||
}
|
||||
// Method to parse a futures spread, e.g. "CLX4-CLZ4"
|
||||
public static FutureSpreadContract parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) {
|
||||
public static FutureSpreadContract? parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) {
|
||||
// Split the string by '-' to handle each leg
|
||||
var spreadParts = contract_txt.Split('-');
|
||||
if (spreadParts.Length < 2) {
|
||||
@@ -201,11 +193,11 @@ namespace bts.BlueOrb {
|
||||
#endregion
|
||||
|
||||
public static ContractType get_type(string contract_txt) {
|
||||
FutureTradableContract fc = parse_future_contract(contract_txt);
|
||||
var fc = parse_future_contract(contract_txt);
|
||||
if (fc != null) {
|
||||
return ContractType.Future;
|
||||
}
|
||||
FutureSpreadContract sc = parse_future_spread_contract(contract_txt);
|
||||
var sc = parse_future_spread_contract(contract_txt);
|
||||
if (sc != null) {
|
||||
return ContractType.FutureSpread;
|
||||
}
|
||||
@@ -220,13 +212,13 @@ namespace bts.BlueOrb {
|
||||
}
|
||||
|
||||
public static Tuple<ContractCategory, string> get_symbol_category(string symbol) {
|
||||
FutureTradableContract fc = parse_future_contract(symbol);
|
||||
var fc = parse_future_contract(symbol);
|
||||
if (fc != null) {
|
||||
return Tuple.Create(ContractCategory.FutureString, fc.symbol);
|
||||
return Tuple.Create(ContractCategory.FutureString, fc.Symbol);
|
||||
}
|
||||
var sc = parse_future_spread_contract(symbol);
|
||||
if (sc != null) {
|
||||
return Tuple.Create(ContractCategory.FutureString, sc.symbol);
|
||||
return Tuple.Create(ContractCategory.FutureString, sc.Symbol);
|
||||
}
|
||||
return Tuple.Create(ContractCategory.Symbol, symbol);
|
||||
}
|
||||
|
||||
+13
-10
@@ -2,6 +2,7 @@ namespace bts.BlueOrb {
|
||||
using bts.SilverPine;
|
||||
using BCG;
|
||||
|
||||
using Bts.DataStreamExpressions;
|
||||
|
||||
public class DateTimeRange {
|
||||
public readonly DateTime range_beg;
|
||||
@@ -18,7 +19,7 @@ namespace bts.BlueOrb {
|
||||
}
|
||||
|
||||
public class FutureBarsLoader {
|
||||
public static string get_db_table_name(BarSize bar_size, string symbol, bool is_continous) {
|
||||
public static string? get_db_table_name(BarSize bar_size, string symbol, bool is_continous) {
|
||||
switch (bar_size) {
|
||||
case BarSize.ONE_DAY: {
|
||||
if (is_continous) {
|
||||
@@ -62,7 +63,8 @@ namespace bts.BlueOrb {
|
||||
beg = beg.Date;
|
||||
end = end.Date.Add(new TimeSpan(23, 59, 59));
|
||||
// this returns rows from beg and end inclusive
|
||||
Func<string, SQLiteStream> get_sqlite_str = (table_name) => {
|
||||
Func<string?, SQLiteStream> get_sqlite_str = (table_name) => {
|
||||
ArgumentException.ThrowIfNullOrEmpty(table_name);
|
||||
string range_beg_iso8601 = beg.ToString("yyyy-MM-ddTHH:mm:ss"),
|
||||
range_end_iso8601 = end.ToString("yyyy-MM-ddTHH:mm:ss");
|
||||
string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
|
||||
@@ -70,27 +72,27 @@ namespace bts.BlueOrb {
|
||||
};
|
||||
string symbol = future_contract.symbol();
|
||||
if (future_contract.wildcard_contract != null) {
|
||||
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
|
||||
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
|
||||
ViewStream rs = get_sqlite_str(table_name);
|
||||
//rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
|
||||
mySetRealSource(rs);
|
||||
return;
|
||||
}
|
||||
if (future_contract.wildcard_contract != null) {
|
||||
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
|
||||
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
|
||||
ViewStream rs = get_sqlite_str(table_name);
|
||||
//rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
|
||||
mySetRealSource(rs);
|
||||
return;
|
||||
}
|
||||
if (future_contract.is_continous_contract) {
|
||||
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true);
|
||||
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true);
|
||||
ViewStream rs = get_sqlite_str(table_name);
|
||||
//rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
|
||||
mySetRealSource(rs);
|
||||
return;
|
||||
} else {
|
||||
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
|
||||
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
|
||||
ViewStream rs = get_sqlite_str(table_name);
|
||||
//rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
|
||||
//TODO: filter it
|
||||
@@ -115,7 +117,8 @@ namespace bts.BlueOrb {
|
||||
// it gets the raw table with all contracts for that symbol
|
||||
public FuturesHistorical1MinuteBarsStream(ContractUtils.FutureContract future_contract,
|
||||
DateTimeRange dt_range) {
|
||||
Func<string, SQLiteStream> get_sqlite_str = (table_name) => {
|
||||
Func<string?, SQLiteStream> get_sqlite_str = (table_name) => {
|
||||
ArgumentException.ThrowIfNullOrEmpty(table_name);
|
||||
string range_beg_iso8601 = dt_range.range_beg.ToString("yyyy-MM-ddTHH:mm:ss"),
|
||||
range_end_iso8601 = dt_range.range_end.ToString("yyyy-MM-ddTHH:mm:ss");
|
||||
//string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'";
|
||||
@@ -124,18 +127,18 @@ namespace bts.BlueOrb {
|
||||
};
|
||||
string symbol = future_contract.symbol();
|
||||
if (future_contract.wildcard_contract != null) {
|
||||
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
|
||||
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
|
||||
ViewStream rs = get_sqlite_str(table_name);
|
||||
mySetRealSource(rs);
|
||||
return;
|
||||
}
|
||||
if (future_contract.is_continous_contract) {
|
||||
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true);
|
||||
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true);
|
||||
ViewStream rs = get_sqlite_str(table_name);
|
||||
mySetRealSource(rs);
|
||||
return;
|
||||
} else {
|
||||
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
|
||||
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
|
||||
ViewStream rs = get_sqlite_str(table_name);
|
||||
// TODO: filter
|
||||
rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'");
|
||||
|
||||
+4
-6
@@ -1,6 +1,4 @@
|
||||
namespace bts.BlueOrb {
|
||||
using System;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts.api;
|
||||
@@ -24,23 +22,23 @@
|
||||
// wild card
|
||||
if (fc.is_wildcard_contract) {
|
||||
var wc_contr = fc.wildcard_contract;
|
||||
ViewStream rs = new bts.BlueOrb.FilterStream(parent_str, $"symbol='{wc_contr.symbol}'");
|
||||
ViewStream rs = new FilterStream(parent_str, $"symbol='{wc_contr.Symbol}'");
|
||||
rs = new bts.BlueOrb.FilterStream(parent_str, $"sec_type='{SecurityType.Future}' or sec_type = '{SecurityType.Spread}'");
|
||||
mySetRealSource(rs);
|
||||
return;
|
||||
} else if (fc.is_future_tradable_contract) {
|
||||
var fut_contr = fc.future_tradable_contract;
|
||||
ViewStream rs = new bts.BlueOrb.FilterStream(parent_str, $"sec='{fut_contr.contract_text}'");
|
||||
ViewStream rs = new FilterStream(parent_str, $"sec='{fut_contr.ContractText}'");
|
||||
mySetRealSource(rs);
|
||||
return;
|
||||
} else if (fc.is_spread_tradable_contract) {
|
||||
var spr_contr = fc.future_tradable_contract;
|
||||
ViewStream rs = new bts.BlueOrb.FilterStream(parent_str, $"sec='{spr_contr.contract_text}'");
|
||||
ViewStream rs = new FilterStream(parent_str, $"sec='{spr_contr.ContractText}'");
|
||||
mySetRealSource(rs);
|
||||
return;
|
||||
} else if (fc.is_continous_contract) {
|
||||
var cont_contr = fc.continous_constract;
|
||||
ViewStream rs = new bts.BlueOrb.FilterStream(parent_str, $"symbol='{cont_contr.symbol}' and sec_type='{SecurityType.Future}'");
|
||||
ViewStream rs = new FilterStream(parent_str, $"symbol='{cont_contr.Symbol}' and sec_type='{SecurityType.Future}'");
|
||||
rs = new bts.BlueOrb.GroupByStream(rs, () => { return new string[] { "symbol" }; },
|
||||
(group_rows) => {
|
||||
IBlueRow r = group_rows.ToList().OrderBy(r => r.get_datetime("expiration")).ElementAt(0);
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
namespace bts.BlueOrb {
|
||||
using System;
|
||||
|
||||
using BCG;
|
||||
|
||||
using bts.api;
|
||||
@@ -10,7 +8,7 @@
|
||||
public LiveTradeStream(string target)
|
||||
: this(SecurityType.Future, target) { }
|
||||
public LiveTradeStream(SecurityType sec_type, string target) {
|
||||
ViewStream rs = null;
|
||||
ViewStream? rs = null;
|
||||
switch (sec_type) {
|
||||
case SecurityType.Future:
|
||||
rs = new LiveFuturesTradeStream(target);
|
||||
|
||||
+13
-11
@@ -1,11 +1,11 @@
|
||||
namespace bts.BlueOrb {
|
||||
using BCG;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using BCG;
|
||||
|
||||
[DataStreamUserRespresentation("ReplayFutureLiveBBOStream")]
|
||||
public class ReplayFutureLiveBBOStream : ViewStream, BCG.Utilities.General.IWorker{
|
||||
@@ -14,13 +14,13 @@
|
||||
public DateTime last_firetime_clock = DateTime.MinValue;
|
||||
}
|
||||
private class RowsWithPointer {
|
||||
public List<IBlueRow> rows;
|
||||
public List<IBlueRow>? rows;
|
||||
public int pointer;
|
||||
}
|
||||
|
||||
|
||||
private readonly ViewStreamDatabase _vsd;
|
||||
private readonly ContractUtils.FutureContinousContract _fcc;
|
||||
readonly ViewStreamDatabase? _vsd;
|
||||
readonly ContractUtils.FutureContinousContract? _fcc;
|
||||
private readonly LastFireTime _last_fire_time = new LastFireTime();
|
||||
private RowsWithPointer _rows = new RowsWithPointer();
|
||||
|
||||
@@ -62,20 +62,22 @@
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
var file_path = Path.Combine(SQLiteHelper.location, $"mbp1_{_fcc.symbol}_small.csv".ToLower());
|
||||
Log.notnull(_fcc);
|
||||
var file_path = Path.Combine(SQLiteHelper.location, $"mbp1_{_fcc.Symbol}_small.csv".ToLower());
|
||||
var rows = BCG.CSVReaderStream.LoadFile(file_path);
|
||||
_rows.rows = new(rows.Count);
|
||||
_rows.rows.AddRange(rows.OfType<IBlueRow>().OrderBy(r => r.get<string>("ts_event")));
|
||||
_rows.pointer = 0;
|
||||
_vsd.addWorker(this);
|
||||
ExceptionAssert.NotNull(_vsd).addWorker(this);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
_vsd.removeWorker(this);
|
||||
ExceptionAssert.NotNull(_vsd).removeWorker(this);
|
||||
}
|
||||
|
||||
public void doWork() {
|
||||
IBlueRow cand_row = _rows.rows[_rows.pointer];
|
||||
var rows = ExceptionAssert.NotNull(_rows.rows);
|
||||
IBlueRow cand_row = rows[_rows.pointer];
|
||||
string cand_ts_event = cand_row.get<string>("ts_event");
|
||||
DateTime cand_ts_event_dt = DateTime.Parse(cand_ts_event, null, System.Globalization.DateTimeStyles.RoundtripKind);
|
||||
TimeSpan cand_ts_span_file = cand_ts_event_dt - _last_fire_time.last_fireitme_file;
|
||||
|
||||
+15
-12
@@ -1,11 +1,11 @@
|
||||
namespace bts.BlueOrb {
|
||||
using BCG;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using BCG;
|
||||
|
||||
[DataStreamUserRespresentation("ReplayFutureLiveTradeStream")]
|
||||
public class ReplayFutureLiveTradeStream : ViewStream, BCG.Utilities.General.IWorker {
|
||||
@@ -14,15 +14,15 @@
|
||||
public DateTime last_firetime_clock = DateTime.MinValue;
|
||||
}
|
||||
private class RowsWithPointer {
|
||||
public List<IBlueRow> rows;
|
||||
public List<IBlueRow>? rows;
|
||||
public int pointer;
|
||||
}
|
||||
|
||||
|
||||
private readonly ViewStreamDatabase _vsd;
|
||||
private readonly ContractUtils.FutureContinousContract _fcc;
|
||||
readonly ViewStreamDatabase? _vsd;
|
||||
readonly ContractUtils.FutureContinousContract? _fcc;
|
||||
private readonly LastFireTime _last_fire_time = new LastFireTime();
|
||||
private RowsWithPointer _rows = new RowsWithPointer();
|
||||
readonly RowsWithPointer _rows = new();
|
||||
|
||||
public ReplayFutureLiveTradeStream(string continous_contract) {
|
||||
Func<string> iso_8601_now = () => {
|
||||
@@ -61,21 +61,24 @@
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
var file_path = Path.Combine(SQLiteHelper.location, $"trades_{_fcc.symbol}.csv");
|
||||
Log.notnull(_vsd);
|
||||
Log.notnull(_fcc);
|
||||
var file_path = Path.Combine(SQLiteHelper.location, $"trades_{_fcc.Symbol}.csv");
|
||||
var csv_rd = new CSVReaderStream(file_path, false);
|
||||
var rows = BCG.CSVReaderStream.LoadFile(file_path);
|
||||
_rows.rows = new(rows.Count);
|
||||
_rows.rows.AddRange(rows.OfType<IBlueRow>().OrderBy(r => r.get<string>("ts_event")));
|
||||
_rows.pointer = 0;
|
||||
_vsd.addWorker(this);
|
||||
ExceptionAssert.NotNull(_vsd).addWorker(this);
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
_vsd.removeWorker(this);
|
||||
ExceptionAssert.NotNull(_vsd).removeWorker(this);
|
||||
}
|
||||
|
||||
public void doWork() {
|
||||
IBlueRow cand_row = _rows.rows[_rows.pointer];
|
||||
var rows = ExceptionAssert.NotNull(_rows.rows);
|
||||
IBlueRow cand_row = rows[_rows.pointer];
|
||||
string cand_ts_event = cand_row.get<string>("ts_event");
|
||||
DateTime cand_ts_event_dt = DateTime.Parse(cand_ts_event, null, System.Globalization.DateTimeStyles.RoundtripKind);
|
||||
TimeSpan cand_ts_span_file = cand_ts_event_dt - _last_fire_time.last_fireitme_file;
|
||||
|
||||
@@ -8,7 +8,7 @@ namespace bts {
|
||||
using System.Text.Json.Nodes;
|
||||
[DataStreamUserRespresentation("benzingaNews")]
|
||||
public class BenzingaNewsStream : ViewStream {
|
||||
private IBlueTableSchema _schema = null;
|
||||
IBlueTableSchema? _schema;
|
||||
private int _id = 0;
|
||||
|
||||
|
||||
@@ -63,14 +63,14 @@ namespace bts {
|
||||
}
|
||||
foreach (JsonNode jnode in json_objs) {
|
||||
if (jnode is JsonObject json_object) {
|
||||
string symbol = jnode["symbol"] == null ? String.Empty : jnode["symbol"].ToString();
|
||||
string exchange = jnode["exchange"] == null ? String.Empty : jnode["exchange"].ToString();
|
||||
string symbol = jnode["symbol"]?.ToString() ?? string.Empty;
|
||||
string exchange = jnode["exchange"]?.ToString() ?? string.Empty;
|
||||
MutableBlueRow nr = new MutableBlueRow(_schema, r);
|
||||
nr.setValue("id", _id++);
|
||||
nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
|
||||
nr.setValue("doc_id_internal", $"{r["doc_id"]}_{symbol}");
|
||||
nr.setValue("symbol", symbol);
|
||||
nr.setValue("exchange", exchange);
|
||||
nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
|
||||
nr.setValue("is_primary_exchange", jnode["primary"]?.GetValue<bool>() ?? false);
|
||||
expanded_rows.Add(nr);
|
||||
}
|
||||
}
|
||||
@@ -84,4 +84,4 @@ namespace bts {
|
||||
mySetRealSource(output_stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loaded 100 of 118 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user