diff --git a/src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs index a10bc8236..093b32522 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/FilterStream.cs @@ -1,5 +1,8 @@ namespace bts.BlueOrb { using BCG; + + using Bts.DataStreamCore; + using System; using System.Text.RegularExpressions; @@ -21,22 +24,24 @@ namespace bts.BlueOrb { #region IStreamFilter Members public bool filter(IBlueRow dr) { - bool rv = true; for (int i = 0; i < myCols.Length; i++) { - rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString()); - if (!rv) - return rv; + var val = dr[myCols[i]]; + var str = val is DBNull + ? string.Empty + : (val?.ToString() ?? string.Empty); + if (!myRegExps[i].IsMatch(str)) { + return false; + } } - return rv; + return true; } #endregion } - private BlueTable myLastFiredRows; - private IBlueRowCollection myLastReceivedRows; + BlueTable? myLastFiredRows; + IBlueRowCollection? myLastReceivedRows; private IStreamFilter myFilter; private ViewStream myParentStream; private string myFilterString; - private bool myUseNewFilter = false;// Configuration.get().UseNewFilter; #region constructors public delegate bool OnFilter(IBlueRow r); @@ -52,12 +57,11 @@ namespace bts.BlueOrb { public FilterStream(ViewStream vs, IStreamFilter filter) { - { - myParentStream = vs; - myFilter = filter; - myFilterString = filter.ToString(); - if (filter is IDynamicStreamFilter) - (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange); + myParentStream = vs; + myFilter = filter; + myFilterString = filter.ToString() ?? string.Empty; + if (filter is IDynamicStreamFilter dynFilter) { + dynFilter.onFilterChange += myHandleFilterChange; } } @@ -99,16 +103,20 @@ namespace bts.BlueOrb { if (myLastFiredRows != null) { retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema), new BlueTable(myLastFiredRows.schema)); - foreach (IBlueRow r in arrivedPacket.deletedRows) - if (myLastFiredRows.containsID((int)r["id"])) { + var deletedRows = new Lazy(() => (BlueTable)retVal.deletedRows); + foreach (IBlueRow r in arrivedPacket.deletedRows) { + if (myLastFiredRows.containsID(r.Id())) { myLastFiredRows.deleteRow(r); - (retVal.deletedRows as BlueTable).addRow(r); + deletedRows.Value.addRow(r); } - foreach (IBlueRow r in arrivedPacket.addedRows) + } + var addedRows = new Lazy(() => (BlueTable)retVal.addedRows); + foreach (IBlueRow r in arrivedPacket.addedRows) { if (myFilter.filter(r)) { myLastFiredRows.addRow(r); - (retVal.addedRows as BlueTable).addRow(r); + addedRows.Value.addRow(r); } + } } else { myLastFiredRows = new BlueTable(arrivedPacket.Schema); retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema)); diff --git a/src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs index bbc4de7fd..7a1fb7c4e 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/GroupByStream.cs @@ -1,12 +1,13 @@ -using System; +using BCG; + +using Bts.DataStreamCore; + +using Spark.Core.Diagnostics; + +using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; -using System.Text.Json; - -using bts; -//using bts.SilverPine; -using BCG; namespace bts.BlueOrb { [DataStreamUserRespresentation("GroupByStream")] @@ -14,20 +15,21 @@ namespace bts.BlueOrb { public interface INonIncGroupByEvaluator { // grouped_rows - IBlueRowCollection evaluate(IDictionary> grouped_rows); - string[] get_group_by_cols(); + IBlueRowCollection? evaluate(IDictionary> grouped_rows); + string[]? get_group_by_cols(); } private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator { - Func _groupby_cols_func; + Func _groupby_cols_func; Func, List> _evaluate_func; - public DelNonIncGroupByEvaluator(Func groupby_cols_func, + public DelNonIncGroupByEvaluator(Func groupby_cols_func, Func, List> evaluate_func) { _groupby_cols_func = groupby_cols_func; _evaluate_func = evaluate_func; } - public IBlueRowCollection evaluate(IDictionary> grouped_rows) { - BlueTable rv = null; + + public IBlueRowCollection? evaluate(IDictionary> grouped_rows) { + BlueTable? rv = null; foreach (var kv in grouped_rows) { List g_rows = kv.Value; List out_rows = _evaluate_func.Invoke(g_rows); @@ -39,17 +41,19 @@ namespace bts.BlueOrb { } return rv; } - public string[] get_group_by_cols() { + + public string[]? get_group_by_cols() { return _groupby_cols_func(); } } - private ViewStream _main_stream; - private INonIncGroupByEvaluator _groupby_func; - private BlueRowHasher _row_hasher; - private string[] _groupby_cols; + ViewStream? _main_stream; + INonIncGroupByEvaluator? _groupby_func; + BlueRowHasher? _row_hasher; + string[]? _groupby_cols; private Stopwatch _timer = new(); private int _n_total_fires = 0, _n_expensive_fires = 0; + IDisposable? subscription; public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) { _main_stream = main_stream; @@ -58,7 +62,7 @@ namespace bts.BlueOrb { _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length); } - public GroupByStream(ViewStream main_stream, Func groupby_cols_func, + public GroupByStream(ViewStream main_stream, Func groupby_cols_func, Func, List> evaluate_func) : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { } @@ -68,33 +72,41 @@ namespace bts.BlueOrb { // this is the main constructor that does everything public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) { MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops); - Func groupby_cols_func = () => { - if (String.IsNullOrEmpty(groupby_cols)) - return null; - return groupby_cols.Split(',').Select(x => x.Trim()).ToArray(); - }; Func, List> evaluate_func = (group_rows) => { if (!String.IsNullOrEmpty(optional_sort_col)) group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col)); // this si where the group by aggregation happens - IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by); + IBlueRow? rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by); if (rv == null) return new List(); return new IBlueRow[] { rv }.ToList(); }; ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func); mySetRealSource(rs); + + string[]? groupby_cols_func() { + if (string.IsNullOrEmpty(groupby_cols)) { + return null; + } + return groupby_cols.Split(',').Select(x => x.Trim()).ToArray(); + } } public GroupByStream(ViewStream parent_str, string standard_group_by_text) : this(parent_str, String.Empty, standard_group_by_text) { } + protected override void myOnFirstSubscriber() { + Log.isnull(subscription); + subscription = ExceptionAssert.MemberNotNull(_main_stream).Subscribe(_main_stream_onchange); + } - protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; } - - protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; } + protected override void myOnLastSubscriber() { + subscription?.Dispose(); + subscription = null; + } private void _main_stream_onchange(object nv) { + Log.notnull(_groupby_func); var dp = (IDataPacket)nv; ///grouping together _timer.Restart(); @@ -109,7 +121,7 @@ namespace bts.BlueOrb { // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime)); /// reducing _timer.Restart(); - IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows); + IBlueRowCollection? total_state = _groupby_func.evaluate(grouped_rows); if (total_state == null) // mayeb returna n empty table ? return; diff --git a/src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs index ed485af97..bc66a8a0b 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs @@ -1,8 +1,13 @@ namespace bts.BlueOrb { using BCG; + using Bts.DataStream; + + using Spark.Core.Diagnostics; + using System; using System.Collections.Generic; + using System.Diagnostics.CodeAnalysis; using System.Linq; using RowHash = System.Int32; @@ -11,29 +16,29 @@ public class SimpleJoinStream : ViewStream { public class JoinBookkeepingData { // initialized in cons - private readonly List _join_col_names; - private readonly bool _is_left; + readonly string[] _join_col_names; private readonly BlueRowHasher _hasher; // initialized on first udpate - private IBlueTableSchema _schema; - private int[] _join_cols; + IBlueTableSchema? _schema; + int[]? _join_cols; private IDictionary> _hash_row_map = new Dictionary>(); - public JoinBookkeepingData(ViewStream vs, List join_cols, bool is_left, BlueRowHasher row_hasher) { - _join_col_names = join_cols.Where(c => !String.IsNullOrEmpty(c)).ToList(); - _is_left = is_left; + public JoinBookkeepingData(ViewStream vs, List join_cols, BlueRowHasher row_hasher) { + _join_col_names = join_cols.Where(c => !string.IsNullOrEmpty(c)).ToArray(); _hasher = row_hasher; } - public IBlueTableSchema schema => _schema; + [NotNullIfNotNull(nameof(_schema))] + public IBlueTableSchema? schema => _schema; public IDictionary> rows => _hash_row_map; - public List get_matching_rows(int hash) { - List rv = null; - _hash_row_map.TryGetValue(hash, out rv); - return rv; - } + [MemberNotNull(nameof(_schema))] + public IBlueTableSchema RequireSchema() => + ExceptionAssert.MemberNotNull(_schema); + + public List? get_matching_rows(int hash) => + _hash_row_map.GetValueOrDefault(hash); private RowHash _get_hash(IBlueRow r) { if (_join_cols == null || _join_cols.Length == 0) { @@ -45,13 +50,12 @@ public void update_data(IDataPacket dp) { if (!ready) { _schema = dp.newRows.schema; - _join_cols = _join_col_names.Count == 0 ? new int[] { } : _join_col_names.Select(n => _schema[n].colNo).ToArray(); + _join_cols = _schema.ResolveIndices(_join_col_names); } _hash_row_map.Clear(); foreach (IBlueRow r in dp.newRows) { int hash = _get_hash(r); - List hash_rows = null; - if (!_hash_row_map.TryGetValue(hash, out hash_rows)) { + if (!_hash_row_map.TryGetValue(hash, out var hash_rows)) { hash_rows = new List(); _hash_row_map.Add(hash, hash_rows); } @@ -59,6 +63,7 @@ } } + [MemberNotNullWhen(true, nameof(_schema))] public bool ready { get { return _schema != null; @@ -69,8 +74,7 @@ private ViewStream _ls, _rs; private JoinBookkeepingData _ldata, _rdata; - private ViewStreamDatabase _vsd; - private IBlueTableSchema _schema = null; + IBlueTableSchema? _schema; public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) : this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { } @@ -83,8 +87,8 @@ Log.assert(lj_col_list.Count == rj_col_list.Count); int n_cols = lj_col_list.Count; BlueRowHasher hasher = new BlueRowHasher(n_cols); - _ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher); - _rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher); + _ldata = new JoinBookkeepingData(ls, lj_col_list, hasher); + _rdata = new JoinBookkeepingData(rs, rj_col_list, hasher); } protected override void myOnFirstSubscriber() { @@ -109,20 +113,24 @@ _rs.onChange -= _rs_onchange; } + [MemberNotNull(nameof(_schema))] private void _build_schema() { if (_schema != null) return; - List cis = new List(); - cis.Add(new ColumnInfo("id", 0, TypeCode.Int32)); - foreach (ColumnInfo ci in _ldata.schema.columnInfo) { - string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName; + var lschema = Log.notnull(_ldata.schema); + var rschema = Log.notnull(_rdata.schema); + var cis = new List { + new("id", 0, TypeCode.Int32) + }; + foreach (ColumnInfo ci in lschema.columnInfo) { + string col_name = rschema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName; cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType)); } - foreach (ColumnInfo ci in _rdata.schema.columnInfo) { - string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName; + foreach (ColumnInfo ci in rschema.columnInfo) { + string col_name = lschema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName; cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType)); } - _schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray()); + _schema = new BlueTableSchema($"{lschema.tableName}_join_{rschema.tableName}", cis.ToArray()); } private void _form_and_fire() { @@ -131,34 +139,11 @@ _build_schema(); List res_rows = new List(); - Action copy_row = (rval, r, is_left) => { - int offset = 1; - JoinBookkeepingData sdata = _ldata; - if (is_left) { - offset = 1; - sdata = _ldata; - } else { - sdata = _rdata; - offset = 1 + _ldata.schema.columnInfo.Length; - } - for (int i = 0; i < sdata.schema.columnInfo.Length; i++) { - rval[offset + i] = r[i]; - } - }; - Func build_row_array = (id, l_row) => { - object[] rv = new object[_schema.columnNames.Length]; - for (int i = 0; i < _schema.columnNames.Length; i++) { - rv[i] = DBNull.Value; - } - rv[0] = id; - copy_row(rv, l_row, true); - return rv; - }; // this is where the join happens foreach (var kv in _ldata.rows) { RowHash hash = kv.Key; List l_rows = kv.Value; - List r_rows = _rdata.get_matching_rows(hash); + List? r_rows = _rdata.get_matching_rows(hash); foreach (IBlueRow lr in l_rows) { if (r_rows != null) { @@ -179,6 +164,31 @@ } var dp = BlueDataUtility.build_dp(rv, myLastValue as IDataPacket); myFireEvent(dp); + + object[] build_row_array(int id, IBlueRow l_row) { + object[] rv = new object[_schema.columnNames.Length]; + for (int i = 0; i < _schema.columnNames.Length; i++) { + rv[i] = DBNull.Value; + } + rv[0] = id; + copy_row(rv, l_row, true); + return rv; + } + + void copy_row(object?[] rval, IBlueRow r, bool is_left) { + int offset = 1; + JoinBookkeepingData sdata = _ldata; + if (is_left) { + offset = 1; + sdata = _ldata; + } else { + sdata = _rdata; + offset = 1 + _ldata.RequireSchema().columnInfo.Length; + } + for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++) { + rval[offset + i] = r[i]; + } + } } } diff --git a/src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs b/src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs index 2b0b70d16..81e5052e0 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/MultiRowOpHelper.cs @@ -45,8 +45,8 @@ namespace bts.BlueOrb { } // aggregates multiple rows into one - ie does group_by compute - public static IBlueRow group_by_compute(List rows, MultiRowOpHelper parameters) { - if (rows.Count() == 0) { + public static IBlueRow? group_by_compute(List rows, MultiRowOpHelper parameters) { + if (rows.Count == 0) { return null; } @@ -73,8 +73,8 @@ namespace bts.BlueOrb { } // aggregates multiple rows into one, used by MultiRowRollingMathStream - public static IBlueRow group_by_compute(List rows, int start_idx, int n_rows, MultiRowOpHelper parameters) { - if (rows.Count() == 0) { + public static IBlueRow? group_by_compute(List rows, int start_idx, int n_rows, MultiRowOpHelper parameters) { + if (rows.Count == 0) { return null; } IBlueRow ir = rows.ElementAt(start_idx); @@ -104,8 +104,8 @@ namespace bts.BlueOrb { // rolling_rows are the rolling window of rows. the 0th row is the one thats gettin // calculated. rows 1 - n-1 are its window rows. the schema is based on the operations. - public IBlueRow rolling_calc(List rolling_rows) { - if (rolling_rows.Count() == 0) { + public IBlueRow? rolling_calc(List rolling_rows) { + if (rolling_rows.Count == 0) { return null; } IBlueRow ir = rolling_rows.ElementAt(0); @@ -156,7 +156,6 @@ namespace bts.BlueOrb { double first = col_vals.First(); double last = col_vals.Last(); return ((first - last) / last) * 100.0; - return col_vals.First() - col_vals.Last(); } default: // TODO diff --git a/src/DataStreamSuite/DataStream/BlueOrb/MultiRowRollingMathStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/MultiRowRollingMathStream.cs index 378a7b155..56d236b07 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/MultiRowRollingMathStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/MultiRowRollingMathStream.cs @@ -1,8 +1,7 @@ namespace bts.BlueOrb { using BCG; - using System.Collections.Generic; - using System.Net; + using System; [DataStreamUserRespresentation("MultiRowRollingMathStream")] public class MultiRowRollingMathStream : ViewStream { @@ -26,9 +25,10 @@ private void _parent_str_onchange(object dp_obj) { var dp = (IDataPacket)dp_obj; var rows = dp.newRows.to_list(); - BlueTable out_table = null; + BlueTable? out_table = null; for (int i = 0; i < rows.Count; i++) { - var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op); + var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op) + ?? throw new InvalidOperationException("group_by_compute returned null row"); if (out_table == null) out_table = new BlueTable(out_r.schema); out_table.addRow(out_r); diff --git a/src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs b/src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs index 1e23ea149..a1dde5cc4 100644 --- a/src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs +++ b/src/DataStreamSuite/DataStream/BlueOrb/ProjectStream.cs @@ -1,5 +1,8 @@ namespace bts.BlueOrb { using BCG; + + using Spark.Core.Diagnostics; + using System; using System.Collections; using System.Collections.Generic; @@ -54,7 +57,7 @@ namespace bts.BlueOrb { public bool IsReadOnly { get { return true; } } - public object this[object key] { + public object? this[object key] { get { if (key is string) return this[(string)key]; @@ -63,16 +66,15 @@ namespace bts.BlueOrb { set { throw new NotSupportedException("The IBlueRow is readonly"); } } - public object this[int index] { + public object? this[int index] { get { return this[schema.columnInfo[index].colName]; } } - public object this[string col_name] { + public object? this[string col_name] { get { - string mapped_to_col = null; - if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) { + if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out var mapped_to_col)) { return _row[mapped_to_col]; } return _row[col_name]; @@ -89,7 +91,8 @@ namespace bts.BlueOrb { } } - public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); } + public void Add(object key, object? value) => + throw new NotSupportedException(); public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); } @@ -114,7 +117,7 @@ namespace bts.BlueOrb { private ViewStream _stream; private string _project_expression; - private ProjectData _project_data = null; + ProjectData? _project_data; //public ProjectData public NewProjectStream(ViewStream vs, string project_expression) { _stream = vs; @@ -128,9 +131,8 @@ namespace bts.BlueOrb { return _project_data.out_schema; } - public IBlueRow map(IBlueRow row) { - return new NewProjectRow(_project_data, row); - } + public IBlueRow map(IBlueRow row) => + new NewProjectRow(ExceptionAssert.MemberNotNull(_project_data), row); // this is where most of the logic is, in terms of translating cols private static ProjectData _create_project_data(IBlueTableSchema input_schema, diff --git a/src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs b/src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs index 51e612d75..f2915f592 100644 --- a/src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs +++ b/src/DataStreamSuite/DataStream/Offloading/RemoteRequest.cs @@ -2,9 +2,6 @@ namespace bts { using System.IO; using BCG; - using System.IO; - using System.Threading.Tasks.Dataflow; - public class RemoteRequest { public readonly string stream_expression; public readonly string published_table_name; @@ -28,4 +25,4 @@ namespace bts { } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs index 93dfc89f5..98d767a6e 100644 --- a/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/AddColumns/AddColumnsStream.cs @@ -1,33 +1,28 @@ +using Spark.Core.Collections; + using System; using System.Collections.Generic; +using System.Linq; namespace BCG { public class AddColumnsStream : ViewStream { - public static Tuple parse_single_tok(string exp) { + public static (string Name, Type Type, object Value) parse_single_tok(string exp) { string[] toks = exp.Split(':'); string col_name = toks[0].Trim(); string col_value = toks[1].Trim(); Tuple tup = BlueDataUtility.guess_type(col_value); - return Tuple.Create(col_name, tup.Item1, tup.Item2); + return (col_name, tup.Item1, tup.Item2); } - public static List> parse_exp(string exp) { - string[] toks = exp.Split(","); - List> rv = new List>(); - foreach (string s in toks) { - var tup = parse_single_tok(s.Trim()); - rv.Add(tup); - } - return rv; - } + public static List<(string Name, Type Type, object Value)> parse_exp(string exp) => + exp.Split(",").ToList(t => parse_single_tok(t.Trim())); - private Tuple _new_col; - private List> _new_col_map; - private IBlueTableSchema _out_schema; + readonly List<(string Name, Type Type, object Value)> _new_col_map; + IBlueTableSchema? _out_schema; public AddColumnsStream(ViewStream vs, string col_def) { _new_col_map = parse_exp(col_def); - ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r)); + ViewStream rs = new MapStream(vs, _get_schema, _map); mySetRealSource(rs); } @@ -44,7 +39,7 @@ namespace BCG { private IBlueRow _map(IBlueRow r) { MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r); foreach (var m in _new_col_map) { - rv.setValue(m.Item1, m.Item3); + rv.setValue(m.Name, m.Value); } return rv; } diff --git a/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs b/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs index b62281538..c3face083 100644 --- a/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/AxisTransposition/YXTranspositionStream.cs @@ -1,9 +1,10 @@ -using System; +using mbts; + +using Spark.Core.Diagnostics; + +using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; -using mbts; namespace BCG { @@ -42,8 +43,8 @@ namespace BCG } } - public delegate void IncrementValues(object[] current_values, object[] values_to_inc); - public delegate string GetColumnPrefix(object[] transposing_col_values); + public delegate void IncrementValues(object?[] current_values, object?[] values_to_inc); + public delegate string GetColumnPrefix(object?[] transposing_col_values); public delegate string GetColumnSuffix(string col_name); private GetColumnSuffix _get_column_suffix = new GetColumnSuffix((cn) => { return cn; }); @@ -51,33 +52,31 @@ namespace BCG private IncrementValues _increment_values; private GetColumnPrefix _get_column_prefix; - private IBlueTableSchema _schema; + IBlueTableSchema? _schema; private string[] _group_by_cols; private Pair[] _cols_per_transposed; private string[] _transposing_cols; - private int[] _transposing_col_indices; + int[]? _transposing_col_indices; - private Comparison> _transpsed_col_value_comparer_wrapper; + Comparison>? _transpsed_col_value_comparer_wrapper; public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols, - IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison transposed_col_value_comparer) + IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison transposed_col_value_comparer) : this(stream, transposed_cols_group, cols_per_transposed, group_by_cols, increment_values_del, get_column_prefix_del, get_column_suffix_del, transposed_col_value_comparer, true) { } public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols, - IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison transposed_col_value_comparer, bool only_fire_once) + IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison transposed_col_value_comparer, bool only_fire_once) { if (get_column_suffix_del != null) _get_column_suffix = get_column_suffix_del; if (transposed_col_value_comparer != null) { - _transpsed_col_value_comparer_wrapper = new Comparison>((p1,p2) => - { - return transposed_col_value_comparer(p1.second, p2.second); - }); + _transpsed_col_value_comparer_wrapper = new((p1, p2) => + transposed_col_value_comparer(ExceptionAssert.NotNull(p1.second), ExceptionAssert.NotNull(p2.second))); } _group_by_cols = group_by_cols.Split(' '); _transposing_cols = transposed_cols_group.Split(' '); @@ -106,7 +105,7 @@ namespace BCG col_info.Add(new ColumnInfo(p.first, col_index++, p.second)); foreach (string tc_prefix in tc_prefixes) foreach (Pair p in _cols_per_transposed) - col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first), col_index++, p.second)); + col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, p.second)); IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name); if (_schema == null) _schema = schema; @@ -130,19 +129,14 @@ namespace BCG public IBlueRowCollection evaluate(IDictionary> grouped_rows) { - Dictionary> all_tc_infos_and_values = new Dictionary>(); - Dictionary group_by_cols_value_dict = new Dictionary(); + var all_tc_infos_and_values = new Dictionary>(); + var group_by_cols_value_dict = new Dictionary(); BlueRowHasher hasher = new BlueRowHasher(_transposing_cols.Length); Pair[] group_by_col_info = new Pair[_group_by_cols.Length]; string table_name = string.Empty; if (grouped_rows.Count > 0) { - IBlueTableSchema schema = null; - foreach (KeyValuePair> kvp in grouped_rows) - { - schema = kvp.Value[0].schema; - break; - } + var schema = grouped_rows.Values.First()[0].schema; table_name = schema.tableName; _transposing_col_indices = BlueDataUtility.get_column_indices(schema, _transposing_cols); int[] group_by_indices = BlueDataUtility.get_column_indices(schema, _group_by_cols); @@ -155,23 +149,22 @@ namespace BCG else return new BlueTable(new BlueTableSchema(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }, table_name)); Dictionary tc_prefixes = new Dictionary(); - List> tc_values = new List>(); + var tc_values = new List>(); foreach (KeyValuePair> kvp in grouped_rows) { //Dictionary of trans_col_group_hash -> pair: first=trans_col_group_values second=per_transposing_col_values - Dictionary incrementing_values = new Dictionary(); + var incrementing_values = new Dictionary(); foreach (IBlueRow r in kvp.Value) { int hash = hasher.get_hash(r, _transposing_col_indices); - object[] values; - if (!incrementing_values.TryGetValue(hash, out values)) + if (!incrementing_values.TryGetValue(hash, out var values)) { incrementing_values.Add(hash, _get_per_transposing_col_values(r)); string prefix = _get_column_prefix(_get_transposing_col_values(r)); if (!tc_prefixes.ContainsKey(hash)) { tc_prefixes.Add(hash, prefix); - tc_values.Add(new Pair(hash, _get_transposing_col_values(r))); + tc_values.Add(new(hash, _get_transposing_col_values(r))); } } else @@ -194,18 +187,18 @@ namespace BCG _set_schema(prefixes, group_by_col_info, table_name); BlueTable result = new BlueTable(_schema); int id = 1; - foreach (KeyValuePair> kvp in all_tc_infos_and_values) + foreach (var kvp in all_tc_infos_and_values) { MutableBlueRow mbr = new MutableBlueRow(_schema); mbr.setValue("id", id++); for (int i = 0; i < _group_by_cols.Length; i++) mbr.setValue(_group_by_cols[i], group_by_cols_value_dict[kvp.Key][i]); - foreach (KeyValuePair tc_cols in kvp.Value) + foreach (var tc_cols in kvp.Value) { string prefix = tc_prefixes[tc_cols.Key]; for (int i = 0; i < _cols_per_transposed.Length; i++) { - string column = _get_full_column_name(prefix, _cols_per_transposed[i].first); + string column = _get_full_column_name(prefix, _cols_per_transposed[i].first ?? string.Empty); if(mbr.schema.columnNames.Contains(column)) mbr.setValue(column, tc_cols.Value[i]); } @@ -217,28 +210,25 @@ namespace BCG #endregion - private object[] _get_transposing_col_values(IBlueRow row) - { - object[] values = new object[_transposing_cols.Length]; + object?[] _get_transposing_col_values(IBlueRow row) { + var values = new object?[_transposing_cols.Length]; for (int i = 0; i < _transposing_cols.Length; i++) values[i] = row[_transposing_cols[i]]; return values; } - private object[] _get_per_transposing_col_values(IBlueRow row) - { - object[] values = new object[_cols_per_transposed.Length]; + object?[] _get_per_transposing_col_values(IBlueRow row) { + var values = new object?[_cols_per_transposed.Length]; for (int i = 0; i < _cols_per_transposed.Length; i++) - values[i] = row[_cols_per_transposed[i].first]; + values[i] = row[ExceptionAssert.NotNull(_cols_per_transposed[i].first)]; return values; } - private object[] _get_group_by_cols_values(IBlueRow row) - { - object[] values = new object[_group_by_cols.Length]; + object?[] _get_group_by_cols_values(IBlueRow row) { + var values = new object?[_group_by_cols.Length]; for (int i = 0; i < values.Length; i++) values[i] = row[_group_by_cols[i]]; return values; } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Derivative/DerivativeStream.cs b/src/DataStreamSuite/DataStream/Primitives/Derivative/DerivativeStream.cs index 1bcaedc48..b3ec179e9 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Derivative/DerivativeStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Derivative/DerivativeStream.cs @@ -1,22 +1,19 @@ -using System; +using Spark.Core.Diagnostics; + +using System; using System.Collections.Generic; -using System.Linq; -using System.Text; namespace BCG { public class DerivativeStream : ViewStream { - - private IBlueTableSchema mySchema; + IBlueTableSchema? mySchema; private string myDerivandColumn; private string myOutputColumn; private string myKeyColumn; private DateTime myLastPacketArrival = DateTime.MinValue; private ViewStream myDataSource; - - private IDictionary myPreviousValues; - + Dictionary? myPreviousValues; public DerivativeStream(ViewStream input, string keyColumn, string columnToDifferentiate, string outputColName) { @@ -64,12 +61,13 @@ namespace BCG protected IBlueRow myMakeRow(IBlueRow r, double timeScale) { + ExceptionAssert.NotNull(mySchema, myPreviousValues); object[] outValues = new object[mySchema.columnInfo.Length]; r.Values.CopyTo(outValues, 0); - object derivand = r[myDerivandColumn]; - double lastValue = default(double); - if (myPreviousValues.TryGetValue(r[myKeyColumn].ToString(), out lastValue) && derivand != null && derivand != DBNull.Value) + var derivand = r[myDerivandColumn]; + var key = getRowKey(r); + if (myPreviousValues.TryGetValue(key, out double lastValue) && derivand != null && derivand != DBNull.Value) outValues[mySchema.columnInfo.Length - 1] = (Convert.ToDouble(derivand) - lastValue) * timeScale; else outValues[mySchema.columnInfo.Length - 1] = 0.0; @@ -90,10 +88,13 @@ namespace BCG myLastPacketArrival = DateTime.Now; foreach (IBlueRow row in dp.newRows) { - object derivand = row[myDerivandColumn]; + var derivand = row[myDerivandColumn]; if (derivand != null && derivand != DBNull.Value) - myPreviousValues[row[myKeyColumn].ToString()] = Convert.ToDouble(derivand); + myPreviousValues[getRowKey(row)] = Convert.ToDouble(derivand); } } + + string getRowKey(IBlueRow row) => + row[myKeyColumn]?.ToString() ?? throw new ArgumentException("Row key is null", nameof(row)); } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs b/src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs index b25e2aec0..f4fe4a439 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs @@ -1,7 +1,9 @@ -using System; -using System.Collections; +using mbts; + +using Spark.Core.Collections; + +using System; using System.Collections.Generic; -using mbts; namespace BCG { @@ -12,9 +14,9 @@ namespace BCG { ViewStream myLStream, myRStream; - DataPacket myLastR = null, myLastL = null; - - BlueTable myLastFired = null; + DataPacket? myLastR; + DataPacket? myLastL; + BlueTable? myLastFired; double myThreshold = .00005; @@ -24,14 +26,14 @@ namespace BCG public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) : this(null, v1, v2, thresh, ColumnsToMatch) { } - public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch) + public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch) { myThreshold = thresh; myLStream = v1; myRStream = v2; myColumnsToMatch = columnsToMatch; } - public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch) + public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, params string[] columnsToMatch) : this(db,v1,v2,.00005,columnsToMatch) { } @@ -50,32 +52,9 @@ namespace BCG myRStream.onChange -= myRChange; } - struct ListComparer : IEqualityComparer> - { - public bool Equals(List x, List y) - { - if (x.Count != y.Count) - return false; - - for (int i = 0; i < x.Count; ++i) - if (!x[i].Equals(y[i])) - return false; - return true; - } - - public int GetHashCode(List obj) - { - int ret = 0; - for(int i = 0; i < obj.Count; ++i) - ret ^= obj[i].GetHashCode(); - return ret; - } - } - void myHandleCallbacks() { - DataPacket packet = null; - + DataPacket? packet; if (myLastL == null || myLastR == null) { packet = myLastL == null ? myLastR : myLastL; @@ -88,7 +67,7 @@ namespace BCG BlueTable table = new BlueTable(myLastL.newRows.schema); - Dictionary, Pair> rowPairs = new Dictionary, Pair>(new ListComparer()); + var rowPairs = new Dictionary, Pair>(EnumerableComparer.Create()); foreach(BlueRow row in myLastL.newRows) { @@ -118,11 +97,7 @@ namespace BCG } catch (Exception) { } } - - Pair pair; - rowPairs.TryGetValue(hash, out pair); - - + rowPairs.TryGetValue(hash, out var pair); if (pair == null) pair = new Pair(null, row); else @@ -138,8 +113,7 @@ namespace BCG if (pair.first == null || pair.second == null) { - BlueRow row = pair.first == null ? pair.second : pair.first; - + var row = pair.first ?? pair.second ?? throw new InvalidOperationException("Both rows in rowPair are null"); Dictionary newRow = new Dictionary(); foreach (ColumnInfo info in row.schema.columnInfo) { @@ -211,4 +185,4 @@ namespace BCG } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/DateBasedFilter.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/DateBasedFilter.cs index dec70b4de..b92b6ff29 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Filter/DateBasedFilter.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/DateBasedFilter.cs @@ -1,8 +1,7 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using BCG.Utilities.General; +using BCG.Utilities.General; + +using System; + namespace BCG { public class DateBasedFilter: IDynamicStreamFilter, IWorker @@ -26,20 +25,14 @@ namespace BCG myReference = myTodaysDate.AddDays(-1 * myNDays); } - public event FilterChangeDelegate onFilterChange; + public event FilterChangeDelegate? onFilterChange; - public bool filter(IBlueRow r) - { - if (DBNull.Value.Equals(r[myDateColumn])) - { - //return false; - return true; //it can be considered as our assumption whenever datecolumn is db null we return true - } - DateTime t = (DateTime)r[myDateColumn]; - bool returnVallue = t < myReference ? false : true; - return returnVallue; - } - + public bool filter(IBlueRow r) => + r[myDateColumn] switch { + DBNull or null => true, + DateTime t => t >= myReference, + _ => throw new ArgumentException("Row has invalid date column value", nameof(r)), + }; public void doWork() { @@ -47,7 +40,7 @@ namespace BCG { myTodaysDate = DateTime.Today; myReference = myTodaysDate.AddDays(-1 * myNDays); - onFilterChange(); + onFilterChange?.Invoke(); } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/DayRollingFilter.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/DayRollingFilter.cs index 8512bb50d..6f7503790 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Filter/DayRollingFilter.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/DayRollingFilter.cs @@ -1,8 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using BCG.Utilities.General; +using BCG.Utilities.General; + +using System; namespace BCG { @@ -26,7 +24,7 @@ namespace BCG #region IDynamicStreamFilter Members - public event FilterChangeDelegate onFilterChange; + public event FilterChangeDelegate? onFilterChange; #endregion @@ -34,7 +32,10 @@ namespace BCG public bool filter(IBlueRow r) { - DateTime rowDate = DBNull.Value.Equals (r[myDateColName]) ? DateTime.Now.Date : ((DateTime)r[myDateColName]).Date; + DateTime rowDate = r[myDateColName] switch { + DBNull or null => DateTime.Now.Date, + object obj => (DateTime)obj, + }; DateTime today = DateTime.Now.Date; return rowDate > today ? true : !(today.Subtract(rowDate).Days >= myNPastTodayDays); } @@ -48,7 +49,7 @@ namespace BCG if (DateTime.Now.Date != myLastCheckedTime.Date) { myLastCheckedTime = DateTime.Now; - onFilterChange(); + onFilterChange?.Invoke(); } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs index d46a0a8a6..8260b97a1 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs @@ -1,18 +1,23 @@ using bts; -using Bts.DataStream; using Bts.DataStream.Primitives.Filter; +using Bts.DataStreamCore; -using mbts.config; +using Spark.Core.Diagnostics; +using Spark.Core.Disposables; using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; using System.Text.RegularExpressions; namespace BCG.StreamWorld.Addition { [DataStreamUserRespresentation(false)] public class FilterStream : ViewStream { private ViewStream myBaseStream; - private BlueTable myBaseTable, myStateTable; + BlueTable? myBaseTable; + BlueTable? myStateTable; private IStreamFilter myFilter; public FilterStream(ViewStream vs) @@ -25,8 +30,8 @@ namespace BCG.StreamWorld.Addition { myBaseStream = vs; myFilter = filter; vs.ErrorChanged += OnError; - if (filter is IDynamicStreamFilter) - (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange); + if (filter is IDynamicStreamFilter dfilter) + dfilter.onFilterChange += myHandleFilterChange; } @@ -36,8 +41,8 @@ namespace BCG.StreamWorld.Addition { protected override void myOnLastSubscriber() { myBaseStream.onChange -= new ChangeDelegate(myHandleChanges); - myBaseTable.clear(); - myStateTable.clear(); + myBaseTable?.clear(); + myStateTable?.clear(); } bool filterRow(IBlueRow r) { @@ -54,16 +59,12 @@ namespace BCG.StreamWorld.Addition { try { var dp = (IDataPacket)newValue; var schema = dp.Schema; - if (myStateTable == null) { - bts.Log.assert(myBaseTable == null); - myStateTable = new BlueTable(schema); - myBaseTable = new BlueTable(schema); - } + init(schema); var dels = new BlueTable(schema); var adds = new BlueTable(schema); foreach (IBlueRow r in dp.deletedRows) { myBaseTable.deleteRow(r); - if (myStateTable.containsID((int)r["id"])) { + if (myStateTable.containsID(r.Id())) { dels.addRow(r); myStateTable.deleteRow(r); } @@ -86,6 +87,7 @@ namespace BCG.StreamWorld.Addition { set { myFilter = value; if (myBaseTable != null) { + ExceptionAssert.NotNull(myStateTable); IBlueRowCollection oldState = new BlueTable(myStateTable); myStateTable.clear(); // If the filter throws an exception, we can rethrow it as a StreamFilterException @@ -113,6 +115,17 @@ namespace BCG.StreamWorld.Addition { //public string filterString { set { filter = new BooleanExpressionFilter(value); } } private void myHandleFilterChange() { filter = myFilter; } + + [MemberNotNull(nameof(myStateTable), nameof(myBaseTable))] + void init(IBlueTableSchema schema) { + if (myStateTable is null) { + Log.isnull(myBaseTable); + myStateTable = new BlueTable(schema); + myBaseTable = new BlueTable(schema); + } else { + Log.notnull(myBaseTable); + } + } } } @@ -136,29 +149,24 @@ namespace BCG } #region IStreamFilter Members - public bool filter(IBlueRow dr) - { - bool rv = true; - for (int i = 0; i < myCols.Length; i++) - { - rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString()); - if (!rv) - return rv; - } - return rv; - } + public bool filter(IBlueRow dr) => + myCols.Zip(myRegExps).All(p => p.Second.IsMatch(dr[p.First] switch { + DBNull or null => string.Empty, + object obj => obj.ToString() ?? string.Empty + })); #endregion } public class FilterStream : ViewStream { - private BlueTable myLastFiredRows; - private IBlueRowCollection myLastReceivedRows; - private IStreamFilter myFilter; - private ViewStream myParentStream; - private string myFilterString; - private BCG.StreamWorld.Addition.FilterStream myFilterStream; - private bool myUseNewFilter = false; + BlueTable? myLastFiredRows; + IBlueRowCollection? myLastReceivedRows; + IStreamFilter? myFilter; + readonly ViewStream? myParentStream; + string? myFilterString; + readonly StreamWorld.Addition.FilterStream? myFilterStream; + readonly bool myUseNewFilter = false; + IDisposable? subscription; #region constructors public delegate bool OnFilter(IBlueRow r); @@ -188,9 +196,10 @@ namespace BCG { myParentStream = vs; myFilter = filter; - myFilterString = filter.ToString(); - if (filter is IDynamicStreamFilter) - (filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange); + myFilterString = filter.ToString() ?? string.Empty; + if (filter is IDynamicStreamFilter dfilter) { + dfilter.onFilterChange += myHandleFilterChange; + } } } @@ -214,13 +223,15 @@ namespace BCG #endregion #region overridden methods - protected override void myOnFirstSubscriber() - { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); } + protected override void myOnFirstSubscriber() { + ExceptionAssert.IsNull(subscription); + subscription = ExceptionAssert.NotNull(myParentStream).Subscribe(myHandleCallbacks); + } protected override void myOnLastSubscriber() { myLastFiredRows = null; - myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks); + Disposable.Dispose(ref subscription); } protected override void myFireEvent(object o) @@ -239,19 +250,20 @@ namespace BCG try { if (myLastFiredRows != null) { - retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema), - new BlueTable(myLastFiredRows.schema)); + var addedRows = new BlueTable(myLastFiredRows.schema); + var deletedRows = new BlueTable(myLastFiredRows.schema); + retVal = new DataPacket(myLastFiredRows, addedRows, deletedRows); foreach (IBlueRow r in arrivedPacket.deletedRows) - if (myLastFiredRows.containsID((int)r["id"])) + if (myLastFiredRows.containsID(r.Id())) { myLastFiredRows.deleteRow(r); - (retVal.deletedRows as BlueTable).addRow(r); + deletedRows.addRow(r); } foreach (IBlueRow r in arrivedPacket.addedRows) if (filterRow(r)) { myLastFiredRows.addRow(r); - (retVal.addedRows as BlueTable).addRow(r); + addedRows.addRow(r); } } else @@ -274,7 +286,7 @@ namespace BCG { if (myUseNewFilter) { - myFilterStream.filter = filter; + requireNewFilter().filter = filter; } else { @@ -303,9 +315,9 @@ namespace BCG get { if (myUseNewFilter) - return myFilterStream.filter; + return requireNewFilter().filter; else - return myFilter; + return requireOldFilter(); } } @@ -318,17 +330,25 @@ namespace BCG private void myHandleFilterChange() { - filter = myFilter; + filter = requireOldFilter(); } bool filterRow(IBlueRow r) { try { - return myFilter.filter(r); + return requireOldFilter().filter(r); } catch (Exception ex) { Log.Gen.error($"FilterStream delegate threw exception: {ex}"); OnError(new StreamFilterException(StreamId, StreamName, null, ex, r)); } return false; } + + [MemberNotNull(nameof(myFilterStream))] + StreamWorld.Addition.FilterStream requireNewFilter() => + ExceptionAssert.MemberNotNull(myFilterStream); + + [MemberNotNull(nameof(myFilter))] + IStreamFilter requireOldFilter() => + ExceptionAssert.MemberNotNull(myFilter); } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/SlidingWindowFilter.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/SlidingWindowFilter.cs index 0d902f6cc..57dfb6bf0 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Filter/SlidingWindowFilter.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/SlidingWindowFilter.cs @@ -1,8 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using BCG.Utilities.General; +using BCG.Utilities.General; + +using System; namespace BCG { @@ -29,7 +27,7 @@ namespace BCG myReference = myCurrentTime.AddSeconds(-1 * myNSecs); } - public event FilterChangeDelegate onFilterChange; + public event FilterChangeDelegate? onFilterChange; public bool filter(IBlueRow r) { @@ -38,9 +36,8 @@ namespace BCG //it can be considered as our assumption whenever datecolumn is db null we return true return true; //dont use this for volparamsched use true } - DateTime t = (DateTime)r[myDateColumn]; - bool returnVallue = t < myReference ? false : true; - return returnVallue; + var t = (DateTime)r[myDateColumn]!; + return t >= myReference; } @@ -51,8 +48,7 @@ namespace BCG myCurrentTime = DateTime.Now; myReference = myCurrentTime.AddSeconds(-1 * myNSecs); myLastTimeFilteredApplied = DateTime.Now; - onFilterChange(); - + onFilterChange?.Invoke(); } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs b/src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs index ef4ba2e1b..51985d205 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs @@ -1,11 +1,13 @@ -using System; +using BCG.Utilities.General; + +using Spark.Core.Diagnostics; + +using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; -using System.Text; -using BCG.Utilities.General; using FuzzyRowComparer = System.Func; -using System.Diagnostics; namespace BCG { @@ -40,12 +42,12 @@ namespace BCG private IComparer myRightCompare; protected FuzzyRowComparer myFuzzyCompare; protected TimeSpan myTimeWindow; - protected IBlueTableSchema myLeftSchema; - protected IBlueTableSchema myRightSchema; - protected IBlueTableSchema myOutputSchema; + protected IBlueTableSchema? myLeftSchema; + protected IBlueTableSchema? myRightSchema; + protected IBlueTableSchema? myOutputSchema; private int myID = 0; - protected TimeWindowTable myStateRows; + protected TimeWindowTable? myStateRows; private DateTime myNextCull; const long MinCullIntervalMS = 1000; @@ -65,7 +67,7 @@ namespace BCG this.myFuzzyCompare = fuzzyCompare; } - public virtual IBlueTableSchema OutputSchema + public virtual IBlueTableSchema? OutputSchema { get { @@ -150,7 +152,7 @@ namespace BCG int count; for (count = 0; count < list.Count; count++) { - if ((DateTime)list[count][timeColumn] > expiry) + if ((DateTime)list[count][timeColumn]! > expiry) { // Stop when we find the first row that we can keep. Since this is sorted all others are also kept. break; @@ -172,16 +174,16 @@ namespace BCG this.myFitness = fitness; } public IBlueRow myRightRow { get; private set; } - public IBlueRow myLeftRow { get; set; } + public IBlueRow? myLeftRow { get; set; } public double myFitness { get; private set; } - public IBlueRow myMergedRow { get; set; } + public IBlueRow? myMergedRow { get; set; } #region IComparable Members - public int CompareTo(RowMatch other) - { - return myFitness.CompareTo(other.myFitness); - } + public int CompareTo(RowMatch? other) => + other is null + ? 1 + : myFitness.CompareTo(other.myFitness); #endregion } @@ -262,12 +264,12 @@ namespace BCG private void HandleCorrection(List corrections, BlueTable adds, BlueTable dels) { + ExceptionAssert.NotNull(myStateRows); // We already emitted correction.MergedRow so we need to redact that and // then emit correction.left as an add since it is no longer matched foreach (RowMatch correction in corrections.Distinct()) { - Debug.Assert(correction.myMergedRow != null); - this.myStateRows.RemoveRow(correction.myMergedRow); + myStateRows.RemoveRow(ExceptionAssert.NotNull(correction.myMergedRow)); adds.addRow(this.MergeRows(correction.myLeftRow, null)); dels.addRow(correction.myMergedRow); } @@ -285,7 +287,7 @@ namespace BCG int r = 0; for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; ) { - if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn]) + if ((DateTime)leftExpired[l][this.myLeftTimeColumn]! <= (DateTime)rightExpired[r][this.myRightTimeColumn]!) { // Advance Left IEnumerable rm = rightMatches[l]; @@ -332,7 +334,7 @@ namespace BCG private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable rightMatches) { - IBlueRow merged = null; + IBlueRow? merged = null; if (rightMatches != null) { foreach (RowMatch match in rightMatches) @@ -366,9 +368,9 @@ namespace BCG /// Override this function to provide a custom row merge. /// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting. /// - protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right) + protected virtual IBlueRow MergeRows(IBlueRow? left, IBlueRow? right) { - Dictionary newRow = new Dictionary(); + Dictionary newRow = new(); if (left == null && right == null) throw new ArgumentException("oh come on."); @@ -393,14 +395,15 @@ namespace BCG } } newRow["id"] = this.myID++; - newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn]; - + newRow[NEW_TIME_COLUMN] = left is not null + ? left[myLeftTimeColumn] + : right![myRightTimeColumn]; return new BlueRow(this.OutputSchema, newRow); } private IEnumerable FindMatches(IBlueRow row, List list, List corrections) { - List matches = null; + List? matches = null; // Search through all rows in list and compare with the fuzzy comparer for (int x = 0; x < list.Count; x++) @@ -421,8 +424,7 @@ namespace BCG matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch); // Mark the best match for this particular right hand match - RowMatch oldMatch; - if (this.myBestMatch.TryGetValue(match, out oldMatch)) + if (this.myBestMatch.TryGetValue(match, out var oldMatch)) { if (currentMatch.myFitness < oldMatch.myFitness) { @@ -441,7 +443,7 @@ namespace BCG } } - return matches; + return matches ?? Enumerable.Empty(); } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/INonIncGroupByEvaluator.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/INonIncGroupByEvaluator.cs index 3b4bfffdf..08f7ea97f 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/INonIncGroupByEvaluator.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/INonIncGroupByEvaluator.cs @@ -1,14 +1,10 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using System.Collections.Generic; namespace BCG { public interface INonIncGroupByEvaluator { - IBlueRowCollection evaluate(IDictionary> grouped_rows); - string[] get_group_by_cols(); + IBlueRowCollection? evaluate(IDictionary> grouped_rows); + string[]? get_group_by_cols(); } } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs index 047c9e3f6..8a2516f20 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs @@ -1,46 +1,51 @@ -using System; +using bts; +using bts.SilverPine; + +using Bts.DataStreamCore; + +using Spark.Core.Diagnostics; +using Spark.Core.Disposables; + +using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; -using System.Text.Json; - -using bts; -using bts.SilverPine; namespace BCG { //[DataStreamUserRespresentation("AggregateStream")] public class NonIncGroupByStream : ViewStream { private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator { - Func _groupby_cols_func; - Func, List> _evaluate_func; - public DelNonIncGroupByEvaluator(Func groupby_cols_func, + Func _groupby_cols_func; + Func, List?> _evaluate_func; + public DelNonIncGroupByEvaluator(Func groupby_cols_func, Func, List> evaluate_func) { _groupby_cols_func = groupby_cols_func; _evaluate_func = evaluate_func; } - public IBlueRowCollection evaluate(IDictionary> grouped_rows) { - BlueTable rv = null; + public IBlueRowCollection? evaluate(IDictionary> grouped_rows) { + BlueTable? rv = null; foreach (var kv in grouped_rows) { List g_rows = kv.Value; - List out_rows = _evaluate_func.Invoke(g_rows); + var out_rows = _evaluate_func(g_rows); if (out_rows != null && out_rows.Count > 0) { rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema); + rv.addRows(out_rows); } - rv.addRows(out_rows); } return rv; } - public string[] get_group_by_cols() { + public string[]? get_group_by_cols() { return _groupby_cols_func(); } } - private ViewStream _main_stream; - private INonIncGroupByEvaluator _groupby_func; - private BlueRowHasher _row_hasher; - private string[] _groupby_cols; + readonly ViewStream? _main_stream; + readonly INonIncGroupByEvaluator? _groupby_func; + BlueRowHasher? _row_hasher; + readonly string[]? _groupby_cols; private Stopwatch _timer = new(); private int _n_total_fires = 0, _n_expensive_fires = 0; + IDisposable? subscription; public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) { _main_stream = main_stream; @@ -49,7 +54,7 @@ namespace BCG { _row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length); } - public NonIncGroupByStream(ViewStream main_stream, Func groupby_cols_func, + public NonIncGroupByStream(ViewStream main_stream, Func groupby_cols_func, Func, List> evaluate_func) : this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { } @@ -58,7 +63,7 @@ namespace BCG { public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) { MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text); - Func groupby_cols_func = () => { + Func groupby_cols_func = () => { return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col }; }; Func, List> evaluate_func = (group_rows) => { @@ -75,11 +80,16 @@ namespace BCG { } - protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; } + protected override void myOnFirstSubscriber() { + ExceptionAssert.IsNull(subscription); + ExceptionAssert.NotNull(_main_stream).Subscribe(_main_stream_onchange); + } - protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; } + protected override void myOnLastSubscriber() => + Disposable.Dispose(ref subscription); private void _main_stream_onchange(object nv) { + ExceptionAssert.NotNull(_groupby_func); var dp = (IDataPacket)nv; ///grouping together _timer.Restart(); @@ -94,7 +104,7 @@ namespace BCG { // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime)); /// reducing _timer.Restart(); - IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows); + var total_state = _groupby_func.evaluate(grouped_rows); _timer.Stop(); if (total_state is null) { Log.Gen.warn("NonIncGroupByStream _groupby_func.evaluate returned null (ignoring last update)."); diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs index 286d25c21..d2f7a373d 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/RowDictionaryTable.cs @@ -1,6 +1,6 @@ using System; -using BCG.Utilities; using System.Collections; +using System.Collections.Generic; namespace BCG { @@ -11,12 +11,10 @@ namespace BCG /// public class RowDictionaryTable : IBlueRowCollection { - protected IDictionary myRowCollections = new Hashtable (); - protected BlueTable myLinearCollection; - protected string[] myCols; + protected readonly Dictionary myRowCollections = new(); + protected readonly BlueTable myLinearCollection; + protected readonly string[] myCols; protected IBlueTableSchema mySchema; - - protected RowDictionaryTable () {} public RowDictionaryTable (IBlueRowCollection rows, params string[] cols) : this (rows.schema, cols) @@ -40,7 +38,7 @@ namespace BCG public virtual void addRow (IBlueRow row) { - BlueTable rows = (BlueTable)getGroup (row); + var rows = getMutableGroup(row); if (rows == null) { //i.e we dont have any group for this row => we need to create a new group @@ -53,7 +51,7 @@ namespace BCG public virtual void removeRow (IBlueRow row) { - BlueTable rowGroup = (BlueTable)getGroup (row); + var rowGroup = requireMutableGroup(row); // BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0); rowGroup.deleteRow (row); myLinearCollection.deleteRow (row); @@ -65,7 +63,7 @@ namespace BCG public void removeGroup (IBlueRow representativeRow) { - IBlueRowCollection rowGroup = getGroup (representativeRow); + var rowGroup = requireGroup(representativeRow); // BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0); foreach (IBlueRow row in rowGroup) myLinearCollection.deleteRow (row); @@ -85,24 +83,31 @@ namespace BCG /// //public IBlueRowCollection getGroup (IBlueRow row) {return getGroup ( - public IBlueRowCollection getGroup (IBlueRow r) {return getGroup (myRowToString (r));} + public IBlueRowCollection? getGroup (IBlueRow r) => + getGroup(myRowToString(r)); public ICollection getGroups () {return myRowCollections.Values;} - private bool myAreRowsEqual (IBlueRow r1, IBlueRow r2) - { - bool retVal = true; - foreach (string col in myCols) - retVal &= r1[col].Equals(r2[col]); - return retVal; - } + public IBlueRowCollection? getGroup(string repString) => + getMutableGroup(repString); - public IBlueRowCollection getGroup (string repString) - { - IBlueRowCollection retVal = myRowCollections[repString] != null ? - (IBlueRowCollection) myRowCollections[repString] : null; - return retVal; - } + BlueTable? getMutableGroup(IBlueRow row) => + getMutableGroup(myRowToString(row)); + + BlueTable? getMutableGroup(string group) => + myRowCollections.GetValueOrDefault(group); + + IBlueRowCollection requireGroup(IBlueRow row) => + requireGroup(myRowToString(row)); + + IBlueRowCollection requireGroup(string group) => + getGroup(group) ?? throw new ArgumentException("No table for group", nameof(group)); + + BlueTable requireMutableGroup(IBlueRow row) => + requireMutableGroup(myRowToString(row)); + + BlueTable requireMutableGroup(string group) => + getMutableGroup(group) ?? throw new ArgumentException("No table for group", nameof(group)); private string myRowToString (IBlueRow r) {return BlueDataUtility.rowToString (r, myCols);} diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinForeignStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinForeignStream.cs index 51bf7fdbb..f4c8ac101 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinForeignStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinForeignStream.cs @@ -1,5 +1,3 @@ -using System; - namespace BCG { [DataStreamUserRespresentation (false)] @@ -17,6 +15,7 @@ namespace BCG protected override void myInitFiredPacket () { + requireSchema(); myLastFiredPacket = new DataPacket (); myLastFiredPacket.newRows = new JoinForeignTable (mySchema); myLastFiredPacket.addedRows = new BlueTable (mySchema); @@ -25,6 +24,10 @@ namespace BCG public override void onTableClear (bool isLeft) { + if (myLastFiredPacket is null) { + return; + } + requireSchema(); //incase of natural join, left or right wont matter, the end result zeroes out myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; myLastFiredPacket.newRows = new JoinForeignTable (mySchema); diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinStream.cs index 0a88b7e5f..b1f02e1d5 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/InternalJoinStream.cs @@ -1,5 +1,7 @@ +using Spark.Core.Diagnostics; + using System; -using System.Collections; +using System.Diagnostics.CodeAnalysis; namespace BCG { @@ -13,12 +15,10 @@ namespace BCG protected JoinOperand myLeftOperand, myRightOperand; protected int myLeftJoinColOrdinal, myRightJoinColOrdinal; protected ViewStream myLeftStream, myRightStream; - protected JoinTableSchema mySchema; - protected DataPacket myLastFiredPacket; + protected JoinTableSchema? mySchema; + protected DataPacket? myLastFiredPacket; protected bool myOnceFired = false; - public IBlueTableSchema schema {get{return mySchema;}} - public InternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight) { myLeftStream = vsLeft; @@ -27,6 +27,9 @@ namespace BCG myRightOperand = new JoinOperand (jcRight, false); } + public IBlueTableSchema schema => + requireSchema(); + protected override void myOnFirstSubscriber() { myLeftOperand.master = this; @@ -62,29 +65,32 @@ namespace BCG { myLeftJoinColOrdinal = myLeftOperand.joinColOridinal; myRightJoinColOrdinal = myRightOperand.joinColOridinal; + myLeftOperand.RequireInitialized(); + myRightOperand.RequireInitialized(); mySchema = new JoinTableSchema (myLeftOperand.operandSchema, myLeftOperand.joinCol, myRightOperand.operandSchema, myRightOperand.joinCol); myInitFiredPacket (); } } + [MemberNotNull(nameof(mySchema), nameof(myLastFiredPacket))] protected virtual void myInitFiredPacket () { + var schema = requireSchema(); myLastFiredPacket = new DataPacket (); - myLastFiredPacket.newRows = new JoinTable (mySchema); - myLastFiredPacket.addedRows = new BlueTable (mySchema); - myLastFiredPacket.deletedRows = new BlueTable (mySchema); + myLastFiredPacket.newRows = new JoinTable (schema); + myLastFiredPacket.addedRows = new BlueTable (schema); + myLastFiredPacket.deletedRows = new BlueTable (schema); } - public IBlueTableSchema getSchema (bool isLeft) - { - IBlueTableSchema retVal; - retVal = isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema; - return retVal; - } + public IBlueTableSchema? getSchema (bool isLeft) => + isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema; public void endSession () { + if (myLastFiredPacket is null) { + return; + } if (!myOnceFired) { myOnceFired = true; @@ -97,13 +103,17 @@ namespace BCG protected virtual void myInit () { - myLeftOperand.init (); - myRightOperand.init (); + myLeftOperand.Reset (); + myRightOperand.Reset (); myLastFiredPacket = null; } public virtual void onTableClear (bool isLeft) { + if (myLastFiredPacket is null) { + return; + } + requireSchema(); //incase of natural join, left or right wont matter, the end result zeroes out myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; myLastFiredPacket.newRows = new JoinTable (mySchema); @@ -112,15 +122,19 @@ namespace BCG public virtual void deleteRows (IBlueRowCollection rows, bool isLeft) { + var last = myLastFiredPacket; + if (last is null) { + return; + } JoinOperand thisOp, otherOp; int joinColOrdinal; myAssignAccordingly (out thisOp, out otherOp, out joinColOrdinal, isLeft); foreach (IBlueRow row in rows) { thisOp.deleteRow (row); - if (otherOp.getMatchingRows (row[joinColOrdinal]).Count == 0) + if (otherOp.getMatchingRows(getJoinKey(row, joinColOrdinal)).Count == 0) continue; //row does not participate in join - ((JoinTable)(myLastFiredPacket.newRows)).deleteRow (isLeft, row, myLastFiredPacket); + ((JoinTable)last.newRows).deleteRow(isLeft, row, last); } } @@ -135,10 +149,12 @@ namespace BCG otherOp = myLeftOperand; joinColOrdinal = myRightJoinColOrdinal; } + var schema = requireSchema(); + myLastFiredPacket ??= new DataPacket(new BlueTable(schema), new BlueTable(schema), new BlueTable(schema)); foreach (IBlueRow row in rows) { thisOp.addRow (row); - RowList matchedNodes = otherOp.getMatchingRows (row[joinColOrdinal]); + RowList matchedNodes = otherOp.getMatchingRows(getJoinKey(row, joinColOrdinal)); if (matchedNodes.Count == 0) continue; //this row did not participate in the join if (isLeft) @@ -146,10 +162,22 @@ namespace BCG ((JoinTable)myLastFiredPacket.newRows).addRow (row, matchedNode.row, myLastFiredPacket); else foreach (RowNode matchedNode in matchedNodes) - ((JoinTable)myLastFiredPacket.newRows).addRow (matchedNode.row, row, myLastFiredPacket); + ((JoinTable)myLastFiredPacket.newRows).addRow(ExceptionAssert.NotNull(matchedNode.row), row, myLastFiredPacket); } } + [MemberNotNull(nameof(mySchema))] + protected JoinTableSchema requireSchema() => + ExceptionAssert.NotNull(mySchema); + + [MemberNotNull(nameof(myLastFiredPacket))] + protected DataPacket requireLastFiredPacket() { + if (myLastFiredPacket is null) { + myInitFiredPacket(); + } + return myLastFiredPacket; + } + private void myAssignAccordingly (out JoinOperand thisOp, out JoinOperand otherOp, out int joinColOrdinal, bool isLeft) { if (isLeft) @@ -166,6 +194,9 @@ namespace BCG } } + object getJoinKey(IBlueRow row, int joinColOrdinal) => + row[joinColOrdinal] ?? throw new ArgumentException("Row has null join key", nameof(row)); + /**************************************TEST METHOD(S)*********************************************/ /// /// Used for testing purposes only. This enables the test to work without a DataSource (PubServer, diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinDeletionIndex.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinDeletionIndex.cs index 0db00f9ec..97b0b7469 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/JoinDeletionIndex.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinDeletionIndex.cs @@ -1,5 +1,6 @@ -using System; -using System.Collections; +using Spark.Core.Diagnostics; + +using System.Collections.Generic; namespace BCG { @@ -8,7 +9,7 @@ namespace BCG /// internal class JoinDeletionIndex { - IDictionary myIndex = new Hashtable (); + Dictionary myIndex = new(); private string myOpIdName ; private bool myIsLeft; @@ -28,10 +29,8 @@ namespace BCG /// JoinRow to add public void add (JoinRow row) { - object o = row[myOpIdName]; - if (myIndex[o] == null) - myIndex[o] = new RowList (); - RowNode nodeAdded = ((RowList)myIndex[o]).add (row); + object o = ExceptionAssert.NotNull(row[myOpIdName]); + RowNode nodeAdded = myIndex.GetOrAdd(o, _ => new()).add(row); if (myIsLeft) row.leftNode = nodeAdded; else @@ -43,7 +42,10 @@ namespace BCG /// To remove a node, if node does not exist exception is thrown /// /// - public void remove (RowNode node) {((RowList)myIndex[node.row[myOpIdName]]).remove (node);} + public void remove (RowNode node) { + var nodeId = ExceptionAssert.NotNull(node.row?[myOpIdName]); + myIndex[nodeId].remove(node); + } public RowList getMatchingRows (object key) { RowList retVal = (RowList)myIndex[key]; diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinForeignTable.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinForeignTable.cs index 9bf32b527..094f46de3 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/JoinForeignTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinForeignTable.cs @@ -1,19 +1,18 @@ -using System; -using BCG.Utilities; - using bts; +using Bts.DataStreamCore; + namespace BCG { internal class JoinForeignTable : JoinTable { public JoinForeignTable (JoinTableSchema schema) : base (schema) {} - public override void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp) - { - JoinRow newRow = new JoinRow (leftRow, rightRow, mySchema); - Log.assert(myIdIndex[leftRow["id"]] == null); - myIdIndex[leftRow["id"]] = newRow; + public override void addRow(IBlueRow leftRow, IBlueRow? rightRow, IDataPacket dp) { + var newRow = new JoinRow(leftRow, rightRow, mySchema); + int leftId = leftRow.Id(); + Log.assert(myIdIndex[leftId] == null); + myIdIndex[leftId] = newRow; ((BlueTable)(dp.addedRows)).addRow (newRow); myLeftIndex.add (newRow); myRightIndex.add (newRow); diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinNameResolver.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinNameResolver.cs index b56d4c5ed..dd278a78d 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/JoinNameResolver.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinNameResolver.cs @@ -1,11 +1,12 @@ -using System; -using System.Collections; -using BCG.Utilities; - using bts; using Bts.ColumnMapExpressions; +using Spark.Core.Diagnostics; + +using System; +using System.Collections; + namespace BCG { /// @@ -37,7 +38,8 @@ namespace BCG private JoinTableSchema myJoinSchema; private IBlueTableSchema myLeftSchema, myRightSchema; private string myLeftColExpression, myRightColExpression; - private string[] myUserRules, myResolvedNames; + string[] myUserRules; + string[]? myResolvedNames; private IDictionary myFinalMap = new Hashtable (); @@ -87,16 +89,18 @@ namespace BCG /// private void myFormUserRules () { + var leftExp = ColumnMapExpression.Parse(myLeftColExpression) ?? throw new InvalidOperationException("Failed to parse left column expression"); + var rightExp = ColumnMapExpression.Parse(myRightColExpression) ?? throw new InvalidOperationException("Failed to parse right column expression"); if (DataStreamLoadFileConfig.instance.useStaticParser) { - myFormUserRules(true, ColumnMapExpression.Parse(myLeftColExpression)); - myFormUserRules(false, ColumnMapExpression.Parse(myRightColExpression)); + myFormUserRules(true, leftExp); + myFormUserRules(false, rightExp); } else { // TODO (maybe): Actually use non-static parser (unclear what advantages it will have) - myFormUserRules(true, ColumnMapExpression.Parse(myLeftColExpression)); - myFormUserRules(false, ColumnMapExpression.Parse(myRightColExpression)); + myFormUserRules(true, leftExp); + myFormUserRules(false, rightExp); } } @@ -115,13 +119,7 @@ namespace BCG OriginalNameInfo oni = myOriginalColInfo[i]; if (oni.isLeft == isLeft) { - if (userMap[oni.colName] != null) - { - if ((string)userMap[oni.colName] != oni.colName) - myUserRules[i] = (string)userMap[oni.colName]; - else - myUserRules[i] = oni.colName; - } + myUserRules[i] = userMap[oni.colName] as string ?? oni.colName; } } else @@ -131,10 +129,13 @@ namespace BCG OriginalNameInfo oni = myOriginalColInfo[i]; if (oni.isLeft == isLeft) { - if (userMap[oni.colName] == null) + var colName = userMap[oni.colName] as string; + // myUserRules[i] = userMap[oni.colName] as string + if (colName is null) { myUserRules[i] = oni.colName; - else if ((string)userMap[oni.colName] != oni.colName) - myUserRules[i] = (string)userMap[oni.colName]; + } else if (colName != oni.colName) { + myUserRules[i] = colName; + } } } } @@ -165,26 +166,12 @@ namespace BCG private void myFormFinalMap () { + ExceptionAssert.NotNull(myResolvedNames); foreach (string resolvedName in myResolvedNames) if (resolvedName != null) myFinalMap [resolvedName] = myJoinNames[Array.IndexOf (myResolvedNames, resolvedName)]; } - - /// - /// Utility method to invert a hashtable - /// - /// - /// - - private IDictionary reciprocateDictionary (IDictionary arg) - { - IDictionary retVal = new Hashtable (); - foreach (DictionaryEntry de in arg) - retVal[de.Value] = de.Key; - return retVal; - } - /// /// Utility method to find all occurances of an obkect in an IList /// @@ -194,9 +181,11 @@ namespace BCG private ArrayList myFindAllInCollection (IList list, object obj) { ArrayList retVal = new ArrayList (); - for (int i = 0; i < list.Count; i++) - if (list[i] != null && list[i].Equals (obj)) + for (int i = 0; i < list.Count; i++) { + if (Equals(list[i], obj)) { retVal.Add (i); + } + } return retVal; } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperand.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperand.cs index 961f203bf..9244c1720 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperand.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperand.cs @@ -1,6 +1,7 @@ +using Spark.Core.Diagnostics; + using System; -using System.Collections; -using BCG.Utilities; +using System.Diagnostics.CodeAnalysis; namespace BCG { @@ -12,32 +13,45 @@ namespace BCG { private string myJoinCol; IDataPacket? myLastArrivedPacket; - private JoinOperand myOtherOperand; + JoinOperand? myOtherOperand; private bool myIsLeft; - private JoinOperandRowIndex myRowIndex; - private InternalJoinStream myMaster; - private IBlueTableSchema myOpSchema; - + JoinOperandRowIndex? myRowIndex; + InternalJoinStream? myMaster; + public JoinOperand (string joinCol, bool isLeft) { myJoinCol = joinCol; myIsLeft = isLeft; } + [MemberNotNullWhen(true, nameof(myMaster), nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))] + bool isInitialized { + get { + if (myLastArrivedPacket is null) { + return false; + } + ExceptionAssert.NotNull(myMaster, myLastArrivedPacket, operandSchema, myOtherOperand, myRowIndex); + return true; + } + } + internal void changeHandler (object o) { var dp = (IDataPacket)o; - if (myLastArrivedPacket != null) + if (isInitialized) { mySubsequentChangeHandler (dp); - else + } else { myFirstChangeHandler(dp); + } } - void myFirstChangeHandler(IDataPacket arrivedDp) { + [MemberNotNull(nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))] + bool myFirstChangeHandler(IDataPacket arrivedDp) { + ExceptionAssert.NotNull(myMaster); myLastArrivedPacket = arrivedDp; - myOpSchema = myLastArrivedPacket.newRows.schema; + operandSchema = myLastArrivedPacket.newRows.schema; myOtherOperand = myMaster.getOtherOperand (this); - myRowIndex = new JoinOperandRowIndex (myOpSchema[myJoinCol].colNo, myOpSchema["id"].colNo); + myRowIndex = new JoinOperandRowIndex (operandSchema[myJoinCol].colNo, operandSchema["id"].colNo); if (myOtherOperand.hasReceivedData) { //the other operand has already received data, so the join would @@ -48,13 +62,16 @@ namespace BCG myMaster.addRows (myLastArrivedPacket.newRows, myIsLeft); myMaster.endSession (); } - else + else { myRowIndex = new JoinOperandRowIndex (myLastArrivedPacket.newRows, - myOpSchema[myJoinCol].colNo, - myOpSchema["id"].colNo); + operandSchema[myJoinCol].colNo, + operandSchema["id"].colNo); + } + return true; } void mySubsequentChangeHandler (IDataPacket arrivedDp) { + RequireInitialized(); myLastArrivedPacket = arrivedDp; if (!myOtherOperand.hasReceivedData) { @@ -83,29 +100,46 @@ namespace BCG myMaster.endSession (); } - internal void addRow (IBlueRow row) {myRowIndex.add (row);} + internal void addRow (IBlueRow row) => + ExceptionAssert.NotNull(myRowIndex).add(row); - internal void deleteRow (IBlueRow row) {myRowIndex.remove (row);} + internal void deleteRow (IBlueRow row) { + RequireInitialized(); + myRowIndex.remove (row); + } - internal IBlueTableSchema operandSchema {get{return myOpSchema;}} + internal IBlueTableSchema? operandSchema { get; private set; } internal InternalJoinStream master {set{myMaster = value;}} internal string joinCol {get{return myJoinCol;}} - internal int joinColOridinal {get{return myOpSchema[myJoinCol].colNo;}} + internal int joinColOridinal { + get { + RequireInitialized(); + return operandSchema[myJoinCol].colNo; + } + } - internal RowList getMatchingRows (object key) {return (RowList) myRowIndex.getMatchingRows (key);} + internal RowList getMatchingRows (object key) { + RequireInitialized(); + return myRowIndex.getMatchingRows(key); + } internal bool hasReceivedData {get{return myLastArrivedPacket != null;}} - internal void init () - { + internal void Reset() { ///should be brought back to the the state before the first callback ///(ie the state after construction) myRowIndex = null; myLastArrivedPacket = null; } - + + [MemberNotNull(nameof(myMaster), nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))] + internal void RequireInitialized() { + if (!isInitialized) { + throw new InvalidOperationException("Internal state not properly initialized"); + } + } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperandRowIndex.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperandRowIndex.cs index 3c22cd209..4a6eb992e 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperandRowIndex.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperandRowIndex.cs @@ -1,6 +1,6 @@ -using System; -using System.Collections; -using BCG.Utilities; +using Spark.Core.Diagnostics; + +using System.Collections.Generic; namespace BCG { @@ -11,8 +11,10 @@ namespace BCG /// internal class JoinOperandRowIndex { - private IDictionary myJoinColDict = new Hashtable (), myIdDict = new Hashtable (); - private int myJoinColOrdinal, myIdColOrdinal; + readonly Dictionary myJoinColDict = new(); + readonly Dictionary myIdDict = new(); + readonly int myJoinColOrdinal; + readonly int myIdColOrdinal; /// /// @@ -34,19 +36,14 @@ namespace BCG public void add (IBlueRow row) { - RowList list; - if (myJoinColDict[row[myJoinColOrdinal]] == null) - myJoinColDict[row[myJoinColOrdinal]] = list = new RowList (); - else - list = (RowList)myJoinColDict[row[myJoinColOrdinal]]; - myIdDict[row[myIdColOrdinal]] = list.add (row); + myIdDict[getId(row)] = myJoinColDict.GetOrAdd(getJoinKey(row), _ => new()).add(row); } public void remove (IBlueRow row) { - RowNode node = (RowNode)myIdDict[row[myIdColOrdinal]]; - ((RowList)myJoinColDict[row[myJoinColOrdinal]]).remove (node); - myIdDict.Remove (row[myIdColOrdinal]); + var id = getId(row); + myJoinColDict[getJoinKey(row)].remove(myIdDict[id]); + myIdDict.Remove(id); } public RowList getMatchingRows (object key) @@ -62,5 +59,11 @@ namespace BCG myJoinColDict.Clear (); myIdDict.Clear (); } + + object getJoinKey(IBlueRow row) => + ExceptionAssert.NotNull(row[myJoinColOrdinal]); + + int getId(IBlueRow row) => + (int)row[myIdColOrdinal]!; } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinRow.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinRow.cs index e4f8425b5..6c144dd68 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/JoinRow.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinRow.cs @@ -1,17 +1,21 @@ -using System; -using BCG.Utilities; -using System.Collections; using bts; +using Bts.DataStreamCore; + +using System; +using System.Collections; + namespace BCG { internal class JoinRow : IBlueRow { - private RowNode myLeftNode, myRightNode; - protected IBlueRow myLeftRow, myRightRow; + RowNode? myLeftNode; + RowNode? myRightNode; + protected IBlueRow myLeftRow; + protected IBlueRow? myRightRow; protected int myId; protected JoinTableSchema mySchema; - private object[] myValues; + private object?[] myValues; #region Constructor(s) @@ -21,8 +25,8 @@ namespace BCG /// left row of the join /// right row of the join /// schema of the joined row - public JoinRow (IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema) - : this ((int)leftRow["id"], leftRow, rightRow, schema) {} + public JoinRow (IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema) + : this(leftRow.Id(), leftRow, rightRow, schema) { } /// /// Constructs a row assigning it the ID provioded as the first argument @@ -31,8 +35,7 @@ namespace BCG /// left row of the join /// right row of the join /// schema of the joined row - public JoinRow (int id, IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema) - { + public JoinRow (int id, IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema) { myId = id; myLeftRow = leftRow; myRightRow = rightRow; @@ -48,13 +51,13 @@ namespace BCG public bool isPartialJoin{get{return myRightRow == null;}} - public RowNode leftNode + public RowNode? leftNode { get{return myLeftNode;} set{myLeftNode = value;} } - public RowNode rightNode + public RowNode? rightNode { get{return myRightNode;} set{myRightNode = value;} @@ -85,7 +88,7 @@ namespace BCG /// The key can either be the string corrsponding to the column name or /// the order of the column /// - public virtual object this [object key] + public virtual object? this[object key] { get { @@ -96,7 +99,7 @@ namespace BCG set {throw new NotSupportedException ("The IBlueRow is read only");} } - public object this [int index] + public object? this[int index] { get { @@ -119,7 +122,8 @@ namespace BCG } } - public object this [string colName] {get{return this[mySchema[colName].colNo];}} + public object? this[string colName] => + this[mySchema[colName].colNo]; public ICollection Keys{get{return mySchema.columnNames;}} @@ -133,7 +137,8 @@ namespace BCG } } - public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");} + public void Add (object key, object? value) => + throw new NotSupportedException ("The IBlueRow is readonly"); public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");} diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs index 5a72e617a..1990663a7 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs @@ -1,20 +1,26 @@ -using System; -using System.Data; -using System.Collections; -using System.Text.RegularExpressions; -using BCG.Utilities; -using BCG.Utilities.General; using bts; +using Bts.DataStreamCore; + +using Spark.Core.Diagnostics; +using Spark.Core.Disposables; + +using System; +using System.Collections; + namespace BCG { public class JoinStream : ViewStream { - protected InternalJoinStream? myIjs; + protected readonly InternalJoinStream? myIjs; + protected readonly ColumnMap? myLeftCm; + protected readonly ColumnMap? myRightCm; + protected readonly bool myIsProject = true; protected ProjectStream? myPs; - protected ColumnMap? myLeftCm, myRightCm; - protected bool myIsProject = true; - private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin; + + readonly bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin; + IDisposable? internalSubscription; + IDisposable? projectSubscription; protected JoinStream () {} public JoinStream (ViewStream v1, string joinCol1, ColumnMap cm1, @@ -111,47 +117,61 @@ namespace BCG { if (myPs == null) { - JoinNameResolver resolver = new JoinNameResolver ((JoinTableSchema)myIjs.schema, myIjs.getSchema (true), myLeftCm.columnExpression, - myIjs.getSchema (false), myRightCm.columnExpression); + ExceptionAssert.NotNull(myIjs, myLeftCm, myRightCm); + var leftSchema = myIjs.getSchema(true) ?? throw new InvalidOperationException("Left schema is null"); + var rightSchema = myIjs.getSchema(true) ?? throw new InvalidOperationException("Right schema is null"); + var joinSchema = (JoinTableSchema)myIjs.schema; + JoinNameResolver resolver = new JoinNameResolver( + joinSchema, leftSchema, myLeftCm.columnExpression, rightSchema, myRightCm.columnExpression); IDictionary map = resolver.getMap (); map["id"] = "id"; + Disposable.Dispose(ref projectSubscription); myPs = new ProjectStream (myIjs, map); - myPs.onChange += new DataStream.ChangeDelegate (myProjectChangeHandler); + projectSubscription = myPs.Subscribe(myProjectChangeHandler); return; } else Log.assert(false, "This call back should not have been called"); } - else - myFireEvent ((o as ICloneable).Clone()); + else { + myFireEvent(cloneDataPacket(o)); + } } private void myProjectChangeHandler (object o) { - if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1) - myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler); - myFireEvent ((o as ICloneable).Clone()); - + ExceptionAssert.NotNull(myIjs); + if (myIjs.numberOfSubscribers > 1) { + Disposable.Dispose(ref internalSubscription); + } + myFireEvent(cloneDataPacket(o)); } protected override void myOnFirstSubscriber () { - myIjs.onChange += new DataStream.ChangeDelegate (myInternalJoinChangeHandler); + Log.isnull(internalSubscription); + internalSubscription = ExceptionAssert.NotNull(myIjs).Subscribe(myInternalJoinChangeHandler); } protected override void myOnLastSubscriber () { - // how does this relate to the -= in myProjectChangeHandler? - myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler); - + Disposable.Dispose(ref internalSubscription); // if there is a project after the ijs we need to // unsubscribe to it otherwise references will be kept and // a memory leak will occur - if (myPs != null) - myPs.onChange -= new DataStream.ChangeDelegate (myProjectChangeHandler); + Disposable.Dispose(ref projectSubscription); myPs = null; } + object cloneDataPacket(object dpObj) { + var rv = (dpObj as ICloneable)?.Clone(); + if (rv is null) { + var dp = (IDataPacket)dpObj; + rv = new DataPacket(new BlueTable(dp.newRows), new BlueTable(dp.addedRows), new BlueTable(dp.deletedRows)); + } + return rv; + } + /**************************************TEST METHOD(S)*********************************************/ /// /// Used for testing purposes only. This enables the test to work without a DataSource (PubServer, @@ -159,7 +179,9 @@ namespace BCG /// /// /// - public void test (DataPacket dp, bool isLeft) {myIjs.test (dp, isLeft);} + public void test (DataPacket dp, bool isLeft) { + ExceptionAssert.NotNull(myIjs).test(dp, isLeft); + } /***********************************************************************************************/ } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs index a2d88c1ea..71c6f47cd 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs @@ -1,14 +1,17 @@ +using Bts.DataStreamCore; + +using Spark.Core.Diagnostics; + using System; using System.Collections; -using System.IO; -using BCG.Utilities.General; +using System.Collections.Generic; namespace BCG { internal class JoinTable : IBlueRowCollection { protected JoinTableSchema mySchema; - protected IDictionary myIdIndex = new Hashtable (); + protected readonly Dictionary myIdIndex = new(); protected JoinDeletionIndex myLeftIndex, myRightIndex; private int myNewRowId; @@ -19,19 +22,16 @@ namespace BCG myRightIndex = new JoinDeletionIndex (mySchema.getRenamedName ("id", false), false); } - public IBlueRow this [int id] - { - get - { - if (myIdIndex[id] == null) - throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName); - return (IBlueRow)myIdIndex[id]; + public IBlueRow this [int id] { + get { + var rowObj = myIdIndex[id] + ?? throw new BlueDataException($"ID {id} does not exist in table {mySchema.tableName}"); + return (IBlueRow)rowObj; } } #region JoinTable members - public virtual void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp) - { + public virtual void addRow(IBlueRow leftRow, IBlueRow? rightRow, IDataPacket dp) { JoinRow newRow = new JoinRow (myNewRowId, leftRow, rightRow, mySchema); myIdIndex[myNewRowId ++] = newRow; ((BlueTable)(dp.addedRows)).addRow (newRow); @@ -40,7 +40,7 @@ namespace BCG myRightIndex.add (newRow); } - public virtual void deleteRow (bool isLeft, IBlueRow deletedRow, DataPacket dp) + public virtual void deleteRow (bool isLeft, IBlueRow deletedRow, IDataPacket dp) { JoinDeletionIndex thisIndex = myRightIndex, otherIndex = myLeftIndex; if (isLeft) @@ -48,17 +48,17 @@ namespace BCG thisIndex = myLeftIndex; otherIndex = myRightIndex; } - RowList matchingNodes = thisIndex.getMatchingRows (deletedRow["id"]); + RowList matchingNodes = thisIndex.getMatchingRows(deletedRow.Id()); foreach (RowNode node in matchingNodes) { - JoinRow row = (JoinRow)node.row; - RowNode correspondingNode = isLeft ? row.rightNode : row.leftNode; + var row = ExceptionAssert.Argument.Is(node.row); + RowNode? correspondingNode = isLeft ? row.rightNode : row.leftNode; if (correspondingNode != null) otherIndex.remove (correspondingNode); //in case of partial left joinss - myIdIndex.Remove (row["id"]); + myIdIndex.Remove(row.Id()); ((BlueTable)dp.deletedRows).addRow (row); } - thisIndex.removeAll (deletedRow["id"]); + thisIndex.removeAll(deletedRow.Id()); } #endregion #region ICollection members @@ -68,7 +68,11 @@ namespace BCG public object SyncRoot{get{return this;}} - public void CopyTo (Array array, int index) {myIdIndex.Values.CopyTo (array, index);} + public void CopyTo (Array array, int index) { + foreach (var row in myIdIndex.Values) { + array.SetValue(row, index++); + } + } #endregion #region IEnumerable members diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinTableSchema.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinTableSchema.cs index 74ae14431..7404a1b64 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/JoinTableSchema.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinTableSchema.cs @@ -1,9 +1,8 @@ -using System; -using System.Collections; -using BCG.Utilities; - using bts; +using System.Collections.Generic; +using System.Linq; + namespace BCG { /// @@ -13,10 +12,11 @@ namespace BCG public class JoinTableSchema : BlueTableSchema { private string myLeftJc = "", myRightJc = "", myLeftId = "", myRightId = ""; - private IDictionary myLeftNameToAlias = new Hashtable (); - private IDictionary myRightNameToAlias = new Hashtable (); - private IDictionary myAliasToLeftName = new Hashtable(), - myAliasToRightName = new Hashtable(); + readonly Dictionary myLeftNameToAlias = new(); + readonly Dictionary myRightNameToAlias = new(); + readonly Dictionary myAliasToLeftName = new(); + readonly Dictionary myAliasToRightName = new(); + /// /// Constructs the schema of the Jointable with the schema of the left and right tables. Incase of /// a name being there in both the shemas it prefixes the name with "L." or "R." accordingly. Column "id" @@ -85,9 +85,9 @@ namespace BCG renamed = false; } myIndexByColName["id"] = new ColumnInfo("id", myIndexByColName.Count, typeof(int)); - myIndexByColOrder[myIndexByColOrder.Length - 1] = (ColumnInfo)myIndexByColName["id"]; - myAliasToLeftName = ProjectHelper.invertDic(myLeftNameToAlias); - myAliasToRightName = ProjectHelper.invertDic(myRightNameToAlias); + myIndexByColOrder[myIndexByColOrder.Length - 1] = myIndexByColName["id"]; + myAliasToLeftName = myLeftNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key); + myAliasToRightName = myRightNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key); } public string leftJoinCol {get{return myLeftJc;}} diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs index c579d4616..ccab9ebb9 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs @@ -1,17 +1,21 @@ using bts; using bts.collections; +using Bts.DataStreamCore; + using mbts; +using Spark.Core.Diagnostics; +using Spark.Core.Disposables; + using System; using System.Collections.Generic; using System.Diagnostics; -using System.Linq; -using System.Security.Cryptography; +using System.Diagnostics.CodeAnalysis; namespace BCG { public class NonIncJoinBookkeeper : IDisposable { - public Action on_new_data; + public event Action? on_new_data; private readonly NonIncJoinSpec _join_spec; private readonly int _join_position; private readonly int _start_index_in_output; @@ -20,13 +24,14 @@ namespace BCG { private readonly bool _is_tag_along; private int _n_total_fires = 0, _n_expensive_fires = 0; - private IBlueTableSchema _schema = null; + IBlueTableSchema? _schema; private IDictionary _row_map = new Dictionary(); private List> _row_list = new List>(); - private IBlueRow _solitary_row = null; + IBlueRow? _solitary_row; - private int[] _join_col_indices = null, - _data_col_indices = null; + int[]? _join_col_indices; + int[]? _data_col_indices; + IDisposable? subscription; public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) { _join_spec = the_join_spec; @@ -70,7 +75,8 @@ namespace BCG { } public void init() { - _join_spec.stream.onChange += _on_stream_cb; + ExceptionAssert.MemberIsNull(subscription); + subscription = ExceptionAssert.MemberNotNull(_join_spec.stream).Subscribe(_on_stream_cb); } public void _on_stream_cb(object dp_obj) { @@ -78,12 +84,7 @@ namespace BCG { var timer = Stopwatch.StartNew(); var dp = (IDataPacket)dp_obj; _fill_missing_data(dp); - if (_schema == null) { - _schema = dp.Schema; - _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names); - _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names); - } - + init(dp); if (_is_left_most) { _row_list.Clear(); foreach (IBlueRow r in dp.newRows) @@ -108,44 +109,68 @@ namespace BCG { on_new_data(); } + [MemberNotNullWhen(true, nameof(_schema))] public bool is_ready { get { return _schema != null; } } public void constribute_cols_to_schema(List cis) { + requireReady(); for (int i = 0; i < _join_spec.data_col_names.Length; i++) cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType)); } public List> get_list() { return _row_list; } - public void copy_master_row_to_output_row(int row_index, object[] output_row) { - IBlueRow this_row = _row_list[row_index].second; - BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output); - } + public void copy_master_row_to_output_row(int row_index, object?[] output_row) => + copyRow(ExceptionAssert.NotNull(_row_list[row_index].second), output_row); - public void copy_matching_row_to_output_row(int row_hash, object[] output_row) { + public void copy_matching_row_to_output_row(int row_hash, object?[] output_row) { if (_is_tag_along) { if (_solitary_row != null) - BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output); + copyRow(_solitary_row, output_row); else copy_dbnull_to_output_row(output_row); } else { - IBlueRow this_row = null; - if (_row_map.TryGetValue(row_hash, out this_row)) - BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output); + if (_row_map.TryGetValue(row_hash, out var this_row)) + copyRow(this_row, output_row); else copy_dbnull_to_output_row(output_row); } } - private void copy_dbnull_to_output_row(object[] output_row) { + void copy_dbnull_to_output_row(object?[] output_row) { + requireDataColIndices(); for (int i = 0; i < _data_col_indices.Length; i++) output_row[i + _start_index_in_output] = DBNull.Value; } #region IDisposable - public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; } + public void Dispose() => + Disposable.Dispose(ref subscription); #endregion + + void copyRow(IBlueRow src, object?[] dst) { + requireDataColIndices(); + BlueDataUtility.copy_to_row(src, _data_col_indices, dst, _start_index_in_output); + } + + [MemberNotNull(nameof(_schema), nameof(_join_col_indices), nameof(_data_col_indices))] + void init(IDataPacket dp) { + _schema ??= dp.Schema; + _join_col_indices ??= BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names); + _data_col_indices ??= BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names); + } + + [MemberNotNull(nameof(_data_col_indices))] + int[] requireDataColIndices() => + ExceptionAssert.NotNull(_data_col_indices); + + [MemberNotNull(nameof(_schema))] + void requireReady() { + if (!is_ready) { + throw new InvalidOperationException("Not ready"); + } + } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs index 432dbd539..b3e949501 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs @@ -1,22 +1,25 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using mbts; -using mbts; +using System; +using System.Collections.Generic; namespace BCG { // this a collection of data, theres no joining logic in here public class NonIncJoinSpec { - public ViewStream stream; + public ViewStream? stream; // join col names are the names on which the streams are joined (for equality), // data col names are the names in the original stream // projected col names are their new names, ie if its x:l_x, then x is in the // data_col_names and l_x in projected cols - public string[] join_col_names, data_col_names, projected_col_names; + public string[] join_col_names; + public string[] data_col_names; + public string[] projected_col_names; - public NonIncJoinSpec() { } + public NonIncJoinSpec() { + join_col_names = Array.Empty(); + data_col_names = Array.Empty(); + projected_col_names = Array.Empty(); + } public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names) : this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs index cd8872b9b..2ba22ad42 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs @@ -5,18 +5,19 @@ using mbts; using System; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; namespace BCG { [DataStreamUserRespresentation("fastJoin")] public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker { private readonly NonIncJoinSpec[] _join_specs; - private readonly INonIncJoinInlineMapper _output_mapper; + readonly INonIncJoinInlineMapper? _output_mapper; private readonly BlueRowHasher _row_hasher; private readonly TimeSpan _throttle_period; private readonly ViewStreamDatabase _vsd; private List _book_keeping_data = new List(); - private IBlueTableSchema _output_schema = null; + IBlueTableSchema? _output_schema; private DateTime _last_fire_time = DateTime.MinValue; private bool _is_dirty = false; @@ -35,10 +36,10 @@ namespace BCG { params Triplet[] join_specs) : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { } - public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) + public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper? output_mapper, int throttles_ms = 0) : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { } - public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) { + public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper? output_mapper, int throttles_ms = 0) { _join_specs = join_specs; _output_mapper = output_mapper; _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length); @@ -88,6 +89,7 @@ namespace BCG { return rv; } + [MemberNotNull(nameof(_output_schema))] private void _construct_schema() { if (_output_schema == null) { List cis = new List(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }); diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/RowList.cs b/src/DataStreamSuite/DataStream/Primitives/Join/RowList.cs index c79697571..f7c7c9c60 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/RowList.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/RowList.cs @@ -1,3 +1,5 @@ +using Spark.Core.Diagnostics; + using System; using System.Collections; @@ -9,7 +11,7 @@ namespace BCG /// internal class RowList : ICollection { - private RowNode myFirstPtr; + RowNode? myFirstPtr; private int myCount = 0; #region ICollection methods @@ -67,19 +69,19 @@ namespace BCG } else if (node.isFirst) { - myFirstPtr = myFirstPtr.next; - myFirstPtr.previous = null; + myFirstPtr = ExceptionAssert.NotNull(myFirstPtr).next; + ExceptionAssert.NotNull(myFirstPtr).previous = null; return; } else if (node.isLast) { - node.previous.next = null; + ExceptionAssert.NotNull(node.previous).next = null; return; } else if (!node.isFirst && !node.isLast) { - node.previous.next = node.next; - node.next.previous = node.previous; + ExceptionAssert.NotNull(node.previous).next = node.next; + ExceptionAssert.NotNull(node.next).previous = node.previous; return; } else @@ -91,14 +93,16 @@ namespace BCG internal class RowListEnumerator : IEnumerator { - private RowNode myPointer; + RowNode? myPointer; private RowList myList; private int myPos = -1; public RowListEnumerator (RowList list) {myList = list;} - public object Current {get{return myPointer;}} + public object? Current => myPointer; public bool MoveNext() { - myPointer = myPos == -1 ? myList.myFirstPtr : myPointer.next; + myPointer = myPos == -1 + ? myList.myFirstPtr + : ExceptionAssert.NotNull(myPointer).next; myPos ++; return myPointer != null; } @@ -108,28 +112,24 @@ namespace BCG internal class RowNode { - private IBlueRow myRow; - private RowNode myNext; - private RowNode myPrevious; - private RowList myList; - - public RowNode (IBlueRow row){myRow = row;} - public RowNode (IBlueRow row, RowList list) : this (row) {myList = list;} - public bool isLast {get{return myNext == null;}} - public bool isFirst {get{return myPrevious == null;}} - public RowList list {get{return myList;}} - public IBlueRow row - { - get{return myRow;} - set{myRow = value;} - } - public RowNode next { - get{return myNext;} - set{myNext = value;} - } - public RowNode previous { - get{return myPrevious;} - set{myPrevious = value;} - } + public RowNode (IBlueRow row) => + this.row = row; + + public RowNode (IBlueRow row, RowList list) + : this(row) => + this.list = list; + + public bool isLast => next is null; + public bool isFirst => previous is null; + public RowList? list { get; } + public IBlueRow? row { get; set; } + public RowNode? next { get; set; } + public RowNode? previous { get; set; } + + public RowList RequireList() => + ExceptionAssert.MemberNotNull(list); + + public IBlueRow RequireRow() => + ExceptionAssert.MemberNotNull(row); } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs index cb6940cb0..a2034ae85 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs @@ -1,125 +1,17 @@ +using Bts.DataStreamCore; + +using mbts; + +using Spark.Core.Collections; +using Spark.Core.Diagnostics; + using System; using System.Collections; using System.Collections.Generic; -using mbts; -using Spark.Core.Collections; +using System.Diagnostics.CodeAnalysis; namespace BCG { - internal class SingleRowJoinDicionary : IDictionary - { - private TKey myKey; - private TValue myValue; - - #region IDictionary Members - - public void Add(TKey key, TValue value) - { - myKey = key; - myValue = value; - } - - public bool ContainsKey(TKey key) - { - return myKey.Equals(key); - } - - public ICollection Keys - { - get { return new TKey[] { myKey }; } - } - - public bool Remove(TKey key) - { - bool rv = myKey.Equals(key); - myKey = default(TKey); - return rv; - } - - public bool TryGetValue(TKey key, out TValue value) - { - bool rv = false; - value = (rv = myKey.Equals(key)) ? myValue : default(TValue); - return rv; - } - - public ICollection Values - { - get { throw new Exception("The method or operation is not implemented."); } - } - - public TValue this[TKey key] - { - get - { - throw new Exception("The method or operation is not implemented."); - } - set - { - throw new Exception("The method or operation is not implemented."); - } - } - - #endregion - - #region ICollection> Members - - public void Add(KeyValuePair item) - { - throw new Exception("The method or operation is not implemented."); - } - - public void Clear() - { - throw new Exception("The method or operation is not implemented."); - } - - public bool Contains(KeyValuePair item) - { - throw new Exception("The method or operation is not implemented."); - } - - public void CopyTo(KeyValuePair[] array, int arrayIndex) - { - throw new Exception("The method or operation is not implemented."); - } - - public int Count - { - get { throw new Exception("The method or operation is not implemented."); } - } - - public bool IsReadOnly - { - get { throw new Exception("The method or operation is not implemented."); } - } - - public bool Remove(KeyValuePair item) - { - throw new Exception("The method or operation is not implemented."); - } - - #endregion - - #region IEnumerable> Members - - public IEnumerator> GetEnumerator() - { - throw new Exception("The method or operation is not implemented."); - } - - #endregion - - #region IEnumerable Members - - IEnumerator IEnumerable.GetEnumerator() - { - throw new Exception("The method or operation is not implemented."); - } - - #endregion - } - #region Helper class to deal with stream projections public class JoinProjectHelper @@ -167,7 +59,7 @@ namespace BCG { Pair p = myAliasToOriginal[col]; isLeft = p.second; - return p.first; + return ExceptionAssert.NotNull(p.first); } } @@ -178,10 +70,11 @@ namespace BCG public class SuperJoinRow : IBlueRow { private JoinProjectHelper myHelper; - private IBlueRow myLeftRow, myRightRow; + IBlueRow myLeftRow; + IBlueRow? myRightRow; int myID; - public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow rightRow, JoinProjectHelper joinProjectHelper) + public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow? rightRow, JoinProjectHelper joinProjectHelper) { myID = id; myLeftRow = leftRow; @@ -189,21 +82,22 @@ namespace BCG myHelper = joinProjectHelper; } - public int leftID { get { return (int)myLeftRow["id"]; } } + public int leftID => myLeftRow.Id(); - public int rightID { get { return (int)myRightRow["id"]; } } + public int rightID => myRightRow.Id(); public IBlueRow leftRow { get { return myLeftRow; } } - public IBlueRow rightRow { get { return myRightRow; } } + public IBlueRow? rightRow => myRightRow; #region IBlueRow Members public IBlueTableSchema schema { get { return myHelper.schema; } } - public object this[int index] { get { return this[schema[index].colName]; } } + public object? this[int index] => + this[schema[index].colName]; - public object this[string colName] + public object? this[string colName] { get { @@ -229,7 +123,7 @@ namespace BCG public bool Contains(object key) { - return myLeftRow.Contains(key) || myRightRow.Contains(key); + return myLeftRow.Contains(key) || (myRightRow?.Contains(key) ?? false); } public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } @@ -242,11 +136,11 @@ namespace BCG public void Remove(object key) { throw new NotSupportedException("IBlueRow is readonly"); } - public System.Collections.ICollection Values + public ICollection Values { get { - List rv = new List(); + var rv = new List(Keys.Count); foreach (string col in Keys) rv.Add(this[col]); return rv; @@ -291,9 +185,10 @@ namespace BCG myLeftUnmatchedRowsDels = new Dictionary(); private bool myManageUnmappedLeftRows = false; protected string myLeftProj = String.Empty, myRightProj = String.Empty; - protected IBlueTableSchema myLeftSchema, myRightSchema; - protected JoinProjectHelper myJoinProjectHelper; - protected BlueTable myState; + protected IBlueTableSchema? myLeftSchema; + protected IBlueTableSchema? myRightSchema; + protected JoinProjectHelper? myJoinProjectHelper; + protected BlueTable? myState; protected int myID; protected ViewStream myLeftStream, myRightStream; protected string myLeftKeyCol, myRightKeyCol; @@ -357,12 +252,8 @@ namespace BCG private void myHandleStreamChanges(IDataPacket dp, bool isLeft) { myInitSchema(dp.Schema, isLeft); - BlueTable dels = null, adds = null; - if (myState != null) - { - dels = new BlueTable(myState.schema); - adds = new BlueTable(myState.schema); - } + var dels = new BlueTable(myState.schema); + var adds = new BlueTable(myState.schema); myDeleteRows(dp.deletedRows, isLeft, myState, dels); myAddRows(dp.addedRows, isLeft, myState, adds); if (myState != null) @@ -373,6 +264,7 @@ namespace BCG } } + [MemberNotNull(nameof(myState))] private void myInitSchema(IBlueTableSchema schema, bool isLeft) { if (myState == null) @@ -402,6 +294,7 @@ namespace BCG private void myAddRows(IBlueRowCollection addedRows, bool isLeft, BlueTable state, BlueTable addedRowsOut) { + ExceptionAssert.NotNull(myJoinProjectHelper); object thisKeyCol = isLeft ? myLeftKeyCol : myRightKeyCol, otherSideKeyCol = isLeft ? myRightKeyCol : myLeftKeyCol; Dictionary> thisKeyToRows = isLeft ? myLeftKeyToLeftRows : myRightKeyToRightRows, @@ -410,8 +303,8 @@ namespace BCG otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows; foreach (IBlueRow r in addedRows) { - object currentKey = r[thisKeyCol]; - int currentID = (int)r["id"]; + object currentKey = ExceptionAssert.NotNull(r[thisKeyCol]); + int currentID = r.Id(); ///add it to 1. if (!thisKeyToRows.ContainsKey(currentKey)) thisKeyToRows.Add(currentKey, new Dictionary()); @@ -422,14 +315,14 @@ namespace BCG ICollection matchingRows = otherSideKeyToRows[currentKey].Values; foreach (IBlueRow mr in matchingRows) { - int mrID = (int)mr["id"]; + int mrID = mr.Id(); SuperJoinRow rr = new SuperJoinRow(myID++, isLeft ? r : mr, isLeft ? mr : r, myJoinProjectHelper); state.addRow(rr); addedRowsOut.addRow(rr); ///add to 3. if (!thisIDToResultRows.ContainsKey(currentID)) thisIDToResultRows.Add(currentID, new Dictionary()); - thisIDToResultRows[currentID].Add((int)rr["id"], rr); + thisIDToResultRows[currentID].Add(rr.Id(), rr); ///add to 4. if (!otherIDToResultRows.ContainsKey(mrID)) { @@ -437,7 +330,7 @@ namespace BCG if (myManageUnmappedLeftRows && !isLeft) myTakeCareOfUnmatachedLeftRows(mr, false); } - otherIDToResultRows[mrID].Add((int)rr["id"], rr); + otherIDToResultRows[mrID].Add(rr.Id(), rr); } } else @@ -458,22 +351,22 @@ namespace BCG otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows; foreach (IBlueRow r in deletedRows) { - int id = (int)r["id"]; - object key = r[thisKeyCol]; + int id = r.Id(); + object key = ExceptionAssert.NotNull(r[thisKeyCol]); ///delete it from 1. thisKeyToRows[key].Remove(id); if (thisKeyToRows[key].Count == 0) thisKeyToRows.Remove(key); ///form return rows and delete from 3. - if (thisIDToResultRows.ContainsKey(id)) + if (thisIDToResultRows.TryGetValue(id, out IDictionary? removedResultRowMap)) { - ICollection removedResultRows = thisIDToResultRows[id].Values as ICollection; + var removedResultRows = (ICollection)removedResultRowMap.Values; thisIDToResultRows.Remove(id); foreach (SuperJoinRow rr in removedResultRows) { ///delete from 4. int otherId = isLeft ? rr.rightID : rr.leftID; - otherIDToResultRows[otherId].Remove((int)rr["id"]); + otherIDToResultRows[otherId].Remove(rr.Id()); if (otherIDToResultRows[otherId].Count == 0) otherIDToResultRows.Remove(otherId); delRowsOut.addRow(rr); @@ -493,7 +386,7 @@ namespace BCG private void myTakeCareOfUnmatachedLeftRows(IBlueRow leftRow, bool add) { - int lid = (int)leftRow["id"]; + int lid = leftRow.Id(); if (add) myLeftUnmatchedRowsAdds.Add(lid, leftRow); else @@ -501,7 +394,7 @@ namespace BCG if (myLeftUnmatchedRowsAdds.ContainsKey(lid)) myLeftUnmatchedRowsAdds.Remove(lid); else - myLeftUnmatchedRowsDels.Add((int)leftRow["id"], leftRow); + myLeftUnmatchedRowsDels.Add(lid, leftRow); } } } @@ -526,38 +419,45 @@ namespace BCG } void myHandleJoinResults(Dictionary unmatchedLeftRowsDels, Dictionary unmatchedLeftRowAdds, IDataPacket resultPacket) { - BlueTable dels = resultPacket.deletedRows as BlueTable, - adds = resultPacket.addedRows as BlueTable; + ExceptionAssert.NotNull(myJoinProjectHelper); + var dels = resultPacket.deletedRows; + var adds = resultPacket.addedRows; - foreach (IBlueRow lr in unmatchedLeftRowsDels.Values) - { - ///an lr will be here if its actually deleted or - ///a right row is added, so now it has a partner - ///and is no longer unmatched. - ///in both cases the half row that this lr - ///created needs to be removed from this - ///class's state and added to dels - ///it means a right is added - int lid = (int)lr["id"]; - SuperJoinRow rr = myLeftIDToHalfRows[lid]; - myLeftIDToHalfRows.Remove(lid); - myState.deleteRow(rr); - dels.addRow(rr); + if (unmatchedLeftRowsDels.Count > 0) { + ExceptionAssert.NotNull(myState); + var mutableDels = (BlueTable)dels; + foreach (IBlueRow lr in unmatchedLeftRowsDels.Values) { + ///an lr will be here if its actually deleted or + ///a right row is added, so now it has a partner + ///and is no longer unmatched. + ///in both cases the half row that this lr + ///created needs to be removed from this + ///class's state and added to dels + ///it means a right is added + int lid = lr.Id(); + SuperJoinRow rr = myLeftIDToHalfRows[lid]; + myLeftIDToHalfRows.Remove(lid); + myState.deleteRow(rr); + mutableDels.addRow(rr); + } } - foreach (IBlueRow lr in unmatchedLeftRowAdds.Values) - { - ///an lr will be here if its added with no - ///matching right, or a right matching row - ///is gone so it is unmatched now. - ///in both cases a half row needs to be - ///added to this class's state and added - ///to adds - int lid = (int)lr["id"]; - SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper); - myLeftIDToHalfRows.Add(lid, rr); - myState.addRow(rr); - adds.addRow(rr); + if (unmatchedLeftRowAdds.Count > 0) { + ExceptionAssert.NotNull(myState); + var mutableAdds = (BlueTable)adds; + foreach (IBlueRow lr in unmatchedLeftRowAdds.Values) { + ///an lr will be here if its added with no + ///matching right, or a right matching row + ///is gone so it is unmatched now. + ///in both cases a half row needs to be + ///added to this class's state and added + ///to adds + int lid = lr.Id(); + SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper); + myLeftIDToHalfRows.Add(lid, rr); + myState.addRow(rr); + mutableAdds.addRow(rr); + } } base.myFireEvent(new DataPacket(myState, adds, dels)); } diff --git a/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinStream.cs index d90d402c6..e04c21b20 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinStream.cs @@ -1,4 +1,9 @@ -using System.Collections; +using Bts.DataStreamCore; + +using Spark.Core.Diagnostics; + +using System; +using System.Collections.Generic; namespace BCG { @@ -12,7 +17,8 @@ namespace BCG protected IDataPacket? myLeftDataPacket; protected IDataPacket? myRightDataPacket; //Container to contain all the left rows that are orphaned during a deletion operation - private IDictionary myLeftOrphanedRows; + Dictionary? myLeftOrphanedRows; + public LeftInternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight) : base (vsLeft, jcLeft, vsRight, jcRight) {} @@ -61,7 +67,9 @@ namespace BCG private void myTestAndFire (bool isLeft) { JoinOperand jo = isLeft ? myLeftOperand : myRightOperand; - var dp = isLeft ? myLeftDataPacket : myRightDataPacket; + var dp = isLeft + ? ExceptionAssert.NotNull(myLeftDataPacket) + : ExceptionAssert.NotNull(myRightDataPacket); //to test whether both have received call backs if (myLeftOperand.hasReceivedData) jo.changeHandler (dp); @@ -79,11 +87,13 @@ namespace BCG /// public override void addRows (IBlueRowCollection rows, bool isLeft) { + ExceptionAssert.NotNull(myLastFiredPacket); foreach (IBlueRow row in rows) myAddRow (row, isLeft); if (!isLeft) - foreach (RowNode node in myLeftOrphanedRows.Values) - ((JoinTable)myLastFiredPacket.newRows).addRow (node.row, null, myLastFiredPacket); + foreach (RowNode node in ExceptionAssert.NotNull(myLeftOrphanedRows).Values) { + ((JoinTable)myLastFiredPacket.newRows).addRow(node.RequireRow(), null, myLastFiredPacket); + } } /// @@ -93,17 +103,19 @@ namespace BCG /// public override void deleteRows (IBlueRowCollection rows, bool isLeft) { - myLeftOrphanedRows = new Hashtable (); + myLeftOrphanedRows = new(); foreach (IBlueRow row in rows) myDeleteRow (row, isLeft); } private void myAddRow (IBlueRow row, bool isLeft) { + ExceptionAssert.NotNull(myLastFiredPacket); if (isLeft) { myLeftOperand.addRow (row); - RowList matchedRows = myRightOperand.getMatchingRows (row[myLeftJoinColOrdinal]); + var joinKey = ExceptionAssert.NotNull(row[myLeftJoinColOrdinal]); + RowList matchedRows = myRightOperand.getMatchingRows(joinKey); if (matchedRows.Count == 0) ((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket); else @@ -112,22 +124,25 @@ namespace BCG } else { - RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]); - int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count; + var joinKey = ExceptionAssert.NotNull(row[myRightJoinColOrdinal]); + RowList matchedRows = myLeftOperand.getMatchingRows(joinKey); + int rightMatchingRowCount = myRightOperand.getMatchingRows(joinKey).Count; myRightOperand.addRow (row); - int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count; + int newRightMatchingRowCount = myRightOperand.getMatchingRows(joinKey).Count; if (matchedRows.Count != 0){ foreach (RowNode matchedNode in matchedRows) { + ExceptionAssert.NotNull(matchedNode.row); //if the left rows were orphaned in this transaction (ie in this callback), they should //just be removed from the orphanedlist and we dont need to worry about the consistency of //the jointable as a new row will replace it if (rightMatchingRowCount == 0 && newRightMatchingRowCount > 0) { - if (myLeftOrphanedRows[matchedNode.row["id"]] == null) - ((JoinTable)myLastFiredPacket.newRows).deleteRow (true, matchedNode.row, myLastFiredPacket); - else - myLeftOrphanedRows.Remove (matchedNode.row["id"]); + ExceptionAssert.NotNull(myLeftOrphanedRows); + int matchedId = matchedNode.row.Id(); + if (!myLeftOrphanedRows.Remove(matchedId)) { + ((JoinTable)myLastFiredPacket.newRows).deleteRow(true, matchedNode.row, myLastFiredPacket); + } } //they were orphaned by a previous transaction and hence the orphans need to be //removed from the Jointable and the complete row added @@ -140,41 +155,52 @@ namespace BCG private void myDeleteRow (IBlueRow row, bool isLeft) { + var last = requireLastFiredPacket(); if (isLeft) { myLeftOperand.deleteRow (row); - ((JoinTable)myLastFiredPacket.newRows).deleteRow (true, row, myLastFiredPacket); + ((JoinTable)last.newRows).deleteRow (true, row, myLastFiredPacket); } else { - RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]); - int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count; + var rightJoinKey = row[myRightJoinColOrdinal] ?? throw new InvalidOperationException("RightJoinKey is null"); + RowList matchedRows = myLeftOperand.getMatchingRows(rightJoinKey); + int rightMatchingRowCount = myRightOperand.getMatchingRows(rightJoinKey).Count; myRightOperand.deleteRow (row); - int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count; + int newRightMatchingRowCount = myRightOperand.getMatchingRows(rightJoinKey).Count; if (matchedRows.Count != 0) { - if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0) + if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0) { + ExceptionAssert.MemberNotNull(myLeftOrphanedRows); //ie all the matchedRows were joined with 'this' right row (method arg), and hence they need to //be handled specially, ie joined with nulls, if there are no right rows with this //joincolumn value being inserted. + ExceptionAssert.NotNull(myLeftOrphanedRows); foreach (RowNode matchedNode in matchedRows) { // BlueEventLog.logAssert (myLeftOrphanedRows[matchedNode.row["id"]] == null); - myLeftOrphanedRows[matchedNode.row["id"]] = matchedNode; + myLeftOrphanedRows[matchedNode.row.Id()] = matchedNode; } - ((JoinTable)myLastFiredPacket.newRows).deleteRow (false, row, myLastFiredPacket); + } + ((JoinTable)last.newRows).deleteRow (false, row, myLastFiredPacket); } } } public override void onTableClear (bool isLeft) { + if (myLastFiredPacket is null) { + return; + } + requireSchema(); myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; myLastFiredPacket.newRows = new JoinTable (mySchema); myLastFiredPacket.addedRows = new BlueTable (mySchema); - if (!isLeft) + if (!isLeft) { + ExceptionAssert.MemberNotNull(myLeftDataPacket); foreach (IBlueRow row in myLeftDataPacket.newRows) ((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket); + } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/ILightForeignJoinRowList.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/ILightForeignJoinRowList.cs index 5766330e6..1a8c5ea50 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/ILightForeignJoinRowList.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/ILightForeignJoinRowList.cs @@ -30,7 +30,7 @@ namespace BCG /// /// Node containing the referencedRow /// - IBlueRow referencedRow {get;} + IBlueRow? referencedRow { get; } /// /// removes all referenced or referencing nodes (indicated by the bool), p.s. theres only one referenced row diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/INode.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/INode.cs index 7adab3364..916d9724f 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/INode.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/INode.cs @@ -15,12 +15,14 @@ namespace BCG /// /// /// - IBlueRow getRow (int id); + IBlueRow? getRow(int id); + /// /// Gets all the rows in the node /// /// - IBlueRow[] getRows (); + IBlueRow?[] getRows(); + /// /// Replaces the row with the given id, with the row passed as the second arg /// diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRow.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRow.cs index 7e02ff7ec..6bc0a3959 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRow.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRow.cs @@ -1,7 +1,5 @@ using System; using System.Collections; -using BCG.Utilities; -using BCG; namespace BCG { @@ -56,13 +54,18 @@ namespace BCG /// The key can either be the string corrsponding to the column name or /// the order of the column /// - public virtual object this [object key] + public virtual object? this [object key] { - get {return key is string ? this [key as string] : this [(int)key];} + // get {return key is string ? this [key as string] : this [(int)key];} + get => key switch { + string colName => this[colName], + int colIndex => this[colIndex], + _ => throw new ArgumentException("Must be string or int", nameof(key)), + }; set {throw new NotSupportedException ("The IBlueRow is read only");} } - public virtual object this [int index] + public virtual object? this [int index] { get { @@ -77,7 +80,8 @@ namespace BCG } } - public object this [string colName] {get{return this[mySchema[colName].colNo];}} + public object? this [string colName] => + this[mySchema[colName].colNo]; public ICollection Keys{get{return mySchema.columnNames;}} diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRowList.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRowList.cs index fb88d8ebf..b40e3bdd1 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRowList.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRowList.cs @@ -1,6 +1,6 @@ -using System; +using Spark.Core.Diagnostics; + using System.Collections; -using BCG.Utilities; namespace BCG { @@ -70,7 +70,8 @@ namespace BCG } else { - SimpleNode sn = n as SimpleNode; + var sn = (SimpleNode)n; + ExceptionAssert.NotNull(sn.prev); sn.prev.next = sn.next; if (sn.next != null) sn.next.prev = sn.prev; @@ -81,7 +82,7 @@ namespace BCG public int count {get {return myFirst.listCount;}} - public IBlueRow referencedRow {get {return myFirst.row;}} + public IBlueRow? referencedRow => myFirst.row; public void removeAll (bool isReD) { @@ -110,10 +111,11 @@ namespace BCG public class LightForeignJoinRowListEnumerator : IEnumerator { private ILightForeignJoinRowList myList; - private SimpleNode myPtr, myFirst; + private SimpleNode? myPtr; + private SimpleNode myFirst; public LightForeignJoinRowListEnumerator (ILightForeignJoinRowList list, SimpleNode first) {myList = list; myFirst = first;} - public object Current {get{return myPtr.row;}} + public object? Current => myPtr?.row; public bool MoveNext() {return (myPtr = myPtr == null ? myFirst : myPtr.next) != null;} public void Reset() {myPtr = null;} public ILightForeignJoinRowList list {get {return myList;}} diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSimpleCollection.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSimpleCollection.cs index 3f25fd7c0..f8abf5de2 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSimpleCollection.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinSimpleCollection.cs @@ -51,7 +51,7 @@ namespace BCG public void CopyTo (Array array, int index) {throw new NotSupportedException ("");} - public object SyncRoot {get{return null;}} + public object SyncRoot => this; #endregion diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinRow.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinRow.cs index 27d4ff0df..5f4b071aa 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinRow.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinRow.cs @@ -1,5 +1,4 @@ using System; -using BCG.Utilities; namespace BCG { @@ -13,8 +12,7 @@ namespace BCG : base (reDRow, rinGRow, schema) {} - public override object this [int index] - { + public override object? this [int index] { get { if (index == mySchema.columnNames.Length - 1) diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/SimpleNode.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/SimpleNode.cs index 5166c1243..487d917fb 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/SimpleNode.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/SimpleNode.cs @@ -1,5 +1,3 @@ -using System; - namespace BCG { /// @@ -9,15 +7,15 @@ namespace BCG /// public class SimpleNode : INode { - public IBlueRow row; - public SimpleNode next, prev; + public IBlueRow? row; + public SimpleNode? next; + public SimpleNode? prev; public SimpleNode (IBlueRow r) : this (r, null, null) {} - public SimpleNode (IBlueRow r, SimpleNode nx, SimpleNode pv) - { + public SimpleNode(IBlueRow r, SimpleNode? nx, SimpleNode? pv) { row = r; next = nx; prev = pv; @@ -25,13 +23,15 @@ namespace BCG #region INode Members - public IBlueRow getRow (int id) {return row;} + public IBlueRow? getRow(int id) => + row; - public IBlueRow[] getRows() {return new IBlueRow[] {row};} + public IBlueRow?[] getRows() => + new[] {row}; public void replace (int id, IBlueRow nr) {row = nr;} #endregion } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs b/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs index 05766e90c..a77397dce 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs @@ -1,14 +1,8 @@ -using System; -using System.Collections; -using System.Collections.Generic; - -using Amazon.Runtime.Internal.Util; - -using BCG; -using BCG.Utilities; - using bts; +using System; +using System.Collections.Generic; + namespace BCG { public class LiteralStream : ViewStream { @@ -42,14 +36,14 @@ namespace BCG { return new BlueRow(map); }; string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries); - BlueTable table = null; + BlueTable? table = null; foreach (string json_string in row_arr) { IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count); get_row(json_string, table == null ? 0 : table.Count); - table = table ?? new BlueTable(r.schema); + table ??= new BlueTable(r.schema); table.addRow(r); } - ViewStream str = new BlueTableStream(table); + ViewStream str = new BlueTableStream(table ?? throw new ArgumentException("No rows specified, unable to determine schema")); str = new SchemaGuesserStream(str); mySetRealSource(str); } diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/DataPacketRecorderStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/DataPacketRecorderStream.cs index 3afbad70c..f65d311d4 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/DataPacketRecorderStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/DataPacketRecorderStream.cs @@ -1,12 +1,12 @@ using BCG; -using System; -using System.IO; -using MongoDB.Bson; using BCG.Utilities.General; -using System.Linq; -using System.Collections.Generic; -using System.Runtime.Intrinsics.X86; +using mbts.utils; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; namespace bts { // listens to the underlying stream, and writes each seriialized packet in a @@ -32,15 +32,9 @@ namespace bts { private void _handle_update(object dp_obj) { var dp = (IDataPacket)dp_obj; - byte[] ser_data = null; - if (_n_updates == 0) { - StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows); - ser_data = _serializer.serialize(new StreamIOMessage(cr_msg)); - } else { - StreamIOMessage.MultipleRemoveAddMessage mra = new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows); - ser_data = _serializer.serialize(new StreamIOMessage(mra)); - } - Log.assert(ser_data != null); + byte[] ser_data = n_updates == 0 + ? _serializer.serialize(new(new StreamIOMessage.CreateMessage(dp.newRows))) + : _serializer.serialize(new(new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows))); _n_updates++; string file_name = $"{_folder}\\{_n_updates}.bin"; _write_to_file(ser_data, file_name); @@ -62,7 +56,7 @@ namespace bts { IStreamMsgDeserilizer _deser = new StreamIODeserializationController(); private int _file_ptr = 0; private List> _file_data = new List>(); - private BlueTable _state = null; + BlueTable? _state; public DataPacketReplayStream(string folder_path, string fire_period) : @@ -98,6 +92,7 @@ namespace bts { break; } case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: { + Log.notnull(_state); var mra = msg.multiple_remove_add_message; foreach (IBlueRow r in mra.removed_rows) _state.deleteRow(r); @@ -139,4 +134,4 @@ namespace bts { }); } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs index dc1c3f1a0..678f4f4fb 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs @@ -1,14 +1,12 @@ using BCG; using BCG.Utilities.General; -using System.Collections.Generic; +using Bts.PubSub.Routing; using System; +using System.Collections.Generic; using System.IO; using System.Linq; -using System.Runtime.CompilerServices; -using bts.api; -using Bts.PubSub.Routing; namespace bts { @@ -17,9 +15,9 @@ namespace bts { public class PayloadMessage { - public readonly string msg_id; + public readonly string? msg_id; public readonly Int64 timestamp = 0; - public readonly byte[] payload = null; + public readonly byte[]? payload = null; public static byte[] create_data(int payload_size, byte payload_byte) { @@ -53,10 +51,10 @@ namespace bts { var str = new MemoryStream(); BinaryWriter bw = new BinaryWriter(str); - bw.Write(msg_id); + bw.Write(msg_id ?? string.Empty); bw.Write(timestamp); - bw.Write(payload.Length); - bw.Write(payload); + bw.Write(payload?.Length ?? 0); + bw.Write(payload ?? Array.Empty()); str.Position = 0; var bytes = str.ToArray(); return bytes; @@ -73,18 +71,15 @@ namespace bts return new PayloadMessage(guid, timestamp_, payload); } } - //public void - private readonly int _msg_size; + private readonly TimeSpan _publish_period; private DateTime _last_publish_time = DateTime.MinValue; - private PublisherCore _pub_core; - private ISubscriberCore _sub_core; + PublisherCore? _pub_core; + ISubscriberCore? _sub_core; private BlueTable _table; - private int _n_pubs = 0; - private readonly int _max_pubs; - private int _n_updates = 0; private ViewStreamDatabase _vsd; - private PayloadMessage _last_sent_msg, _last_recvd_msg; + PayloadMessage? _last_sent_msg; + PayloadMessage? _last_recvd_msg; private const byte _payload_byte = (byte)'A'; private readonly string _tag_name; private readonly int _payload_sz; @@ -131,13 +126,13 @@ namespace bts protected override void myOnLastSubscriber() { _vsd.removeWorker(this); - _sub_core.unsubscribe(_tag_name); + Log.notnull(_sub_core).unsubscribe(_tag_name); } private void _fire_dp_after_send() { + Log.notnull(_last_sent_msg); // as soon as the data is published, its reflected in the stream state - Log.assert(_last_sent_msg != null); BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value); _table.addRow(r); @@ -147,7 +142,8 @@ namespace bts private void _fire_dp_after_recv() { - Log.assert(_last_recvd_msg != null); + Log.notnull(_last_recvd_msg); + Log.notnull(_last_sent_msg); // update the last publish row int id = _table.Count - 1; IBlueRow last_row = _table[id]; @@ -167,28 +163,16 @@ namespace bts private void _publish_next_packet() { + Log.notnull(_pub_core); var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte); _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes); var ser_bytes = _last_sent_msg.serialize(); - _n_pubs++; _pub_core.publish("Blob", _tag_name, ser_bytes); _last_publish_time = DateTime.Now; } public void doWork() { - Func, PayloadMessage> find_packet = (updates) => - { - foreach (var u in updates) - { - var recv_msg = PayloadMessage.deserialize(u.data); - if (recv_msg.msg_id == _last_sent_msg.msg_id) - { - return recv_msg; - } - } - return null; - }; switch (_state) { case IOState.SEND: @@ -202,7 +186,7 @@ namespace bts } break; case IOState.RECV: - var updates = _sub_core.get_updates(TimeSpan.Zero); + var updates = Log.notnull(_sub_core).get_updates(TimeSpan.Zero); updates = updates.Where(u => u.tag == _tag_name); var recv_msg = find_packet(updates); if (recv_msg != null) @@ -214,6 +198,19 @@ namespace bts } break; } + + PayloadMessage? find_packet(IEnumerable updates) { + if (_last_sent_msg is null) { + return null; + } + foreach (var u in updates) { + var recv_msg = PayloadMessage.deserialize(Log.notnull(u.data)); + if (recv_msg.msg_id == _last_sent_msg.msg_id) { + return recv_msg; + } + } + return null; + } } public static long epoch_now() diff --git a/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs b/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs index f3b5efeed..f52832420 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Misc/TestStream.cs @@ -1,5 +1,7 @@ using bts; +using Spark.Core.Diagnostics; + using System; using System.Collections.Generic; using System.IO; @@ -197,7 +199,6 @@ namespace BCG { private TestResult _run_schema_serialization_test() { - var table_name = "test_table"; var excluded_types = new HashSet() { typeof(DateTime), typeof(Decimal) }; IBlueTableSchema schema = create_schema_with_all_types(excluded_types); MemoryStream str = new MemoryStream(); @@ -375,7 +376,7 @@ namespace BCG { StreamIOSerializationController ser_controller = new StreamIOSerializationController(); StreamIODeserializationController deser_controller = new StreamIODeserializationController(); - IBlueTableSchema schema_p = null; + IBlueTableSchema? schema_p = null; var schema_file = @"c:\Temp\tests\schema.bin"; var row_file = @"C:\Temp\tests\row.bin"; var create_op_file = @"C:\Temp\tests\create_op.bin"; @@ -619,7 +620,7 @@ namespace BCG { private static TestResult _run_single_tag_data_test_clean() { // var tagHandlerFactory // create message - var test_name = MethodBase.GetCurrentMethod()?.Name; + var test_name = MethodBase.GetCurrentMethod()?.Name ?? string.Empty; var schema = create_schema_with_all_types(); var tag = "DS_" + schema.tableName; BlueTable table = new BlueTable(schema); @@ -695,14 +696,14 @@ namespace BCG { } - public static PublishMsg build_pub_msg(IBlueRow added_row, IBlueRow removed_row, bool is_create, + public static PublishMsg build_pub_msg(IBlueRow? added_row, IBlueRow? removed_row, bool is_create, ref IStreamIOMsgSerializer serializer) { Func get_schema = () => { - return (added_row != null ? added_row : removed_row).schema; + return (added_row != null ? added_row : ExceptionAssert.NotNull(removed_row)).schema; }; - bts.StreamIOMessage str_io_msg = null; + StreamIOMessage? str_io_msg; if (is_create) { - Log.assert(added_row != null); + ExceptionAssert.NotNull(added_row); var aux_new = new BlueTable(added_row.schema); aux_new.addRow(added_row); var cr_msg = new bts.StreamIOMessage.CreateMessage(aux_new); @@ -729,7 +730,7 @@ namespace BCG { return pub_msg; } - public static IBlueTableSchema create_schema_with_all_types(HashSet excluded_col_map = null) { + public static IBlueTableSchema create_schema_with_all_types(HashSet? excluded_col_map = null) { excluded_col_map = excluded_col_map == null ? new HashSet() : excluded_col_map; var table_name = "test_table"; BlueTableSchema schema = new BlueTableSchema(); @@ -800,7 +801,6 @@ namespace BCG { return BitConverter.ToUInt64(ulongBuffer, 0); default: throw new ArgumentException($"Unsupported TypeCode: {type_code}"); - return null; } } #endregion diff --git a/src/DataStreamSuite/DataStream/Primitives/Mux/MuxStream.cs b/src/DataStreamSuite/DataStream/Primitives/Mux/MuxStream.cs index 3eef03727..2ba420356 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Mux/MuxStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Mux/MuxStream.cs @@ -1,7 +1,4 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; namespace BCG { @@ -16,7 +13,7 @@ namespace BCG private string myInColumn; private string myOutValue; private object[] myMuxedValues; - private BlueTableSchema mySchema; + BlueTableSchema? mySchema; public MuxStream(ViewStream inStream, string outColumn, string outValue, string inColumn, params object[] muxedValues) { @@ -49,7 +46,7 @@ namespace BCG if (Array.IndexOf(myMuxedValues, r[myInColumn]) != -1) // muxable value? r.setValue(myOutColumn, myOutValue); else - r.setValue(myOutColumn, row[myInColumn].ToString()); + r.setValue(myOutColumn, row[myInColumn]?.ToString()); return r; } diff --git a/src/DataStreamSuite/DataStream/Primitives/NullToConstant/NullToConstantMapper.cs b/src/DataStreamSuite/DataStream/Primitives/NullToConstant/NullToConstantMapper.cs index 3be4d3ba9..53f9d3926 100644 --- a/src/DataStreamSuite/DataStream/Primitives/NullToConstant/NullToConstantMapper.cs +++ b/src/DataStreamSuite/DataStream/Primitives/NullToConstant/NullToConstantMapper.cs @@ -1,11 +1,8 @@ +using mbts; + using System; using System.Collections; -using BCG; -using BCG.Utilities; -using System.Collections.Generic; -using mbts; - namespace BCG { public class NullToConstantMapper : IStreamMapper @@ -27,17 +24,11 @@ namespace BCG #region IDynamicStreamMapper Members - public IBlueRow map(IBlueRow row) - { - if(row[myField].Equals(DBNull.Value)) - { - Hashtable newRow = new Hashtable(row); - newRow[myField]= myConstant; - return new BlueRow(row.schema, newRow); - } - else - return row; - } + public IBlueRow map(IBlueRow row) => + row[myField] switch { + null or DBNull => new BlueRow(row.schema, new Hashtable(row) { { myField, myConstant } }), + _ => row, + }; #endregion @@ -69,10 +60,11 @@ namespace BCG Hashtable newRow = new Hashtable(row); foreach(ColumnInfo info in row.schema.columnInfo) { - if(row[info.colName].Equals(DBNull.Value)) - { - change = true; - newRow[info.colName] = Convert.ChangeType(myConstant, info.colType); + switch (row[info.colName]) { + case null or DBNull: + change = true; + newRow[info.colName] = Convert.ChangeType(myConstant, info.colType); + break; } } if(change) diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ChangeTableNameStream.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ChangeTableNameStream.cs index ef684ceec..fb3aa5afd 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Project/ChangeTableNameStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ChangeTableNameStream.cs @@ -19,15 +19,15 @@ namespace BCG public IBlueTableSchema schema { get { return mySchema; } } - public object this[int index] { get { return myRow[index]; } } + public object? this[int index] { get { return myRow[index]; } } - public object this[string colName] { get { return myRow[colName]; } } + public object? this[string colName] { get { return myRow[colName]; } } #endregion #region IDictionary Members - public void Add(object key, object value) + public void Add(object key, object? value) { throw new Exception("The method or operation is not implemented."); } @@ -54,7 +54,7 @@ namespace BCG public ICollection Values { get { return myRow.Values; } } - public object this[object key] + public object? this[object key] { get { return myRow[key]; } set { throw new Exception("The method or operation is not implemented."); } @@ -82,7 +82,7 @@ namespace BCG } private string myNewName; - private IBlueTableSchema mySchema; + IBlueTableSchema? mySchema; public ChangeTableNameStream (ViewStream vs, string name) { diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ColumnMap.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ColumnMap.cs index 7bdfd4a90..0b957f6df 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Project/ColumnMap.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ColumnMap.cs @@ -1,9 +1,9 @@ +using Bts.ColumnMapExpressions; + +using Spark.Core.Diagnostics; + using System; using System.Collections; -using System.Data; -using System.Text.RegularExpressions; -using BCG.Utilities; -using Bts.ColumnMapExpressions; namespace BCG { @@ -56,7 +56,7 @@ namespace BCG { IDictionary retVal = new Hashtable (); foreach (DictionaryEntry de in dic) - retVal[de.Value] = de.Key; + retVal[ExceptionAssert.NotNull(de.Value)] = de.Key; return retVal; } diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectHelper.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectHelper.cs index c4cf98e75..12dadc850 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectHelper.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectHelper.cs @@ -1,13 +1,11 @@ using Bts.ColumnMapExpressions; -using Bts.DataStream; -using mbts.config; +using Spark.Core.Diagnostics; using System; using System.Collections; using System.Collections.Generic; using System.Linq; -// using BCG.Evaluator.ColumnMapExpression; namespace BCG { @@ -26,7 +24,7 @@ namespace BCG /// original(unprojected) schema /// expression defining the mapping /// table name of the new schema - internal ProjectHelper (IBlueTableSchema isch, string exp, string tableName) + internal ProjectHelper (IBlueTableSchema isch, string exp, string? tableName) : this (isch, myGetDictionary (isch, exp), tableName) {} @@ -36,7 +34,7 @@ namespace BCG /// dictionary mapping unprojected names to their projected names, dropped cols should not be there /// table name of the new schema internal ProjectHelper ( - IBlueTableSchema isch, IDictionary origToAlias, string tableName) + IBlueTableSchema isch, IDictionary origToAlias, string? tableName) { ArrayList colMapAl = new ArrayList (), colInfoAl = new ArrayList (); @@ -61,7 +59,7 @@ namespace BCG schema = new BlueTableSchema (tableName == null ? isch.tableName : tableName, colInfoAl.ToArray (typeof(ColumnInfo)) as ColumnInfo[]); - colMap = colMapAl.ToArray (typeof(int)) as int[]; + colMap = (int[])colMapAl.ToArray(typeof(int)); } /// @@ -74,8 +72,8 @@ namespace BCG private static IDictionary myGetDictionary (IBlueTableSchema isch, string exp) { IDictionary origToAlias = new Dictionary(); - - var e = ColumnMapExpression.Parse(exp); + var e = ColumnMapExpression.Parse(exp) + ?? throw new ArgumentException("Failed to parse ColumnMapExpression", nameof(exp)); if (e.IsInclusive) { origToAlias = e.Columns.ToDictionary(); origToAlias["id"] = "id"; @@ -95,7 +93,7 @@ namespace BCG { IDictionary retVal = new Hashtable (dic.Count); foreach (DictionaryEntry de in dic) - retVal.Add (de.Value, de.Key); + retVal.Add(ExceptionAssert.NotNull(de.Value), de.Key); return retVal; } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs index bca970bae..f7de48306 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs @@ -1,6 +1,10 @@ namespace BCG { - using Spark.Core.Collections; + using Bts.DataStreamCore; + using Spark.Core.Collections; + using Spark.Core.Diagnostics; + + using System; using System.Collections; using System.Collections.Generic; using System.Linq; @@ -8,10 +12,13 @@ namespace BCG { [DataStreamUserRespresentation("oldproject")] public class ProjectStream : ViewStream { - private ProjectHelper myHelper; - private string myColExp, myTableName; - private ViewStream myVs; - private IDictionary myDic; + private ProjectHelper? myHelper; + readonly string? myColExp; + readonly string? myTableName; + readonly ViewStream? myVs; + private IDictionary? myDic; + IDisposable? subscription; + /// /// /// stream to susbsribe to as the input stream @@ -52,14 +59,13 @@ namespace BCG { myDic = dic.ToDictionary(kv => kv.Value, kv => kv.Key); } - protected override void myOnFirstSubscriber() - { - myVs.onChange += new BCG.DataStream.ChangeDelegate (myHandleChange); - } + protected override void myOnFirstSubscriber() => + subscription = ExceptionAssert.MemberNotNull(myVs).Subscribe(myHandleChange); protected override void myOnLastSubscriber() { - myVs.onChange -= new BCG.DataStream.ChangeDelegate (myHandleChange);; + subscription?.Dispose(); + subscription = null; } internal void myHandleChange (object newValue) @@ -79,7 +85,7 @@ namespace BCG { else { //first time a callback is received myHelper = myDic == null ? - new ProjectHelper (dp.Schema, myColExp, myTableName) : + new ProjectHelper(dp.Schema, ExceptionAssert.MemberNotNull(myColExp), myTableName) : new ProjectHelper (dp.Schema, myDic, myTableName); IBlueRowCollection temp; diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedRow.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedRow.cs index 36569aa4e..652e8664a 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedRow.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedRow.cs @@ -1,6 +1,5 @@ using System; using System.Collections; -using BCG.Utilities; namespace BCG { @@ -22,9 +21,11 @@ namespace BCG #region IBlueRow Members - public object this [string colName] {get {return this[myHelper.schema[colName].colNo];}} + public object? this [string colName] => + this[myHelper.schema[colName].colNo]; - public object this [int index] {get {return myRow[myHelper.colMap[index]];}} + public object? this [int index] => + myRow[myHelper.colMap[index]]; public IBlueTableSchema schema {get {return myHelper.schema;}} @@ -36,9 +37,13 @@ namespace BCG public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator (this);} - object System.Collections.IDictionary.this[object key] + object? IDictionary.this[object key] { - get {return key is int ? this[(int)key] : this[key as string];} + get => key switch { + int i => this[i], + string s => this[s], + _ => throw new ArgumentException("Must be int or string", nameof(key)), + }; set {throw new ApplicationException ("Is read only");} } @@ -58,7 +63,8 @@ namespace BCG } } - public void Add (object key, object value) {throw new ApplicationException ("Is read only");} + public void Add(object key, object? value) => + throw new NotSupportedException(); public ICollection Keys {get {return myHelper.schema.columnNames;}} @@ -78,7 +84,7 @@ namespace BCG array.SetValue (this[ci.colNo], index ++); } - public object SyncRoot {get {return null;}} + public object SyncRoot => this; #endregion diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedTable.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedTable.cs index 628f54272..9c75fbd5c 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectedTable.cs @@ -1,3 +1,5 @@ +using Bts.DataStreamCore; + using System; using System.Collections; @@ -42,7 +44,7 @@ namespace BCG public void CopyTo (Array array, int index) { foreach (IBlueRow r in myRows) - array.SetValue (this[(int)r["id"]], index++); + array.SetValue(this[r.Id()], index++); } @@ -72,14 +74,10 @@ namespace BCG public void Reset() {myMaster.Reset();} - public object Current - { - get - { - return myMaster.Current == null ? null : - new ProjectedRow (myMaster.Current as IBlueRow, myHelper); - } - } + public object? Current => + myMaster.Current is null + ? null + : new ProjectedRow((IBlueRow)myMaster.Current, myHelper); public bool MoveNext() {return myMaster.MoveNext ();} diff --git a/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs index 3b475b051..2ec56f090 100644 --- a/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs +++ b/src/DataStreamSuite/DataStream/Primitives/SQL/SQLite/SQLiteHelper.cs @@ -1,25 +1,16 @@ using System; +using System.Collections.Generic; using System.Data; using System.Data.SQLite; -using System.Diagnostics; -using System.Collections; using System.IO; -using System.Text; - - -using BCG; -using System.Linq; -using System.Data.Entity; -using System.Collections.Generic; -using System.Data.SQLite; using System.Runtime.InteropServices; - - namespace bts.BlueOrb{ public static class SQLiteHelper { - public static string location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:\\BlueOrb_Data\\" : Path.Combine(Environment.GetEnvironmentVariable("HOME"), "BlueOrb_Data"); + public static string location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? "C:\\BlueOrb_Data\\" + : Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? string.Empty, "BlueOrb_Data"); public static string db_name = "stream_data.db"; public static DataTable execute_select_query(string connection_string, string query) { diff --git a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs index f5dcfea44..3130f3771 100644 --- a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs @@ -1,6 +1,9 @@ +using Spark.Core.Diagnostics; + using System; -using System.IO; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.IO; namespace BCG { [DataStreamUserRespresentation("CSVSaver")] @@ -77,7 +80,7 @@ namespace BCG { if (col != "id") { string colText = String.Empty; if (r[col] != null && r[col] != DBNull.Value) - colText = r[col].ToString(); + colText = r[col]?.ToString() ?? string.Empty; if (r.schema[col].colType == typeof(string)) colText = "\"" + colText + "\""; rv += (i++ == 0 ? colText : "," + colText); @@ -135,13 +138,14 @@ namespace BCG { } public class CSVReaderStreamMapper : IStreamMapper { - private IBlueTableSchema mySchema; + IBlueTableSchema? mySchema; private readonly string[] myOutputSchemaStringArray; public CSVReaderStreamMapper(string outputSchema) { myOutputSchemaStringArray = outputSchema.Split(' '); } + [MemberNotNull(nameof(mySchema))] public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) { if (mySchema == null) { ColumnInfo[] ci = inputSchema.columnInfo; @@ -161,6 +165,7 @@ namespace BCG { } public IBlueRow map(IBlueRow row) { + ExceptionAssert.NotNull(mySchema); object[] vals = new object[row.schema.columnInfo.Length]; row.Values.CopyTo(vals, 0); for (int i = 0; i < mySchema.columnInfo.Length; i++) { diff --git a/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs b/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs index 77dfecd42..702b675ec 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Sort/SortedTable.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using Bts.ColumnSortExpressions; +using Bts.DataStreamCore; namespace BCG; @@ -54,7 +55,7 @@ public class SortedTable : IBlueRowCollection { for (int i = 0; i < this.rows.Count; i++) { if (newRow.CompareTo(this.rows[i]) <= 0) { this.rows.Insert(i, newRow); - rowIndex[(int)newRow["id"]] = newRow; + rowIndex[newRow.Id()] = newRow; if (++addedRowsIndex == addedRows.Count) { return; } @@ -65,17 +66,17 @@ public class SortedTable : IBlueRowCollection { for (int i = addedRowsIndex; i < addedRows.Count; i++) { newRow = addedRows.getRowByIndex(i); this.rows.Add(newRow); - rowIndex[(int)newRow["id"]] = newRow; + rowIndex[newRow.Id()] = newRow; } } } public void deleteRows(IBlueRowCollection deletedRows) { foreach (IBlueRow deletedRow in deletedRows) { - var row = rowIndex[(int)deletedRow["id"]!] as SortRow + var row = rowIndex[deletedRow.Id()] as SortRow ?? throw new InvalidOperationException("Index did not return IBlueRow"); rows.Remove(row); - rowIndex.Remove((int)row["id"]!); + rowIndex.Remove(row.Id()); } } @@ -101,7 +102,7 @@ public class SortedTable : IBlueRowCollection { SortRow newRow = comparer is null ? new(row, sortCols) : new(row, comparer); this.rows.Add(newRow); - rowIndex[(int)row["id"]!] = newRow; + rowIndex[row.Id()] = newRow; } this.rows.Sort(); } diff --git a/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextReaderStream.cs b/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextReaderStream.cs index 62b4c9f0b..c8c309bc6 100644 --- a/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextReaderStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextReaderStream.cs @@ -19,7 +19,8 @@ namespace BCG StreamReader reader = new StreamReader(myFile); #region get schema - IBlueTableSchema schema = null; string schemaStr; + IBlueTableSchema? schema; + string? schemaStr; if ((schemaStr = reader.ReadLine()) == null) { reader.Close(); @@ -33,7 +34,7 @@ namespace BCG BlueTable adds = new BlueTable(schema); BlueTable dels = new BlueTable(schema); - string rowStr; + string? rowStr; while ((rowStr = reader.ReadLine()) != null) { adds.addRow(myGetRow(schema, rowStr)); diff --git a/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextWriterStream.cs b/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextWriterStream.cs index 1082e49fd..ef4750196 100644 --- a/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextWriterStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextWriterStream.cs @@ -101,7 +101,7 @@ namespace BCG rowStr += DELIM; } - rowStr += row[columnName].ToString(); + rowStr += row[columnName]?.ToString(); } writer.WriteLine(rowStr); @@ -144,7 +144,7 @@ namespace BCG private int myWriteFreqSecs; private DateTime myNextWrite; private bool myGotUpdate; - private string mySchemaStr; + private string? mySchemaStr; #endregion } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs index 24e6facb5..3a2bad50b 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Time/DateStandardizationStream.cs @@ -20,7 +20,7 @@ namespace BCG { [DataStreamUserRespresentation("dateStandardize")] public class DateStandardizationStream : ViewStream, IStreamMapper { - private IBlueTableSchema mySchema; + private IBlueTableSchema? mySchema; private DateConversion myConversion; private string myConversionColName; private readonly Dictionary _date_conversion_to_type_map = new Dictionary() @@ -60,8 +60,7 @@ namespace BCG { List cis = new List(); foreach (ColumnInfo c in inputSchema.columnInfo) { if (c.colName == myConversionColName) { - Type convType; - if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType)) { + if (!_date_conversion_to_type_map.TryGetValue(myConversion, out var convType)) { //BCG.Utilities.BlueEventLog.logEvent("Please update the _date_conversion_to_type_map in DateStandardizationStream.cs.", BCG.Utilities.BlueEventLevel.Error); convType = typeof(DateTime); } @@ -87,7 +86,7 @@ namespace BCG { #endregion - public static object mapDateTime(object obj_in, DateConversion conv) { + public static object mapDateTime(object? obj_in, DateConversion conv) { if (obj_in == DBNull.Value || obj_in == null) { return DBNull.Value; } else { @@ -118,7 +117,7 @@ namespace BCG { return day_period.TotalSeconds; } case DateConversion.ISO8601: { - string iso8601Date = obj_in.ToString(); + var iso8601Date = obj_in.ToString() ?? string.Empty; return DateTime.Parse(iso8601Date, null, System.Globalization.DateTimeStyles.RoundtripKind); } case DateConversion.EpochToDatetime: { @@ -135,4 +134,4 @@ namespace BCG { } } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/LatencyStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/LatencyStream.cs index 226408212..d01d4b17a 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Time/LatencyStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Time/LatencyStream.cs @@ -1,9 +1,6 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using System.Collections; +using System.Collections.Generic; namespace BCG { @@ -11,7 +8,7 @@ namespace BCG { private ViewStream _src; private string _time_stamp_field; - private IBlueTableSchema _schema; + IBlueTableSchema? _schema; private string _latency_field; public LatencyStream(ViewStream input, string time_stamp_field, string latency_field) @@ -38,10 +35,11 @@ namespace BCG public IBlueRow map(IBlueRow row) { var ht = new Hashtable(row); - if (row[_time_stamp_field].Equals(DBNull.Value)) - ht[_latency_field] = DBNull.Value; - else - ht[_latency_field] = (DateTime.Now - (DateTime)row[_time_stamp_field]).TotalMilliseconds; + ht[_latency_field] = row[_time_stamp_field] switch { + null or DBNull => DBNull.Value, + DateTime ts => (DateTime.Now - ts).TotalMilliseconds, + _ => throw new ArgumentException("Row timestamp must be DateTime", nameof(row)) + }; return new BlueRow(_schema, ht); } diff --git a/src/DataStreamSuite/DataStream/Primitives/TimeAverage/TimeAverageStream.cs b/src/DataStreamSuite/DataStream/Primitives/TimeAverage/TimeAverageStream.cs index f770af1ad..a3ed8322f 100644 --- a/src/DataStreamSuite/DataStream/Primitives/TimeAverage/TimeAverageStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/TimeAverage/TimeAverageStream.cs @@ -1,14 +1,15 @@ -using System; +using Spark.Core.Diagnostics; + +using System; using System.Collections.Generic; using System.Linq; -using System.Text; namespace BCG { public class TimeAverageStream : ViewStream { - private IDictionary> myKeyToTimeToValue; + Dictionary>? myKeyToTimeToValue; private ViewStream myToAverage; private string myCol_Key, myCol_Time, myCol_Value; private double myAveragePeriodSeconds; @@ -42,8 +43,8 @@ namespace BCG BlueTable result = new BlueTable(dp.Schema); foreach (IBlueRow r in dp.newRows) { - string key = r[myCol_Key].ToString(); - DateTime dt = (DateTime) r[myCol_Time]; + string key = r[myCol_Key]?.ToString() ?? string.Empty; + DateTime dt = (DateTime)r[myCol_Time]!; double value = Convert.ToDouble(r[myCol_Value]); object[] rowValues = new object[r.Count]; @@ -67,9 +68,8 @@ namespace BCG private double myUpdateValue(string key, DateTime time, double value) { - - SortedList seriesValues; - if (!myKeyToTimeToValue.TryGetValue(key, out seriesValues)) + ExceptionAssert.MemberNotNull(myKeyToTimeToValue); + if (!myKeyToTimeToValue.TryGetValue(key, out var seriesValues)) { seriesValues = new SortedList(); myKeyToTimeToValue[key] = seriesValues; diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs b/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs index 5ac1c2bbd..b9a10be62 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs @@ -1,5 +1,8 @@ +using Bts.DataStreamCore; + using System; using System.Collections; +using System.Collections.Generic; namespace BCG { @@ -10,9 +13,9 @@ namespace BCG internal class AppendCollection : IBlueRowCollection { private int myId = 0; - private IDictionary myLeftIndex = new Hashtable (), - myRightIndex = new Hashtable (); - private IBlueTableSchema mySchema; + readonly Dictionary myLeftIndex = new(); + readonly Dictionary myRightIndex = new(); + IBlueTableSchema? mySchema; public AppendCollection () {} @@ -24,15 +27,15 @@ namespace BCG IDictionary index = isLeft ? myLeftIndex : myRightIndex; foreach (IBlueRow r in dp.deletedRows) { - IBlueRow dr = index[r["id"]] as IBlueRow; + var dr = index[r.Id()] as IBlueRow; dRows.addRow (dr); - index.Remove (r["id"]); + index.Remove(r.Id()); } foreach (IBlueRow r in dp.addedRows) { IBlueRow ar = new AppendRow (r, myId++, mySchema); aRows.addRow (ar); - index.Add (r["id"], ar); + index.Add(r.Id(), ar); } return new DataPacket (this, aRows, dRows); } @@ -43,7 +46,8 @@ namespace BCG public bool containsID (int id) {throw new ApplicationException ("IBlueRow is readonly");} - public IBlueTableSchema schema {get {return mySchema;}} + public IBlueTableSchema schema => + mySchema ?? throw new InvalidOperationException("Schema is unknown (no rows added)"); #endregion @@ -60,7 +64,7 @@ namespace BCG array.SetValue (r, i++); } - public object SyncRoot {get {return null;}} + public object SyncRoot => this; #endregion @@ -87,7 +91,7 @@ namespace BCG public void Reset() {myEnum = myLeft.GetEnumerator ();} - public object Current {get {return myEnum.Current;}} + public object? Current {get {return myEnum.Current;}} public bool MoveNext() { diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/AppendRow.cs b/src/DataStreamSuite/DataStream/Primitives/Union/AppendRow.cs index c7322bfb1..082580962 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Union/AppendRow.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Union/AppendRow.cs @@ -23,12 +23,12 @@ namespace BCG #region IBlueRow Members - public object this [string colName] + public object? this [string colName] { get {return colName.Equals("id") ? myId : myRow[colName];} } - public object this [int index] + public object? this [int index] { get {return this[schema[index].colName];} } @@ -43,10 +43,13 @@ namespace BCG public System.Collections.IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator (this);} - object System.Collections.IDictionary.this[object key] - { - get {return key is string ? this[key as string] : this [(int)key];} - set {throw new ApplicationException ("IBlueRow is readonly");} + object? IDictionary.this[object key] { + get => key switch { + string colName => this[colName], + int colIndex => this[colIndex], + _ => throw new ArgumentException("Must be string or int", nameof(key)), + }; + set => throw new NotSupportedException(); } public void Remove (object key) {throw new ApplicationException ("IBlueRow is readonly");} @@ -65,7 +68,8 @@ namespace BCG } } - public void Add (object key, object value) {throw new ApplicationException ("IBlueRow is readonly");} + public void Add(object key, object? value) => + throw new NotSupportedException(); public System.Collections.ICollection Keys {get {return schema.columnNames;}} diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs b/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs index b332d7fff..4fa0dc7ab 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs @@ -29,7 +29,7 @@ namespace BCG { public IBlueTableSchema schema { get { return myRow.schema; } } - public object this[int index] { + public object? this[int index] { get { if (myRow.schema[index].colName == "id") return myID; @@ -38,15 +38,18 @@ namespace BCG { } } - public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } } + public object? this[string colName] => + colName == "id" ? myID : myRow[colName]; #endregion #region IDictionary Members - public void Add(object key, object value) { throw new Exception("Row is read only."); } + public void Add(object key, object? value) => + throw new NotSupportedException(); - public void Clear() { throw new Exception("Row is read only."); } + public void Clear() => + throw new NotSupportedException(); public bool Contains(object key) { return myRow.Contains(key); } @@ -68,9 +71,13 @@ namespace BCG { } } - public object this[object key] { - get { return key is string ? this[key as string] : this[(int)key]; } - set { throw new Exception("The method or operation is not implemented."); } + public object? this[object key] { + get => key switch { + string colName => this[colName], + int colIndex => this[colIndex], + _ => throw new ArgumentException("Must be string or int", nameof(key)), + }; + set => throw new NotSupportedException(); } #endregion @@ -112,18 +119,18 @@ namespace BCG { myIDMap.Add(tableNo, new Dictionary()); IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id"); myRows.addRow(nr); - myIDMap[tableNo].Add((int)r["id"], nr); + myIDMap[tableNo].Add(r.Id(), nr); return nr; } public IBlueRow removeRow(IBlueRow r, int tableNo) { - IBlueRow dr = myIDMap[tableNo][(int)r["id"]]; + IBlueRow dr = myIDMap[tableNo][r.Id()]; myRows.deleteRow(dr); - myIDMap[tableNo].Remove((int)r["id"]); + myIDMap[tableNo].Remove(r.Id()); return dr; } - public IBlueRowCollection removeTable(int tableNo) { + public IBlueRowCollection? removeTable(int tableNo) { if (!myIDMap.ContainsKey(tableNo)) return null; else { @@ -170,7 +177,7 @@ namespace BCG { #endregion } private bool mySubscriptionActive = false; - private MutableAppendTable myTable; + MutableAppendTable? myTable; private int myMovingIndex; bool isFirstUpdate = true; ImmutableDictionary myStreamIndex = @@ -222,7 +229,7 @@ namespace BCG { if (mySubscriptionActive) { sub?.Dispose(); if (myTable != null) { - IBlueRowCollection delRows = myTable.removeTable(kvp.Key); + IBlueRowCollection? delRows = myTable.removeTable(kvp.Key); if (delRows != null) myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows)); } diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/UnionRow.cs b/src/DataStreamSuite/DataStream/Primitives/Union/UnionRow.cs index 22fd96c2d..248fb0e2e 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Union/UnionRow.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Union/UnionRow.cs @@ -99,7 +99,8 @@ namespace BCG } } - public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");} + public void Add(object key, object? value) => + throw new NotSupportedException(); public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");} diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowDictionaryEnumerator.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowDictionaryEnumerator.cs index 03ed1d973..5f2167c4c 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowDictionaryEnumerator.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueRowDictionaryEnumerator.cs @@ -1,4 +1,4 @@ -using System; +#nullable enable using System.Collections; namespace BCG @@ -10,14 +10,7 @@ namespace BCG public class BlueRowDictionaryEnumerator : BlueRowEnumerator, IDictionaryEnumerator { public BlueRowDictionaryEnumerator (IBlueRow row) : base (row) {} - public override object Current {get{return Entry;}} - #region IDictionaryEnumerator members - public object Key {get{return myRow.schema[myPos].colName;}} - - public object Value {get{return myRow[myPos];}} - - public DictionaryEntry Entry {get{return new DictionaryEntry (Key, Value);}} - #endregion + public override object? Current => Entry; } } diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchema.cs b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchema.cs index 4b3f3fa6c..86905d6a0 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchema.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/BlueTableSchema.cs @@ -1,13 +1,14 @@ -using System; -using System.Data; -using System.Collections; -using System.IO; -using System.Reflection; -using System.Collections.Generic; -using System.Linq; - using BCG.Utilities.General; +using Spark.Core.Collections; + +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Linq; +using System.Reflection; + namespace BCG { /// /// The BlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created @@ -117,7 +118,7 @@ namespace BCG { // } } - protected Hashtable myIndexByColName = new Hashtable(); + protected Dictionary myIndexByColName = new(); protected ColumnInfo[] myIndexByColOrder; private string[] myTableColNames; protected string myTableName; @@ -187,18 +188,8 @@ namespace BCG { /// /// The returned Array is sorted according to the Column No /// - public string[] columnNames { - get { - if (myTableColNames == null) { - myTableColNames = new string[myIndexByColName.Count]; - foreach (DictionaryEntry de in myIndexByColName) { - ColumnInfo temp = (ColumnInfo)de.Value; - myTableColNames[temp.colNo] = temp.colName; - } - } - return myTableColNames; - } - } + public string[] columnNames => + myTableColNames ??= myIndexByColName.Values.ToIndexedArray(c => (c.colNo, c.colName)); /// /// The returned Array is sorted according to the Column No diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs index 62e9b4f17..fafe943ed 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs @@ -1,3 +1,4 @@ +#nullable enable using System; using System.Text.Json.Serialization; @@ -6,10 +7,10 @@ namespace BCG { /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable /// [Serializable] - public class ColumnInfo : IComparable { + public class ColumnInfo : IComparable, IComparable, IEquatable { //the default column info does not cach values in a lookup table, therefore it is in group 0 - public ColumnInfo(string colName, int colNo, Type colType, object colValue = null) { + public ColumnInfo(string colName, int colNo, Type colType, object? colValue = null) { myColName = colName; myColNo = colNo; myColType = colType; @@ -27,8 +28,14 @@ namespace BCG { type == typeof(string) || type == typeof(DateTime); - public int CompareTo(object obj) { - int otherColNo = ((ColumnInfo)(obj)).colNo; + public int CompareTo(object? obj) => + CompareTo(obj as ColumnInfo); + + public int CompareTo(ColumnInfo? other) { + if (other is null) { + return 1; + } + int otherColNo = other.colNo; if (myColNo < otherColNo) return -1; else if (myColNo > otherColNo) @@ -36,18 +43,21 @@ namespace BCG { return 0; } - public override bool Equals(object obj) { - if (obj is ColumnInfo) { - ColumnInfo other = obj as ColumnInfo; - return (colNo == other.colNo) && (colName == other.colName) && (colType == other.colType); - } - return false; - } + public override bool Equals(object? obj) => + Equals(obj as ColumnInfo); + + public bool Equals(ColumnInfo? other) => + other is not null + && colNo == other.colNo + && colName == other.colName + && colType == other.colType; public override string ToString() { return colName + " " + colNo + " " + colType; } + public override int GetHashCode() => + HashCode.Combine(colNo, colName, colType); // for backwards compatibility to allow build public bool isConstant {get{return myColValue != null;}} @@ -55,7 +65,7 @@ namespace BCG { private int myColNo; private string myColName; private Type myColType; - private object myColValue; + private object? myColValue; [JsonPropertyName("colNo")] public int colNo { get { return myColNo; } } @@ -67,6 +77,6 @@ namespace BCG { public Type colType { get { return myColType; } } [JsonIgnore] - public object colValue {get{return myColValue;}} + public object? colValue {get{return myColValue;}} } } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamIOCommon.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamIOCommon.cs index 6f93ea702..366f7f168 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamIOCommon.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamIOCommon.cs @@ -1,24 +1,39 @@ -using System; +#nullable enable using BCG; -using System.Collections; + +using Spark.Core.Collections; + +using System; using System.IO; // common place for all serialization interfaces namespace bts { // this is the data that is sent across the write for stream IO - public class StreamIOMessage { + public class StreamIOMessage : IEquatable { public enum StreamOp { CREATE = 0, // removes and adds multiple rows // other operations can be expressed with this MULTIPLE_REMOVE_ADD = 1, } - public class CreateMessage { + + public class CreateMessage : IEquatable { public IBlueRowCollection rows; public CreateMessage(IBlueRowCollection rows_) { rows = rows_; } + + public bool Equals(CreateMessage? other) => + other is not null + && BlueDataUtility.compute_equality(rows, other.rows); + + public override bool Equals(object? obj) => + Equals(obj as CreateMessage); + + public override int GetHashCode() => + rows.Count; + public override string ToString() { string rv = "Create message "; if (rows == null) { @@ -30,21 +45,27 @@ namespace bts { } return rv; } - public override bool Equals(object obj) { - if (obj != null && obj is CreateMessage) { - var other = obj as CreateMessage; - return BlueDataUtility.compute_equality(rows, other.rows); - } - return false; - } } - public class MultipleRemoveAddMessage { + + public class MultipleRemoveAddMessage : IEquatable { public IBlueRowCollection removed_rows; public IBlueRowCollection added_rows; public MultipleRemoveAddMessage(IBlueRowCollection removed_rows_, IBlueRowCollection added_rows_) { removed_rows = removed_rows_; added_rows = added_rows_; } + + public bool Equals(MultipleRemoveAddMessage? other) => + other is not null + && BlueDataUtility.compute_equality(removed_rows, other.removed_rows) + && BlueDataUtility.compute_equality(added_rows, other.added_rows); + + public override bool Equals(object? obj) => + Equals(obj as MultipleRemoveAddMessage); + + public override int GetHashCode() => + HashCode.Combine(removed_rows.Count, added_rows.Count); + public override string ToString() { string rv = "MultipleRemoveAdd removed: "; foreach (IBlueRow r in removed_rows) { @@ -56,56 +77,46 @@ namespace bts { } return rv; } - public override bool Equals(object obj) { - if (obj != null && obj is MultipleRemoveAddMessage) { - MultipleRemoveAddMessage other = obj as MultipleRemoveAddMessage; - return BlueDataUtility.compute_equality(removed_rows, other.removed_rows) && - BlueDataUtility.compute_equality(added_rows, other.added_rows); - } - return false; - } } public StreamOp op; - public readonly CreateMessage create_message = null; - public readonly MultipleRemoveAddMessage multiple_remove_add_message = null; + public readonly CreateMessage? create_message; + public readonly MultipleRemoveAddMessage? multiple_remove_add_message; public StreamIOMessage(CreateMessage create_message_) { op = StreamOp.CREATE; create_message = create_message_; - multiple_remove_add_message = null; } public StreamIOMessage(MultipleRemoveAddMessage multiple_remove_add_message_) { op = StreamOp.MULTIPLE_REMOVE_ADD; - create_message = null; multiple_remove_add_message = multiple_remove_add_message_; } + public bool Equals(StreamIOMessage? other) => + other is not null + && op == other.op + && op switch { + StreamOp.CREATE => NullCheckEqualityComparer.Equals(create_message, other.create_message), + StreamOp.MULTIPLE_REMOVE_ADD => NullCheckEqualityComparer.Equals(multiple_remove_add_message, other.multiple_remove_add_message), + _ => false, + }; + + public override bool Equals(object? obj) => + Equals(obj as StreamIOMessage); + + public override int GetHashCode() => + HashCode.Combine(op, create_message, multiple_remove_add_message); + public override string ToString() { switch (op) { case StreamIOMessage.StreamOp.CREATE: - return create_message.ToString(); + return create_message?.ToString() ?? string.Empty; case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: - return multiple_remove_add_message.ToString(); + return multiple_remove_add_message?.ToString() ?? string.Empty; } bts.Log.assert(false); return String.Empty; } - public override bool Equals(object obj) { - if (obj != null && obj is StreamIOMessage) { - StreamIOMessage other = obj as StreamIOMessage; - if (op == other.op) { - switch (op) { - case StreamOp.CREATE: - return create_message.Equals(other.create_message); - case StreamOp.MULTIPLE_REMOVE_ADD: - return multiple_remove_add_message.Equals(other.multiple_remove_add_message); - } - } - } - return false; - } - } #region serialization // This helps putting stream callbacks (op + data) on the wire, to send it to the pubsubserver. @@ -134,4 +145,4 @@ namespace bts { } #endregion -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs index 8d4727f03..10b466234 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamSerializationUtils.cs @@ -1,12 +1,10 @@ +#nullable enable namespace bts { using BCG; using System; using System.Collections.Generic; using System.IO; - using System.Reactive.Subjects; - using System.Reflection.Metadata; - using System.Runtime.Serialization.Formatters.Binary; using System.Text; public static class StreamSerializationUtils { @@ -122,17 +120,24 @@ namespace bts { } public static object deserialize_column(ref BinaryReader br, Type t) { - Func reader_func; - Log.assert(_col_deser_map.TryGetValue(t, out reader_func)); - return reader_func(br); + if (_col_deser_map.TryGetValue(t, out var deserialize)) { + return deserialize(br); + } + var msg = $"Deserialization function not defined for type '{t}'"; + Log.assert(false, msg); + throw new ArgumentException(msg, nameof(t)); } public static void serialize_row_column(BinaryWriter bw, IBlueRow r, int col_no) { var ci = r.schema.columnInfo[col_no]; var type = ci.colType; - Action ser_action; - Log.assert(_col_ser_map.TryGetValue(type, out ser_action)); - ser_action(bw, r, col_no); + if (_col_ser_map.TryGetValue(type, out var serialize)) { + serialize(bw, r, col_no); + return; + } + var msg = $"Serialization function not defined for type '{type}' (failed on column '{ci.colName}')"; + Log.assert(false, msg); + throw new ArgumentException(msg, nameof(r)); } public static ColumnInfo deserialize_column_info(ref BinaryReader br) { @@ -226,4 +231,4 @@ namespace bts { } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStreamTests/PubSubMessages/PuSubMessagesTests.cs b/src/DataStreamSuite/DataStreamTests/PubSubMessages/PuSubMessagesTests.cs index 72f27a9b3..0db2fb9bf 100644 --- a/src/DataStreamSuite/DataStreamTests/PubSubMessages/PuSubMessagesTests.cs +++ b/src/DataStreamSuite/DataStreamTests/PubSubMessages/PuSubMessagesTests.cs @@ -1,6 +1,4 @@ -using NUnit.Framework; - -using System.Reactive.Disposables; +using Spark.Core.Diagnostics; namespace bts { [TestFixture] @@ -41,7 +39,7 @@ namespace bts { sub_msg_desered.construct_deserialize(str); Assert.That(sub_msg_desered.connection, Is.EqualTo(sub_msg.connection)); Assert.That(sub_msg_desered.subscribing_app, Is.EqualTo(sub_msg.subscribing_app)); - compare_arrays(sub_msg.tags, sub_msg_desered.tags); + compare_arrays(sub_msg.tags, ExceptionAssert.NotNull(sub_msg_desered.tags)); Assert.That(sub_msg_desered, Is.EqualTo(sub_msg)); } @@ -60,7 +58,7 @@ namespace bts { sub_msg_desered.construct_deserialize(str); Assert.That(sub_msg_desered.connection, Is.EqualTo(unsub_msg.connection)); Assert.That(sub_msg_desered.subscribing_app, Is.EqualTo(unsub_msg.subscribing_app)); - compare_arrays(unsub_msg.tags, sub_msg_desered.tags); + compare_arrays(unsub_msg.tags, ExceptionAssert.NotNull(sub_msg_desered.tags)); } [Test] @@ -87,7 +85,7 @@ namespace bts { Assert.That(pub_msg_desered.tag_type, Is.EqualTo(pub_msg.tag_type)); Assert.That(pub_msg_desered.is_compressed, Is.EqualTo(pub_msg.is_compressed)); Assert.That(pub_msg_desered.publish_time_epoch_ms, Is.EqualTo(pub_msg.publish_time_epoch_ms)); - Assert.That(pub_msg_desered.data_to_publish.Length, Is.EqualTo(pub_msg.data_to_publish.Length)); + Assert.That(ExceptionAssert.NotNull(pub_msg_desered.data_to_publish).Length, Is.EqualTo(pub_msg.data_to_publish.Length)); compare_arrays(pub_msg.data_to_publish, pub_msg_desered.data_to_publish); } @@ -105,7 +103,7 @@ namespace bts { ps_server_msg_desered.construct_deserialize(str); Assert.That(ps_server_msg_desered.tag, Is.EqualTo(ps_server_msg.tag)); Assert.That(ps_server_msg_desered.tag_type, Is.EqualTo(ps_server_msg.tag_type)); - compare_arrays(ps_server_msg.data, ps_server_msg_desered.data); + compare_arrays(ps_server_msg.data, ExceptionAssert.NotNull(ps_server_msg_desered.data)); } [Test] diff --git a/src/DataStreamSuite/PubSub/Client/Communication/PubSubServerConnectionFactory.cs b/src/DataStreamSuite/PubSub/Client/Communication/PubSubServerConnectionFactory.cs index ef030af16..035b9149a 100644 --- a/src/DataStreamSuite/PubSub/Client/Communication/PubSubServerConnectionFactory.cs +++ b/src/DataStreamSuite/PubSub/Client/Communication/PubSubServerConnectionFactory.cs @@ -20,8 +20,7 @@ namespace BCG { } public IPubSubServerConnection get_connection(RemoteConnectionInfo server_rci) { - IPubSubServerConnection rv = null; - if (!_conn_map.TryGetValue(server_rci, out rv)) { + if (!_conn_map.TryGetValue(server_rci, out var rv)) { rv = _build_connection(server_rci); _conn_map.Add(server_rci, rv); } diff --git a/src/DataStreamSuite/PubSub/Client/InternalTags/TagPubSubInfo.cs b/src/DataStreamSuite/PubSub/Client/InternalTags/TagPubSubInfo.cs index c4e4b3db1..2cf3beaf8 100644 --- a/src/DataStreamSuite/PubSub/Client/InternalTags/TagPubSubInfo.cs +++ b/src/DataStreamSuite/PubSub/Client/InternalTags/TagPubSubInfo.cs @@ -1,9 +1,11 @@ +using bts; + +using mbts; + using System; using System.Collections.Generic; -using System.Text; +using System.Diagnostics.CodeAnalysis; using System.IO; -//using BCG.Utilities; -using mbts; using System.Linq; namespace BCG { @@ -14,8 +16,8 @@ namespace BCG { /// public class TagPubSubInfo { public string tag; - public IDictionary subscribers, - publishers; + public IDictionary subscribers; + public IDictionary publishers; public ICollection> dead_publishers; public string pubsub_server; @@ -58,6 +60,8 @@ namespace BCG { } bw.Write(dead_publishers.Count); foreach (Pair p in dead_publishers) { + Log.notnull(p.first); + Log.notnull(p.second); bw.Write(p.first); bw.Write(p.second); } @@ -91,6 +95,7 @@ namespace BCG { /// constructs itself from the stream /// /// + [MemberNotNull(nameof(tag), nameof(subscribers), nameof(publishers), nameof(dead_publishers), nameof(pubsub_server))] private void _read_from(Stream s) { BinaryReader br = new BinaryReader(s); tag = br.ReadString(); diff --git a/src/DataStreamSuite/PubSub/Client/ThreadDrivenMasterWorker.cs b/src/DataStreamSuite/PubSub/Client/ThreadDrivenMasterWorker.cs index 2a4a2125b..72402d233 100644 --- a/src/DataStreamSuite/PubSub/Client/ThreadDrivenMasterWorker.cs +++ b/src/DataStreamSuite/PubSub/Client/ThreadDrivenMasterWorker.cs @@ -1,11 +1,10 @@ -using System; -using System.Collections.Generic; -using System.Text; -using System.Threading; -// using BCG.Utilities; - using BCG.Utilities.General; +using BCG.Utilities.General; + using mbts; +using System; +using System.Threading; + namespace BCG { /// @@ -62,7 +61,7 @@ namespace BCG // w.ToString(), myWorkers.Count), PS3LogLevels.connection); myWorkers.Remove(w); } - foreach (KeyValuePair> p in myJustAdded) + foreach (var p in myJustAdded) { // BlueEventLog.logEvent(String.Format("ThreadDrivenMasterWorker : adding worker {0} , worker count before adding this = {1}", // p.Key.ToString(), myWorkers.Count), PS3LogLevels.connection); @@ -78,7 +77,7 @@ namespace BCG { // BlueEventLog.logEvent(ex); if (ex is OutOfMemoryException) - throw ex; + throw; } } } diff --git a/src/DataStreamSuite/PubSub/IGenericFormatter.cs b/src/DataStreamSuite/PubSub/IGenericFormatter.cs index e5810b0ef..3e2e29ba2 100644 --- a/src/DataStreamSuite/PubSub/IGenericFormatter.cs +++ b/src/DataStreamSuite/PubSub/IGenericFormatter.cs @@ -1,11 +1,10 @@ -using System; using System.IO; namespace BCG { public interface IGenericFormatter { - T Deserialize (Stream serializationStream); - void Serialize (Stream serializationStream, T graph); + T Deserialize (Stream serializationStream) where T : notnull; + void Serialize (Stream serializationStream, T graph) where T : notnull; } } diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs index d62661e9c..e821a14a7 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs @@ -2,17 +2,20 @@ namespace bts; using BCG; +using Spark.Core.Diagnostics; + using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Net; using System.Net.Sockets; // opens a TCP port and starts listening to connections public class AsyncTCPConnectionAcceptor_APM { private readonly int _port; - private Socket _accept_socket; - private ITCPConnectionHandler _connection_handler; - private HashSet _key_map = new HashSet(); + Socket? _accept_socket; + readonly ITCPConnectionHandler _connection_handler; + readonly HashSet _key_map = new(); public AsyncTCPConnectionAcceptor_APM(int port, ITCPConnectionHandler connection_handler) { Log.Gen.info($"AsyncTCPConnectionAcceptor_APM. cons started on port: {port}"); @@ -25,6 +28,7 @@ public class AsyncTCPConnectionAcceptor_APM { _begin_accepting(); } + [MemberNotNull(nameof(_accept_socket))] void _init() { Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._init started on port: {_port}"); _accept_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); @@ -34,7 +38,9 @@ public class AsyncTCPConnectionAcceptor_APM { _accept_socket.Listen(250); } + [MemberNotNull(nameof(_accept_socket))] void _begin_accepting() { + ArgumentNullException.ThrowIfNull(_accept_socket); var accept_key = Guid.NewGuid().ToString(); Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._begin_accepting port: {_port} accept_key: {accept_key}"); var res = _accept_socket.BeginAccept(new AsyncCallback(_on_accept), accept_key); @@ -46,14 +52,14 @@ public class AsyncTCPConnectionAcceptor_APM { var log_msg = $"AsyncTCPConnectionAcceptor_APM._on_accept state: {res.AsyncState}"; Log.Gen.info(log_msg); Console.WriteLine(log_msg); - string key = res.AsyncState.ToString(); + var key = ExceptionAssert.NotNull(res.AsyncState).ToString() ?? string.Empty; if (_key_map.Contains(key)) { return; } _key_map.Add(key); Socket new_sock; try { - new_sock = _accept_socket.EndAccept(res); + new_sock = ExceptionAssert.NotNull(_accept_socket).EndAccept(res); } catch (Exception ex) { bts.Log.Gen.info("AsyncTCPConnectionAcceptor_APM._on_accept exception thrown by EndAccept ex: {ex}"); System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(250)); diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_AsyncWrapper.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_AsyncWrapper.cs index 856839e5e..96d373930 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_AsyncWrapper.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_AsyncWrapper.cs @@ -5,6 +5,8 @@ using BCG; using System; using System.Collections.Generic; using System.Threading.Tasks; +using System.Diagnostics.CodeAnalysis; +using Spark.Core.Diagnostics; namespace bts { @@ -13,10 +15,10 @@ namespace bts { // it uses TcpListener, instead of raw sockets. public class AsyncTCPConnectionAcceptor_AsyncWrapper { private readonly int _accept_port; - private Socket _accept_socket; + // private Socket _accept_socket; private ITCPConnectionHandler _connection_handler; - private TcpListener _server; - private Action _cbs = (c) => { }; + TcpListener? _server; + // private Action _cbs = (c) => { }; public AsyncTCPConnectionAcceptor_AsyncWrapper(int accept_port, ITCPConnectionHandler connection_handler) { Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper.cons started on port: {accept_port}"); @@ -25,9 +27,10 @@ namespace bts { } public void resgiter_callback(Action cb) { - _cbs += cb; + // _cbs += cb; } + [MemberNotNull(nameof(_server))] public void start() { Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._init started on port: {_accept_port}"); _server = new TcpListener(IPAddress.Any, _accept_port); @@ -36,6 +39,7 @@ namespace bts { } private void _accept_connections_forever() { + ExceptionAssert.MemberNotNull(_server); Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port}"); while (true) { // this is a blocking call @@ -55,4 +59,4 @@ namespace bts { } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs index 1f8dd3b57..76b18bc4a 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs @@ -2,13 +2,13 @@ namespace bts { using BCG; using BCG.Utilities.General; + + using Spark.Core.Diagnostics; + using System; using System.IO; using System.Net.Sockets; - using System.Threading.Tasks; using System.Threading; - using System.Net; - using System.Text; // this ITCPConnection uses the TCPClient class instead of a raw socket (in the hope, that it would configure the socket correctly) @@ -34,7 +34,7 @@ namespace bts private const int HB_MSG_SIGNAL = 1; private TcpClient _tcp_client; private NetworkStream _nw_stream; - private ITCPMessageHandler _msg_handler; + ITCPMessageHandler? _msg_handler; private bool _is_alive = true; ConnectionID _connection_id; @@ -191,7 +191,7 @@ namespace bts private class OneRecvMessage { public int msg_type; - public byte[] data; + public byte[]? data; public OneRecvMessage(int msg_type_, byte[] data_) { msg_type = msg_type_; @@ -211,7 +211,7 @@ namespace bts public override string ToString() { - string rv = $"OneMessage msg_type: {msg_type} data_len: {data.Length}"; + string rv = $"OneMessage msg_type: {msg_type} data_len: {data?.Length ?? 0}"; return rv; } } @@ -237,6 +237,7 @@ namespace bts private void _read_msgs_forever() { + ExceptionAssert.MemberNotNull(_msg_handler); while (_is_alive) { try @@ -270,7 +271,7 @@ namespace bts public class ErrorMessage { public readonly SocketError? socket_error = null; - public readonly Exception exception = null; + public readonly Exception? exception = null; public readonly bool is_exception = false; public readonly string context = String.Empty; public ErrorMessage(SocketError socket_error_, string context_) @@ -313,9 +314,9 @@ namespace bts { String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket"); if (ex is OutOfMemoryException) - throw ex; + throw; } - _msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _connection_id.peer)); + _msg_handler?.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _connection_id.peer)); } } } @@ -325,8 +326,10 @@ namespace bts lock (this) { Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}"); - if (err_msg.oom) + if (err_msg.oom) { + Log.notnull(err_msg.exception); throw err_msg.exception; + } Dispose(); } } @@ -336,4 +339,4 @@ namespace bts return _connection_id.ToString(); } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_APM.cs b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_APM.cs index ebbda0262..db40f16de 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_APM.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_APM.cs @@ -1,10 +1,10 @@ -using System.Net.Sockets; -using System.Net; - using BCG; -using System; -using System.Collections.Generic; +using Spark.Core.Diagnostics; + +using System; +using System.Net; +using System.Net.Sockets; namespace bts { // Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests @@ -42,7 +42,7 @@ namespace bts { private static void _on_connect(IAsyncResult res) { Log.Gen.info($"AsyncTCPConnector_APM._on_connect res: {res.AsyncState}"); - SocketConnectionArgs sc_args = res.AsyncState as SocketConnectionArgs; + var sc_args = ExceptionAssert.Cast(res.AsyncState).To(); try { sc_args.sock.EndConnect(res); } catch (SocketException socket_ex) { diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/RemoteConnectionInfo.cs b/src/DataStreamSuite/PubSub/ManagedTCP/RemoteConnectionInfo.cs index 5bfcd6dee..a581a3927 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/RemoteConnectionInfo.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/RemoteConnectionInfo.cs @@ -1,12 +1,11 @@ +using bts; + +using Bts.Net; + using System; using System.Collections.Generic; -using System.Text; -using System.Net; -using bts.utils; -using System.Net.NetworkInformation; -using bts; -using Bts.Net; using System.Linq; +using System.Net; namespace BCG { public class RemoteConnectionInfo : IEquatable { @@ -106,13 +105,7 @@ namespace BCG { return ip.GetHashCode() ^ port.GetHashCode(); } - public override bool Equals(object? obj) { - if (obj == null) { - return false; - } - RemoteConnectionInfo other = obj as RemoteConnectionInfo; - return other != null && other.ip == ip && other.port == port; - } - + public override bool Equals(object? obj) => + obj is RemoteConnectionInfo other && other.ip == ip && other.port == port; } } diff --git a/src/DataStreamSuite/PubSub/ManagedTCP/TCPSocketUtils.cs b/src/DataStreamSuite/PubSub/ManagedTCP/TCPSocketUtils.cs index 15a58dfc4..a316d497f 100644 --- a/src/DataStreamSuite/PubSub/ManagedTCP/TCPSocketUtils.cs +++ b/src/DataStreamSuite/PubSub/ManagedTCP/TCPSocketUtils.cs @@ -1,14 +1,14 @@ +using bts; + +using Spark.Core.Diagnostics; + using System; using System.IO; -using System.Linq; using System.Net; using System.Net.Sockets; -using System.Runtime.CompilerServices; -using System.Text; -using System.Threading.Tasks; -using bts; -using System.Threading; using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; namespace BCG { // utility functions for TCP socket async communications. Does the following @@ -50,7 +50,9 @@ namespace BCG { Marshal.Copy(ptr, inOptionValues, 0, size); Marshal.FreeHGlobal(ptr); - server_socket.IOControl(IOControlCode.KeepAliveValues, inOptionValues, null); + if (OperatingSystem.IsWindows()) { + server_socket.IOControl(IOControlCode.KeepAliveValues, inOptionValues, null); + } } // Set other socket options as needed @@ -67,14 +69,14 @@ namespace BCG { } public static RemoteConnectionInfo get_peer_address(Socket sock) { - IPEndPoint ip_ep = sock.RemoteEndPoint as IPEndPoint; + var ip_ep = ExceptionAssert.Cast(sock.RemoteEndPoint).To(); int port = ip_ep.Port; string ip = ip_ep.Address.ToString(); return new RemoteConnectionInfo(ip, port); } public static RemoteConnectionInfo get_self_address(Socket socket) { - IPEndPoint ip_ep = socket.LocalEndPoint as IPEndPoint; + var ip_ep = ExceptionAssert.Cast(socket.LocalEndPoint).To(); int port = ip_ep.Port; string ip = ip_ep.Address.ToString(); return new RemoteConnectionInfo(ip, port); @@ -101,7 +103,9 @@ namespace BCG { Marshal.Copy(ptr, inOptionValues, 0, size); Marshal.FreeHGlobal(ptr); - client_socket.IOControl(IOControlCode.KeepAliveValues, inOptionValues, null); + if (OperatingSystem.IsWindows()) { + client_socket.IOControl(IOControlCode.KeepAliveValues, inOptionValues, null); + } } // Set other socket options as needed @@ -126,7 +130,7 @@ namespace BCG { Socket listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); listener.Bind(ip_end_point); listener.Listen(10); - Socket client_socket = null; + Socket? client_socket; while (true) { try { // keeps on accepting connections @@ -135,16 +139,16 @@ namespace BCG { handler.onConnection(new_conn); } catch (Exception ex) { bts.Log.Gen.error($"TCPConnection.accept ex: {ex} trying again"); - Task.Delay(wait_after_error); + await Task.Delay(wait_after_error); } } } // connects to the server and ona successful connection, calls con__handler.on_connection_async with an ITCPConnection. // In case of failures to connect it waits for a given amount of time before retrying - public static async Task connect_to_server_async(string ip, int server_port, ITCPConnectionHandler conn_handler, + public static async Task connect_to_server_async(string ip, int server_port, ITCPConnectionHandler conn_handler, TimeSpan wait_after_failed_attempt, int n_max_attempts = 1024) { - Socket client_socket = null; + Socket? client_socket = null; for (int i = 0; i < n_max_attempts; i++) { try { string log_conn = $"{ip}:{server_port}"; @@ -158,7 +162,7 @@ namespace BCG { break; } catch (Exception ex) { bts.Log.Gen.error($"AsyncTCPConnection2.connect_to_server threw ex: {ex}, will reattempt now"); - Task.Delay(wait_after_failed_attempt); + await Task.Delay(wait_after_failed_attempt); } } return client_socket; @@ -281,4 +285,4 @@ namespace BCG { #endregion -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/PubSub/Misc/BlockingQueue.cs b/src/DataStreamSuite/PubSub/Misc/BlockingQueue.cs index 0b54f3375..eb4113d60 100644 --- a/src/DataStreamSuite/PubSub/Misc/BlockingQueue.cs +++ b/src/DataStreamSuite/PubSub/Misc/BlockingQueue.cs @@ -2,9 +2,6 @@ using System; using System.Collections; using System.Threading; -using BCG; -using BCG.Utilities; - namespace BCG { [Serializable] @@ -36,13 +33,10 @@ namespace BCG } } - public object dequeue() - { - return dequeue(Timeout.Infinite); - } + public object? dequeue() => + dequeue(Timeout.Infinite); - public object dequeue (int millisecondsToWait) - { + public object? dequeue (int millisecondsToWait) { for (;;) lock (this) { @@ -60,13 +54,13 @@ namespace BCG } } - public object[] dequeueMultipleElements(int millisecondsToWait, int maxElements) + public object?[] dequeueMultipleElements(int millisecondsToWait, int maxElements) { ArrayList rv = new ArrayList(); int n = getCount(); for (int i = 0; i < n; i++) { - object o = dequeue(0); + var o = dequeue(0); if (o != null) { rv.Add(o); @@ -78,7 +72,7 @@ namespace BCG int timeRemaining = millisecondsToWait; while (true) { - object o = dequeue(timeRemaining); + var o = dequeue(timeRemaining); if (o != null) rv.Add(o); timeRemaining = millisecondsToWait - Convert.ToInt32 (((TimeSpan)(DateTime.Now - startTime)).TotalMilliseconds); @@ -87,10 +81,8 @@ namespace BCG } } - public object[] dequeueMultipleElements(TimeSpan ts, int maxElements) - { - return dequeueMultipleElements(Convert.ToInt32 (ts.TotalMilliseconds), maxElements); - } + public object?[] dequeueMultipleElements(TimeSpan ts, int maxElements) => + dequeueMultipleElements(Convert.ToInt32 (ts.TotalMilliseconds), maxElements); public int getCount() { diff --git a/src/DataStreamSuite/PubSub/Misc/ClientInfo.cs b/src/DataStreamSuite/PubSub/Misc/ClientInfo.cs index 27f8f2034..97231b251 100644 --- a/src/DataStreamSuite/PubSub/Misc/ClientInfo.cs +++ b/src/DataStreamSuite/PubSub/Misc/ClientInfo.cs @@ -1,8 +1,5 @@ using System; using System.Text; -using BCG.Utilities.General; - - namespace BCG { @@ -43,7 +40,7 @@ namespace BCG set { myStartTime = value; } } - private string myProcessString; + string? myProcessString; public string processString { get @@ -84,7 +81,7 @@ namespace BCG myStartTime = System.Diagnostics.Process.GetCurrentProcess().StartTime; } - private static ClientInfo myLocalClient; + static ClientInfo? myLocalClient; private static object myLock = new object(); public static ClientInfo thisClient @@ -135,4 +132,4 @@ namespace BCG return builder.ToString(); } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarBuffer.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarBuffer.cs index e9396168e..3291244fc 100644 --- a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarBuffer.cs +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarBuffer.cs @@ -33,8 +33,8 @@ // exception statement from your version. using System; +using System.Diagnostics.CodeAnalysis; using System.IO; -using System.Text; namespace BCG.Utilities.SharpZipLib.Tar { @@ -55,10 +55,10 @@ namespace BCG.Utilities.SharpZipLib.Tar { public static readonly int DEFAULT_RCDSIZE = 512; public static readonly int DEFAULT_BLKSIZE = DEFAULT_RCDSIZE * 20; - Stream inputStream; - Stream outputStream; + Stream? inputStream; + Stream? outputStream; - byte[] blockBuffer; + byte[]? blockBuffer; int currBlkIdx; int currRecIdx; int blockSize; @@ -110,6 +110,7 @@ namespace BCG.Utilities.SharpZipLib.Tar { /// /// Initialization common to all constructors. /// + [MemberNotNull(nameof(blockBuffer))] void Initialize(int blockSize, int recordSize) { this.debug = false; @@ -197,7 +198,7 @@ namespace BCG.Utilities.SharpZipLib.Tar { /// /// The record data. /// - public byte[] ReadRecord() + public byte[]? ReadRecord() { if (this.debug) { Console.Error.WriteLine( "ReadRecord: recIdx = " + this.currRecIdx + " blkIdx = " + this.currBlkIdx ); @@ -214,7 +215,7 @@ namespace BCG.Utilities.SharpZipLib.Tar { } byte[] result = new byte[this.recordSize]; - + requireBlockBuffer(); Array.Copy(this.blockBuffer, (this.currRecIdx * this.recordSize), result, 0, this.recordSize ); this.currRecIdx++; return result; @@ -238,34 +239,37 @@ namespace BCG.Utilities.SharpZipLib.Tar { int offset = 0; int bytesNeeded = this.blockSize; - for (; bytesNeeded > 0 ;) { - long numBytes = this.inputStream.Read(this.blockBuffer, offset, bytesNeeded); - - // - // NOTE - // We have fit EOF, and the block is not full! - // - // This is a broken archive. It does not follow the standard - // blocking algorithm. However, because we are generous, and - // it requires little effort, we will simply ignore the error - // and continue as if the entire block were read. This does - // not appear to break anything upstream. We used to return - // false in this case. - // - // Thanks to 'Yohann.Roussel@alcatel.fr' for this fix. - // - if (numBytes <= 0) { - break; - } - - offset += (int)numBytes; - bytesNeeded -= (int)numBytes; - if (numBytes != this.blockSize) { - if (this.debug) { - Console.Error.WriteLine("ReadBlock: INCOMPLETE READ " + numBytes + " of " + this.blockSize + " bytes read."); - } - } - } + if (bytesNeeded > 0) { + requireBlockBuffer(); + do { + long numBytes = this.inputStream.Read(this.blockBuffer, offset, bytesNeeded); + + // + // NOTE + // We have fit EOF, and the block is not full! + // + // This is a broken archive. It does not follow the standard + // blocking algorithm. However, because we are generous, and + // it requires little effort, we will simply ignore the error + // and continue as if the entire block were read. This does + // not appear to break anything upstream. We used to return + // false in this case. + // + // Thanks to 'Yohann.Roussel@alcatel.fr' for this fix. + // + if (numBytes <= 0) { + break; + } + + offset += (int)numBytes; + bytesNeeded -= (int)numBytes; + if (numBytes != this.blockSize) { + if (this.debug) { + Console.Error.WriteLine("ReadBlock: INCOMPLETE READ " + numBytes + " of " + this.blockSize + " bytes read."); + } + } + } while(bytesNeeded > 0); + } this.currBlkIdx++; return true; @@ -318,6 +322,7 @@ namespace BCG.Utilities.SharpZipLib.Tar { if (this.currRecIdx >= this.recsPerBlock) { this.WriteBlock(); } + requireBlockBuffer(); Array.Copy(record, 0, this.blockBuffer, (this.currRecIdx * this.recordSize), this.recordSize ); this.currRecIdx++; } @@ -350,7 +355,7 @@ namespace BCG.Utilities.SharpZipLib.Tar { if (this.currRecIdx >= this.recsPerBlock) { this.WriteBlock(); } - + requireBlockBuffer(); Array.Copy(buf, offset, this.blockBuffer, (this.currRecIdx * this.recordSize), this.recordSize ); this.currRecIdx++; @@ -368,7 +373,7 @@ namespace BCG.Utilities.SharpZipLib.Tar { if (this.outputStream == null) { throw new System.IO.IOException("no output stream defined"); } - + requireBlockBuffer(); this.outputStream.Write(this.blockBuffer, 0, this.blockSize); this.outputStream.Flush(); @@ -418,6 +423,13 @@ namespace BCG.Utilities.SharpZipLib.Tar { inputStream = null; } } + + [MemberNotNull(nameof(blockBuffer))] + void requireBlockBuffer() { + if (blockBuffer is null) { + throw new InvalidOperationException("blockBuffer is null"); + } + } } } /* The original Java file had this header: diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarInputStream.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarInputStream.cs index 14c48e681..38e202c9b 100644 --- a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarInputStream.cs +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Tar/TarInputStream.cs @@ -32,7 +32,6 @@ // obligated to do so. If you do not wish to do so, delete this // exception statement from your version. -#nullable disable using System; using System.IO; using System.Text; @@ -53,11 +52,11 @@ namespace BCG.Utilities.SharpZipLib.Tar { protected int entrySize; protected int entryOffset; - protected byte[] readBuf; + protected byte[]? readBuf; protected TarBuffer buffer; - protected TarEntry currEntry; - protected IEntryFactory eFactory; + protected TarEntry? currEntry; + protected IEntryFactory? eFactory; Stream inputStream; /// @@ -308,7 +307,7 @@ namespace BCG.Utilities.SharpZipLib.Tar { this.readBuf = null; } - byte[] headerBuf = this.buffer.ReadRecord(); + byte[]? headerBuf = this.buffer.ReadRecord(); if (headerBuf == null) { if (this.debug) { @@ -327,6 +326,9 @@ namespace BCG.Utilities.SharpZipLib.Tar { if (this.hasHitEOF) { this.currEntry = null; } else { + if (headerBuf is null) { + throw new InvalidOperationException("headerBuf is null"); + } try { if (this.eFactory == null) { this.currEntry = new TarEntry(headerBuf); @@ -422,13 +424,9 @@ namespace BCG.Utilities.SharpZipLib.Tar { } while (numToRead > 0) { - byte[] rec = this.buffer.ReadRecord(); - if (rec == null) { - // Unexpected EOF! - throw new IOException("unexpected EOF with " + numToRead + " bytes unread"); - } - - int sz = numToRead; + byte[]? rec = buffer.ReadRecord() + ?? throw new IOException("unexpected EOF with " + numToRead + " bytes unread"); + int sz = numToRead; int recLen = rec.Length; if (recLen > sz) { diff --git a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipOutputStream.cs b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipOutputStream.cs index 97676b06c..26712b88e 100644 --- a/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipOutputStream.cs +++ b/src/DataStreamSuite/PubSub/Misc/CompressionUtilities/SharpZipLib/Zip/ZipOutputStream.cs @@ -44,6 +44,8 @@ using BCG.Utilities.SharpZipLib.Checksums; using BCG.Utilities.SharpZipLib.Zip.Compression; using BCG.Utilities.SharpZipLib.Zip.Compression.Streams; +using Spark.Core.Diagnostics; + namespace BCG.Utilities.SharpZipLib.Zip { /// @@ -96,9 +98,9 @@ namespace BCG.Utilities.SharpZipLib.Zip { /// public class ZipOutputStream : DeflaterOutputStream { - private ArrayList entries = new ArrayList(); - private Crc32 crc = new Crc32(); - private ZipEntry curEntry = null; + private ArrayList? entries = new(); + private Crc32 crc = new(); + private ZipEntry? curEntry = null; private CompressionMethod curMethod; private int size; @@ -308,10 +310,7 @@ namespace BCG.Utilities.SharpZipLib.Zip { if (name.Length > 0xFFFF) { throw new ZipException("Name too long."); } - byte[] extra = entry.ExtraData; - if (extra == null) { - extra = new byte[0]; - } + byte[] extra = entry.ExtraData ?? Array.Empty(); if (extra.Length > 0xFFFF) { throw new ZipException("Extra data too long."); } @@ -397,7 +396,7 @@ namespace BCG.Utilities.SharpZipLib.Zip { } } - entries.Add(curEntry); + ExceptionAssert.MemberNotNull(entries).Add(curEntry); curEntry = null; } @@ -468,10 +467,7 @@ namespace BCG.Utilities.SharpZipLib.Zip { if (name.Length > 0xffff) { throw new ZipException("Name too long."); } - byte[] extra = entry.ExtraData; - if (extra == null) { - extra = new byte[0]; - } + byte[] extra = entry.ExtraData ?? Array.Empty(); string strComment = entry.Comment; byte[] comment = strComment != null ? ZipConstants.ConvertToArray(strComment) : new byte[0]; if (comment.Length > 0xffff) { diff --git a/src/DataStreamSuite/PubSub/Misc/GenericFormatter.cs b/src/DataStreamSuite/PubSub/Misc/GenericFormatter.cs index 60b003d3f..de65cca4b 100644 --- a/src/DataStreamSuite/PubSub/Misc/GenericFormatter.cs +++ b/src/DataStreamSuite/PubSub/Misc/GenericFormatter.cs @@ -1,4 +1,3 @@ -using System; using System.IO; using System.Runtime.Serialization; using System.Runtime.Serialization.Formatters.Binary; @@ -10,14 +9,11 @@ namespace BCG { IFormatter myFormatter = new F (); - public T Deserialize (Stream serializationStream) - { - return (T) myFormatter.Deserialize (serializationStream); - } - public void Serialize (Stream serializationStream, T graph) - { + public T Deserialize (Stream serializationStream) where T : notnull => + (T)myFormatter.Deserialize (serializationStream); + + public void Serialize (Stream serializationStream, T graph) where T : notnull => myFormatter.Serialize (serializationStream, graph); - } } public class GenericBinaryFormatter : GenericFormatter { } diff --git a/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs b/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs index 6f19b0df5..19f479698 100644 --- a/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs +++ b/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs @@ -17,12 +17,13 @@ namespace BCG { myLastLogTime = DateTime.MinValue; protected TimeSpan myPeriod = new TimeSpan(0, 0, 0, 0, 100), myLogPeriod = new TimeSpan(0, 1, 0); - protected Dictionary> myWorkers = new Dictionary>(), - myJustAdded = new Dictionary>(), - myJustRemoved = new Dictionary>(); + protected readonly Dictionary?> myWorkers = new(); + protected readonly Dictionary?> myJustAdded = new(); + protected readonly Dictionary?> myJustRemoved = new(); readonly Stopwatch timer = new(); - private static MasterWorker myThreadDriven = null, - myDoWorkDriven = null; + static MasterWorker? myThreadDriven; + static MasterWorker? myDoWorkDriven; + internal MasterWorker() { myInit(); } @@ -35,7 +36,6 @@ namespace BCG { // TODO myThreadDriven = myThreadDriven ?? new ThreadDrivenMasterWorker(); return myThreadDriven; - return null; } else { myDoWorkDriven = myDoWorkDriven ?? new MasterWorker(); return myDoWorkDriven; @@ -61,10 +61,10 @@ namespace BCG { public virtual void doWork() { - foreach (KeyValuePair> p in myJustAdded) { + foreach (var p in myJustAdded) { myWorkers.Add(p.Key, p.Value); } - foreach (KeyValuePair> p in myJustRemoved) { + foreach (var p in myJustRemoved) { myWorkers.Remove(p.Key); } myJustAdded.Clear(); diff --git a/src/DataStreamSuite/PubSub/Misc/Pair.cs b/src/DataStreamSuite/PubSub/Misc/Pair.cs index e75b18688..b7c11fa4b 100644 --- a/src/DataStreamSuite/PubSub/Misc/Pair.cs +++ b/src/DataStreamSuite/PubSub/Misc/Pair.cs @@ -1,12 +1,5 @@ -//using ProtoBuf; - using System; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -//using System.Windows; -//using System.Windows.Media; namespace mbts { @@ -18,7 +11,7 @@ namespace mbts public S? second; public Pair() { } - public Pair(F f, S s) + public Pair(F? f, S? s) { first = f; second = s; diff --git a/src/DataStreamSuite/PubSub/Server/SingleTagConnectionData.cs b/src/DataStreamSuite/PubSub/Server/SingleTagConnectionData.cs index b4080953b..3ce19c5e2 100644 --- a/src/DataStreamSuite/PubSub/Server/SingleTagConnectionData.cs +++ b/src/DataStreamSuite/PubSub/Server/SingleTagConnectionData.cs @@ -1,13 +1,11 @@ -using System; -using System.Collections.Generic; -using System.Text; - -using BCG.Utilities; - using bts; using mbts; +using Spark.Core.Diagnostics; + +using System.Collections.Generic; + namespace BCG { /// /// SingleTagPubSubMap consists of all the state w.r.t to @@ -79,6 +77,7 @@ namespace BCG { public bool handle_sub(SubscribeMsg sub_msg) { Log.Gen.info($"SingleTagConnectionData.handle_sub sub_msg: {sub_msg}"); + ExceptionAssert.NotNull(sub_msg.connection); if (_sub_conn_to_app_map.ContainsKey(sub_msg.connection)) { bts.Log.Gen.info($"tag: {_tag} has already been subscribed by connection: {sub_msg.connection}"); return false; @@ -89,6 +88,7 @@ namespace BCG { } public bool handle_unsub(UnsubscribeMsg unsub_msg) { + ExceptionAssert.NotNull(unsub_msg.connection); if (_sub_conn_to_app_map.ContainsKey(unsub_msg.connection)) { _sub_conn_to_app_map.Remove(unsub_msg.connection); return true; @@ -123,6 +123,7 @@ namespace BCG { } public void handle_pub(PublishMsg pub_msg, out bool new_publisher) { + ExceptionAssert.NotNull(pub_msg.connection); if (_active_pub_conn_to_app_map.ContainsKey(pub_msg.connection)) { new_publisher = false; } else { @@ -131,4 +132,4 @@ namespace BCG { } } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/PubSub/Testing/SimpleTCPServer.cs b/src/DataStreamSuite/PubSub/Testing/SimpleTCPServer.cs index a99086c35..d036a7673 100644 --- a/src/DataStreamSuite/PubSub/Testing/SimpleTCPServer.cs +++ b/src/DataStreamSuite/PubSub/Testing/SimpleTCPServer.cs @@ -1,6 +1,9 @@ +using bts; + +using Spark.Core.Diagnostics; + using System; using System.Collections.Generic; -using bts; using System.Threading; namespace BCG { @@ -40,15 +43,13 @@ namespace BCG { return; } RemoteConnectionInfo rci = msg.remoteConnectionInfo; - ITCPConnection this_conn; - bool found = _conn_map.TryGetValue(rci, out this_conn); - if (!found) { + if (!_conn_map.TryGetValue(rci, out var this_conn)) { Log.Gen.error($"Could not find connection: {rci}"); return; } Log.Gen.info($"\n\n\nSimpleTCPServerAsync.onData msgs_received: {_n_msgs_received} msg: {msg} ci: {rci} NOW echoing it back\n\n\n"); if (msg.messageType == RemoteMessageType.Data) { - this_conn.send(msg.message); + this_conn.send(ExceptionAssert.NotNull(msg.message)); } else { if (msg.messageType == RemoteMessageType.Disconnection) { var ci = msg.remoteConnectionInfo; diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/ContractUtils.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/ContractUtils.cs index cdb35395f..af11afb07 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/ContractUtils.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/ContractUtils.cs @@ -1,103 +1,102 @@ -using bts.SilverPine; - namespace bts.BlueOrb { using bts; //using bts.api; //using bts.spark; using System; - - using ZstdSharp.Unsafe; + using System.Diagnostics.CodeAnalysis; public static class ContractUtils { - public class FutureTradableContract { - public string symbol; - public DateTime expiration; - public Exchange exchange; - public string contract_text; - } + public record FutureTradableContract( + string Symbol, + DateTime Expiration, + Exchange Exchange, + string? ContractText); + public class FutureSpreadContract { - public List> legs; - public string symbol { + public List> legs = new(2); + public string Symbol { get { - return legs == null || legs.Count == 0 ? string.Empty : legs.ElementAt(0).Item2.symbol; + return legs == null || legs.Count == 0 ? string.Empty : legs.ElementAt(0).Item2.Symbol; } } - public string contract_text; - } - public class FutureContinousContract { - public string symbol; - public Exchange exchange; + public string? ContractText; } + + public record FutureContinousContract(string Symbol, Exchange Exchange); + // this is for all future conytracts belonging to a symbol - public class FutureWildcardContract { - public string symbol; - public Exchange exchange; - } + public record FutureWildcardContract(string Symbol, Exchange Exchange); + public class FutureContract { - public FutureWildcardContract wildcard_contract = null; - public FutureTradableContract future_tradable_contract = null; - public FutureSpreadContract spread_tradable_contract = null; - public FutureContinousContract continous_constract = null; + public FutureWildcardContract? wildcard_contract; + public FutureTradableContract? future_tradable_contract; + public FutureSpreadContract? spread_tradable_contract; + public FutureContinousContract? continous_constract; public FutureContract(string txt) { wildcard_contract = parse_wildcard_contract(txt); if (wildcard_contract != null) return; - FutureTradableContract fc = parse_future_contract(txt); + var fc = parse_future_contract(txt); if (fc != null) { future_tradable_contract = fc; return; } - FutureSpreadContract sc = parse_future_spread_contract(txt); + var sc = parse_future_spread_contract(txt); if (sc != null) { spread_tradable_contract = sc; return; } // this is not totally true, since ParseFutureSpreadContract does not // parse ICS spreads - continous_constract = new FutureContinousContract() { symbol = txt }; + continous_constract = new(txt, Exchange.CME); } + + [MemberNotNullWhen(true, nameof(continous_constract))] public bool is_continous_contract => continous_constract != null; + + [MemberNotNullWhen(true, nameof(wildcard_contract))] public bool is_wildcard_contract => wildcard_contract != null; + + [MemberNotNullWhen(true, nameof(future_tradable_contract))] public bool is_future_tradable_contract => future_tradable_contract != null; + + [MemberNotNullWhen(true, nameof(spread_tradable_contract))] public bool is_spread_tradable_contract => spread_tradable_contract != null; public string symbol() { if (wildcard_contract != null) - return wildcard_contract.symbol; + return wildcard_contract.Symbol; if (continous_constract != null) { - return continous_constract.symbol; + return continous_constract.Symbol; } else if (future_tradable_contract != null) { - return future_tradable_contract.symbol; + return future_tradable_contract.Symbol; } else if (spread_tradable_contract != null) { - return spread_tradable_contract.symbol; + return spread_tradable_contract.Symbol; } Log.assert(false); return string.Empty; } - public string contract_text { + public string? contract_text { get { if (future_tradable_contract != null) - return future_tradable_contract.contract_text; + return future_tradable_contract.ContractText; if (spread_tradable_contract != null) - return spread_tradable_contract.contract_text; + return spread_tradable_contract.ContractText; return null; } } } #region Parsing - public static FutureWildcardContract parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) { + public static FutureWildcardContract? parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) { if (contract_txt.EndsWith("*")) { contract_txt = contract_txt.Replace("*", string.Empty); - return new FutureWildcardContract() { - symbol = contract_txt, - exchange = exchange - }; + return new FutureWildcardContract(contract_txt, exchange); } return null; } - public static FutureTradableContract parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) { + public static FutureTradableContract? parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) { // Validate the future string length if (!contract_txt.All(c => char.IsLetterOrDigit(c))) return null; @@ -143,17 +142,10 @@ namespace bts.BlueOrb { // Create the FutureContract instance and set its properties DateTime expiration = new DateTime(year, month, 1); - var futureContract = new FutureTradableContract { - symbol = symbol, - expiration = expiration, - exchange = exchange, - contract_text = contract_txt - }; - - return futureContract; + return new(symbol, expiration, exchange, contract_txt); } // Method to parse a futures spread, e.g. "CLX4-CLZ4" - public static FutureSpreadContract parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) { + public static FutureSpreadContract? parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) { // Split the string by '-' to handle each leg var spreadParts = contract_txt.Split('-'); if (spreadParts.Length < 2) { @@ -201,11 +193,11 @@ namespace bts.BlueOrb { #endregion public static ContractType get_type(string contract_txt) { - FutureTradableContract fc = parse_future_contract(contract_txt); + var fc = parse_future_contract(contract_txt); if (fc != null) { return ContractType.Future; } - FutureSpreadContract sc = parse_future_spread_contract(contract_txt); + var sc = parse_future_spread_contract(contract_txt); if (sc != null) { return ContractType.FutureSpread; } @@ -220,13 +212,13 @@ namespace bts.BlueOrb { } public static Tuple get_symbol_category(string symbol) { - FutureTradableContract fc = parse_future_contract(symbol); + var fc = parse_future_contract(symbol); if (fc != null) { - return Tuple.Create(ContractCategory.FutureString, fc.symbol); + return Tuple.Create(ContractCategory.FutureString, fc.Symbol); } var sc = parse_future_spread_contract(symbol); if (sc != null) { - return Tuple.Create(ContractCategory.FutureString, sc.symbol); + return Tuple.Create(ContractCategory.FutureString, sc.Symbol); } return Tuple.Create(ContractCategory.Symbol, symbol); } diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricalBarsStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricalBarsStream.cs index db434cd7c..4933ae193 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricalBarsStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricalBarsStream.cs @@ -2,6 +2,7 @@ namespace bts.BlueOrb { using bts.SilverPine; using BCG; + using Bts.DataStreamExpressions; public class DateTimeRange { public readonly DateTime range_beg; @@ -18,7 +19,7 @@ namespace bts.BlueOrb { } public class FutureBarsLoader { - public static string get_db_table_name(BarSize bar_size, string symbol, bool is_continous) { + public static string? get_db_table_name(BarSize bar_size, string symbol, bool is_continous) { switch (bar_size) { case BarSize.ONE_DAY: { if (is_continous) { @@ -62,7 +63,8 @@ namespace bts.BlueOrb { beg = beg.Date; end = end.Date.Add(new TimeSpan(23, 59, 59)); // this returns rows from beg and end inclusive - Func get_sqlite_str = (table_name) => { + Func get_sqlite_str = (table_name) => { + ArgumentException.ThrowIfNullOrEmpty(table_name); string range_beg_iso8601 = beg.ToString("yyyy-MM-ddTHH:mm:ss"), range_end_iso8601 = end.ToString("yyyy-MM-ddTHH:mm:ss"); string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'"; @@ -70,27 +72,27 @@ namespace bts.BlueOrb { }; string symbol = future_contract.symbol(); if (future_contract.wildcard_contract != null) { - string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false); + var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false); ViewStream rs = get_sqlite_str(table_name); //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime); mySetRealSource(rs); return; } if (future_contract.wildcard_contract != null) { - string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false); + var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false); ViewStream rs = get_sqlite_str(table_name); //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime); mySetRealSource(rs); return; } if (future_contract.is_continous_contract) { - string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true); + var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true); ViewStream rs = get_sqlite_str(table_name); //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime); mySetRealSource(rs); return; } else { - string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false); + var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false); ViewStream rs = get_sqlite_str(table_name); //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime); //TODO: filter it @@ -115,7 +117,8 @@ namespace bts.BlueOrb { // it gets the raw table with all contracts for that symbol public FuturesHistorical1MinuteBarsStream(ContractUtils.FutureContract future_contract, DateTimeRange dt_range) { - Func get_sqlite_str = (table_name) => { + Func get_sqlite_str = (table_name) => { + ArgumentException.ThrowIfNullOrEmpty(table_name); string range_beg_iso8601 = dt_range.range_beg.ToString("yyyy-MM-ddTHH:mm:ss"), range_end_iso8601 = dt_range.range_end.ToString("yyyy-MM-ddTHH:mm:ss"); //string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'"; @@ -124,18 +127,18 @@ namespace bts.BlueOrb { }; string symbol = future_contract.symbol(); if (future_contract.wildcard_contract != null) { - string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false); + var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false); ViewStream rs = get_sqlite_str(table_name); mySetRealSource(rs); return; } if (future_contract.is_continous_contract) { - string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true); + var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true); ViewStream rs = get_sqlite_str(table_name); mySetRealSource(rs); return; } else { - string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false); + var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false); ViewStream rs = get_sqlite_str(table_name); // TODO: filter rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'"); diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesContractStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesContractStream.cs index 4a16164b9..b4edb0e64 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesContractStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveFuturesContractStream.cs @@ -1,6 +1,4 @@ namespace bts.BlueOrb { - using System; - using BCG; using bts.api; @@ -24,23 +22,23 @@ // wild card if (fc.is_wildcard_contract) { var wc_contr = fc.wildcard_contract; - ViewStream rs = new bts.BlueOrb.FilterStream(parent_str, $"symbol='{wc_contr.symbol}'"); + ViewStream rs = new FilterStream(parent_str, $"symbol='{wc_contr.Symbol}'"); rs = new bts.BlueOrb.FilterStream(parent_str, $"sec_type='{SecurityType.Future}' or sec_type = '{SecurityType.Spread}'"); mySetRealSource(rs); return; } else if (fc.is_future_tradable_contract) { var fut_contr = fc.future_tradable_contract; - ViewStream rs = new bts.BlueOrb.FilterStream(parent_str, $"sec='{fut_contr.contract_text}'"); + ViewStream rs = new FilterStream(parent_str, $"sec='{fut_contr.ContractText}'"); mySetRealSource(rs); return; } else if (fc.is_spread_tradable_contract) { var spr_contr = fc.future_tradable_contract; - ViewStream rs = new bts.BlueOrb.FilterStream(parent_str, $"sec='{spr_contr.contract_text}'"); + ViewStream rs = new FilterStream(parent_str, $"sec='{spr_contr.ContractText}'"); mySetRealSource(rs); return; } else if (fc.is_continous_contract) { var cont_contr = fc.continous_constract; - ViewStream rs = new bts.BlueOrb.FilterStream(parent_str, $"symbol='{cont_contr.symbol}' and sec_type='{SecurityType.Future}'"); + ViewStream rs = new FilterStream(parent_str, $"symbol='{cont_contr.Symbol}' and sec_type='{SecurityType.Future}'"); rs = new bts.BlueOrb.GroupByStream(rs, () => { return new string[] { "symbol" }; }, (group_rows) => { IBlueRow r = group_rows.ToList().OrderBy(r => r.get_datetime("expiration")).ElementAt(0); diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveTradeStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveTradeStream.cs index a38e5cce1..9758603dc 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveTradeStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/LiveTradeStream.cs @@ -1,6 +1,4 @@ namespace bts.BlueOrb { - using System; - using BCG; using bts.api; @@ -10,7 +8,7 @@ public LiveTradeStream(string target) : this(SecurityType.Future, target) { } public LiveTradeStream(SecurityType sec_type, string target) { - ViewStream rs = null; + ViewStream? rs = null; switch (sec_type) { case SecurityType.Future: rs = new LiveFuturesTradeStream(target); diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveBBOStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveBBOStream.cs index 911ededf8..13907a3d1 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveBBOStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveBBOStream.cs @@ -1,11 +1,11 @@ namespace bts.BlueOrb { + using BCG; + + using Spark.Core.Diagnostics; + using System; using System.Collections.Generic; using System.Linq; - using System.Text; - using System.Threading.Tasks; - - using BCG; [DataStreamUserRespresentation("ReplayFutureLiveBBOStream")] public class ReplayFutureLiveBBOStream : ViewStream, BCG.Utilities.General.IWorker{ @@ -14,13 +14,13 @@ public DateTime last_firetime_clock = DateTime.MinValue; } private class RowsWithPointer { - public List rows; + public List? rows; public int pointer; } - private readonly ViewStreamDatabase _vsd; - private readonly ContractUtils.FutureContinousContract _fcc; + readonly ViewStreamDatabase? _vsd; + readonly ContractUtils.FutureContinousContract? _fcc; private readonly LastFireTime _last_fire_time = new LastFireTime(); private RowsWithPointer _rows = new RowsWithPointer(); @@ -62,20 +62,22 @@ } protected override void myOnFirstSubscriber() { - var file_path = Path.Combine(SQLiteHelper.location, $"mbp1_{_fcc.symbol}_small.csv".ToLower()); + Log.notnull(_fcc); + var file_path = Path.Combine(SQLiteHelper.location, $"mbp1_{_fcc.Symbol}_small.csv".ToLower()); var rows = BCG.CSVReaderStream.LoadFile(file_path); _rows.rows = new(rows.Count); _rows.rows.AddRange(rows.OfType().OrderBy(r => r.get("ts_event"))); _rows.pointer = 0; - _vsd.addWorker(this); + ExceptionAssert.NotNull(_vsd).addWorker(this); } protected override void myOnLastSubscriber() { - _vsd.removeWorker(this); + ExceptionAssert.NotNull(_vsd).removeWorker(this); } public void doWork() { - IBlueRow cand_row = _rows.rows[_rows.pointer]; + var rows = ExceptionAssert.NotNull(_rows.rows); + IBlueRow cand_row = rows[_rows.pointer]; string cand_ts_event = cand_row.get("ts_event"); DateTime cand_ts_event_dt = DateTime.Parse(cand_ts_event, null, System.Globalization.DateTimeStyles.RoundtripKind); TimeSpan cand_ts_span_file = cand_ts_event_dt - _last_fire_time.last_fireitme_file; diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveTradeStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveTradeStream.cs index eaed74de3..9700e3849 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveTradeStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Live/ReplayFutureLiveTradeStream.cs @@ -1,11 +1,11 @@ namespace bts.BlueOrb { + using BCG; + + using Spark.Core.Diagnostics; + using System; using System.Collections.Generic; using System.Linq; - using System.Text; - using System.Threading.Tasks; - - using BCG; [DataStreamUserRespresentation("ReplayFutureLiveTradeStream")] public class ReplayFutureLiveTradeStream : ViewStream, BCG.Utilities.General.IWorker { @@ -14,15 +14,15 @@ public DateTime last_firetime_clock = DateTime.MinValue; } private class RowsWithPointer { - public List rows; + public List? rows; public int pointer; } - private readonly ViewStreamDatabase _vsd; - private readonly ContractUtils.FutureContinousContract _fcc; + readonly ViewStreamDatabase? _vsd; + readonly ContractUtils.FutureContinousContract? _fcc; private readonly LastFireTime _last_fire_time = new LastFireTime(); - private RowsWithPointer _rows = new RowsWithPointer(); + readonly RowsWithPointer _rows = new(); public ReplayFutureLiveTradeStream(string continous_contract) { Func iso_8601_now = () => { @@ -61,21 +61,24 @@ } protected override void myOnFirstSubscriber() { - var file_path = Path.Combine(SQLiteHelper.location, $"trades_{_fcc.symbol}.csv"); + Log.notnull(_vsd); + Log.notnull(_fcc); + var file_path = Path.Combine(SQLiteHelper.location, $"trades_{_fcc.Symbol}.csv"); var csv_rd = new CSVReaderStream(file_path, false); var rows = BCG.CSVReaderStream.LoadFile(file_path); _rows.rows = new(rows.Count); _rows.rows.AddRange(rows.OfType().OrderBy(r => r.get("ts_event"))); _rows.pointer = 0; - _vsd.addWorker(this); + ExceptionAssert.NotNull(_vsd).addWorker(this); } protected override void myOnLastSubscriber() { - _vsd.removeWorker(this); + ExceptionAssert.NotNull(_vsd).removeWorker(this); } public void doWork() { - IBlueRow cand_row = _rows.rows[_rows.pointer]; + var rows = ExceptionAssert.NotNull(_rows.rows); + IBlueRow cand_row = rows[_rows.pointer]; string cand_ts_event = cand_row.get("ts_event"); DateTime cand_ts_event_dt = DateTime.Parse(cand_ts_event, null, System.Globalization.DateTimeStyles.RoundtripKind); TimeSpan cand_ts_span_file = cand_ts_event_dt - _last_fire_time.last_fireitme_file; diff --git a/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsStream.cs b/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsStream.cs index 2b0868bef..fc1e66aa7 100644 --- a/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsStream.cs @@ -8,7 +8,7 @@ namespace bts { using System.Text.Json.Nodes; [DataStreamUserRespresentation("benzingaNews")] public class BenzingaNewsStream : ViewStream { - private IBlueTableSchema _schema = null; + IBlueTableSchema? _schema; private int _id = 0; @@ -63,14 +63,14 @@ namespace bts { } foreach (JsonNode jnode in json_objs) { if (jnode is JsonObject json_object) { - string symbol = jnode["symbol"] == null ? String.Empty : jnode["symbol"].ToString(); - string exchange = jnode["exchange"] == null ? String.Empty : jnode["exchange"].ToString(); + string symbol = jnode["symbol"]?.ToString() ?? string.Empty; + string exchange = jnode["exchange"]?.ToString() ?? string.Empty; MutableBlueRow nr = new MutableBlueRow(_schema, r); nr.setValue("id", _id++); - nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}"); + nr.setValue("doc_id_internal", $"{r["doc_id"]}_{symbol}"); nr.setValue("symbol", symbol); nr.setValue("exchange", exchange); - nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString())); + nr.setValue("is_primary_exchange", jnode["primary"]?.GetValue() ?? false); expanded_rows.Add(nr); } } @@ -84,4 +84,4 @@ namespace bts { mySetRealSource(output_stream); } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsSymbolStateStream.cs b/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsSymbolStateStream.cs index 944c9b73a..4ef8bc058 100644 --- a/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsSymbolStateStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsSymbolStateStream.cs @@ -15,8 +15,8 @@ namespace bts { this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) { } - public IBlueRowCollection evaluate(IDictionary> grouped_rows) { - BlueTable rv = null; + public IBlueRowCollection? evaluate(IDictionary> grouped_rows) { + BlueTable? rv = null; foreach (var kv in grouped_rows) { var symbol_rows = kv.Value; IBlueRow state_row = BlueDataUtility.find_max(symbol_rows, "created_at"); @@ -30,4 +30,4 @@ namespace bts { return new string[] { "symbol" }; } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs index e2e35a99a..a8ec9fe56 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs @@ -14,7 +14,7 @@ public class SpatialArbStatsStream : ViewStream{ private ViewStream _spatial_pairs_stream; private ViewStreamDatabase _vsd; - private ViewStream _tot_str = null; + ViewStream? _tot_str = null; private HashSet _key_map = new HashSet(); private ViewStream _bbo_stream; private readonly TimeSpan _window_sz, _sampling_period_sz; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceEvaluator.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceEvaluator.cs index 606ae305e..93d110d09 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceEvaluator.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceEvaluator.cs @@ -1,30 +1,8 @@ -using System; -using System.Collections.Generic; -using System.Linq; using MathNet.Numerics.Statistics; using MathNet.Numerics.LinearRegression; -using System.Collections.Generic; -using System.Linq; -using MathNet.Numerics.Statistics; using MathNet.Numerics.LinearAlgebra; -using Amazon.Runtime; -using System; -using System.Collections.Generic; -using System.Linq; -using MathNet.Numerics.Statistics; -using System; -using System.Collections.Generic; -using System.Linq; -using MathNet.Numerics.Statistics; -using System; -using System.Collections.Generic; -using System.Linq; -using MathNet.Numerics.Statistics; -// stats calculator -using System; -using System.Collections.Generic; -using System.Linq; -using MathNet.Numerics.Statistics; + +using Spark.Core.Diagnostics; /// /// Provides evaluation metrics to assess the quality of a synthetic spread for mean-reversion trading. @@ -35,13 +13,13 @@ public static class SpreadPerformanceEvaluator public static class SpreadStatistics { public class SpreadStatsResult { - public Dictionary bid_zscore_distribution { get; set; } - public Dictionary ask_zscore_distribution { get; set; } - public Dictionary mid_zscore_distribution { get; set; } + public Dictionary? bid_zscore_distribution { get; set; } + public Dictionary? ask_zscore_distribution { get; set; } + public Dictionary? mid_zscore_distribution { get; set; } - public DescriptiveStatistics bid_stats { get; set; } - public DescriptiveStatistics ask_stats { get; set; } - public DescriptiveStatistics mid_stats { get; set; } + public DescriptiveStatistics? bid_stats { get; set; } + public DescriptiveStatistics? ask_stats { get; set; } + public DescriptiveStatistics? mid_stats { get; set; } } private static readonly string[] _zscore_labels = new[] @@ -105,9 +83,10 @@ public static class SpreadPerformanceEvaluator } public static double score_spread(SpreadStatsResult result) { - var ask_bins = result.ask_zscore_distribution; - var bid_bins = result.bid_zscore_distribution; - var mid_bins = result.mid_zscore_distribution; + var ask_bins = ExceptionAssert.NotNull(result.ask_zscore_distribution); + var bid_bins = ExceptionAssert.NotNull(result.bid_zscore_distribution); + var mid_bins = ExceptionAssert.NotNull(result.mid_zscore_distribution); + var mid_stats = ExceptionAssert.NotNull(result.mid_stats); double buy_opportunity = ask_bins["Z < -2σ"] + ask_bins["-2σ ≤ Z < -1σ"]; double sell_opportunity = bid_bins["Z ≥ 2σ"] + bid_bins["1σ ≤ Z < 2σ"]; @@ -117,7 +96,7 @@ public static class SpreadPerformanceEvaluator double above_mean = mid_bins["Z ≥ 2σ"] + mid_bins["1σ ≤ Z < 2σ"] + mid_bins["0σ ≤ Z < 1σ"]; double symmetry_score = 1.0 - Math.Abs(below_mean - above_mean) / (below_mean + above_mean + 1e-6); - double amplitude_score = result.mid_stats.StandardDeviation; + double amplitude_score = mid_stats.StandardDeviation; double skew_penalty = 1.0 / (1.0 + Math.Abs(result.mid_stats.Skewness)); double kurtosis_bonus = Math.Min(result.mid_stats.Kurtosis, 6.0) / 6.0; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs index 897f1de46..194e63e1b 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs @@ -1,7 +1,10 @@ namespace bts.BlueOrb { using BCG; + using Bts.DataStreamCore; + using Spark.Api.MarketData; + using Spark.Core.Diagnostics; using System.Diagnostics; @@ -9,11 +12,13 @@ namespace bts.BlueOrb { // IMarketDataClient. Until then, hide it from DSQA. [DataStreamUserRespresentation(false)] public class SpreadPerformanceStream : ViewStream { - ViewStream _spread_price_stream = null; + ViewStream? _spread_price_stream; private readonly IBlueTableSchema _schema; private readonly int _window_secs; private readonly double _buy_threshold, _sell_threshold, _flat_threshold; private ViewStream _bbo_stream; + IDisposable? subscription; + public SpreadPerformanceStream( string s1, string s2, @@ -84,11 +89,13 @@ namespace bts.BlueOrb { } protected override void myOnFirstSubscriber() { - _spread_price_stream.onChange += _handle_spread_prices; + ExceptionAssert.IsNull(subscription); + subscription = ExceptionAssert.NotNull(_spread_price_stream).Subscribe(_handle_spread_prices); } protected override void myOnLastSubscriber() { - _spread_price_stream.onChange -= _handle_spread_prices; + subscription?.Dispose(); + subscription = null; } private void _handle_spread_prices(object dp_obj) { diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs index 07f0675f4..79153ca5f 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadStatsStream.cs @@ -2,6 +2,7 @@ using BCG; using Spark.Api.MarketData; + using Spark.Core.Diagnostics; using System; using System.Collections.Generic; @@ -100,36 +101,40 @@ var result = SpreadPerformanceEvaluator.SpreadStatistics.analyze_spread_quotes(spread_prices); (uint n_buy_signals, double cumm_buy_price, uint n_sell_signals, double cumm_sell_price) = SpreadPerformanceEvaluator.EvaluateExcursionCount(spread_prices.Select(t => Tuple.Create(t.Item2, t.Item3, t.Item4, t.Item5)).ToList(), 2.0); + var bidStats = ExceptionAssert.NotNull(result.bid_stats); + var askStats = ExceptionAssert.NotNull(result.ask_stats); + var bidZscoreDist = ExceptionAssert.NotNull(result.bid_zscore_distribution); + var askZscoreDist = ExceptionAssert.NotNull(result.ask_zscore_distribution); IBlueRow nr = new BlueRow(_schema, 0, _spread, $"n_ticks: {spread_prices.Count} t0: {t0.ToLongTimeString()} -> t1: {t1.ToLongTimeString()} ({tot_secs}) secs", - result.bid_stats.Mean, - result.ask_stats.Mean, - result.bid_stats.StandardDeviation, - result.ask_stats.StandardDeviation, - result.bid_stats.Skewness, - result.ask_stats.Skewness, - result.bid_stats.Kurtosis, - result.ask_stats.Kurtosis, - result.bid_stats.Maximum, - result.ask_stats.Maximum, - result.bid_stats.Minimum, - result.ask_stats.Minimum, + bidStats.Mean, + askStats.Mean, + bidStats.StandardDeviation, + askStats.StandardDeviation, + bidStats.Skewness, + askStats.Skewness, + bidStats.Kurtosis, + askStats.Kurtosis, + bidStats.Maximum, + askStats.Maximum, + bidStats.Minimum, + askStats.Minimum, // bid distribution - result.bid_zscore_distribution["Z < -2σ"], - result.bid_zscore_distribution["-2σ ≤ Z < -1σ"], - result.bid_zscore_distribution["-1σ ≤ Z < 0σ"], - result.bid_zscore_distribution["0σ ≤ Z < 1σ"], - result.bid_zscore_distribution["1σ ≤ Z < 2σ"], - result.bid_zscore_distribution["Z ≥ 2σ"], + bidZscoreDist["Z < -2σ"], + bidZscoreDist["-2σ ≤ Z < -1σ"], + bidZscoreDist["-1σ ≤ Z < 0σ"], + bidZscoreDist["0σ ≤ Z < 1σ"], + bidZscoreDist["1σ ≤ Z < 2σ"], + bidZscoreDist["Z ≥ 2σ"], // ask distributions - result.ask_zscore_distribution["Z < -2σ"], - result.ask_zscore_distribution["-2σ ≤ Z < -1σ"], - result.ask_zscore_distribution["-1σ ≤ Z < 0σ"], - result.ask_zscore_distribution["0σ ≤ Z < 1σ"], - result.ask_zscore_distribution["1σ ≤ Z < 2σ"], - result.ask_zscore_distribution["Z ≥ 2σ"], + askZscoreDist["Z < -2σ"], + askZscoreDist["-2σ ≤ Z < -1σ"], + askZscoreDist["-1σ ≤ Z < 0σ"], + askZscoreDist["0σ ≤ Z < 1σ"], + askZscoreDist["1σ ≤ Z < 2σ"], + askZscoreDist["Z ≥ 2σ"], n_buy_signals, cumm_buy_price, n_sell_signals, diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs index 984a8e5a6..8b5f924da 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/EventsStream.cs @@ -1,15 +1,15 @@ namespace bts.BlueOrb { using BCG; + using System.IO; - using System.Text; using System.Runtime.InteropServices; [DataStreamUserRespresentation("Events")] public class EventsStream : ViewStream { - public EventsStream(string event_type) : this(ViewStreamDatabase.instance, event_type) { } - - public EventsStream(ViewStreamDatabase vsd, string event_type) { - string path = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:\\BlueOrb_Data\\":Path.Combine(Environment.GetEnvironmentVariable("HOME") , "BlueOrb_Data"); + public EventsStream(string event_type) { + string path = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? "C:\\BlueOrb_Data\\" + : Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? string.Empty , "BlueOrb_Data"); string query = "Data Source="+ Path.Combine(path,"stream_data.db")+";Version=3; select * from events"; bts.Log.Gen.info($"EventsStream path: {path}, query: {query}"); ViewStream rs = diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs index 9d23bbfb6..31cc3382d 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs @@ -24,9 +24,9 @@ namespace bts.BlueOrb{ mySetRealSource(new NonIncGroupByStream(rs, this)); } - public IBlueRowCollection evaluate(IDictionary> grouped_rows) { + public IBlueRowCollection? evaluate(IDictionary> grouped_rows) { Func is_future = (c) => ContractUtils.get_category(c) == ContractUtils.ContractCategory.FutureString; - BlueTable rv = null; + BlueTable? rv = null; Func, IBlueRow> reduce = (rows) => { var future_rows = rows.Where(r => is_future(r.get("contract"))); IBlueRow rv = future_rows.ElementAt(0); @@ -41,7 +41,7 @@ namespace bts.BlueOrb{ if (rows == null || rows.Count == 0) continue; IBlueRow rr = reduce(rows); - rv = rv ?? new BlueTable(rr.schema); + rv ??= new BlueTable(rr.schema); rv.addRow(rr); } return rv; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs index 45cf6862d..7558a81ef 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs @@ -23,8 +23,7 @@ namespace bts.BlueOrb { private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500); private DateTime _last_poll_time = DateTime.MinValue; readonly IWorkScheduler workScheduler; - private int _id = 0; - IReadOnlyDictionary _cached_state = null; + IReadOnlyDictionary? _cached_state; IMarketDataClient _md_client; IDisposable? subscription; private bool _dirty = false; @@ -87,7 +86,7 @@ namespace bts.BlueOrb { bts.api.SecurityType sec_type = sec.security_type(); object exp = DBNull.Value; if (sec_type == bts.api.SecurityType.Future || sec_type == bts.api.SecurityType.Option) - exp = sec.security_expiration(); + exp = sec.security_expiration() ?? throw new InvalidOperationException($"Security has invalid expiration (id={skey})"); object strike = DBNull.Value; if (sec_type == bts.api.SecurityType.Option) strike = sec.strike(); @@ -97,7 +96,7 @@ namespace bts.BlueOrb { IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, sec.Currency, sec.ContractSpec.CashPerPoint, exp, strike, cp); new_state.addRow(r); } - DataPacket dp = null; + DataPacket? dp; if (myLastValue == null) { dp = new DataPacket(new_state, new_state, new BlueTable(_instrument_schema)); } else { diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs index e5b501fe2..120f5ad91 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs @@ -167,10 +167,10 @@ namespace bts.BlueOrb { return String.Empty; } - private Source _sourse = Source.SQL; // get from config - readonly string _location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? "C:\\BlueOrb_Data\\" - : Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? string.Empty, "BlueOrb_Data"); + // private Source _sourse = Source.SQL; // get from config + // readonly string _location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + // ? "C:\\BlueOrb_Data\\" + // : Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? string.Empty, "BlueOrb_Data"); private readonly string _db_name = "stream_data.db"; // the symbol is used ot retreive the correct table, the data flows as-is from the table // with some name / type changes diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs index 540e03893..259dea136 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs @@ -20,11 +20,9 @@ namespace BCG { private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500); private DateTime _last_poll_time = DateTime.MinValue; private ViewStreamDatabase _vsd; - private int _id = 0; - IReadOnlyDictionary _cached_state = null; + IReadOnlyDictionary? _cached_state = null; IMarketDataClient _md_client; private bool _dirty = false; - private IDisposable _sub = null; private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo", Tuple.Create("id", TypeCode.Int32), Tuple.Create("sec_key", TypeCode.UInt32), @@ -48,7 +46,7 @@ namespace BCG { } private void _log_updates(TableUpdate update) { - int i = 0; + // int i = 0; // foreach (BookTuple bt in update.added) { // bts.Log.Gen.info($"PriceStatsStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}"); // } @@ -68,7 +66,7 @@ namespace BCG { protected override void myOnLastSubscriber() { _vsd.removeWorker(this); - Disposable.dispose(ref _sub); + Disposable.Dispose(ref subscription); } private void _scan() { @@ -85,7 +83,7 @@ namespace BCG { IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), today, pstats.open, pstats.close, pstats.settlement); new_state.addRow(r); } - DataPacket dp = null; + DataPacket dp; if (myLastValue == null) { dp = new DataPacket(new_state, new_state, new BlueTable(_pstats_scheam)); } else { diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs index b4c5cab36..6fab2a0cd 100644 --- a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs +++ b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs @@ -4,6 +4,8 @@ using BCG.Utilities.General; using Bts.PubSub; using Bts.SharedStreams.Tables; +using Spark.Core.Diagnostics; + namespace bts { /// /// Gets the subscription and publication state for one @@ -46,12 +48,11 @@ namespace bts { var updates = _sub_core.get_updates(TimeSpan.Zero); //this is where the pubsub updates are converted to rows if (updates.Count() > 0) { - int id = 0; foreach (var u in updates) { if (u.tag != _tag) { continue; } - var tpsi_list = TagPubSubInfo.deserialize(u.data); + var tpsi_list = TagPubSubInfo.deserialize(ExceptionAssert.NotNull(u.data)); foreach (var tsi in tpsi_list) { _tag_map[tsi.tag] = tsi; } diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/TableMetaDataStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/TableMetaDataStream.cs index 43e299114..7c1f632d0 100644 --- a/src/DataStreamSuite/SharedStreams/System/PS3/TableMetaDataStream.cs +++ b/src/DataStreamSuite/SharedStreams/System/PS3/TableMetaDataStream.cs @@ -11,7 +11,7 @@ namespace bts private class TableState { public ViewStream table_stream; - public IBlueTableSchema schema = null; + public IBlueTableSchema? schema = null; public bool processed = false; public TableState(ViewStream table_stream_) { @@ -100,8 +100,7 @@ namespace bts { Log.assert(tag_name.StartsWith("DS_")); string table_name = tag_name.Remove(0, 3); - TableState ts = null; - if (!_table_map.TryGetValue(table_name, out ts)) + if (!_table_map.TryGetValue(table_name, out var ts)) { ts = new TableState(new TableStream(table_name, _vsd)); _table_map.Add(table_name, ts); diff --git a/src/cs/Bts.Spark.Core/Collections/EnumerableComparer.cs b/src/cs/Bts.Spark.Core/Collections/EnumerableComparer.cs new file mode 100644 index 000000000..b338861fa --- /dev/null +++ b/src/cs/Bts.Spark.Core/Collections/EnumerableComparer.cs @@ -0,0 +1,40 @@ +#nullable enable +namespace Spark.Core.Collections; + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; + +public static class EnumerableComparer { + public const int DefaultMaxHashItemCount = 16; + + public static EnumerableComparer Create(int maxHashItemCount = DefaultMaxHashItemCount) => + new(maxHashItemCount); +} + +public class EnumerableComparer : IEqualityComparer> { + readonly int maxHashItemCount; + + public EnumerableComparer(int maxHashItemCount = EnumerableComparer.DefaultMaxHashItemCount) => + this.maxHashItemCount = maxHashItemCount; + + public bool Equals(IEnumerable? x, IEnumerable? y) { + if (x is null) { + return y is null; + } + if (y is null) { + return false; + } + return x.SequenceEqual(y); + } + + public int GetHashCode([DisallowNull] IEnumerable obj) { + var hash = new HashCode(); + using var en = obj.GetEnumerator(); + for (int i = 0; i < maxHashItemCount && en.MoveNext(); i++) { + hash.Add(en.Current); + } + return hash.ToHashCode(); + } +} diff --git a/src/cs/Bts.Spark.Core/Collections/Extensions/CollectionExtensions.cs b/src/cs/Bts.Spark.Core/Collections/Extensions/CollectionExtensions.cs index 89755fca5..64d57d8c0 100644 --- a/src/cs/Bts.Spark.Core/Collections/Extensions/CollectionExtensions.cs +++ b/src/cs/Bts.Spark.Core/Collections/Extensions/CollectionExtensions.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; namespace Spark.Core.Collections { public static class CollectionExtensions { @@ -20,6 +21,32 @@ namespace Spark.Core.Collections { collection.AddRange(items); } + public static TResult[] ToArray(this ICollection collection, Func selector) { + var rv = new TResult[collection.Count]; + int i = 0; + foreach (var v in collection) { + rv[i++] = selector(v); + } + return rv; + } + + public static TResult[] ToIndexedArray(this ICollection collection, Func selector) { + var rv = new TResult[collection.Count]; + foreach (var v in collection) { + var (resIdx, res) = selector(v); + rv[resIdx] = res; + } + return rv; + } + + public static List ToList(this ICollection collection, Func selector) { + var results = new List(collection.Count); + foreach (var v in collection) { + results.Add(selector(v)); + } + return results; + } + public static bool within_range(this ICollection collection, int index) => index >= 0 && index < collection.Count; } diff --git a/src/cs/Bts.Spark.Core/Collections/Extensions/DictionaryExtensions.cs b/src/cs/Bts.Spark.Core/Collections/Extensions/DictionaryExtensions.cs index 862898763..7fad737af 100644 --- a/src/cs/Bts.Spark.Core/Collections/Extensions/DictionaryExtensions.cs +++ b/src/cs/Bts.Spark.Core/Collections/Extensions/DictionaryExtensions.cs @@ -122,7 +122,7 @@ namespace System.Collections.Generic { } #nullable enable - public static TValue? GetValueOrDefault(this Dictionary? dictionary, TKey key) where TKey : notnull { + public static TValue? GetValueOrDefault(this Dictionary? dictionary, TKey key) where TKey : notnull { if (dictionary is null) { throw new ArgumentNullException(nameof(dictionary)); } diff --git a/src/cs/Bts.Spark.Core/Collections/NullCheckEqualityComparer.cs b/src/cs/Bts.Spark.Core/Collections/NullCheckEqualityComparer.cs new file mode 100644 index 000000000..b0aa068a4 --- /dev/null +++ b/src/cs/Bts.Spark.Core/Collections/NullCheckEqualityComparer.cs @@ -0,0 +1,38 @@ +#nullable enable +namespace Spark.Core.Collections; + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +public static class NullCheckEqualityComparer { + public static NullCheckEqualityComparer Default() => + NullCheckEqualityComparer.Default; + + public static NullCheckEqualityComparer Create(IEqualityComparer comparer) => + new(comparer); + + public static bool Equals(T? x, T? y) => + NullCheckEqualityComparer.Default.Equals(x, y); +} + +public class NullCheckEqualityComparer : IEqualityComparer { + public static readonly NullCheckEqualityComparer Default = new(); + + readonly IEqualityComparer comparer; + + public NullCheckEqualityComparer() + : this(EqualityComparer.Default) { } + + public NullCheckEqualityComparer(IEqualityComparer comparer) => + this.comparer = comparer; + + public bool Equals(T? x, T? y) { + if (x is null) { + return y is null; + } + return y is not null && comparer.Equals(x, y); + } + + public int GetHashCode([DisallowNull] T obj) => + comparer.GetHashCode(obj); +} diff --git a/src/cs/Bts.Spark.Core/Diagnostics/ExceptionAssert.cs b/src/cs/Bts.Spark.Core/Diagnostics/ExceptionAssert.cs index 3e08d12c4..a0e1a784a 100644 --- a/src/cs/Bts.Spark.Core/Diagnostics/ExceptionAssert.cs +++ b/src/cs/Bts.Spark.Core/Diagnostics/ExceptionAssert.cs @@ -16,9 +16,55 @@ namespace Spark.Core.Diagnostics { public static void IsNull([MaybeNull]T? obj, [CallerArgumentExpression(nameof(obj))] string? name = null) => Argument.IsNull(obj, name); + public static void MemberIsNull(T memberValue, [CallerArgumentExpression(nameof(memberValue))] string? memberName = null) => + InvalidOperation.MemberIsNull(memberValue, memberName); + + public static T MemberNotNull([NotNull] T? obj, [CallerArgumentExpression(nameof(obj))] string? name = null) => + InvalidOperation.MemberNotNull(obj, name); + public static T NotNull([NotNull] T? obj, [CallerArgumentExpression(nameof(obj))] string? name = null) => Argument.NotNull(obj, name); + public static (T1, T2) NotNull( + [NotNull] T1? obj1, + [NotNull] T2? obj2, + [CallerArgumentExpression(nameof(obj1))] string? name1 = null, + [CallerArgumentExpression(nameof(obj2))] string? name2 = null) => + (Argument.NotNull(obj1, name1), Argument.NotNull(obj2, name2)); + + public static (T1, T2, T3) NotNull( + [NotNull] T1? obj1, + [NotNull] T2? obj2, + [NotNull] T3? obj3, + [CallerArgumentExpression(nameof(obj1))] string? name1 = null, + [CallerArgumentExpression(nameof(obj2))] string? name2 = null, + [CallerArgumentExpression(nameof(obj3))] string? name3 = null) => + (Argument.NotNull(obj1, name1), Argument.NotNull(obj2, name2), Argument.NotNull(obj3, name3)); + + public static (T1, T2, T3, T4) NotNull( + [NotNull] T1? obj1, + [NotNull] T2? obj2, + [NotNull] T3? obj3, + [NotNull] T4? obj4, + [CallerArgumentExpression(nameof(obj1))] string? name1 = null, + [CallerArgumentExpression(nameof(obj2))] string? name2 = null, + [CallerArgumentExpression(nameof(obj3))] string? name3 = null, + [CallerArgumentExpression(nameof(obj4))] string? name4 = null) => + (Argument.NotNull(obj1, name1), Argument.NotNull(obj2, name2), Argument.NotNull(obj3, name3), Argument.NotNull(obj4, name4)); + + public static (T1, T2, T3, T4, T5) NotNull( + [NotNull] T1? obj1, + [NotNull] T2? obj2, + [NotNull] T3? obj3, + [NotNull] T4? obj4, + [NotNull] T5? obj5, + [CallerArgumentExpression(nameof(obj1))] string? name1 = null, + [CallerArgumentExpression(nameof(obj2))] string? name2 = null, + [CallerArgumentExpression(nameof(obj3))] string? name3 = null, + [CallerArgumentExpression(nameof(obj4))] string? name4 = null, + [CallerArgumentExpression(nameof(obj5))] string? name5 = null) => + (Argument.NotNull(obj1, name1), Argument.NotNull(obj2, name2), Argument.NotNull(obj3, name3), Argument.NotNull(obj4, name4), Argument.NotNull(obj5, name5)); + public static class Index { public static void IsValid(int index, int length, bool canInsert = false) { if (canInsert) { @@ -202,8 +248,14 @@ namespace Spark.Core.Diagnostics { throw new InvalidOperationException(message); } - public static T MemberNotNull([NotNull] T? memberValue, [CallerArgumentExpression(nameof(memberValue))] string? propertyName = null) => - memberValue ?? throw new InvalidOperationException($"{propertyName} must not be null."); + public static void MemberIsNull(T memberValue, [CallerArgumentExpression(nameof(memberValue))] string? memberName = null) { + if (memberValue is not null) { + throw new InvalidOperationException($"{memberName} must be null"); + } + } + + public static T MemberNotNull([NotNull] T? memberValue, [CallerArgumentExpression(nameof(memberValue))] string? memberName = null) => + memberValue ?? throw new InvalidOperationException($"{memberName} must not be null."); public static void MemberNotNull(params (object value, string name)[] properties) { foreach ((object value, string name) prop in properties) diff --git a/src/cs/Bts.Spark.Core/Disposables/Disposable.cs b/src/cs/Bts.Spark.Core/Disposables/Disposable.cs index a30122a0a..bd51adf9a 100644 --- a/src/cs/Bts.Spark.Core/Disposables/Disposable.cs +++ b/src/cs/Bts.Spark.Core/Disposables/Disposable.cs @@ -50,6 +50,7 @@ namespace Spark.Core.Disposables { d2?.Dispose(); }); + // TODO: replace refs to this fn with 'Dispose' (the PascalCase version) public static void dispose(ref T? disposable) where T : class, IDisposable { var tmp = disposable; /// set to null before actually disposing it to prevent some @@ -58,6 +59,14 @@ namespace Spark.Core.Disposables { tmp?.Dispose(); } + public static void Dispose(ref T? disposable) where T : class, IDisposable { + var tmp = disposable; + /// set to null before actually disposing it to prevent some + /// strange re-entrancy bug + disposable = null; + tmp?.Dispose(); + } + public static void dispose_thread_safe(ref T? disposable) where T : class, IDisposable => Interlocked.CompareExchange(ref disposable, null, disposable)?.Dispose();