cleanup: Fix more build warnings in DataStreamSuite

Alot more
This commit is contained in:
John Colvin committed 2026-01-11 15:15:13 -05:00
1 parent 989d103eef
commit bcd207f89e
118 files changed
+1812 -1597

No files matched your search

@@ -1,5 +1,8 @@
namespace bts.BlueOrb {
using BCG;
using Bts.DataStreamCore;
using System;
using System.Text.RegularExpressions;
@@ -21,22 +24,24 @@ namespace bts.BlueOrb {
#region IStreamFilter Members
public bool filter(IBlueRow dr) {
bool rv = true;
for (int i = 0; i < myCols.Length; i++) {
rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
if (!rv)
return rv;
var val = dr[myCols[i]];
var str = val is DBNull
? string.Empty
: (val?.ToString() ?? string.Empty);
if (!myRegExps[i].IsMatch(str)) {
return false;
}
}
return rv;
return true;
}
#endregion
}
private BlueTable myLastFiredRows;
private IBlueRowCollection myLastReceivedRows;
BlueTable? myLastFiredRows;
IBlueRowCollection? myLastReceivedRows;
private IStreamFilter myFilter;
private ViewStream myParentStream;
private string myFilterString;
private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
#region constructors
public delegate bool OnFilter(IBlueRow r);
@@ -52,12 +57,11 @@ namespace bts.BlueOrb {
public FilterStream(ViewStream vs, IStreamFilter filter) {
{
myParentStream = vs;
myFilter = filter;
myFilterString = filter.ToString();
if (filter is IDynamicStreamFilter)
(filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
myParentStream = vs;
myFilter = filter;
myFilterString = filter.ToString() ?? string.Empty;
if (filter is IDynamicStreamFilter dynFilter) {
dynFilter.onFilterChange += myHandleFilterChange;
}
}
@@ -99,16 +103,20 @@ namespace bts.BlueOrb {
if (myLastFiredRows != null) {
retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
new BlueTable(myLastFiredRows.schema));
foreach (IBlueRow r in arrivedPacket.deletedRows)
if (myLastFiredRows.containsID((int)r["id"])) {
var deletedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.deletedRows);
foreach (IBlueRow r in arrivedPacket.deletedRows) {
if (myLastFiredRows.containsID(r.Id())) {
myLastFiredRows.deleteRow(r);
(retVal.deletedRows as BlueTable).addRow(r);
deletedRows.Value.addRow(r);
}
foreach (IBlueRow r in arrivedPacket.addedRows)
}
var addedRows = new Lazy<BlueTable>(() => (BlueTable)retVal.addedRows);
foreach (IBlueRow r in arrivedPacket.addedRows) {
if (myFilter.filter(r)) {
myLastFiredRows.addRow(r);
(retVal.addedRows as BlueTable).addRow(r);
addedRows.Value.addRow(r);
}
}
} else {
myLastFiredRows = new BlueTable(arrivedPacket.Schema);
retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
@@ -1,12 +1,13 @@
using System;
using BCG;
using Bts.DataStreamCore;
using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text.Json;
using bts;
//using bts.SilverPine;
using BCG;
namespace bts.BlueOrb {
[DataStreamUserRespresentation("GroupByStream")]
@@ -14,20 +15,21 @@ namespace bts.BlueOrb {
public interface INonIncGroupByEvaluator {
// grouped_rows
IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
string[] get_group_by_cols();
IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
string[]? get_group_by_cols();
}
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
Func<string[]> _groupby_cols_func;
Func<string[]?> _groupby_cols_func;
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func,
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
_groupby_cols_func = groupby_cols_func;
_evaluate_func = evaluate_func;
}
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
BlueTable rv = null;
public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
BlueTable? rv = null;
foreach (var kv in grouped_rows) {
List<IBlueRow> g_rows = kv.Value;
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
@@ -39,17 +41,19 @@ namespace bts.BlueOrb {
}
return rv;
}
public string[] get_group_by_cols() {
public string[]? get_group_by_cols() {
return _groupby_cols_func();
}
}
private ViewStream _main_stream;
private INonIncGroupByEvaluator _groupby_func;
private BlueRowHasher _row_hasher;
private string[] _groupby_cols;
ViewStream? _main_stream;
INonIncGroupByEvaluator? _groupby_func;
BlueRowHasher? _row_hasher;
string[]? _groupby_cols;
private Stopwatch _timer = new();
private int _n_total_fires = 0, _n_expensive_fires = 0;
IDisposable? subscription;
public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
_main_stream = main_stream;
@@ -58,7 +62,7 @@ namespace bts.BlueOrb {
_row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
}
public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
public GroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func,
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
@@ -68,33 +72,41 @@ namespace bts.BlueOrb {
// this is the main constructor that does everything
public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
Func<string[]> groupby_cols_func = () => {
if (String.IsNullOrEmpty(groupby_cols))
return null;
return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
};
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
if (!String.IsNullOrEmpty(optional_sort_col))
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
// this si where the group by aggregation happens
IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
IBlueRow? rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
if (rv == null)
return new List<IBlueRow>();
return new IBlueRow[] { rv }.ToList();
};
ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
mySetRealSource(rs);
string[]? groupby_cols_func() {
if (string.IsNullOrEmpty(groupby_cols)) {
return null;
}
return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
}
}
public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
this(parent_str, String.Empty, standard_group_by_text) {
}
protected override void myOnFirstSubscriber() {
Log.isnull(subscription);
subscription = ExceptionAssert.MemberNotNull(_main_stream).Subscribe(_main_stream_onchange);
}
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
protected override void myOnLastSubscriber() {
subscription?.Dispose();
subscription = null;
}
private void _main_stream_onchange(object nv) {
Log.notnull(_groupby_func);
var dp = (IDataPacket)nv;
///grouping together
_timer.Restart();
@@ -109,7 +121,7 @@ namespace bts.BlueOrb {
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
/// reducing
_timer.Restart();
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
IBlueRowCollection? total_state = _groupby_func.evaluate(grouped_rows);
if (total_state == null)
// mayeb returna n empty table ?
return;
@@ -1,8 +1,13 @@
namespace bts.BlueOrb {
using BCG;
using Bts.DataStream;
using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using RowHash = System.Int32;
@@ -11,29 +16,29 @@
public class SimpleJoinStream : ViewStream {
public class JoinBookkeepingData {
// initialized in cons
private readonly List<string> _join_col_names;
private readonly bool _is_left;
readonly string[] _join_col_names;
private readonly BlueRowHasher _hasher;
// initialized on first udpate
private IBlueTableSchema _schema;
private int[] _join_cols;
IBlueTableSchema? _schema;
int[]? _join_cols;
private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
_join_col_names = join_cols.Where(c => !String.IsNullOrEmpty(c)).ToList();
_is_left = is_left;
public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher) {
_join_col_names = join_cols.Where(c => !string.IsNullOrEmpty(c)).ToArray();
_hasher = row_hasher;
}
public IBlueTableSchema schema => _schema;
[NotNullIfNotNull(nameof(_schema))]
public IBlueTableSchema? schema => _schema;
public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
public List<IBlueRow> get_matching_rows(int hash) {
List<IBlueRow> rv = null;
_hash_row_map.TryGetValue(hash, out rv);
return rv;
}
[MemberNotNull(nameof(_schema))]
public IBlueTableSchema RequireSchema() =>
ExceptionAssert.MemberNotNull(_schema);
public List<IBlueRow>? get_matching_rows(int hash) =>
_hash_row_map.GetValueOrDefault(hash);
private RowHash _get_hash(IBlueRow r) {
if (_join_cols == null || _join_cols.Length == 0) {
@@ -45,13 +50,12 @@
public void update_data(IDataPacket dp) {
if (!ready) {
_schema = dp.newRows.schema;
_join_cols = _join_col_names.Count == 0 ? new int[] { } : _join_col_names.Select(n => _schema[n].colNo).ToArray();
_join_cols = _schema.ResolveIndices(_join_col_names);
}
_hash_row_map.Clear();
foreach (IBlueRow r in dp.newRows) {
int hash = _get_hash(r);
List<IBlueRow> hash_rows = null;
if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
if (!_hash_row_map.TryGetValue(hash, out var hash_rows)) {
hash_rows = new List<IBlueRow>();
_hash_row_map.Add(hash, hash_rows);
}
@@ -59,6 +63,7 @@
}
}
[MemberNotNullWhen(true, nameof(_schema))]
public bool ready {
get {
return _schema != null;
@@ -69,8 +74,7 @@
private ViewStream _ls, _rs;
private JoinBookkeepingData _ldata, _rdata;
private ViewStreamDatabase _vsd;
private IBlueTableSchema _schema = null;
IBlueTableSchema? _schema;
public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
@@ -83,8 +87,8 @@
Log.assert(lj_col_list.Count == rj_col_list.Count);
int n_cols = lj_col_list.Count;
BlueRowHasher hasher = new BlueRowHasher(n_cols);
_ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
_rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
_ldata = new JoinBookkeepingData(ls, lj_col_list, hasher);
_rdata = new JoinBookkeepingData(rs, rj_col_list, hasher);
}
protected override void myOnFirstSubscriber() {
@@ -109,20 +113,24 @@
_rs.onChange -= _rs_onchange;
}
[MemberNotNull(nameof(_schema))]
private void _build_schema() {
if (_schema != null)
return;
List<ColumnInfo> cis = new List<ColumnInfo>();
cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
var lschema = Log.notnull(_ldata.schema);
var rschema = Log.notnull(_rdata.schema);
var cis = new List<ColumnInfo> {
new("id", 0, TypeCode.Int32)
};
foreach (ColumnInfo ci in lschema.columnInfo) {
string col_name = rschema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
}
foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
foreach (ColumnInfo ci in rschema.columnInfo) {
string col_name = lschema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
}
_schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
_schema = new BlueTableSchema($"{lschema.tableName}_join_{rschema.tableName}", cis.ToArray());
}
private void _form_and_fire() {
@@ -131,34 +139,11 @@
_build_schema();
List<object[]> res_rows = new List<object[]>();
Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
int offset = 1;
JoinBookkeepingData sdata = _ldata;
if (is_left) {
offset = 1;
sdata = _ldata;
} else {
sdata = _rdata;
offset = 1 + _ldata.schema.columnInfo.Length;
}
for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
rval[offset + i] = r[i];
}
};
Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
object[] rv = new object[_schema.columnNames.Length];
for (int i = 0; i < _schema.columnNames.Length; i++) {
rv[i] = DBNull.Value;
}
rv[0] = id;
copy_row(rv, l_row, true);
return rv;
};
// this is where the join happens
foreach (var kv in _ldata.rows) {
RowHash hash = kv.Key;
List<IBlueRow> l_rows = kv.Value;
List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
List<IBlueRow>? r_rows = _rdata.get_matching_rows(hash);
foreach (IBlueRow lr in l_rows) {
if (r_rows != null) {
@@ -179,6 +164,31 @@
}
var dp = BlueDataUtility.build_dp(rv, myLastValue as IDataPacket);
myFireEvent(dp);
object[] build_row_array(int id, IBlueRow l_row) {
object[] rv = new object[_schema.columnNames.Length];
for (int i = 0; i < _schema.columnNames.Length; i++) {
rv[i] = DBNull.Value;
}
rv[0] = id;
copy_row(rv, l_row, true);
return rv;
}
void copy_row(object?[] rval, IBlueRow r, bool is_left) {
int offset = 1;
JoinBookkeepingData sdata = _ldata;
if (is_left) {
offset = 1;
sdata = _ldata;
} else {
sdata = _rdata;
offset = 1 + _ldata.RequireSchema().columnInfo.Length;
}
for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++) {
rval[offset + i] = r[i];
}
}
}
}
@@ -45,8 +45,8 @@ namespace bts.BlueOrb {
}
// aggregates multiple rows into one - ie does group_by compute
public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
if (rows.Count() == 0) {
public static IBlueRow? group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
if (rows.Count == 0) {
return null;
}
@@ -73,8 +73,8 @@ namespace bts.BlueOrb {
}
// aggregates multiple rows into one, used by MultiRowRollingMathStream
public static IBlueRow group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
if (rows.Count() == 0) {
public static IBlueRow? group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
if (rows.Count == 0) {
return null;
}
IBlueRow ir = rows.ElementAt(start_idx);
@@ -104,8 +104,8 @@ namespace bts.BlueOrb {
// rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
// calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
if (rolling_rows.Count() == 0) {
public IBlueRow? rolling_calc(List<IBlueRow> rolling_rows) {
if (rolling_rows.Count == 0) {
return null;
}
IBlueRow ir = rolling_rows.ElementAt(0);
@@ -156,7 +156,6 @@ namespace bts.BlueOrb {
double first = col_vals.First();
double last = col_vals.Last();
return ((first - last) / last) * 100.0;
return col_vals.First() - col_vals.Last();
}
default:
// TODO
@@ -1,8 +1,7 @@
namespace bts.BlueOrb {
using BCG;
using System.Collections.Generic;
using System.Net;
using System;
[DataStreamUserRespresentation("MultiRowRollingMathStream")]
public class MultiRowRollingMathStream : ViewStream {
@@ -26,9 +25,10 @@
private void _parent_str_onchange(object dp_obj) {
var dp = (IDataPacket)dp_obj;
var rows = dp.newRows.to_list();
BlueTable out_table = null;
BlueTable? out_table = null;
for (int i = 0; i < rows.Count; i++) {
var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op);
var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op)
?? throw new InvalidOperationException("group_by_compute returned null row");
if (out_table == null)
out_table = new BlueTable(out_r.schema);
out_table.addRow(out_r);
@@ -1,5 +1,8 @@
namespace bts.BlueOrb {
using BCG;
using Spark.Core.Diagnostics;
using System;
using System.Collections;
using System.Collections.Generic;
@@ -54,7 +57,7 @@ namespace bts.BlueOrb {
public bool IsReadOnly { get { return true; } }
public object this[object key] {
public object? this[object key] {
get {
if (key is string)
return this[(string)key];
@@ -63,16 +66,15 @@ namespace bts.BlueOrb {
set { throw new NotSupportedException("The IBlueRow is readonly"); }
}
public object this[int index] {
public object? this[int index] {
get {
return this[schema.columnInfo[index].colName];
}
}
public object this[string col_name] {
public object? this[string col_name] {
get {
string mapped_to_col = null;
if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out var mapped_to_col)) {
return _row[mapped_to_col];
}
return _row[col_name];
@@ -89,7 +91,8 @@ namespace bts.BlueOrb {
}
}
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
public void Add(object key, object? value) =>
throw new NotSupportedException();
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
@@ -114,7 +117,7 @@ namespace bts.BlueOrb {
private ViewStream _stream;
private string _project_expression;
private ProjectData _project_data = null;
ProjectData? _project_data;
//public ProjectData
public NewProjectStream(ViewStream vs, string project_expression) {
_stream = vs;
@@ -128,9 +131,8 @@ namespace bts.BlueOrb {
return _project_data.out_schema;
}
public IBlueRow map(IBlueRow row) {
return new NewProjectRow(_project_data, row);
}
public IBlueRow map(IBlueRow row) =>
new NewProjectRow(ExceptionAssert.MemberNotNull(_project_data), row);
// this is where most of the logic is, in terms of translating cols
private static ProjectData _create_project_data(IBlueTableSchema input_schema,
@@ -2,9 +2,6 @@ namespace bts {
using System.IO;
using BCG;
using System.IO;
using System.Threading.Tasks.Dataflow;
public class RemoteRequest {
public readonly string stream_expression;
public readonly string published_table_name;
@@ -28,4 +25,4 @@ namespace bts {
}
}
}
}
@@ -1,33 +1,28 @@
using Spark.Core.Collections;
using System;
using System.Collections.Generic;
using System.Linq;
namespace BCG {
public class AddColumnsStream : ViewStream {
public static Tuple<string, Type, object> parse_single_tok(string exp) {
public static (string Name, Type Type, object Value) parse_single_tok(string exp) {
string[] toks = exp.Split(':');
string col_name = toks[0].Trim();
string col_value = toks[1].Trim();
Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
return Tuple.Create(col_name, tup.Item1, tup.Item2);
return (col_name, tup.Item1, tup.Item2);
}
public static List<Tuple<string, Type, object>> parse_exp(string exp) {
string[] toks = exp.Split(",");
List<Tuple<string, Type, object>> rv = new List<Tuple<string, Type, object>>();
foreach (string s in toks) {
var tup = parse_single_tok(s.Trim());
rv.Add(tup);
}
return rv;
}
public static List<(string Name, Type Type, object Value)> parse_exp(string exp) =>
exp.Split(",").ToList(t => parse_single_tok(t.Trim()));
private Tuple<string, Type, object> _new_col;
private List<Tuple<string, Type, object>> _new_col_map;
private IBlueTableSchema _out_schema;
readonly List<(string Name, Type Type, object Value)> _new_col_map;
IBlueTableSchema? _out_schema;
public AddColumnsStream(ViewStream vs, string col_def) {
_new_col_map = parse_exp(col_def);
ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
ViewStream rs = new MapStream(vs, _get_schema, _map);
mySetRealSource(rs);
}
@@ -44,7 +39,7 @@ namespace BCG {
private IBlueRow _map(IBlueRow r) {
MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
foreach (var m in _new_col_map) {
rv.setValue(m.Item1, m.Item3);
rv.setValue(m.Name, m.Value);
}
return rv;
}
@@ -1,9 +1,10 @@
using System;
using mbts;
using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using mbts;
namespace BCG
{
@@ -42,8 +43,8 @@ namespace BCG
}
}
public delegate void IncrementValues(object[] current_values, object[] values_to_inc);
public delegate string GetColumnPrefix(object[] transposing_col_values);
public delegate void IncrementValues(object?[] current_values, object?[] values_to_inc);
public delegate string GetColumnPrefix(object?[] transposing_col_values);
public delegate string GetColumnSuffix(string col_name);
private GetColumnSuffix _get_column_suffix = new GetColumnSuffix((cn) => { return cn; });
@@ -51,33 +52,31 @@ namespace BCG
private IncrementValues _increment_values;
private GetColumnPrefix _get_column_prefix;
private IBlueTableSchema _schema;
IBlueTableSchema? _schema;
private string[] _group_by_cols;
private Pair<string, Type>[] _cols_per_transposed;
private string[] _transposing_cols;
private int[] _transposing_col_indices;
int[]? _transposing_col_indices;
private Comparison<Pair<int, object[]>> _transpsed_col_value_comparer_wrapper;
Comparison<Pair<int, object?[]>>? _transpsed_col_value_comparer_wrapper;
public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer)
IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer)
: this(stream, transposed_cols_group, cols_per_transposed, group_by_cols, increment_values_del,
get_column_prefix_del, get_column_suffix_del, transposed_col_value_comparer, true)
{ }
public YXTranspositionStream(ViewStream stream, string transposed_cols_group, string cols_per_transposed, string group_by_cols,
IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object[]> transposed_col_value_comparer, bool only_fire_once)
IncrementValues increment_values_del, GetColumnPrefix get_column_prefix_del, GetColumnSuffix get_column_suffix_del, Comparison<object?[]> transposed_col_value_comparer, bool only_fire_once)
{
if (get_column_suffix_del != null)
_get_column_suffix = get_column_suffix_del;
if (transposed_col_value_comparer != null)
{
_transpsed_col_value_comparer_wrapper = new Comparison<Pair<int, object[]>>((p1,p2) =>
{
return transposed_col_value_comparer(p1.second, p2.second);
});
_transpsed_col_value_comparer_wrapper = new((p1, p2) =>
transposed_col_value_comparer(ExceptionAssert.NotNull(p1.second), ExceptionAssert.NotNull(p2.second)));
}
_group_by_cols = group_by_cols.Split(' ');
_transposing_cols = transposed_cols_group.Split(' ');
@@ -106,7 +105,7 @@ namespace BCG
col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
foreach (string tc_prefix in tc_prefixes)
foreach (Pair<string, Type> p in _cols_per_transposed)
col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first), col_index++, p.second));
col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, p.second));
IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
if (_schema == null)
_schema = schema;
@@ -130,19 +129,14 @@ namespace BCG
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows)
{
Dictionary<int, Dictionary<int, object[]>> all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object[]>>();
Dictionary<int, object[]> group_by_cols_value_dict = new Dictionary<int, object[]>();
var all_tc_infos_and_values = new Dictionary<int, Dictionary<int, object?[]>>();
var group_by_cols_value_dict = new Dictionary<int, object?[]>();
BlueRowHasher hasher = new BlueRowHasher(_transposing_cols.Length);
Pair<string, Type>[] group_by_col_info = new Pair<string, Type>[_group_by_cols.Length];
string table_name = string.Empty;
if (grouped_rows.Count > 0)
{
IBlueTableSchema schema = null;
foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
{
schema = kvp.Value[0].schema;
break;
}
var schema = grouped_rows.Values.First()[0].schema;
table_name = schema.tableName;
_transposing_col_indices = BlueDataUtility.get_column_indices(schema, _transposing_cols);
int[] group_by_indices = BlueDataUtility.get_column_indices(schema, _group_by_cols);
@@ -155,23 +149,22 @@ namespace BCG
else
return new BlueTable(new BlueTableSchema(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }, table_name));
Dictionary<int, string> tc_prefixes = new Dictionary<int, string>();
List<Pair<int, object[]>> tc_values = new List<Pair<int, object[]>>();
var tc_values = new List<Pair<int, object?[]>>();
foreach (KeyValuePair<int, List<IBlueRow>> kvp in grouped_rows)
{
//Dictionary of trans_col_group_hash -> pair: first=trans_col_group_values second=per_transposing_col_values
Dictionary<int, object[]> incrementing_values = new Dictionary<int, object[]>();
var incrementing_values = new Dictionary<int, object?[]>();
foreach (IBlueRow r in kvp.Value)
{
int hash = hasher.get_hash(r, _transposing_col_indices);
object[] values;
if (!incrementing_values.TryGetValue(hash, out values))
if (!incrementing_values.TryGetValue(hash, out var values))
{
incrementing_values.Add(hash, _get_per_transposing_col_values(r));
string prefix = _get_column_prefix(_get_transposing_col_values(r));
if (!tc_prefixes.ContainsKey(hash))
{
tc_prefixes.Add(hash, prefix);
tc_values.Add(new Pair<int,object[]>(hash, _get_transposing_col_values(r)));
tc_values.Add(new(hash, _get_transposing_col_values(r)));
}
}
else
@@ -194,18 +187,18 @@ namespace BCG
_set_schema(prefixes, group_by_col_info, table_name);
BlueTable result = new BlueTable(_schema);
int id = 1;
foreach (KeyValuePair<int, Dictionary<int, object[]>> kvp in all_tc_infos_and_values)
foreach (var kvp in all_tc_infos_and_values)
{
MutableBlueRow mbr = new MutableBlueRow(_schema);
mbr.setValue("id", id++);
for (int i = 0; i < _group_by_cols.Length; i++)
mbr.setValue(_group_by_cols[i], group_by_cols_value_dict[kvp.Key][i]);
foreach (KeyValuePair<int, object[]> tc_cols in kvp.Value)
foreach (var tc_cols in kvp.Value)
{
string prefix = tc_prefixes[tc_cols.Key];
for (int i = 0; i < _cols_per_transposed.Length; i++)
{
string column = _get_full_column_name(prefix, _cols_per_transposed[i].first);
string column = _get_full_column_name(prefix, _cols_per_transposed[i].first ?? string.Empty);
if(mbr.schema.columnNames.Contains(column))
mbr.setValue(column, tc_cols.Value[i]);
}
@@ -217,28 +210,25 @@ namespace BCG
#endregion
private object[] _get_transposing_col_values(IBlueRow row)
{
object[] values = new object[_transposing_cols.Length];
object?[] _get_transposing_col_values(IBlueRow row) {
var values = new object?[_transposing_cols.Length];
for (int i = 0; i < _transposing_cols.Length; i++)
values[i] = row[_transposing_cols[i]];
return values;
}
private object[] _get_per_transposing_col_values(IBlueRow row)
{
object[] values = new object[_cols_per_transposed.Length];
object?[] _get_per_transposing_col_values(IBlueRow row) {
var values = new object?[_cols_per_transposed.Length];
for (int i = 0; i < _cols_per_transposed.Length; i++)
values[i] = row[_cols_per_transposed[i].first];
values[i] = row[ExceptionAssert.NotNull(_cols_per_transposed[i].first)];
return values;
}
private object[] _get_group_by_cols_values(IBlueRow row)
{
object[] values = new object[_group_by_cols.Length];
object?[] _get_group_by_cols_values(IBlueRow row) {
var values = new object?[_group_by_cols.Length];
for (int i = 0; i < values.Length; i++)
values[i] = row[_group_by_cols[i]];
return values;
}
}
}
}
@@ -1,22 +1,19 @@
using System;
using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BCG
{
public class DerivativeStream : ViewStream
{
private IBlueTableSchema mySchema;
IBlueTableSchema? mySchema;
private string myDerivandColumn;
private string myOutputColumn;
private string myKeyColumn;
private DateTime myLastPacketArrival = DateTime.MinValue;
private ViewStream myDataSource;
private IDictionary<string, double> myPreviousValues;
Dictionary<string, double>? myPreviousValues;
public DerivativeStream(ViewStream input, string keyColumn, string columnToDifferentiate, string outputColName)
{
@@ -64,12 +61,13 @@ namespace BCG
protected IBlueRow myMakeRow(IBlueRow r, double timeScale)
{
ExceptionAssert.NotNull(mySchema, myPreviousValues);
object[] outValues = new object[mySchema.columnInfo.Length];
r.Values.CopyTo(outValues, 0);
object derivand = r[myDerivandColumn];
double lastValue = default(double);
if (myPreviousValues.TryGetValue(r[myKeyColumn].ToString(), out lastValue) && derivand != null && derivand != DBNull.Value)
var derivand = r[myDerivandColumn];
var key = getRowKey(r);
if (myPreviousValues.TryGetValue(key, out double lastValue) && derivand != null && derivand != DBNull.Value)
outValues[mySchema.columnInfo.Length - 1] = (Convert.ToDouble(derivand) - lastValue) * timeScale;
else
outValues[mySchema.columnInfo.Length - 1] = 0.0;
@@ -90,10 +88,13 @@ namespace BCG
myLastPacketArrival = DateTime.Now;
foreach (IBlueRow row in dp.newRows)
{
object derivand = row[myDerivandColumn];
var derivand = row[myDerivandColumn];
if (derivand != null && derivand != DBNull.Value)
myPreviousValues[row[myKeyColumn].ToString()] = Convert.ToDouble(derivand);
myPreviousValues[getRowKey(row)] = Convert.ToDouble(derivand);
}
}
string getRowKey(IBlueRow row) =>
row[myKeyColumn]?.ToString() ?? throw new ArgumentException("Row key is null", nameof(row));
}
}
@@ -1,7 +1,9 @@
using System;
using System.Collections;
using mbts;
using Spark.Core.Collections;
using System;
using System.Collections.Generic;
using mbts;
namespace BCG
{
@@ -12,9 +14,9 @@ namespace BCG
{
ViewStream myLStream, myRStream;
DataPacket myLastR = null, myLastL = null;
BlueTable myLastFired = null;
DataPacket? myLastR;
DataPacket? myLastL;
BlueTable? myLastFired;
double myThreshold = .00005;
@@ -24,14 +26,14 @@ namespace BCG
public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) :
this(null, v1, v2, thresh, ColumnsToMatch) { }
public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
{
myThreshold = thresh;
myLStream = v1;
myRStream = v2;
myColumnsToMatch = columnsToMatch;
}
public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
public DiffStream(ViewStreamDatabase? db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
: this(db,v1,v2,.00005,columnsToMatch)
{
}
@@ -50,32 +52,9 @@ namespace BCG
myRStream.onChange -= myRChange;
}
struct ListComparer : IEqualityComparer<List<object>>
{
public bool Equals(List<object> x, List<object> y)
{
if (x.Count != y.Count)
return false;
for (int i = 0; i < x.Count; ++i)
if (!x[i].Equals(y[i]))
return false;
return true;
}
public int GetHashCode(List<object> obj)
{
int ret = 0;
for(int i = 0; i < obj.Count; ++i)
ret ^= obj[i].GetHashCode();
return ret;
}
}
void myHandleCallbacks()
{
DataPacket packet = null;
DataPacket? packet;
if (myLastL == null || myLastR == null)
{
packet = myLastL == null ? myLastR : myLastL;
@@ -88,7 +67,7 @@ namespace BCG
BlueTable table = new BlueTable(myLastL.newRows.schema);
Dictionary<List<object>, Pair<BlueRow, BlueRow>> rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(new ListComparer());
var rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(EnumerableComparer.Create<object>());
foreach(BlueRow row in myLastL.newRows)
{
@@ -118,11 +97,7 @@ namespace BCG
}
catch (Exception) { }
}
Pair<BlueRow, BlueRow> pair;
rowPairs.TryGetValue(hash, out pair);
rowPairs.TryGetValue(hash, out var pair);
if (pair == null)
pair = new Pair<BlueRow, BlueRow>(null, row);
else
@@ -138,8 +113,7 @@ namespace BCG
if (pair.first == null || pair.second == null)
{
BlueRow row = pair.first == null ? pair.second : pair.first;
var row = pair.first ?? pair.second ?? throw new InvalidOperationException("Both rows in rowPair are null");
Dictionary<string, object> newRow = new Dictionary<string, object>();
foreach (ColumnInfo info in row.schema.columnInfo)
{
@@ -211,4 +185,4 @@ namespace BCG
}
}
}
@@ -1,8 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using BCG.Utilities.General;
using BCG.Utilities.General;
using System;
namespace BCG
{
public class DateBasedFilter: IDynamicStreamFilter, IWorker
@@ -26,20 +25,14 @@ namespace BCG
myReference = myTodaysDate.AddDays(-1 * myNDays);
}
public event FilterChangeDelegate onFilterChange;
public event FilterChangeDelegate? onFilterChange;
public bool filter(IBlueRow r)
{
if (DBNull.Value.Equals(r[myDateColumn]))
{
//return false;
return true; //it can be considered as our assumption whenever datecolumn is db null we return true
}
DateTime t = (DateTime)r[myDateColumn];
bool returnVallue = t < myReference ? false : true;
return returnVallue;
}
public bool filter(IBlueRow r) =>
r[myDateColumn] switch {
DBNull or null => true,
DateTime t => t >= myReference,
_ => throw new ArgumentException("Row has invalid date column value", nameof(r)),
};
public void doWork()
{
@@ -47,7 +40,7 @@ namespace BCG
{
myTodaysDate = DateTime.Today;
myReference = myTodaysDate.AddDays(-1 * myNDays);
onFilterChange();
onFilterChange?.Invoke();
}
}
}
@@ -1,8 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using BCG.Utilities.General;
using BCG.Utilities.General;
using System;
namespace BCG
{
@@ -26,7 +24,7 @@ namespace BCG
#region IDynamicStreamFilter Members
public event FilterChangeDelegate onFilterChange;
public event FilterChangeDelegate? onFilterChange;
#endregion
@@ -34,7 +32,10 @@ namespace BCG
public bool filter(IBlueRow r)
{
DateTime rowDate = DBNull.Value.Equals (r[myDateColName]) ? DateTime.Now.Date : ((DateTime)r[myDateColName]).Date;
DateTime rowDate = r[myDateColName] switch {
DBNull or null => DateTime.Now.Date,
object obj => (DateTime)obj,
};
DateTime today = DateTime.Now.Date;
return rowDate > today ? true : !(today.Subtract(rowDate).Days >= myNPastTodayDays);
}
@@ -48,7 +49,7 @@ namespace BCG
if (DateTime.Now.Date != myLastCheckedTime.Date)
{
myLastCheckedTime = DateTime.Now;
onFilterChange();
onFilterChange?.Invoke();
}
}
@@ -1,18 +1,23 @@
using bts;
using Bts.DataStream;
using Bts.DataStream.Primitives.Filter;
using Bts.DataStreamCore;
using mbts.config;
using Spark.Core.Diagnostics;
using Spark.Core.Disposables;
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Text.RegularExpressions;
namespace BCG.StreamWorld.Addition {
[DataStreamUserRespresentation(false)]
public class FilterStream : ViewStream {
private ViewStream myBaseStream;
private BlueTable myBaseTable, myStateTable;
BlueTable? myBaseTable;
BlueTable? myStateTable;
private IStreamFilter myFilter;
public FilterStream(ViewStream vs)
@@ -25,8 +30,8 @@ namespace BCG.StreamWorld.Addition {
myBaseStream = vs;
myFilter = filter;
vs.ErrorChanged += OnError;
if (filter is IDynamicStreamFilter)
(filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
if (filter is IDynamicStreamFilter dfilter)
dfilter.onFilterChange += myHandleFilterChange;
}
@@ -36,8 +41,8 @@ namespace BCG.StreamWorld.Addition {
protected override void myOnLastSubscriber() {
myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
myBaseTable.clear();
myStateTable.clear();
myBaseTable?.clear();
myStateTable?.clear();
}
bool filterRow(IBlueRow r) {
@@ -54,16 +59,12 @@ namespace BCG.StreamWorld.Addition {
try {
var dp = (IDataPacket)newValue;
var schema = dp.Schema;
if (myStateTable == null) {
bts.Log.assert(myBaseTable == null);
myStateTable = new BlueTable(schema);
myBaseTable = new BlueTable(schema);
}
init(schema);
var dels = new BlueTable(schema);
var adds = new BlueTable(schema);
foreach (IBlueRow r in dp.deletedRows) {
myBaseTable.deleteRow(r);
if (myStateTable.containsID((int)r["id"])) {
if (myStateTable.containsID(r.Id())) {
dels.addRow(r);
myStateTable.deleteRow(r);
}
@@ -86,6 +87,7 @@ namespace BCG.StreamWorld.Addition {
set {
myFilter = value;
if (myBaseTable != null) {
ExceptionAssert.NotNull(myStateTable);
IBlueRowCollection oldState = new BlueTable(myStateTable);
myStateTable.clear();
// If the filter throws an exception, we can rethrow it as a StreamFilterException
@@ -113,6 +115,17 @@ namespace BCG.StreamWorld.Addition {
//public string filterString { set { filter = new BooleanExpressionFilter(value); } }
private void myHandleFilterChange() { filter = myFilter; }
[MemberNotNull(nameof(myStateTable), nameof(myBaseTable))]
void init(IBlueTableSchema schema) {
if (myStateTable is null) {
Log.isnull(myBaseTable);
myStateTable = new BlueTable(schema);
myBaseTable = new BlueTable(schema);
} else {
Log.notnull(myBaseTable);
}
}
}
}
@@ -136,29 +149,24 @@ namespace BCG
}
#region IStreamFilter Members
public bool filter(IBlueRow dr)
{
bool rv = true;
for (int i = 0; i < myCols.Length; i++)
{
rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
if (!rv)
return rv;
}
return rv;
}
public bool filter(IBlueRow dr) =>
myCols.Zip(myRegExps).All(p => p.Second.IsMatch(dr[p.First] switch {
DBNull or null => string.Empty,
object obj => obj.ToString() ?? string.Empty
}));
#endregion
}
public class FilterStream : ViewStream
{
private BlueTable myLastFiredRows;
private IBlueRowCollection myLastReceivedRows;
private IStreamFilter myFilter;
private ViewStream myParentStream;
private string myFilterString;
private BCG.StreamWorld.Addition.FilterStream myFilterStream;
private bool myUseNewFilter = false;
BlueTable? myLastFiredRows;
IBlueRowCollection? myLastReceivedRows;
IStreamFilter? myFilter;
readonly ViewStream? myParentStream;
string? myFilterString;
readonly StreamWorld.Addition.FilterStream? myFilterStream;
readonly bool myUseNewFilter = false;
IDisposable? subscription;
#region constructors
public delegate bool OnFilter(IBlueRow r);
@@ -188,9 +196,10 @@ namespace BCG
{
myParentStream = vs;
myFilter = filter;
myFilterString = filter.ToString();
if (filter is IDynamicStreamFilter)
(filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
myFilterString = filter.ToString() ?? string.Empty;
if (filter is IDynamicStreamFilter dfilter) {
dfilter.onFilterChange += myHandleFilterChange;
}
}
}
@@ -214,13 +223,15 @@ namespace BCG
#endregion
#region overridden methods
protected override void myOnFirstSubscriber()
{ myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
protected override void myOnFirstSubscriber() {
ExceptionAssert.IsNull(subscription);
subscription = ExceptionAssert.NotNull(myParentStream).Subscribe(myHandleCallbacks);
}
protected override void myOnLastSubscriber()
{
myLastFiredRows = null;
myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
Disposable.Dispose(ref subscription);
}
protected override void myFireEvent(object o)
@@ -239,19 +250,20 @@ namespace BCG
try {
if (myLastFiredRows != null)
{
retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
new BlueTable(myLastFiredRows.schema));
var addedRows = new BlueTable(myLastFiredRows.schema);
var deletedRows = new BlueTable(myLastFiredRows.schema);
retVal = new DataPacket(myLastFiredRows, addedRows, deletedRows);
foreach (IBlueRow r in arrivedPacket.deletedRows)
if (myLastFiredRows.containsID((int)r["id"]))
if (myLastFiredRows.containsID(r.Id()))
{
myLastFiredRows.deleteRow(r);
(retVal.deletedRows as BlueTable).addRow(r);
deletedRows.addRow(r);
}
foreach (IBlueRow r in arrivedPacket.addedRows)
if (filterRow(r))
{
myLastFiredRows.addRow(r);
(retVal.addedRows as BlueTable).addRow(r);
addedRows.addRow(r);
}
}
else
@@ -274,7 +286,7 @@ namespace BCG
{
if (myUseNewFilter)
{
myFilterStream.filter = filter;
requireNewFilter().filter = filter;
}
else
{
@@ -303,9 +315,9 @@ namespace BCG
get
{
if (myUseNewFilter)
return myFilterStream.filter;
return requireNewFilter().filter;
else
return myFilter;
return requireOldFilter();
}
}
@@ -318,17 +330,25 @@ namespace BCG
private void myHandleFilterChange()
{
filter = myFilter;
filter = requireOldFilter();
}
bool filterRow(IBlueRow r) {
try {
return myFilter.filter(r);
return requireOldFilter().filter(r);
} catch (Exception ex) {
Log.Gen.error($"FilterStream delegate threw exception: {ex}");
OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
}
return false;
}
[MemberNotNull(nameof(myFilterStream))]
StreamWorld.Addition.FilterStream requireNewFilter() =>
ExceptionAssert.MemberNotNull(myFilterStream);
[MemberNotNull(nameof(myFilter))]
IStreamFilter requireOldFilter() =>
ExceptionAssert.MemberNotNull(myFilter);
}
}
@@ -1,8 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using BCG.Utilities.General;
using BCG.Utilities.General;
using System;
namespace BCG
{
@@ -29,7 +27,7 @@ namespace BCG
myReference = myCurrentTime.AddSeconds(-1 * myNSecs);
}
public event FilterChangeDelegate onFilterChange;
public event FilterChangeDelegate? onFilterChange;
public bool filter(IBlueRow r)
{
@@ -38,9 +36,8 @@ namespace BCG
//it can be considered as our assumption whenever datecolumn is db null we return true
return true; //dont use this for volparamsched use true
}
DateTime t = (DateTime)r[myDateColumn];
bool returnVallue = t < myReference ? false : true;
return returnVallue;
var t = (DateTime)r[myDateColumn]!;
return t >= myReference;
}
@@ -51,8 +48,7 @@ namespace BCG
myCurrentTime = DateTime.Now;
myReference = myCurrentTime.AddSeconds(-1 * myNSecs);
myLastTimeFilteredApplied = DateTime.Now;
onFilterChange();
onFilterChange?.Invoke();
}
}
}
@@ -1,11 +1,13 @@
using System;
using BCG.Utilities.General;
using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using BCG.Utilities.General;
using FuzzyRowComparer = System.Func<BCG.IBlueRow, BCG.IBlueRow, double>;
using System.Diagnostics;
namespace BCG
{
@@ -40,12 +42,12 @@ namespace BCG
private IComparer<IBlueRow> myRightCompare;
protected FuzzyRowComparer myFuzzyCompare;
protected TimeSpan myTimeWindow;
protected IBlueTableSchema myLeftSchema;
protected IBlueTableSchema myRightSchema;
protected IBlueTableSchema myOutputSchema;
protected IBlueTableSchema? myLeftSchema;
protected IBlueTableSchema? myRightSchema;
protected IBlueTableSchema? myOutputSchema;
private int myID = 0;
protected TimeWindowTable myStateRows;
protected TimeWindowTable? myStateRows;
private DateTime myNextCull;
const long MinCullIntervalMS = 1000;
@@ -65,7 +67,7 @@ namespace BCG
this.myFuzzyCompare = fuzzyCompare;
}
public virtual IBlueTableSchema OutputSchema
public virtual IBlueTableSchema? OutputSchema
{
get
{
@@ -150,7 +152,7 @@ namespace BCG
int count;
for (count = 0; count < list.Count; count++)
{
if ((DateTime)list[count][timeColumn] > expiry)
if ((DateTime)list[count][timeColumn]! > expiry)
{
// Stop when we find the first row that we can keep. Since this is sorted all others are also kept.
break;
@@ -172,16 +174,16 @@ namespace BCG
this.myFitness = fitness;
}
public IBlueRow myRightRow { get; private set; }
public IBlueRow myLeftRow { get; set; }
public IBlueRow? myLeftRow { get; set; }
public double myFitness { get; private set; }
public IBlueRow myMergedRow { get; set; }
public IBlueRow? myMergedRow { get; set; }
#region IComparable<RowMatch> Members
public int CompareTo(RowMatch other)
{
return myFitness.CompareTo(other.myFitness);
}
public int CompareTo(RowMatch? other) =>
other is null
? 1
: myFitness.CompareTo(other.myFitness);
#endregion
}
@@ -262,12 +264,12 @@ namespace BCG
private void HandleCorrection(List<RowMatch> corrections, BlueTable adds, BlueTable dels)
{
ExceptionAssert.NotNull(myStateRows);
// We already emitted correction.MergedRow so we need to redact that and
// then emit correction.left as an add since it is no longer matched
foreach (RowMatch correction in corrections.Distinct())
{
Debug.Assert(correction.myMergedRow != null);
this.myStateRows.RemoveRow(correction.myMergedRow);
myStateRows.RemoveRow(ExceptionAssert.NotNull(correction.myMergedRow));
adds.addRow(this.MergeRows(correction.myLeftRow, null));
dels.addRow(correction.myMergedRow);
}
@@ -285,7 +287,7 @@ namespace BCG
int r = 0;
for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; )
{
if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn])
if ((DateTime)leftExpired[l][this.myLeftTimeColumn]! <= (DateTime)rightExpired[r][this.myRightTimeColumn]!)
{
// Advance Left
IEnumerable<RowMatch> rm = rightMatches[l];
@@ -332,7 +334,7 @@ namespace BCG
private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable<RowMatch> rightMatches)
{
IBlueRow merged = null;
IBlueRow? merged = null;
if (rightMatches != null)
{
foreach (RowMatch match in rightMatches)
@@ -366,9 +368,9 @@ namespace BCG
/// Override this function to provide a custom row merge.
/// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting.
/// </summary>
protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right)
protected virtual IBlueRow MergeRows(IBlueRow? left, IBlueRow? right)
{
Dictionary<string, object> newRow = new Dictionary<string, object>();
Dictionary<string, object?> newRow = new();
if (left == null && right == null) throw new ArgumentException("oh come on.");
@@ -393,14 +395,15 @@ namespace BCG
}
}
newRow["id"] = this.myID++;
newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn];
newRow[NEW_TIME_COLUMN] = left is not null
? left[myLeftTimeColumn]
: right![myRightTimeColumn];
return new BlueRow(this.OutputSchema, newRow);
}
private IEnumerable<RowMatch> FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
{
List<RowMatch> matches = null;
List<RowMatch>? matches = null;
// Search through all rows in list and compare with the fuzzy comparer
for (int x = 0; x < list.Count; x++)
@@ -421,8 +424,7 @@ namespace BCG
matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch);
// Mark the best match for this particular right hand match
RowMatch oldMatch;
if (this.myBestMatch.TryGetValue(match, out oldMatch))
if (this.myBestMatch.TryGetValue(match, out var oldMatch))
{
if (currentMatch.myFitness < oldMatch.myFitness)
{
@@ -441,7 +443,7 @@ namespace BCG
}
}
return matches;
return matches ?? Enumerable.Empty<RowMatch>();
}
}
}
@@ -1,14 +1,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Collections.Generic;
namespace BCG
{
public interface INonIncGroupByEvaluator
{
IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
string[] get_group_by_cols();
IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
string[]? get_group_by_cols();
}
}
@@ -1,46 +1,51 @@
using System;
using bts;
using bts.SilverPine;
using Bts.DataStreamCore;
using Spark.Core.Diagnostics;
using Spark.Core.Disposables;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text.Json;
using bts;
using bts.SilverPine;
namespace BCG {
//[DataStreamUserRespresentation("AggregateStream")]
public class NonIncGroupByStream : ViewStream {
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
Func<string[]> _groupby_cols_func;
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
Func<string[]?> _groupby_cols_func;
Func<List<IBlueRow>, List<IBlueRow>?> _evaluate_func;
public DelNonIncGroupByEvaluator(Func<string[]?> groupby_cols_func,
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
_groupby_cols_func = groupby_cols_func;
_evaluate_func = evaluate_func;
}
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
BlueTable rv = null;
public IBlueRowCollection? evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
BlueTable? rv = null;
foreach (var kv in grouped_rows) {
List<IBlueRow> g_rows = kv.Value;
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
var out_rows = _evaluate_func(g_rows);
if (out_rows != null && out_rows.Count > 0) {
rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
rv.addRows(out_rows);
}
rv.addRows(out_rows);
}
return rv;
}
public string[] get_group_by_cols() {
public string[]? get_group_by_cols() {
return _groupby_cols_func();
}
}
private ViewStream _main_stream;
private INonIncGroupByEvaluator _groupby_func;
private BlueRowHasher _row_hasher;
private string[] _groupby_cols;
readonly ViewStream? _main_stream;
readonly INonIncGroupByEvaluator? _groupby_func;
BlueRowHasher? _row_hasher;
readonly string[]? _groupby_cols;
private Stopwatch _timer = new();
private int _n_total_fires = 0, _n_expensive_fires = 0;
IDisposable? subscription;
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
_main_stream = main_stream;
@@ -49,7 +54,7 @@ namespace BCG {
_row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
}
public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
public NonIncGroupByStream(ViewStream main_stream, Func<string[]?> groupby_cols_func,
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
@@ -58,7 +63,7 @@ namespace BCG {
public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col) {
MultiRowOpHelper std_group_by = MultiRowOpHelper.parse(standard_group_by_text);
Func<string[]> groupby_cols_func = () => {
Func<string[]?> groupby_cols_func = () => {
return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
};
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
@@ -75,11 +80,16 @@ namespace BCG {
}
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
protected override void myOnFirstSubscriber() {
ExceptionAssert.IsNull(subscription);
ExceptionAssert.NotNull(_main_stream).Subscribe(_main_stream_onchange);
}
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
protected override void myOnLastSubscriber() =>
Disposable.Dispose(ref subscription);
private void _main_stream_onchange(object nv) {
ExceptionAssert.NotNull(_groupby_func);
var dp = (IDataPacket)nv;
///grouping together
_timer.Restart();
@@ -94,7 +104,7 @@ namespace BCG {
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
/// reducing
_timer.Restart();
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
var total_state = _groupby_func.evaluate(grouped_rows);
_timer.Stop();
if (total_state is null) {
Log.Gen.warn("NonIncGroupByStream _groupby_func.evaluate returned null (ignoring last update).");
@@ -1,6 +1,6 @@
using System;
using BCG.Utilities;
using System.Collections;
using System.Collections.Generic;
namespace BCG
{
@@ -11,12 +11,10 @@ namespace BCG
/// </summary>
public class RowDictionaryTable : IBlueRowCollection
{
protected IDictionary myRowCollections = new Hashtable ();
protected BlueTable myLinearCollection;
protected string[] myCols;
protected readonly Dictionary<string, BlueTable> myRowCollections = new();
protected readonly BlueTable myLinearCollection;
protected readonly string[] myCols;
protected IBlueTableSchema mySchema;
protected RowDictionaryTable () {}
public RowDictionaryTable (IBlueRowCollection rows, params string[] cols)
: this (rows.schema, cols)
@@ -40,7 +38,7 @@ namespace BCG
public virtual void addRow (IBlueRow row)
{
BlueTable rows = (BlueTable)getGroup (row);
var rows = getMutableGroup(row);
if (rows == null)
{
//i.e we dont have any group for this row => we need to create a new group
@@ -53,7 +51,7 @@ namespace BCG
public virtual void removeRow (IBlueRow row)
{
BlueTable rowGroup = (BlueTable)getGroup (row);
var rowGroup = requireMutableGroup(row);
// BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0);
rowGroup.deleteRow (row);
myLinearCollection.deleteRow (row);
@@ -65,7 +63,7 @@ namespace BCG
public void removeGroup (IBlueRow representativeRow)
{
IBlueRowCollection rowGroup = getGroup (representativeRow);
var rowGroup = requireGroup(representativeRow);
// BlueEventLog.logAssert (rowGroup != null && rowGroup.Count > 0);
foreach (IBlueRow row in rowGroup)
myLinearCollection.deleteRow (row);
@@ -85,24 +83,31 @@ namespace BCG
/// <returns></returns>
//public IBlueRowCollection getGroup (IBlueRow row) {return getGroup (
public IBlueRowCollection getGroup (IBlueRow r) {return getGroup (myRowToString (r));}
public IBlueRowCollection? getGroup (IBlueRow r) =>
getGroup(myRowToString(r));
public ICollection getGroups () {return myRowCollections.Values;}
private bool myAreRowsEqual (IBlueRow r1, IBlueRow r2)
{
bool retVal = true;
foreach (string col in myCols)
retVal &= r1[col].Equals(r2[col]);
return retVal;
}
public IBlueRowCollection? getGroup(string repString) =>
getMutableGroup(repString);
public IBlueRowCollection getGroup (string repString)
{
IBlueRowCollection retVal = myRowCollections[repString] != null ?
(IBlueRowCollection) myRowCollections[repString] : null;
return retVal;
}
BlueTable? getMutableGroup(IBlueRow row) =>
getMutableGroup(myRowToString(row));
BlueTable? getMutableGroup(string group) =>
myRowCollections.GetValueOrDefault(group);
IBlueRowCollection requireGroup(IBlueRow row) =>
requireGroup(myRowToString(row));
IBlueRowCollection requireGroup(string group) =>
getGroup(group) ?? throw new ArgumentException("No table for group", nameof(group));
BlueTable requireMutableGroup(IBlueRow row) =>
requireMutableGroup(myRowToString(row));
BlueTable requireMutableGroup(string group) =>
getMutableGroup(group) ?? throw new ArgumentException("No table for group", nameof(group));
private string myRowToString (IBlueRow r) {return BlueDataUtility.rowToString (r, myCols);}
@@ -1,5 +1,3 @@
using System;
namespace BCG
{
[DataStreamUserRespresentation (false)]
@@ -17,6 +15,7 @@ namespace BCG
protected override void myInitFiredPacket ()
{
requireSchema();
myLastFiredPacket = new DataPacket ();
myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
myLastFiredPacket.addedRows = new BlueTable (mySchema);
@@ -25,6 +24,10 @@ namespace BCG
public override void onTableClear (bool isLeft)
{
if (myLastFiredPacket is null) {
return;
}
requireSchema();
//incase of natural join, left or right wont matter, the end result zeroes out
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
@@ -1,5 +1,7 @@
using Spark.Core.Diagnostics;
using System;
using System.Collections;
using System.Diagnostics.CodeAnalysis;
namespace BCG
{
@@ -13,12 +15,10 @@ namespace BCG
protected JoinOperand myLeftOperand, myRightOperand;
protected int myLeftJoinColOrdinal, myRightJoinColOrdinal;
protected ViewStream myLeftStream, myRightStream;
protected JoinTableSchema mySchema;
protected DataPacket myLastFiredPacket;
protected JoinTableSchema? mySchema;
protected DataPacket? myLastFiredPacket;
protected bool myOnceFired = false;
public IBlueTableSchema schema {get{return mySchema;}}
public InternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight)
{
myLeftStream = vsLeft;
@@ -27,6 +27,9 @@ namespace BCG
myRightOperand = new JoinOperand (jcRight, false);
}
public IBlueTableSchema schema =>
requireSchema();
protected override void myOnFirstSubscriber()
{
myLeftOperand.master = this;
@@ -62,29 +65,32 @@ namespace BCG
{
myLeftJoinColOrdinal = myLeftOperand.joinColOridinal;
myRightJoinColOrdinal = myRightOperand.joinColOridinal;
myLeftOperand.RequireInitialized();
myRightOperand.RequireInitialized();
mySchema = new JoinTableSchema (myLeftOperand.operandSchema, myLeftOperand.joinCol,
myRightOperand.operandSchema, myRightOperand.joinCol);
myInitFiredPacket ();
}
}
[MemberNotNull(nameof(mySchema), nameof(myLastFiredPacket))]
protected virtual void myInitFiredPacket ()
{
var schema = requireSchema();
myLastFiredPacket = new DataPacket ();
myLastFiredPacket.newRows = new JoinTable (mySchema);
myLastFiredPacket.addedRows = new BlueTable (mySchema);
myLastFiredPacket.deletedRows = new BlueTable (mySchema);
myLastFiredPacket.newRows = new JoinTable (schema);
myLastFiredPacket.addedRows = new BlueTable (schema);
myLastFiredPacket.deletedRows = new BlueTable (schema);
}
public IBlueTableSchema getSchema (bool isLeft)
{
IBlueTableSchema retVal;
retVal = isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema;
return retVal;
}
public IBlueTableSchema? getSchema (bool isLeft) =>
isLeft ? myLeftOperand.operandSchema : myRightOperand.operandSchema;
public void endSession ()
{
if (myLastFiredPacket is null) {
return;
}
if (!myOnceFired)
{
myOnceFired = true;
@@ -97,13 +103,17 @@ namespace BCG
protected virtual void myInit ()
{
myLeftOperand.init ();
myRightOperand.init ();
myLeftOperand.Reset ();
myRightOperand.Reset ();
myLastFiredPacket = null;
}
public virtual void onTableClear (bool isLeft)
{
if (myLastFiredPacket is null) {
return;
}
requireSchema();
//incase of natural join, left or right wont matter, the end result zeroes out
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
myLastFiredPacket.newRows = new JoinTable (mySchema);
@@ -112,15 +122,19 @@ namespace BCG
public virtual void deleteRows (IBlueRowCollection rows, bool isLeft)
{
var last = myLastFiredPacket;
if (last is null) {
return;
}
JoinOperand thisOp, otherOp;
int joinColOrdinal;
myAssignAccordingly (out thisOp, out otherOp, out joinColOrdinal, isLeft);
foreach (IBlueRow row in rows)
{
thisOp.deleteRow (row);
if (otherOp.getMatchingRows (row[joinColOrdinal]).Count == 0)
if (otherOp.getMatchingRows(getJoinKey(row, joinColOrdinal)).Count == 0)
continue; //row does not participate in join
((JoinTable)(myLastFiredPacket.newRows)).deleteRow (isLeft, row, myLastFiredPacket);
((JoinTable)last.newRows).deleteRow(isLeft, row, last);
}
}
@@ -135,10 +149,12 @@ namespace BCG
otherOp = myLeftOperand;
joinColOrdinal = myRightJoinColOrdinal;
}
var schema = requireSchema();
myLastFiredPacket ??= new DataPacket(new BlueTable(schema), new BlueTable(schema), new BlueTable(schema));
foreach (IBlueRow row in rows)
{
thisOp.addRow (row);
RowList matchedNodes = otherOp.getMatchingRows (row[joinColOrdinal]);
RowList matchedNodes = otherOp.getMatchingRows(getJoinKey(row, joinColOrdinal));
if (matchedNodes.Count == 0)
continue; //this row did not participate in the join
if (isLeft)
@@ -146,10 +162,22 @@ namespace BCG
((JoinTable)myLastFiredPacket.newRows).addRow (row, matchedNode.row, myLastFiredPacket);
else
foreach (RowNode matchedNode in matchedNodes)
((JoinTable)myLastFiredPacket.newRows).addRow (matchedNode.row, row, myLastFiredPacket);
((JoinTable)myLastFiredPacket.newRows).addRow(ExceptionAssert.NotNull(matchedNode.row), row, myLastFiredPacket);
}
}
[MemberNotNull(nameof(mySchema))]
protected JoinTableSchema requireSchema() =>
ExceptionAssert.NotNull(mySchema);
[MemberNotNull(nameof(myLastFiredPacket))]
protected DataPacket requireLastFiredPacket() {
if (myLastFiredPacket is null) {
myInitFiredPacket();
}
return myLastFiredPacket;
}
private void myAssignAccordingly (out JoinOperand thisOp, out JoinOperand otherOp, out int joinColOrdinal, bool isLeft)
{
if (isLeft)
@@ -166,6 +194,9 @@ namespace BCG
}
}
object getJoinKey(IBlueRow row, int joinColOrdinal) =>
row[joinColOrdinal] ?? throw new ArgumentException("Row has null join key", nameof(row));
/**************************************TEST METHOD(S)*********************************************/
/// <summary>
/// Used for testing purposes only. This enables the test to work without a DataSource (PubServer,
@@ -1,5 +1,6 @@
using System;
using System.Collections;
using Spark.Core.Diagnostics;
using System.Collections.Generic;
namespace BCG
{
@@ -8,7 +9,7 @@ namespace BCG
/// </summary>
internal class JoinDeletionIndex
{
IDictionary myIndex = new Hashtable ();
Dictionary<object, RowList> myIndex = new();
private string myOpIdName ;
private bool myIsLeft;
@@ -28,10 +29,8 @@ namespace BCG
/// <param name="row">JoinRow to add</param>
public void add (JoinRow row)
{
object o = row[myOpIdName];
if (myIndex[o] == null)
myIndex[o] = new RowList ();
RowNode nodeAdded = ((RowList)myIndex[o]).add (row);
object o = ExceptionAssert.NotNull(row[myOpIdName]);
RowNode nodeAdded = myIndex.GetOrAdd(o, _ => new()).add(row);
if (myIsLeft)
row.leftNode = nodeAdded;
else
@@ -43,7 +42,10 @@ namespace BCG
/// To remove a node, if node does not exist exception is thrown
/// </summary>
/// <param name="node"></param>
public void remove (RowNode node) {((RowList)myIndex[node.row[myOpIdName]]).remove (node);}
public void remove (RowNode node) {
var nodeId = ExceptionAssert.NotNull(node.row?[myOpIdName]);
myIndex[nodeId].remove(node);
}
public RowList getMatchingRows (object key) {
RowList retVal = (RowList)myIndex[key];
@@ -1,19 +1,18 @@
using System;
using BCG.Utilities;
using bts;
using Bts.DataStreamCore;
namespace BCG
{
internal class JoinForeignTable : JoinTable
{
public JoinForeignTable (JoinTableSchema schema) : base (schema) {}
public override void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp)
{
JoinRow newRow = new JoinRow (leftRow, rightRow, mySchema);
Log.assert(myIdIndex[leftRow["id"]] == null);
myIdIndex[leftRow["id"]] = newRow;
public override void addRow(IBlueRow leftRow, IBlueRow? rightRow, IDataPacket dp) {
var newRow = new JoinRow(leftRow, rightRow, mySchema);
int leftId = leftRow.Id();
Log.assert(myIdIndex[leftId] == null);
myIdIndex[leftId] = newRow;
((BlueTable)(dp.addedRows)).addRow (newRow);
myLeftIndex.add (newRow);
myRightIndex.add (newRow);
@@ -1,11 +1,12 @@
using System;
using System.Collections;
using BCG.Utilities;
using bts;
using Bts.ColumnMapExpressions;
using Spark.Core.Diagnostics;
using System;
using System.Collections;
namespace BCG
{
/// <summary>
@@ -37,7 +38,8 @@ namespace BCG
private JoinTableSchema myJoinSchema;
private IBlueTableSchema myLeftSchema, myRightSchema;
private string myLeftColExpression, myRightColExpression;
private string[] myUserRules, myResolvedNames;
string[] myUserRules;
string[]? myResolvedNames;
private IDictionary myFinalMap = new Hashtable ();
@@ -87,16 +89,18 @@ namespace BCG
/// </summary>
private void myFormUserRules ()
{
var leftExp = ColumnMapExpression.Parse(myLeftColExpression) ?? throw new InvalidOperationException("Failed to parse left column expression");
var rightExp = ColumnMapExpression.Parse(myRightColExpression) ?? throw new InvalidOperationException("Failed to parse right column expression");
if (DataStreamLoadFileConfig.instance.useStaticParser)
{
myFormUserRules(true, ColumnMapExpression.Parse(myLeftColExpression));
myFormUserRules(false, ColumnMapExpression.Parse(myRightColExpression));
myFormUserRules(true, leftExp);
myFormUserRules(false, rightExp);
}
else
{
// TODO (maybe): Actually use non-static parser (unclear what advantages it will have)
myFormUserRules(true, ColumnMapExpression.Parse(myLeftColExpression));
myFormUserRules(false, ColumnMapExpression.Parse(myRightColExpression));
myFormUserRules(true, leftExp);
myFormUserRules(false, rightExp);
}
}
@@ -115,13 +119,7 @@ namespace BCG
OriginalNameInfo oni = myOriginalColInfo[i];
if (oni.isLeft == isLeft)
{
if (userMap[oni.colName] != null)
{
if ((string)userMap[oni.colName] != oni.colName)
myUserRules[i] = (string)userMap[oni.colName];
else
myUserRules[i] = oni.colName;
}
myUserRules[i] = userMap[oni.colName] as string ?? oni.colName;
}
}
else
@@ -131,10 +129,13 @@ namespace BCG
OriginalNameInfo oni = myOriginalColInfo[i];
if (oni.isLeft == isLeft)
{
if (userMap[oni.colName] == null)
var colName = userMap[oni.colName] as string;
// myUserRules[i] = userMap[oni.colName] as string
if (colName is null) {
myUserRules[i] = oni.colName;
else if ((string)userMap[oni.colName] != oni.colName)
myUserRules[i] = (string)userMap[oni.colName];
} else if (colName != oni.colName) {
myUserRules[i] = colName;
}
}
}
}
@@ -165,26 +166,12 @@ namespace BCG
private void myFormFinalMap ()
{
ExceptionAssert.NotNull(myResolvedNames);
foreach (string resolvedName in myResolvedNames)
if (resolvedName != null)
myFinalMap [resolvedName] = myJoinNames[Array.IndexOf (myResolvedNames, resolvedName)];
}
/// <summary>
/// Utility method to invert a hashtable
/// </summary>
/// <param name="arg"></param>
/// <returns></returns>
private IDictionary reciprocateDictionary (IDictionary arg)
{
IDictionary retVal = new Hashtable ();
foreach (DictionaryEntry de in arg)
retVal[de.Value] = de.Key;
return retVal;
}
/// <summary>
/// Utility method to find all occurances of an obkect in an IList
/// </summary>
@@ -194,9 +181,11 @@ namespace BCG
private ArrayList myFindAllInCollection (IList list, object obj)
{
ArrayList retVal = new ArrayList ();
for (int i = 0; i < list.Count; i++)
if (list[i] != null && list[i].Equals (obj))
for (int i = 0; i < list.Count; i++) {
if (Equals(list[i], obj)) {
retVal.Add (i);
}
}
return retVal;
}
@@ -1,6 +1,7 @@
using Spark.Core.Diagnostics;
using System;
using System.Collections;
using BCG.Utilities;
using System.Diagnostics.CodeAnalysis;
namespace BCG
{
@@ -12,32 +13,45 @@ namespace BCG
{
private string myJoinCol;
IDataPacket? myLastArrivedPacket;
private JoinOperand myOtherOperand;
JoinOperand? myOtherOperand;
private bool myIsLeft;
private JoinOperandRowIndex myRowIndex;
private InternalJoinStream myMaster;
private IBlueTableSchema myOpSchema;
JoinOperandRowIndex? myRowIndex;
InternalJoinStream? myMaster;
public JoinOperand (string joinCol, bool isLeft)
{
myJoinCol = joinCol;
myIsLeft = isLeft;
}
[MemberNotNullWhen(true, nameof(myMaster), nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
bool isInitialized {
get {
if (myLastArrivedPacket is null) {
return false;
}
ExceptionAssert.NotNull(myMaster, myLastArrivedPacket, operandSchema, myOtherOperand, myRowIndex);
return true;
}
}
internal void changeHandler (object o)
{
var dp = (IDataPacket)o;
if (myLastArrivedPacket != null)
if (isInitialized) {
mySubsequentChangeHandler (dp);
else
} else {
myFirstChangeHandler(dp);
}
}
void myFirstChangeHandler(IDataPacket arrivedDp) {
[MemberNotNull(nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
bool myFirstChangeHandler(IDataPacket arrivedDp) {
ExceptionAssert.NotNull(myMaster);
myLastArrivedPacket = arrivedDp;
myOpSchema = myLastArrivedPacket.newRows.schema;
operandSchema = myLastArrivedPacket.newRows.schema;
myOtherOperand = myMaster.getOtherOperand (this);
myRowIndex = new JoinOperandRowIndex (myOpSchema[myJoinCol].colNo, myOpSchema["id"].colNo);
myRowIndex = new JoinOperandRowIndex (operandSchema[myJoinCol].colNo, operandSchema["id"].colNo);
if (myOtherOperand.hasReceivedData)
{
//the other operand has already received data, so the join would
@@ -48,13 +62,16 @@ namespace BCG
myMaster.addRows (myLastArrivedPacket.newRows, myIsLeft);
myMaster.endSession ();
}
else
else {
myRowIndex = new JoinOperandRowIndex (myLastArrivedPacket.newRows,
myOpSchema[myJoinCol].colNo,
myOpSchema["id"].colNo);
operandSchema[myJoinCol].colNo,
operandSchema["id"].colNo);
}
return true;
}
void mySubsequentChangeHandler (IDataPacket arrivedDp) {
RequireInitialized();
myLastArrivedPacket = arrivedDp;
if (!myOtherOperand.hasReceivedData)
{
@@ -83,29 +100,46 @@ namespace BCG
myMaster.endSession ();
}
internal void addRow (IBlueRow row) {myRowIndex.add (row);}
internal void addRow (IBlueRow row) =>
ExceptionAssert.NotNull(myRowIndex).add(row);
internal void deleteRow (IBlueRow row) {myRowIndex.remove (row);}
internal void deleteRow (IBlueRow row) {
RequireInitialized();
myRowIndex.remove (row);
}
internal IBlueTableSchema operandSchema {get{return myOpSchema;}}
internal IBlueTableSchema? operandSchema { get; private set; }
internal InternalJoinStream master {set{myMaster = value;}}
internal string joinCol {get{return myJoinCol;}}
internal int joinColOridinal {get{return myOpSchema[myJoinCol].colNo;}}
internal int joinColOridinal {
get {
RequireInitialized();
return operandSchema[myJoinCol].colNo;
}
}
internal RowList getMatchingRows (object key) {return (RowList) myRowIndex.getMatchingRows (key);}
internal RowList getMatchingRows (object key) {
RequireInitialized();
return myRowIndex.getMatchingRows(key);
}
internal bool hasReceivedData {get{return myLastArrivedPacket != null;}}
internal void init ()
{
internal void Reset() {
///should be brought back to the the state before the first callback
///(ie the state after construction)
myRowIndex = null;
myLastArrivedPacket = null;
}
[MemberNotNull(nameof(myMaster), nameof(myLastArrivedPacket), nameof(operandSchema), nameof(myOtherOperand), nameof(myRowIndex))]
internal void RequireInitialized() {
if (!isInitialized) {
throw new InvalidOperationException("Internal state not properly initialized");
}
}
}
}
@@ -1,6 +1,6 @@
using System;
using System.Collections;
using BCG.Utilities;
using Spark.Core.Diagnostics;
using System.Collections.Generic;
namespace BCG
{
@@ -11,8 +11,10 @@ namespace BCG
/// </summary>
internal class JoinOperandRowIndex
{
private IDictionary myJoinColDict = new Hashtable (), myIdDict = new Hashtable ();
private int myJoinColOrdinal, myIdColOrdinal;
readonly Dictionary<object, RowList> myJoinColDict = new();
readonly Dictionary<int, RowNode> myIdDict = new();
readonly int myJoinColOrdinal;
readonly int myIdColOrdinal;
/// <summary>
///
@@ -34,19 +36,14 @@ namespace BCG
public void add (IBlueRow row)
{
RowList list;
if (myJoinColDict[row[myJoinColOrdinal]] == null)
myJoinColDict[row[myJoinColOrdinal]] = list = new RowList ();
else
list = (RowList)myJoinColDict[row[myJoinColOrdinal]];
myIdDict[row[myIdColOrdinal]] = list.add (row);
myIdDict[getId(row)] = myJoinColDict.GetOrAdd(getJoinKey(row), _ => new()).add(row);
}
public void remove (IBlueRow row)
{
RowNode node = (RowNode)myIdDict[row[myIdColOrdinal]];
((RowList)myJoinColDict[row[myJoinColOrdinal]]).remove (node);
myIdDict.Remove (row[myIdColOrdinal]);
var id = getId(row);
myJoinColDict[getJoinKey(row)].remove(myIdDict[id]);
myIdDict.Remove(id);
}
public RowList getMatchingRows (object key)
@@ -62,5 +59,11 @@ namespace BCG
myJoinColDict.Clear ();
myIdDict.Clear ();
}
object getJoinKey(IBlueRow row) =>
ExceptionAssert.NotNull(row[myJoinColOrdinal]);
int getId(IBlueRow row) =>
(int)row[myIdColOrdinal]!;
}
}
@@ -1,17 +1,21 @@
using System;
using BCG.Utilities;
using System.Collections;
using bts;
using Bts.DataStreamCore;
using System;
using System.Collections;
namespace BCG
{
internal class JoinRow : IBlueRow
{
private RowNode myLeftNode, myRightNode;
protected IBlueRow myLeftRow, myRightRow;
RowNode? myLeftNode;
RowNode? myRightNode;
protected IBlueRow myLeftRow;
protected IBlueRow? myRightRow;
protected int myId;
protected JoinTableSchema mySchema;
private object[] myValues;
private object?[] myValues;
#region Constructor(s)
@@ -21,8 +25,8 @@ namespace BCG
/// <param name="leftRow">left row of the join</param>
/// <param name="rightRow">right row of the join</param>
/// <param name="schema">schema of the joined row</param>
public JoinRow (IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
: this ((int)leftRow["id"], leftRow, rightRow, schema) {}
public JoinRow (IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema)
: this(leftRow.Id(), leftRow, rightRow, schema) { }
/// <summary>
/// Constructs a row assigning it the ID provioded as the first argument
@@ -31,8 +35,7 @@ namespace BCG
/// <param name="leftRow">left row of the join</param>
/// <param name="rightRow">right row of the join</param>
/// <param name="schema">schema of the joined row</param>
public JoinRow (int id, IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
{
public JoinRow (int id, IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema) {
myId = id;
myLeftRow = leftRow;
myRightRow = rightRow;
@@ -48,13 +51,13 @@ namespace BCG
public bool isPartialJoin{get{return myRightRow == null;}}
public RowNode leftNode
public RowNode? leftNode
{
get{return myLeftNode;}
set{myLeftNode = value;}
}
public RowNode rightNode
public RowNode? rightNode
{
get{return myRightNode;}
set{myRightNode = value;}
@@ -85,7 +88,7 @@ namespace BCG
/// The key can either be the string corrsponding to the column name or
/// the order of the column
/// </summary>
public virtual object this [object key]
public virtual object? this[object key]
{
get
{
@@ -96,7 +99,7 @@ namespace BCG
set {throw new NotSupportedException ("The IBlueRow is read only");}
}
public object this [int index]
public object? this[int index]
{
get
{
@@ -119,7 +122,8 @@ namespace BCG
}
}
public object this [string colName] {get{return this[mySchema[colName].colNo];}}
public object? this[string colName] =>
this[mySchema[colName].colNo];
public ICollection Keys{get{return mySchema.columnNames;}}
@@ -133,7 +137,8 @@ namespace BCG
}
}
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
public void Add (object key, object? value) =>
throw new NotSupportedException ("The IBlueRow is readonly");
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
@@ -1,20 +1,26 @@
using System;
using System.Data;
using System.Collections;
using System.Text.RegularExpressions;
using BCG.Utilities;
using BCG.Utilities.General;
using bts;
using Bts.DataStreamCore;
using Spark.Core.Diagnostics;
using Spark.Core.Disposables;
using System;
using System.Collections;
namespace BCG
{
public class JoinStream : ViewStream
{
protected InternalJoinStream? myIjs;
protected readonly InternalJoinStream? myIjs;
protected readonly ColumnMap? myLeftCm;
protected readonly ColumnMap? myRightCm;
protected readonly bool myIsProject = true;
protected ProjectStream? myPs;
protected ColumnMap? myLeftCm, myRightCm;
protected bool myIsProject = true;
private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
readonly bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
IDisposable? internalSubscription;
IDisposable? projectSubscription;
protected JoinStream () {}
public JoinStream (ViewStream v1, string joinCol1, ColumnMap cm1,
@@ -111,47 +117,61 @@ namespace BCG
{
if (myPs == null)
{
JoinNameResolver resolver = new JoinNameResolver ((JoinTableSchema)myIjs.schema, myIjs.getSchema (true), myLeftCm.columnExpression,
myIjs.getSchema (false), myRightCm.columnExpression);
ExceptionAssert.NotNull(myIjs, myLeftCm, myRightCm);
var leftSchema = myIjs.getSchema(true) ?? throw new InvalidOperationException("Left schema is null");
var rightSchema = myIjs.getSchema(true) ?? throw new InvalidOperationException("Right schema is null");
var joinSchema = (JoinTableSchema)myIjs.schema;
JoinNameResolver resolver = new JoinNameResolver(
joinSchema, leftSchema, myLeftCm.columnExpression, rightSchema, myRightCm.columnExpression);
IDictionary map = resolver.getMap ();
map["id"] = "id";
Disposable.Dispose(ref projectSubscription);
myPs = new ProjectStream (myIjs, map);
myPs.onChange += new DataStream.ChangeDelegate (myProjectChangeHandler);
projectSubscription = myPs.Subscribe(myProjectChangeHandler);
return;
}
else
Log.assert(false, "This call back should not have been called");
}
else
myFireEvent ((o as ICloneable).Clone());
else {
myFireEvent(cloneDataPacket(o));
}
}
private void myProjectChangeHandler (object o)
{
if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1)
myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
myFireEvent ((o as ICloneable).Clone());
ExceptionAssert.NotNull(myIjs);
if (myIjs.numberOfSubscribers > 1) {
Disposable.Dispose(ref internalSubscription);
}
myFireEvent(cloneDataPacket(o));
}
protected override void myOnFirstSubscriber ()
{
myIjs.onChange += new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
Log.isnull(internalSubscription);
internalSubscription = ExceptionAssert.NotNull(myIjs).Subscribe(myInternalJoinChangeHandler);
}
protected override void myOnLastSubscriber ()
{
// how does this relate to the -= in myProjectChangeHandler?
myIjs.onChange -= new DataStream.ChangeDelegate (myInternalJoinChangeHandler);
Disposable.Dispose(ref internalSubscription);
// if there is a project after the ijs we need to
// unsubscribe to it otherwise references will be kept and
// a memory leak will occur
if (myPs != null)
myPs.onChange -= new DataStream.ChangeDelegate (myProjectChangeHandler);
Disposable.Dispose(ref projectSubscription);
myPs = null;
}
object cloneDataPacket(object dpObj) {
var rv = (dpObj as ICloneable)?.Clone();
if (rv is null) {
var dp = (IDataPacket)dpObj;
rv = new DataPacket(new BlueTable(dp.newRows), new BlueTable(dp.addedRows), new BlueTable(dp.deletedRows));
}
return rv;
}
/**************************************TEST METHOD(S)*********************************************/
/// <summary>
/// Used for testing purposes only. This enables the test to work without a DataSource (PubServer,
@@ -159,7 +179,9 @@ namespace BCG
/// </summary>
/// <param name="leftPacket"></param>
/// <param name="rightPacket"></param>
public void test (DataPacket dp, bool isLeft) {myIjs.test (dp, isLeft);}
public void test (DataPacket dp, bool isLeft) {
ExceptionAssert.NotNull(myIjs).test(dp, isLeft);
}
/***********************************************************************************************/
}
}
@@ -1,14 +1,17 @@
using Bts.DataStreamCore;
using Spark.Core.Diagnostics;
using System;
using System.Collections;
using System.IO;
using BCG.Utilities.General;
using System.Collections.Generic;
namespace BCG
{
internal class JoinTable : IBlueRowCollection
{
protected JoinTableSchema mySchema;
protected IDictionary myIdIndex = new Hashtable ();
protected readonly Dictionary<int, IBlueRow> myIdIndex = new();
protected JoinDeletionIndex myLeftIndex, myRightIndex;
private int myNewRowId;
@@ -19,19 +22,16 @@ namespace BCG
myRightIndex = new JoinDeletionIndex (mySchema.getRenamedName ("id", false), false);
}
public IBlueRow this [int id]
{
get
{
if (myIdIndex[id] == null)
throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
return (IBlueRow)myIdIndex[id];
public IBlueRow this [int id] {
get {
var rowObj = myIdIndex[id]
?? throw new BlueDataException($"ID {id} does not exist in table {mySchema.tableName}");
return (IBlueRow)rowObj;
}
}
#region JoinTable members
public virtual void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp)
{
public virtual void addRow(IBlueRow leftRow, IBlueRow? rightRow, IDataPacket dp) {
JoinRow newRow = new JoinRow (myNewRowId, leftRow, rightRow, mySchema);
myIdIndex[myNewRowId ++] = newRow;
((BlueTable)(dp.addedRows)).addRow (newRow);
@@ -40,7 +40,7 @@ namespace BCG
myRightIndex.add (newRow);
}
public virtual void deleteRow (bool isLeft, IBlueRow deletedRow, DataPacket dp)
public virtual void deleteRow (bool isLeft, IBlueRow deletedRow, IDataPacket dp)
{
JoinDeletionIndex thisIndex = myRightIndex, otherIndex = myLeftIndex;
if (isLeft)
@@ -48,17 +48,17 @@ namespace BCG
thisIndex = myLeftIndex;
otherIndex = myRightIndex;
}
RowList matchingNodes = thisIndex.getMatchingRows (deletedRow["id"]);
RowList matchingNodes = thisIndex.getMatchingRows(deletedRow.Id());
foreach (RowNode node in matchingNodes)
{
JoinRow row = (JoinRow)node.row;
RowNode correspondingNode = isLeft ? row.rightNode : row.leftNode;
var row = ExceptionAssert.Argument.Is<JoinRow>(node.row);
RowNode? correspondingNode = isLeft ? row.rightNode : row.leftNode;
if (correspondingNode != null)
otherIndex.remove (correspondingNode); //in case of partial left joinss
myIdIndex.Remove (row["id"]);
myIdIndex.Remove(row.Id());
((BlueTable)dp.deletedRows).addRow (row);
}
thisIndex.removeAll (deletedRow["id"]);
thisIndex.removeAll(deletedRow.Id());
}
#endregion
#region ICollection members
@@ -68,7 +68,11 @@ namespace BCG
public object SyncRoot{get{return this;}}
public void CopyTo (Array array, int index) {myIdIndex.Values.CopyTo (array, index);}
public void CopyTo (Array array, int index) {
foreach (var row in myIdIndex.Values) {
array.SetValue(row, index++);
}
}
#endregion
#region IEnumerable members
@@ -1,9 +1,8 @@
using System;
using System.Collections;
using BCG.Utilities;
using bts;
using System.Collections.Generic;
using System.Linq;
namespace BCG
{
/// <summary>
@@ -13,10 +12,11 @@ namespace BCG
public class JoinTableSchema : BlueTableSchema
{
private string myLeftJc = "", myRightJc = "", myLeftId = "", myRightId = "";
private IDictionary myLeftNameToAlias = new Hashtable ();
private IDictionary myRightNameToAlias = new Hashtable ();
private IDictionary myAliasToLeftName = new Hashtable(),
myAliasToRightName = new Hashtable();
readonly Dictionary<string, string> myLeftNameToAlias = new();
readonly Dictionary<string, string> myRightNameToAlias = new();
readonly Dictionary<string, string> myAliasToLeftName = new();
readonly Dictionary<string, string> myAliasToRightName = new();
/// <summary>
/// Constructs the schema of the Jointable with the schema of the left and right tables. Incase of
/// a name being there in both the shemas it prefixes the name with "L." or "R." accordingly. Column "id"
@@ -85,9 +85,9 @@ namespace BCG
renamed = false;
}
myIndexByColName["id"] = new ColumnInfo("id", myIndexByColName.Count, typeof(int));
myIndexByColOrder[myIndexByColOrder.Length - 1] = (ColumnInfo)myIndexByColName["id"];
myAliasToLeftName = ProjectHelper.invertDic(myLeftNameToAlias);
myAliasToRightName = ProjectHelper.invertDic(myRightNameToAlias);
myIndexByColOrder[myIndexByColOrder.Length - 1] = myIndexByColName["id"];
myAliasToLeftName = myLeftNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key);
myAliasToRightName = myRightNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key);
}
public string leftJoinCol {get{return myLeftJc;}}
@@ -1,17 +1,21 @@
using bts;
using bts.collections;
using Bts.DataStreamCore;
using mbts;
using Spark.Core.Diagnostics;
using Spark.Core.Disposables;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Security.Cryptography;
using System.Diagnostics.CodeAnalysis;
namespace BCG {
public class NonIncJoinBookkeeper : IDisposable {
public Action on_new_data;
public event Action? on_new_data;
private readonly NonIncJoinSpec _join_spec;
private readonly int _join_position;
private readonly int _start_index_in_output;
@@ -20,13 +24,14 @@ namespace BCG {
private readonly bool _is_tag_along;
private int _n_total_fires = 0, _n_expensive_fires = 0;
private IBlueTableSchema _schema = null;
IBlueTableSchema? _schema;
private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
private IBlueRow _solitary_row = null;
IBlueRow? _solitary_row;
private int[] _join_col_indices = null,
_data_col_indices = null;
int[]? _join_col_indices;
int[]? _data_col_indices;
IDisposable? subscription;
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
_join_spec = the_join_spec;
@@ -70,7 +75,8 @@ namespace BCG {
}
public void init() {
_join_spec.stream.onChange += _on_stream_cb;
ExceptionAssert.MemberIsNull(subscription);
subscription = ExceptionAssert.MemberNotNull(_join_spec.stream).Subscribe(_on_stream_cb);
}
public void _on_stream_cb(object dp_obj) {
@@ -78,12 +84,7 @@ namespace BCG {
var timer = Stopwatch.StartNew();
var dp = (IDataPacket)dp_obj;
_fill_missing_data(dp);
if (_schema == null) {
_schema = dp.Schema;
_join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
_data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
}
init(dp);
if (_is_left_most) {
_row_list.Clear();
foreach (IBlueRow r in dp.newRows)
@@ -108,44 +109,68 @@ namespace BCG {
on_new_data();
}
[MemberNotNullWhen(true, nameof(_schema))]
public bool is_ready { get { return _schema != null; } }
public void constribute_cols_to_schema(List<ColumnInfo> cis) {
requireReady();
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
}
public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
public void copy_master_row_to_output_row(int row_index, object[] output_row) {
IBlueRow this_row = _row_list[row_index].second;
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
}
public void copy_master_row_to_output_row(int row_index, object?[] output_row) =>
copyRow(ExceptionAssert.NotNull(_row_list[row_index].second), output_row);
public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
public void copy_matching_row_to_output_row(int row_hash, object?[] output_row) {
if (_is_tag_along) {
if (_solitary_row != null)
BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
copyRow(_solitary_row, output_row);
else
copy_dbnull_to_output_row(output_row);
} else {
IBlueRow this_row = null;
if (_row_map.TryGetValue(row_hash, out this_row))
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
if (_row_map.TryGetValue(row_hash, out var this_row))
copyRow(this_row, output_row);
else
copy_dbnull_to_output_row(output_row);
}
}
private void copy_dbnull_to_output_row(object[] output_row) {
void copy_dbnull_to_output_row(object?[] output_row) {
requireDataColIndices();
for (int i = 0; i < _data_col_indices.Length; i++)
output_row[i + _start_index_in_output] = DBNull.Value;
}
#region IDisposable
public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
public void Dispose() =>
Disposable.Dispose(ref subscription);
#endregion
void copyRow(IBlueRow src, object?[] dst) {
requireDataColIndices();
BlueDataUtility.copy_to_row(src, _data_col_indices, dst, _start_index_in_output);
}
[MemberNotNull(nameof(_schema), nameof(_join_col_indices), nameof(_data_col_indices))]
void init(IDataPacket dp) {
_schema ??= dp.Schema;
_join_col_indices ??= BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
_data_col_indices ??= BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
}
[MemberNotNull(nameof(_data_col_indices))]
int[] requireDataColIndices() =>
ExceptionAssert.NotNull(_data_col_indices);
[MemberNotNull(nameof(_schema))]
void requireReady() {
if (!is_ready) {
throw new InvalidOperationException("Not ready");
}
}
}
}
@@ -1,22 +1,25 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using mbts;
using mbts;
using System;
using System.Collections.Generic;
namespace BCG {
// this a collection of data, theres no joining logic in here
public class NonIncJoinSpec {
public ViewStream stream;
public ViewStream? stream;
// join col names are the names on which the streams are joined (for equality),
// data col names are the names in the original stream
// projected col names are their new names, ie if its x:l_x, then x is in the
// data_col_names and l_x in projected cols
public string[] join_col_names, data_col_names, projected_col_names;
public string[] join_col_names;
public string[] data_col_names;
public string[] projected_col_names;
public NonIncJoinSpec() { }
public NonIncJoinSpec() {
join_col_names = Array.Empty<string>();
data_col_names = Array.Empty<string>();
projected_col_names = Array.Empty<string>();
}
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
: this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
@@ -5,18 +5,19 @@ using mbts;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
namespace BCG {
[DataStreamUserRespresentation("fastJoin")]
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
private readonly NonIncJoinSpec[] _join_specs;
private readonly INonIncJoinInlineMapper _output_mapper;
readonly INonIncJoinInlineMapper? _output_mapper;
private readonly BlueRowHasher _row_hasher;
private readonly TimeSpan _throttle_period;
private readonly ViewStreamDatabase _vsd;
private List<NonIncJoinBookkeeper> _book_keeping_data = new List<NonIncJoinBookkeeper>();
private IBlueTableSchema _output_schema = null;
IBlueTableSchema? _output_schema;
private DateTime _last_fire_time = DateTime.MinValue;
private bool _is_dirty = false;
@@ -35,10 +36,10 @@ namespace BCG {
params Triplet<ViewStream, string, string>[] join_specs)
: this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper? output_mapper, int throttles_ms = 0)
: this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper? output_mapper, int throttles_ms = 0) {
_join_specs = join_specs;
_output_mapper = output_mapper;
_row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -88,6 +89,7 @@ namespace BCG {
return rv;
}
[MemberNotNull(nameof(_output_schema))]
private void _construct_schema() {
if (_output_schema == null) {
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
@@ -1,3 +1,5 @@
using Spark.Core.Diagnostics;
using System;
using System.Collections;
@@ -9,7 +11,7 @@ namespace BCG
/// </summary>
internal class RowList : ICollection
{
private RowNode myFirstPtr;
RowNode? myFirstPtr;
private int myCount = 0;
#region ICollection methods
@@ -67,19 +69,19 @@ namespace BCG
}
else if (node.isFirst)
{
myFirstPtr = myFirstPtr.next;
myFirstPtr.previous = null;
myFirstPtr = ExceptionAssert.NotNull(myFirstPtr).next;
ExceptionAssert.NotNull(myFirstPtr).previous = null;
return;
}
else if (node.isLast)
{
node.previous.next = null;
ExceptionAssert.NotNull(node.previous).next = null;
return;
}
else if (!node.isFirst && !node.isLast)
{
node.previous.next = node.next;
node.next.previous = node.previous;
ExceptionAssert.NotNull(node.previous).next = node.next;
ExceptionAssert.NotNull(node.next).previous = node.previous;
return;
}
else
@@ -91,14 +93,16 @@ namespace BCG
internal class RowListEnumerator : IEnumerator
{
private RowNode myPointer;
RowNode? myPointer;
private RowList myList;
private int myPos = -1;
public RowListEnumerator (RowList list) {myList = list;}
public object Current {get{return myPointer;}}
public object? Current => myPointer;
public bool MoveNext()
{
myPointer = myPos == -1 ? myList.myFirstPtr : myPointer.next;
myPointer = myPos == -1
? myList.myFirstPtr
: ExceptionAssert.NotNull(myPointer).next;
myPos ++;
return myPointer != null;
}
@@ -108,28 +112,24 @@ namespace BCG
internal class RowNode
{
private IBlueRow myRow;
private RowNode myNext;
private RowNode myPrevious;
private RowList myList;
public RowNode (IBlueRow row){myRow = row;}
public RowNode (IBlueRow row, RowList list) : this (row) {myList = list;}
public bool isLast {get{return myNext == null;}}
public bool isFirst {get{return myPrevious == null;}}
public RowList list {get{return myList;}}
public IBlueRow row
{
get{return myRow;}
set{myRow = value;}
}
public RowNode next {
get{return myNext;}
set{myNext = value;}
}
public RowNode previous {
get{return myPrevious;}
set{myPrevious = value;}
}
public RowNode (IBlueRow row) =>
this.row = row;
public RowNode (IBlueRow row, RowList list)
: this(row) =>
this.list = list;
public bool isLast => next is null;
public bool isFirst => previous is null;
public RowList? list { get; }
public IBlueRow? row { get; set; }
public RowNode? next { get; set; }
public RowNode? previous { get; set; }
public RowList RequireList() =>
ExceptionAssert.MemberNotNull(list);
public IBlueRow RequireRow() =>
ExceptionAssert.MemberNotNull(row);
}
}
@@ -1,125 +1,17 @@
using Bts.DataStreamCore;
using mbts;
using Spark.Core.Collections;
using Spark.Core.Diagnostics;
using System;
using System.Collections;
using System.Collections.Generic;
using mbts;
using Spark.Core.Collections;
using System.Diagnostics.CodeAnalysis;
namespace BCG
{
internal class SingleRowJoinDicionary<TKey, TValue> : IDictionary<TKey, TValue>
{
private TKey myKey;
private TValue myValue;
#region IDictionary<TKey,TValue> Members
public void Add(TKey key, TValue value)
{
myKey = key;
myValue = value;
}
public bool ContainsKey(TKey key)
{
return myKey.Equals(key);
}
public ICollection<TKey> Keys
{
get { return new TKey[] { myKey }; }
}
public bool Remove(TKey key)
{
bool rv = myKey.Equals(key);
myKey = default(TKey);
return rv;
}
public bool TryGetValue(TKey key, out TValue value)
{
bool rv = false;
value = (rv = myKey.Equals(key)) ? myValue : default(TValue);
return rv;
}
public ICollection<TValue> Values
{
get { throw new Exception("The method or operation is not implemented."); }
}
public TValue this[TKey key]
{
get
{
throw new Exception("The method or operation is not implemented.");
}
set
{
throw new Exception("The method or operation is not implemented.");
}
}
#endregion
#region ICollection<KeyValuePair<TKey,TValue>> Members
public void Add(KeyValuePair<TKey, TValue> item)
{
throw new Exception("The method or operation is not implemented.");
}
public void Clear()
{
throw new Exception("The method or operation is not implemented.");
}
public bool Contains(KeyValuePair<TKey, TValue> item)
{
throw new Exception("The method or operation is not implemented.");
}
public void CopyTo(KeyValuePair<TKey, TValue>[] array, int arrayIndex)
{
throw new Exception("The method or operation is not implemented.");
}
public int Count
{
get { throw new Exception("The method or operation is not implemented."); }
}
public bool IsReadOnly
{
get { throw new Exception("The method or operation is not implemented."); }
}
public bool Remove(KeyValuePair<TKey, TValue> item)
{
throw new Exception("The method or operation is not implemented.");
}
#endregion
#region IEnumerable<KeyValuePair<TKey,TValue>> Members
public IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator()
{
throw new Exception("The method or operation is not implemented.");
}
#endregion
#region IEnumerable Members
IEnumerator IEnumerable.GetEnumerator()
{
throw new Exception("The method or operation is not implemented.");
}
#endregion
}
#region Helper class to deal with stream projections
public class JoinProjectHelper
@@ -167,7 +59,7 @@ namespace BCG
{
Pair<string, bool> p = myAliasToOriginal[col];
isLeft = p.second;
return p.first;
return ExceptionAssert.NotNull(p.first);
}
}
@@ -178,10 +70,11 @@ namespace BCG
public class SuperJoinRow : IBlueRow
{
private JoinProjectHelper myHelper;
private IBlueRow myLeftRow, myRightRow;
IBlueRow myLeftRow;
IBlueRow? myRightRow;
int myID;
public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow rightRow, JoinProjectHelper joinProjectHelper)
public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow? rightRow, JoinProjectHelper joinProjectHelper)
{
myID = id;
myLeftRow = leftRow;
@@ -189,21 +82,22 @@ namespace BCG
myHelper = joinProjectHelper;
}
public int leftID { get { return (int)myLeftRow["id"]; } }
public int leftID => myLeftRow.Id();
public int rightID { get { return (int)myRightRow["id"]; } }
public int rightID => myRightRow.Id();
public IBlueRow leftRow { get { return myLeftRow; } }
public IBlueRow rightRow { get { return myRightRow; } }
public IBlueRow? rightRow => myRightRow;
#region IBlueRow Members
public IBlueTableSchema schema { get { return myHelper.schema; } }
public object this[int index] { get { return this[schema[index].colName]; } }
public object? this[int index] =>
this[schema[index].colName];
public object this[string colName]
public object? this[string colName]
{
get
{
@@ -229,7 +123,7 @@ namespace BCG
public bool Contains(object key)
{
return myLeftRow.Contains(key) || myRightRow.Contains(key);
return myLeftRow.Contains(key) || (myRightRow?.Contains(key) ?? false);
}
public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
@@ -242,11 +136,11 @@ namespace BCG
public void Remove(object key) { throw new NotSupportedException("IBlueRow is readonly"); }
public System.Collections.ICollection Values
public ICollection Values
{
get
{
List<object> rv = new List<object>();
var rv = new List<object?>(Keys.Count);
foreach (string col in Keys)
rv.Add(this[col]);
return rv;
@@ -291,9 +185,10 @@ namespace BCG
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
private bool myManageUnmappedLeftRows = false;
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
protected IBlueTableSchema myLeftSchema, myRightSchema;
protected JoinProjectHelper myJoinProjectHelper;
protected BlueTable myState;
protected IBlueTableSchema? myLeftSchema;
protected IBlueTableSchema? myRightSchema;
protected JoinProjectHelper? myJoinProjectHelper;
protected BlueTable? myState;
protected int myID;
protected ViewStream myLeftStream, myRightStream;
protected string myLeftKeyCol, myRightKeyCol;
@@ -357,12 +252,8 @@ namespace BCG
private void myHandleStreamChanges(IDataPacket dp, bool isLeft)
{
myInitSchema(dp.Schema, isLeft);
BlueTable dels = null, adds = null;
if (myState != null)
{
dels = new BlueTable(myState.schema);
adds = new BlueTable(myState.schema);
}
var dels = new BlueTable(myState.schema);
var adds = new BlueTable(myState.schema);
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
myAddRows(dp.addedRows, isLeft, myState, adds);
if (myState != null)
@@ -373,6 +264,7 @@ namespace BCG
}
}
[MemberNotNull(nameof(myState))]
private void myInitSchema(IBlueTableSchema schema, bool isLeft)
{
if (myState == null)
@@ -402,6 +294,7 @@ namespace BCG
private void myAddRows(IBlueRowCollection addedRows, bool isLeft, BlueTable state, BlueTable addedRowsOut)
{
ExceptionAssert.NotNull(myJoinProjectHelper);
object thisKeyCol = isLeft ? myLeftKeyCol : myRightKeyCol,
otherSideKeyCol = isLeft ? myRightKeyCol : myLeftKeyCol;
Dictionary<object, IDictionary<int, IBlueRow>> thisKeyToRows = isLeft ? myLeftKeyToLeftRows : myRightKeyToRightRows,
@@ -410,8 +303,8 @@ namespace BCG
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
foreach (IBlueRow r in addedRows)
{
object currentKey = r[thisKeyCol];
int currentID = (int)r["id"];
object currentKey = ExceptionAssert.NotNull(r[thisKeyCol]);
int currentID = r.Id();
///add it to 1.
if (!thisKeyToRows.ContainsKey(currentKey))
thisKeyToRows.Add(currentKey, new Dictionary<int, IBlueRow>());
@@ -422,14 +315,14 @@ namespace BCG
ICollection<IBlueRow> matchingRows = otherSideKeyToRows[currentKey].Values;
foreach (IBlueRow mr in matchingRows)
{
int mrID = (int)mr["id"];
int mrID = mr.Id();
SuperJoinRow rr = new SuperJoinRow(myID++, isLeft ? r : mr, isLeft ? mr : r, myJoinProjectHelper);
state.addRow(rr);
addedRowsOut.addRow(rr);
///add to 3.
if (!thisIDToResultRows.ContainsKey(currentID))
thisIDToResultRows.Add(currentID, new Dictionary<int, SuperJoinRow>());
thisIDToResultRows[currentID].Add((int)rr["id"], rr);
thisIDToResultRows[currentID].Add(rr.Id(), rr);
///add to 4.
if (!otherIDToResultRows.ContainsKey(mrID))
{
@@ -437,7 +330,7 @@ namespace BCG
if (myManageUnmappedLeftRows && !isLeft)
myTakeCareOfUnmatachedLeftRows(mr, false);
}
otherIDToResultRows[mrID].Add((int)rr["id"], rr);
otherIDToResultRows[mrID].Add(rr.Id(), rr);
}
}
else
@@ -458,22 +351,22 @@ namespace BCG
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
foreach (IBlueRow r in deletedRows)
{
int id = (int)r["id"];
object key = r[thisKeyCol];
int id = r.Id();
object key = ExceptionAssert.NotNull(r[thisKeyCol]);
///delete it from 1.
thisKeyToRows[key].Remove(id);
if (thisKeyToRows[key].Count == 0)
thisKeyToRows.Remove(key);
///form return rows and delete from 3.
if (thisIDToResultRows.ContainsKey(id))
if (thisIDToResultRows.TryGetValue(id, out IDictionary<int, SuperJoinRow>? removedResultRowMap))
{
ICollection removedResultRows = thisIDToResultRows[id].Values as ICollection;
var removedResultRows = (ICollection)removedResultRowMap.Values;
thisIDToResultRows.Remove(id);
foreach (SuperJoinRow rr in removedResultRows)
{
///delete from 4.
int otherId = isLeft ? rr.rightID : rr.leftID;
otherIDToResultRows[otherId].Remove((int)rr["id"]);
otherIDToResultRows[otherId].Remove(rr.Id());
if (otherIDToResultRows[otherId].Count == 0)
otherIDToResultRows.Remove(otherId);
delRowsOut.addRow(rr);
@@ -493,7 +386,7 @@ namespace BCG
private void myTakeCareOfUnmatachedLeftRows(IBlueRow leftRow, bool add)
{
int lid = (int)leftRow["id"];
int lid = leftRow.Id();
if (add)
myLeftUnmatchedRowsAdds.Add(lid, leftRow);
else
@@ -501,7 +394,7 @@ namespace BCG
if (myLeftUnmatchedRowsAdds.ContainsKey(lid))
myLeftUnmatchedRowsAdds.Remove(lid);
else
myLeftUnmatchedRowsDels.Add((int)leftRow["id"], leftRow);
myLeftUnmatchedRowsDels.Add(lid, leftRow);
}
}
}
@@ -526,38 +419,45 @@ namespace BCG
}
void myHandleJoinResults(Dictionary<int, IBlueRow> unmatchedLeftRowsDels, Dictionary<int, IBlueRow> unmatchedLeftRowAdds, IDataPacket resultPacket) {
BlueTable dels = resultPacket.deletedRows as BlueTable,
adds = resultPacket.addedRows as BlueTable;
ExceptionAssert.NotNull(myJoinProjectHelper);
var dels = resultPacket.deletedRows;
var adds = resultPacket.addedRows;
foreach (IBlueRow lr in unmatchedLeftRowsDels.Values)
{
///an lr will be here if its actually deleted or
///a right row is added, so now it has a partner
///and is no longer unmatched.
///in both cases the half row that this lr
///created needs to be removed from this
///class's state and added to dels
///it means a right is added
int lid = (int)lr["id"];
SuperJoinRow rr = myLeftIDToHalfRows[lid];
myLeftIDToHalfRows.Remove(lid);
myState.deleteRow(rr);
dels.addRow(rr);
if (unmatchedLeftRowsDels.Count > 0) {
ExceptionAssert.NotNull(myState);
var mutableDels = (BlueTable)dels;
foreach (IBlueRow lr in unmatchedLeftRowsDels.Values) {
///an lr will be here if its actually deleted or
///a right row is added, so now it has a partner
///and is no longer unmatched.
///in both cases the half row that this lr
///created needs to be removed from this
///class's state and added to dels
///it means a right is added
int lid = lr.Id();
SuperJoinRow rr = myLeftIDToHalfRows[lid];
myLeftIDToHalfRows.Remove(lid);
myState.deleteRow(rr);
mutableDels.addRow(rr);
}
}
foreach (IBlueRow lr in unmatchedLeftRowAdds.Values)
{
///an lr will be here if its added with no
///matching right, or a right matching row
///is gone so it is unmatched now.
///in both cases a half row needs to be
///added to this class's state and added
///to adds
int lid = (int)lr["id"];
SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
myLeftIDToHalfRows.Add(lid, rr);
myState.addRow(rr);
adds.addRow(rr);
if (unmatchedLeftRowAdds.Count > 0) {
ExceptionAssert.NotNull(myState);
var mutableAdds = (BlueTable)adds;
foreach (IBlueRow lr in unmatchedLeftRowAdds.Values) {
///an lr will be here if its added with no
///matching right, or a right matching row
///is gone so it is unmatched now.
///in both cases a half row needs to be
///added to this class's state and added
///to adds
int lid = lr.Id();
SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
myLeftIDToHalfRows.Add(lid, rr);
myState.addRow(rr);
mutableAdds.addRow(rr);
}
}
base.myFireEvent(new DataPacket(myState, adds, dels));
}
@@ -1,4 +1,9 @@
using System.Collections;
using Bts.DataStreamCore;
using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic;
namespace BCG
{
@@ -12,7 +17,8 @@ namespace BCG
protected IDataPacket? myLeftDataPacket;
protected IDataPacket? myRightDataPacket;
//Container to contain all the left rows that are orphaned during a deletion operation
private IDictionary myLeftOrphanedRows;
Dictionary<int, RowNode>? myLeftOrphanedRows;
public LeftInternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight) :
base (vsLeft, jcLeft, vsRight, jcRight) {}
@@ -61,7 +67,9 @@ namespace BCG
private void myTestAndFire (bool isLeft)
{
JoinOperand jo = isLeft ? myLeftOperand : myRightOperand;
var dp = isLeft ? myLeftDataPacket : myRightDataPacket;
var dp = isLeft
? ExceptionAssert.NotNull(myLeftDataPacket)
: ExceptionAssert.NotNull(myRightDataPacket);
//to test whether both have received call backs
if (myLeftOperand.hasReceivedData)
jo.changeHandler (dp);
@@ -79,11 +87,13 @@ namespace BCG
/// <param name="isLeft"></param>
public override void addRows (IBlueRowCollection rows, bool isLeft)
{
ExceptionAssert.NotNull(myLastFiredPacket);
foreach (IBlueRow row in rows)
myAddRow (row, isLeft);
if (!isLeft)
foreach (RowNode node in myLeftOrphanedRows.Values)
((JoinTable)myLastFiredPacket.newRows).addRow (node.row, null, myLastFiredPacket);
foreach (RowNode node in ExceptionAssert.NotNull(myLeftOrphanedRows).Values) {
((JoinTable)myLastFiredPacket.newRows).addRow(node.RequireRow(), null, myLastFiredPacket);
}
}
/// <summary>
@@ -93,17 +103,19 @@ namespace BCG
/// <param name="isLeft"></param>
public override void deleteRows (IBlueRowCollection rows, bool isLeft)
{
myLeftOrphanedRows = new Hashtable ();
myLeftOrphanedRows = new();
foreach (IBlueRow row in rows)
myDeleteRow (row, isLeft);
}
private void myAddRow (IBlueRow row, bool isLeft)
{
ExceptionAssert.NotNull(myLastFiredPacket);
if (isLeft)
{
myLeftOperand.addRow (row);
RowList matchedRows = myRightOperand.getMatchingRows (row[myLeftJoinColOrdinal]);
var joinKey = ExceptionAssert.NotNull(row[myLeftJoinColOrdinal]);
RowList matchedRows = myRightOperand.getMatchingRows(joinKey);
if (matchedRows.Count == 0)
((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
else
@@ -112,22 +124,25 @@ namespace BCG
}
else
{
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]);
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
var joinKey = ExceptionAssert.NotNull(row[myRightJoinColOrdinal]);
RowList matchedRows = myLeftOperand.getMatchingRows(joinKey);
int rightMatchingRowCount = myRightOperand.getMatchingRows(joinKey).Count;
myRightOperand.addRow (row);
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
int newRightMatchingRowCount = myRightOperand.getMatchingRows(joinKey).Count;
if (matchedRows.Count != 0){
foreach (RowNode matchedNode in matchedRows)
{
ExceptionAssert.NotNull(matchedNode.row);
//if the left rows were orphaned in this transaction (ie in this callback), they should
//just be removed from the orphanedlist and we dont need to worry about the consistency of
//the jointable as a new row will replace it
if (rightMatchingRowCount == 0 && newRightMatchingRowCount > 0)
{
if (myLeftOrphanedRows[matchedNode.row["id"]] == null)
((JoinTable)myLastFiredPacket.newRows).deleteRow (true, matchedNode.row, myLastFiredPacket);
else
myLeftOrphanedRows.Remove (matchedNode.row["id"]);
ExceptionAssert.NotNull(myLeftOrphanedRows);
int matchedId = matchedNode.row.Id();
if (!myLeftOrphanedRows.Remove(matchedId)) {
((JoinTable)myLastFiredPacket.newRows).deleteRow(true, matchedNode.row, myLastFiredPacket);
}
}
//they were orphaned by a previous transaction and hence the orphans need to be
//removed from the Jointable and the complete row added
@@ -140,41 +155,52 @@ namespace BCG
private void myDeleteRow (IBlueRow row, bool isLeft)
{
var last = requireLastFiredPacket();
if (isLeft)
{
myLeftOperand.deleteRow (row);
((JoinTable)myLastFiredPacket.newRows).deleteRow (true, row, myLastFiredPacket);
((JoinTable)last.newRows).deleteRow (true, row, myLastFiredPacket);
}
else
{
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]);
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
var rightJoinKey = row[myRightJoinColOrdinal] ?? throw new InvalidOperationException("RightJoinKey is null");
RowList matchedRows = myLeftOperand.getMatchingRows(rightJoinKey);
int rightMatchingRowCount = myRightOperand.getMatchingRows(rightJoinKey).Count;
myRightOperand.deleteRow (row);
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
int newRightMatchingRowCount = myRightOperand.getMatchingRows(rightJoinKey).Count;
if (matchedRows.Count != 0)
{
if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0)
if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0) {
ExceptionAssert.MemberNotNull(myLeftOrphanedRows);
//ie all the matchedRows were joined with 'this' right row (method arg), and hence they need to
//be handled specially, ie joined with nulls, if there are no right rows with this
//joincolumn value being inserted.
ExceptionAssert.NotNull(myLeftOrphanedRows);
foreach (RowNode matchedNode in matchedRows)
{
// BlueEventLog.logAssert (myLeftOrphanedRows[matchedNode.row["id"]] == null);
myLeftOrphanedRows[matchedNode.row["id"]] = matchedNode;
myLeftOrphanedRows[matchedNode.row.Id()] = matchedNode;
}
((JoinTable)myLastFiredPacket.newRows).deleteRow (false, row, myLastFiredPacket);
}
((JoinTable)last.newRows).deleteRow (false, row, myLastFiredPacket);
}
}
}
public override void onTableClear (bool isLeft)
{
if (myLastFiredPacket is null) {
return;
}
requireSchema();
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
myLastFiredPacket.newRows = new JoinTable (mySchema);
myLastFiredPacket.addedRows = new BlueTable (mySchema);
if (!isLeft)
if (!isLeft) {
ExceptionAssert.MemberNotNull(myLeftDataPacket);
foreach (IBlueRow row in myLeftDataPacket.newRows)
((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
}
}
}
@@ -30,7 +30,7 @@ namespace BCG
/// <summary>
/// Node containing the referencedRow
/// </summary>
IBlueRow referencedRow {get;}
IBlueRow? referencedRow { get; }
/// <summary>
/// removes all referenced or referencing nodes (indicated by the bool), p.s. theres only one referenced row
@@ -15,12 +15,14 @@ namespace BCG
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
IBlueRow getRow (int id);
IBlueRow? getRow(int id);
/// <summary>
/// Gets all the rows in the node
/// </summary>
/// <returns></returns>
IBlueRow[] getRows ();
IBlueRow?[] getRows();
/// <summary>
/// Replaces the row with the given id, with the row passed as the second arg
/// </summary>
@@ -1,7 +1,5 @@
using System;
using System.Collections;
using BCG.Utilities;
using BCG;
namespace BCG
{
@@ -56,13 +54,18 @@ namespace BCG
/// The key can either be the string corrsponding to the column name or
/// the order of the column
/// </summary>
public virtual object this [object key]
public virtual object? this [object key]
{
get {return key is string ? this [key as string] : this [(int)key];}
// get {return key is string ? this [key as string] : this [(int)key];}
get => key switch {
string colName => this[colName],
int colIndex => this[colIndex],
_ => throw new ArgumentException("Must be string or int", nameof(key)),
};
set {throw new NotSupportedException ("The IBlueRow is read only");}
}
public virtual object this [int index]
public virtual object? this [int index]
{
get
{
@@ -77,7 +80,8 @@ namespace BCG
}
}
public object this [string colName] {get{return this[mySchema[colName].colNo];}}
public object? this [string colName] =>
this[mySchema[colName].colNo];
public ICollection Keys{get{return mySchema.columnNames;}}
@@ -1,6 +1,6 @@
using System;
using Spark.Core.Diagnostics;
using System.Collections;
using BCG.Utilities;
namespace BCG
{
@@ -70,7 +70,8 @@ namespace BCG
}
else
{
SimpleNode sn = n as SimpleNode;
var sn = (SimpleNode)n;
ExceptionAssert.NotNull(sn.prev);
sn.prev.next = sn.next;
if (sn.next != null)
sn.next.prev = sn.prev;
@@ -81,7 +82,7 @@ namespace BCG
public int count {get {return myFirst.listCount;}}
public IBlueRow referencedRow {get {return myFirst.row;}}
public IBlueRow? referencedRow => myFirst.row;
public void removeAll (bool isReD)
{
@@ -110,10 +111,11 @@ namespace BCG
public class LightForeignJoinRowListEnumerator : IEnumerator
{
private ILightForeignJoinRowList myList;
private SimpleNode myPtr, myFirst;
private SimpleNode? myPtr;
private SimpleNode myFirst;
public LightForeignJoinRowListEnumerator (ILightForeignJoinRowList list, SimpleNode first)
{myList = list; myFirst = first;}
public object Current {get{return myPtr.row;}}
public object? Current => myPtr?.row;
public bool MoveNext() {return (myPtr = myPtr == null ? myFirst : myPtr.next) != null;}
public void Reset() {myPtr = null;}
public ILightForeignJoinRowList list {get {return myList;}}
@@ -51,7 +51,7 @@ namespace BCG
public void CopyTo (Array array, int index) {throw new NotSupportedException ("");}
public object SyncRoot {get{return null;}}
public object SyncRoot => this;
#endregion
@@ -1,5 +1,4 @@
using System;
using BCG.Utilities;
namespace BCG
{
@@ -13,8 +12,7 @@ namespace BCG
: base (reDRow, rinGRow, schema)
{}
public override object this [int index]
{
public override object? this [int index] {
get
{
if (index == mySchema.columnNames.Length - 1)
@@ -1,5 +1,3 @@
using System;
namespace BCG
{
/// <summary>
@@ -9,15 +7,15 @@ namespace BCG
/// </summary>
public class SimpleNode : INode
{
public IBlueRow row;
public SimpleNode next, prev;
public IBlueRow? row;
public SimpleNode? next;
public SimpleNode? prev;
public SimpleNode (IBlueRow r)
: this (r, null, null)
{}
public SimpleNode (IBlueRow r, SimpleNode nx, SimpleNode pv)
{
public SimpleNode(IBlueRow r, SimpleNode? nx, SimpleNode? pv) {
row = r;
next = nx;
prev = pv;
@@ -25,13 +23,15 @@ namespace BCG
#region INode Members
public IBlueRow getRow (int id) {return row;}
public IBlueRow? getRow(int id) =>
row;
public IBlueRow[] getRows() {return new IBlueRow[] {row};}
public IBlueRow?[] getRows() =>
new[] {row};
public void replace (int id, IBlueRow nr) {row = nr;}
#endregion
}
}
}
@@ -1,14 +1,8 @@
using System;
using System.Collections;
using System.Collections.Generic;
using Amazon.Runtime.Internal.Util;
using BCG;
using BCG.Utilities;
using bts;
using System;
using System.Collections.Generic;
namespace BCG {
public class LiteralStream : ViewStream {
@@ -42,14 +36,14 @@ namespace BCG {
return new BlueRow(map);
};
string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
BlueTable table = null;
BlueTable? table = null;
foreach (string json_string in row_arr) {
IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
get_row(json_string, table == null ? 0 : table.Count);
table = table ?? new BlueTable(r.schema);
table ??= new BlueTable(r.schema);
table.addRow(r);
}
ViewStream str = new BlueTableStream(table);
ViewStream str = new BlueTableStream(table ?? throw new ArgumentException("No rows specified, unable to determine schema"));
str = new SchemaGuesserStream(str);
mySetRealSource(str);
}
@@ -1,12 +1,12 @@
using BCG;
using System;
using System.IO;
using MongoDB.Bson;
using BCG.Utilities.General;
using System.Linq;
using System.Collections.Generic;
using System.Runtime.Intrinsics.X86;
using mbts.utils;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace bts {
// listens to the underlying stream, and writes each seriialized packet in a
@@ -32,15 +32,9 @@ namespace bts {
private void _handle_update(object dp_obj) {
var dp = (IDataPacket)dp_obj;
byte[] ser_data = null;
if (_n_updates == 0) {
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
ser_data = _serializer.serialize(new StreamIOMessage(cr_msg));
} else {
StreamIOMessage.MultipleRemoveAddMessage mra = new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows);
ser_data = _serializer.serialize(new StreamIOMessage(mra));
}
Log.assert(ser_data != null);
byte[] ser_data = n_updates == 0
? _serializer.serialize(new(new StreamIOMessage.CreateMessage(dp.newRows)))
: _serializer.serialize(new(new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
_n_updates++;
string file_name = $"{_folder}\\{_n_updates}.bin";
_write_to_file(ser_data, file_name);
@@ -62,7 +56,7 @@ namespace bts {
IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
private int _file_ptr = 0;
private List<Tuple<string, byte[]>> _file_data = new List<Tuple<string, byte[]>>();
private BlueTable _state = null;
BlueTable? _state;
public DataPacketReplayStream(string folder_path, string fire_period)
:
@@ -98,6 +92,7 @@ namespace bts {
break;
}
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: {
Log.notnull(_state);
var mra = msg.multiple_remove_add_message;
foreach (IBlueRow r in mra.removed_rows)
_state.deleteRow(r);
@@ -139,4 +134,4 @@ namespace bts {
});
}
}
}
}
@@ -1,14 +1,12 @@
using BCG;
using BCG.Utilities.General;
using System.Collections.Generic;
using Bts.PubSub.Routing;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using bts.api;
using Bts.PubSub.Routing;
namespace bts
{
@@ -17,9 +15,9 @@ namespace bts
{
public class PayloadMessage
{
public readonly string msg_id;
public readonly string? msg_id;
public readonly Int64 timestamp = 0;
public readonly byte[] payload = null;
public readonly byte[]? payload = null;
public static byte[] create_data(int payload_size, byte payload_byte)
{
@@ -53,10 +51,10 @@ namespace bts
{
var str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bw.Write(msg_id);
bw.Write(msg_id ?? string.Empty);
bw.Write(timestamp);
bw.Write(payload.Length);
bw.Write(payload);
bw.Write(payload?.Length ?? 0);
bw.Write(payload ?? Array.Empty<byte>());
str.Position = 0;
var bytes = str.ToArray();
return bytes;
@@ -73,18 +71,15 @@ namespace bts
return new PayloadMessage(guid, timestamp_, payload);
}
}
//public void
private readonly int _msg_size;
private readonly TimeSpan _publish_period;
private DateTime _last_publish_time = DateTime.MinValue;
private PublisherCore _pub_core;
private ISubscriberCore _sub_core;
PublisherCore? _pub_core;
ISubscriberCore? _sub_core;
private BlueTable _table;
private int _n_pubs = 0;
private readonly int _max_pubs;
private int _n_updates = 0;
private ViewStreamDatabase _vsd;
private PayloadMessage _last_sent_msg, _last_recvd_msg;
PayloadMessage? _last_sent_msg;
PayloadMessage? _last_recvd_msg;
private const byte _payload_byte = (byte)'A';
private readonly string _tag_name;
private readonly int _payload_sz;
@@ -131,13 +126,13 @@ namespace bts
protected override void myOnLastSubscriber()
{
_vsd.removeWorker(this);
_sub_core.unsubscribe(_tag_name);
Log.notnull(_sub_core).unsubscribe(_tag_name);
}
private void _fire_dp_after_send()
{
Log.notnull(_last_sent_msg);
// as soon as the data is published, its reflected in the stream state
Log.assert(_last_sent_msg != null);
BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
DBNull.Value, DBNull.Value);
_table.addRow(r);
@@ -147,7 +142,8 @@ namespace bts
private void _fire_dp_after_recv()
{
Log.assert(_last_recvd_msg != null);
Log.notnull(_last_recvd_msg);
Log.notnull(_last_sent_msg);
// update the last publish row
int id = _table.Count - 1;
IBlueRow last_row = _table[id];
@@ -167,28 +163,16 @@ namespace bts
private void _publish_next_packet()
{
Log.notnull(_pub_core);
var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
_last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
var ser_bytes = _last_sent_msg.serialize();
_n_pubs++;
_pub_core.publish("Blob", _tag_name, ser_bytes);
_last_publish_time = DateTime.Now;
}
public void doWork()
{
Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
{
foreach (var u in updates)
{
var recv_msg = PayloadMessage.deserialize(u.data);
if (recv_msg.msg_id == _last_sent_msg.msg_id)
{
return recv_msg;
}
}
return null;
};
switch (_state)
{
case IOState.SEND:
@@ -202,7 +186,7 @@ namespace bts
}
break;
case IOState.RECV:
var updates = _sub_core.get_updates(TimeSpan.Zero);
var updates = Log.notnull(_sub_core).get_updates(TimeSpan.Zero);
updates = updates.Where(u => u.tag == _tag_name);
var recv_msg = find_packet(updates);
if (recv_msg != null)
@@ -214,6 +198,19 @@ namespace bts
}
break;
}
PayloadMessage? find_packet(IEnumerable<PubSubServerMessage> updates) {
if (_last_sent_msg is null) {
return null;
}
foreach (var u in updates) {
var recv_msg = PayloadMessage.deserialize(Log.notnull(u.data));
if (recv_msg.msg_id == _last_sent_msg.msg_id) {
return recv_msg;
}
}
return null;
}
}
public static long epoch_now()
@@ -1,5 +1,7 @@
using bts;
using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic;
using System.IO;
@@ -197,7 +199,6 @@ namespace BCG {
private TestResult _run_schema_serialization_test() {
var table_name = "test_table";
var excluded_types = new HashSet<Type>() { typeof(DateTime), typeof(Decimal) };
IBlueTableSchema schema = create_schema_with_all_types(excluded_types);
MemoryStream str = new MemoryStream();
@@ -375,7 +376,7 @@ namespace BCG {
StreamIOSerializationController ser_controller = new StreamIOSerializationController();
StreamIODeserializationController deser_controller = new StreamIODeserializationController();
IBlueTableSchema schema_p = null;
IBlueTableSchema? schema_p = null;
var schema_file = @"c:\Temp\tests\schema.bin";
var row_file = @"C:\Temp\tests\row.bin";
var create_op_file = @"C:\Temp\tests\create_op.bin";
@@ -619,7 +620,7 @@ namespace BCG {
private static TestResult _run_single_tag_data_test_clean() {
// var tagHandlerFactory
// create message
var test_name = MethodBase.GetCurrentMethod()?.Name;
var test_name = MethodBase.GetCurrentMethod()?.Name ?? string.Empty;
var schema = create_schema_with_all_types();
var tag = "DS_" + schema.tableName;
BlueTable table = new BlueTable(schema);
@@ -695,14 +696,14 @@ namespace BCG {
}
public static PublishMsg build_pub_msg(IBlueRow added_row, IBlueRow removed_row, bool is_create,
public static PublishMsg build_pub_msg(IBlueRow? added_row, IBlueRow? removed_row, bool is_create,
ref IStreamIOMsgSerializer serializer) {
Func<IBlueTableSchema> get_schema = () => {
return (added_row != null ? added_row : removed_row).schema;
return (added_row != null ? added_row : ExceptionAssert.NotNull(removed_row)).schema;
};
bts.StreamIOMessage str_io_msg = null;
StreamIOMessage? str_io_msg;
if (is_create) {
Log.assert(added_row != null);
ExceptionAssert.NotNull(added_row);
var aux_new = new BlueTable(added_row.schema);
aux_new.addRow(added_row);
var cr_msg = new bts.StreamIOMessage.CreateMessage(aux_new);
@@ -729,7 +730,7 @@ namespace BCG {
return pub_msg;
}
public static IBlueTableSchema create_schema_with_all_types(HashSet<Type> excluded_col_map = null) {
public static IBlueTableSchema create_schema_with_all_types(HashSet<Type>? excluded_col_map = null) {
excluded_col_map = excluded_col_map == null ? new HashSet<Type>() : excluded_col_map;
var table_name = "test_table";
BlueTableSchema schema = new BlueTableSchema();
@@ -800,7 +801,6 @@ namespace BCG {
return BitConverter.ToUInt64(ulongBuffer, 0);
default:
throw new ArgumentException($"Unsupported TypeCode: {type_code}");
return null;
}
}
#endregion
@@ -1,7 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BCG
{
@@ -16,7 +13,7 @@ namespace BCG
private string myInColumn;
private string myOutValue;
private object[] myMuxedValues;
private BlueTableSchema mySchema;
BlueTableSchema? mySchema;
public MuxStream(ViewStream inStream, string outColumn, string outValue, string inColumn, params object[] muxedValues)
{
@@ -49,7 +46,7 @@ namespace BCG
if (Array.IndexOf(myMuxedValues, r[myInColumn]) != -1) // muxable value?
r.setValue(myOutColumn, myOutValue);
else
r.setValue(myOutColumn, row[myInColumn].ToString());
r.setValue(myOutColumn, row[myInColumn]?.ToString());
return r;
}
@@ -1,11 +1,8 @@
using mbts;
using System;
using System.Collections;
using BCG;
using BCG.Utilities;
using System.Collections.Generic;
using mbts;
namespace BCG
{
public class NullToConstantMapper : IStreamMapper
@@ -27,17 +24,11 @@ namespace BCG
#region IDynamicStreamMapper Members
public IBlueRow map(IBlueRow row)
{
if(row[myField].Equals(DBNull.Value))
{
Hashtable newRow = new Hashtable(row);
newRow[myField]= myConstant;
return new BlueRow(row.schema, newRow);
}
else
return row;
}
public IBlueRow map(IBlueRow row) =>
row[myField] switch {
null or DBNull => new BlueRow(row.schema, new Hashtable(row) { { myField, myConstant } }),
_ => row,
};
#endregion
@@ -69,10 +60,11 @@ namespace BCG
Hashtable newRow = new Hashtable(row);
foreach(ColumnInfo info in row.schema.columnInfo)
{
if(row[info.colName].Equals(DBNull.Value))
{
change = true;
newRow[info.colName] = Convert.ChangeType(myConstant, info.colType);
switch (row[info.colName]) {
case null or DBNull:
change = true;
newRow[info.colName] = Convert.ChangeType(myConstant, info.colType);
break;
}
}
if(change)
@@ -19,15 +19,15 @@ namespace BCG
public IBlueTableSchema schema { get { return mySchema; } }
public object this[int index] { get { return myRow[index]; } }
public object? this[int index] { get { return myRow[index]; } }
public object this[string colName] { get { return myRow[colName]; } }
public object? this[string colName] { get { return myRow[colName]; } }
#endregion
#region IDictionary Members
public void Add(object key, object value)
public void Add(object key, object? value)
{
throw new Exception("The method or operation is not implemented.");
}
@@ -54,7 +54,7 @@ namespace BCG
public ICollection Values { get { return myRow.Values; } }
public object this[object key]
public object? this[object key]
{
get { return myRow[key]; }
set { throw new Exception("The method or operation is not implemented."); }
@@ -82,7 +82,7 @@ namespace BCG
}
private string myNewName;
private IBlueTableSchema mySchema;
IBlueTableSchema? mySchema;
public ChangeTableNameStream (ViewStream vs, string name)
{
@@ -1,9 +1,9 @@
using Bts.ColumnMapExpressions;
using Spark.Core.Diagnostics;
using System;
using System.Collections;
using System.Data;
using System.Text.RegularExpressions;
using BCG.Utilities;
using Bts.ColumnMapExpressions;
namespace BCG
{
@@ -56,7 +56,7 @@ namespace BCG
{
IDictionary retVal = new Hashtable ();
foreach (DictionaryEntry de in dic)
retVal[de.Value] = de.Key;
retVal[ExceptionAssert.NotNull(de.Value)] = de.Key;
return retVal;
}
@@ -1,13 +1,11 @@
using Bts.ColumnMapExpressions;
using Bts.DataStream;
using mbts.config;
using Spark.Core.Diagnostics;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
// using BCG.Evaluator.ColumnMapExpression;
namespace BCG
{
@@ -26,7 +24,7 @@ namespace BCG
/// <param name="isch">original(unprojected) schema</param>
/// <param name="exp">expression defining the mapping</param>
/// <param name="tableName">table name of the new schema</param>
internal ProjectHelper (IBlueTableSchema isch, string exp, string tableName)
internal ProjectHelper (IBlueTableSchema isch, string exp, string? tableName)
: this (isch, myGetDictionary (isch, exp), tableName)
{}
@@ -36,7 +34,7 @@ namespace BCG
/// <param name="origToAlias">dictionary mapping unprojected names to their projected names, dropped cols should not be there</param>
/// <param name="tableName">table name of the new schema</param>
internal ProjectHelper (
IBlueTableSchema isch, IDictionary<string, string> origToAlias, string tableName)
IBlueTableSchema isch, IDictionary<string, string> origToAlias, string? tableName)
{
ArrayList colMapAl = new ArrayList (),
colInfoAl = new ArrayList ();
@@ -61,7 +59,7 @@ namespace BCG
schema = new BlueTableSchema (tableName == null ? isch.tableName : tableName,
colInfoAl.ToArray (typeof(ColumnInfo)) as ColumnInfo[]);
colMap = colMapAl.ToArray (typeof(int)) as int[];
colMap = (int[])colMapAl.ToArray(typeof(int));
}
/// <summary>
@@ -74,8 +72,8 @@ namespace BCG
private static IDictionary<string, string> myGetDictionary (IBlueTableSchema isch, string exp)
{
IDictionary<string, string> origToAlias = new Dictionary<string, string>();
var e = ColumnMapExpression.Parse(exp);
var e = ColumnMapExpression.Parse(exp)
?? throw new ArgumentException("Failed to parse ColumnMapExpression", nameof(exp));
if (e.IsInclusive) {
origToAlias = e.Columns.ToDictionary();
origToAlias["id"] = "id";
@@ -95,7 +93,7 @@ namespace BCG
{
IDictionary retVal = new Hashtable (dic.Count);
foreach (DictionaryEntry de in dic)
retVal.Add (de.Value, de.Key);
retVal.Add(ExceptionAssert.NotNull(de.Value), de.Key);
return retVal;
}
}
@@ -1,6 +1,10 @@
namespace BCG {
using Spark.Core.Collections;
using Bts.DataStreamCore;
using Spark.Core.Collections;
using Spark.Core.Diagnostics;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
@@ -8,10 +12,13 @@ namespace BCG {
[DataStreamUserRespresentation("oldproject")]
public class ProjectStream : ViewStream
{
private ProjectHelper myHelper;
private string myColExp, myTableName;
private ViewStream myVs;
private IDictionary<string, string> myDic;
private ProjectHelper? myHelper;
readonly string? myColExp;
readonly string? myTableName;
readonly ViewStream? myVs;
private IDictionary<string, string>? myDic;
IDisposable? subscription;
/// <summary>
/// </summary>
/// <param name="vs">stream to susbsribe to as the input stream</param>
@@ -52,14 +59,13 @@ namespace BCG {
myDic = dic.ToDictionary(kv => kv.Value, kv => kv.Key);
}
protected override void myOnFirstSubscriber()
{
myVs.onChange += new BCG.DataStream.ChangeDelegate (myHandleChange);
}
protected override void myOnFirstSubscriber() =>
subscription = ExceptionAssert.MemberNotNull(myVs).Subscribe(myHandleChange);
protected override void myOnLastSubscriber()
{
myVs.onChange -= new BCG.DataStream.ChangeDelegate (myHandleChange);;
subscription?.Dispose();
subscription = null;
}
internal void myHandleChange (object newValue)
@@ -79,7 +85,7 @@ namespace BCG {
else
{ //first time a callback is received
myHelper = myDic == null ?
new ProjectHelper (dp.Schema, myColExp, myTableName) :
new ProjectHelper(dp.Schema, ExceptionAssert.MemberNotNull(myColExp), myTableName) :
new ProjectHelper (dp.Schema, myDic, myTableName);
IBlueRowCollection temp;
@@ -1,6 +1,5 @@
using System;
using System.Collections;
using BCG.Utilities;
namespace BCG
{
@@ -22,9 +21,11 @@ namespace BCG
#region IBlueRow Members
public object this [string colName] {get {return this[myHelper.schema[colName].colNo];}}
public object? this [string colName] =>
this[myHelper.schema[colName].colNo];
public object this [int index] {get {return myRow[myHelper.colMap[index]];}}
public object? this [int index] =>
myRow[myHelper.colMap[index]];
public IBlueTableSchema schema {get {return myHelper.schema;}}
@@ -36,9 +37,13 @@ namespace BCG
public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator (this);}
object System.Collections.IDictionary.this[object key]
object? IDictionary.this[object key]
{
get {return key is int ? this[(int)key] : this[key as string];}
get => key switch {
int i => this[i],
string s => this[s],
_ => throw new ArgumentException("Must be int or string", nameof(key)),
};
set {throw new ApplicationException ("Is read only");}
}
@@ -58,7 +63,8 @@ namespace BCG
}
}
public void Add (object key, object value) {throw new ApplicationException ("Is read only");}
public void Add(object key, object? value) =>
throw new NotSupportedException();
public ICollection Keys {get {return myHelper.schema.columnNames;}}
@@ -78,7 +84,7 @@ namespace BCG
array.SetValue (this[ci.colNo], index ++);
}
public object SyncRoot {get {return null;}}
public object SyncRoot => this;
#endregion
@@ -1,3 +1,5 @@
using Bts.DataStreamCore;
using System;
using System.Collections;
@@ -42,7 +44,7 @@ namespace BCG
public void CopyTo (Array array, int index)
{
foreach (IBlueRow r in myRows)
array.SetValue (this[(int)r["id"]], index++);
array.SetValue(this[r.Id()], index++);
}
@@ -72,14 +74,10 @@ namespace BCG
public void Reset() {myMaster.Reset();}
public object Current
{
get
{
return myMaster.Current == null ? null :
new ProjectedRow (myMaster.Current as IBlueRow, myHelper);
}
}
public object? Current =>
myMaster.Current is null
? null
: new ProjectedRow((IBlueRow)myMaster.Current, myHelper);
public bool MoveNext() {return myMaster.MoveNext ();}
@@ -1,25 +1,16 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.SQLite;
using System.Diagnostics;
using System.Collections;
using System.IO;
using System.Text;
using BCG;
using System.Linq;
using System.Data.Entity;
using System.Collections.Generic;
using System.Data.SQLite;
using System.Runtime.InteropServices;
namespace bts.BlueOrb{
public static class SQLiteHelper {
public static string location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:\\BlueOrb_Data\\" : Path.Combine(Environment.GetEnvironmentVariable("HOME"), "BlueOrb_Data");
public static string location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? "C:\\BlueOrb_Data\\"
: Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? string.Empty, "BlueOrb_Data");
public static string db_name = "stream_data.db";
public static DataTable execute_select_query(string connection_string, string query) {
@@ -1,6 +1,9 @@
using Spark.Core.Diagnostics;
using System;
using System.IO;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.IO;
namespace BCG {
[DataStreamUserRespresentation("CSVSaver")]
@@ -77,7 +80,7 @@ namespace BCG {
if (col != "id") {
string colText = String.Empty;
if (r[col] != null && r[col] != DBNull.Value)
colText = r[col].ToString();
colText = r[col]?.ToString() ?? string.Empty;
if (r.schema[col].colType == typeof(string))
colText = "\"" + colText + "\"";
rv += (i++ == 0 ? colText : "," + colText);
@@ -135,13 +138,14 @@ namespace BCG {
}
public class CSVReaderStreamMapper : IStreamMapper {
private IBlueTableSchema mySchema;
IBlueTableSchema? mySchema;
private readonly string[] myOutputSchemaStringArray;
public CSVReaderStreamMapper(string outputSchema) {
myOutputSchemaStringArray = outputSchema.Split(' ');
}
[MemberNotNull(nameof(mySchema))]
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
if (mySchema == null) {
ColumnInfo[] ci = inputSchema.columnInfo;
@@ -161,6 +165,7 @@ namespace BCG {
}
public IBlueRow map(IBlueRow row) {
ExceptionAssert.NotNull(mySchema);
object[] vals = new object[row.schema.columnInfo.Length];
row.Values.CopyTo(vals, 0);
for (int i = 0; i < mySchema.columnInfo.Length; i++) {
@@ -4,6 +4,7 @@ using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using Bts.ColumnSortExpressions;
using Bts.DataStreamCore;
namespace BCG;
@@ -54,7 +55,7 @@ public class SortedTable : IBlueRowCollection {
for (int i = 0; i < this.rows.Count; i++) {
if (newRow.CompareTo(this.rows[i]) <= 0) {
this.rows.Insert(i, newRow);
rowIndex[(int)newRow["id"]] = newRow;
rowIndex[newRow.Id()] = newRow;
if (++addedRowsIndex == addedRows.Count) {
return;
}
@@ -65,17 +66,17 @@ public class SortedTable : IBlueRowCollection {
for (int i = addedRowsIndex; i < addedRows.Count; i++) {
newRow = addedRows.getRowByIndex(i);
this.rows.Add(newRow);
rowIndex[(int)newRow["id"]] = newRow;
rowIndex[newRow.Id()] = newRow;
}
}
}
public void deleteRows(IBlueRowCollection deletedRows) {
foreach (IBlueRow deletedRow in deletedRows) {
var row = rowIndex[(int)deletedRow["id"]!] as SortRow
var row = rowIndex[deletedRow.Id()] as SortRow
?? throw new InvalidOperationException("Index did not return IBlueRow");
rows.Remove(row);
rowIndex.Remove((int)row["id"]!);
rowIndex.Remove(row.Id());
}
}
@@ -101,7 +102,7 @@ public class SortedTable : IBlueRowCollection {
SortRow newRow = comparer is null ?
new(row, sortCols) : new(row, comparer);
this.rows.Add(newRow);
rowIndex[(int)row["id"]!] = newRow;
rowIndex[row.Id()] = newRow;
}
this.rows.Sort();
}
@@ -19,7 +19,8 @@ namespace BCG
StreamReader reader = new StreamReader(myFile);
#region get schema
IBlueTableSchema schema = null; string schemaStr;
IBlueTableSchema? schema;
string? schemaStr;
if ((schemaStr = reader.ReadLine()) == null)
{
reader.Close();
@@ -33,7 +34,7 @@ namespace BCG
BlueTable adds = new BlueTable(schema);
BlueTable dels = new BlueTable(schema);
string rowStr;
string? rowStr;
while ((rowStr = reader.ReadLine()) != null)
{
adds.addRow(myGetRow(schema, rowStr));
@@ -101,7 +101,7 @@ namespace BCG
rowStr += DELIM;
}
rowStr += row[columnName].ToString();
rowStr += row[columnName]?.ToString();
}
writer.WriteLine(rowStr);
@@ -144,7 +144,7 @@ namespace BCG
private int myWriteFreqSecs;
private DateTime myNextWrite;
private bool myGotUpdate;
private string mySchemaStr;
private string? mySchemaStr;
#endregion
}
}
@@ -20,7 +20,7 @@ namespace BCG {
[DataStreamUserRespresentation("dateStandardize")]
public class DateStandardizationStream : ViewStream, IStreamMapper {
private IBlueTableSchema mySchema;
private IBlueTableSchema? mySchema;
private DateConversion myConversion;
private string myConversionColName;
private readonly Dictionary<DateConversion, Type> _date_conversion_to_type_map = new Dictionary<DateConversion, Type>()
@@ -60,8 +60,7 @@ namespace BCG {
List<ColumnInfo> cis = new List<ColumnInfo>();
foreach (ColumnInfo c in inputSchema.columnInfo) {
if (c.colName == myConversionColName) {
Type convType;
if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType)) {
if (!_date_conversion_to_type_map.TryGetValue(myConversion, out var convType)) {
//BCG.Utilities.BlueEventLog.logEvent("Please update the _date_conversion_to_type_map in DateStandardizationStream.cs.", BCG.Utilities.BlueEventLevel.Error);
convType = typeof(DateTime);
}
@@ -87,7 +86,7 @@ namespace BCG {
#endregion
public static object mapDateTime(object obj_in, DateConversion conv) {
public static object mapDateTime(object? obj_in, DateConversion conv) {
if (obj_in == DBNull.Value || obj_in == null) {
return DBNull.Value;
} else {
@@ -118,7 +117,7 @@ namespace BCG {
return day_period.TotalSeconds;
}
case DateConversion.ISO8601: {
string iso8601Date = obj_in.ToString();
var iso8601Date = obj_in.ToString() ?? string.Empty;
return DateTime.Parse(iso8601Date, null, System.Globalization.DateTimeStyles.RoundtripKind);
}
case DateConversion.EpochToDatetime: {
@@ -135,4 +134,4 @@ namespace BCG {
}
}
}
}
}
@@ -1,9 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Collections;
using System.Collections.Generic;
namespace BCG
{
@@ -11,7 +8,7 @@ namespace BCG
{
private ViewStream _src;
private string _time_stamp_field;
private IBlueTableSchema _schema;
IBlueTableSchema? _schema;
private string _latency_field;
public LatencyStream(ViewStream input, string time_stamp_field, string latency_field)
@@ -38,10 +35,11 @@ namespace BCG
public IBlueRow map(IBlueRow row)
{
var ht = new Hashtable(row);
if (row[_time_stamp_field].Equals(DBNull.Value))
ht[_latency_field] = DBNull.Value;
else
ht[_latency_field] = (DateTime.Now - (DateTime)row[_time_stamp_field]).TotalMilliseconds;
ht[_latency_field] = row[_time_stamp_field] switch {
null or DBNull => DBNull.Value,
DateTime ts => (DateTime.Now - ts).TotalMilliseconds,
_ => throw new ArgumentException("Row timestamp must be DateTime", nameof(row))
};
return new BlueRow(_schema, ht);
}
@@ -1,14 +1,15 @@
using System;
using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BCG
{
public class TimeAverageStream : ViewStream
{
private IDictionary<string, SortedList<DateTime, double>> myKeyToTimeToValue;
Dictionary<string, SortedList<DateTime, double>>? myKeyToTimeToValue;
private ViewStream myToAverage;
private string myCol_Key, myCol_Time, myCol_Value;
private double myAveragePeriodSeconds;
@@ -42,8 +43,8 @@ namespace BCG
BlueTable result = new BlueTable(dp.Schema);
foreach (IBlueRow r in dp.newRows)
{
string key = r[myCol_Key].ToString();
DateTime dt = (DateTime) r[myCol_Time];
string key = r[myCol_Key]?.ToString() ?? string.Empty;
DateTime dt = (DateTime)r[myCol_Time]!;
double value = Convert.ToDouble(r[myCol_Value]);
object[] rowValues = new object[r.Count];
@@ -67,9 +68,8 @@ namespace BCG
private double myUpdateValue(string key, DateTime time, double value)
{
SortedList<DateTime, double> seriesValues;
if (!myKeyToTimeToValue.TryGetValue(key, out seriesValues))
ExceptionAssert.MemberNotNull(myKeyToTimeToValue);
if (!myKeyToTimeToValue.TryGetValue(key, out var seriesValues))
{
seriesValues = new SortedList<DateTime, double>();
myKeyToTimeToValue[key] = seriesValues;
@@ -1,5 +1,8 @@
using Bts.DataStreamCore;
using System;
using System.Collections;
using System.Collections.Generic;
namespace BCG
{
@@ -10,9 +13,9 @@ namespace BCG
internal class AppendCollection : IBlueRowCollection
{
private int myId = 0;
private IDictionary myLeftIndex = new Hashtable (),
myRightIndex = new Hashtable ();
private IBlueTableSchema mySchema;
readonly Dictionary<int, IBlueRow> myLeftIndex = new();
readonly Dictionary<int, IBlueRow> myRightIndex = new();
IBlueTableSchema? mySchema;
public AppendCollection () {}
@@ -24,15 +27,15 @@ namespace BCG
IDictionary index = isLeft ? myLeftIndex : myRightIndex;
foreach (IBlueRow r in dp.deletedRows)
{
IBlueRow dr = index[r["id"]] as IBlueRow;
var dr = index[r.Id()] as IBlueRow;
dRows.addRow (dr);
index.Remove (r["id"]);
index.Remove(r.Id());
}
foreach (IBlueRow r in dp.addedRows)
{
IBlueRow ar = new AppendRow (r, myId++, mySchema);
aRows.addRow (ar);
index.Add (r["id"], ar);
index.Add(r.Id(), ar);
}
return new DataPacket (this, aRows, dRows);
}
@@ -43,7 +46,8 @@ namespace BCG
public bool containsID (int id) {throw new ApplicationException ("IBlueRow is readonly");}
public IBlueTableSchema schema {get {return mySchema;}}
public IBlueTableSchema schema =>
mySchema ?? throw new InvalidOperationException("Schema is unknown (no rows added)");
#endregion
@@ -60,7 +64,7 @@ namespace BCG
array.SetValue (r, i++);
}
public object SyncRoot {get {return null;}}
public object SyncRoot => this;
#endregion
@@ -87,7 +91,7 @@ namespace BCG
public void Reset() {myEnum = myLeft.GetEnumerator ();}
public object Current {get {return myEnum.Current;}}
public object? Current {get {return myEnum.Current;}}
public bool MoveNext()
{
@@ -23,12 +23,12 @@ namespace BCG
#region IBlueRow Members
public object this [string colName]
public object? this [string colName]
{
get {return colName.Equals("id") ? myId : myRow[colName];}
}
public object this [int index]
public object? this [int index]
{
get {return this[schema[index].colName];}
}
@@ -43,10 +43,13 @@ namespace BCG
public System.Collections.IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator (this);}
object System.Collections.IDictionary.this[object key]
{
get {return key is string ? this[key as string] : this [(int)key];}
set {throw new ApplicationException ("IBlueRow is readonly");}
object? IDictionary.this[object key] {
get => key switch {
string colName => this[colName],
int colIndex => this[colIndex],
_ => throw new ArgumentException("Must be string or int", nameof(key)),
};
set => throw new NotSupportedException();
}
public void Remove (object key) {throw new ApplicationException ("IBlueRow is readonly");}
@@ -65,7 +68,8 @@ namespace BCG
}
}
public void Add (object key, object value) {throw new ApplicationException ("IBlueRow is readonly");}
public void Add(object key, object? value) =>
throw new NotSupportedException();
public System.Collections.ICollection Keys {get {return schema.columnNames;}}
@@ -29,7 +29,7 @@ namespace BCG {
public IBlueTableSchema schema { get { return myRow.schema; } }
public object this[int index] {
public object? this[int index] {
get {
if (myRow.schema[index].colName == "id")
return myID;
@@ -38,15 +38,18 @@ namespace BCG {
}
}
public object this[string colName] { get { return colName == "id" ? (object)myID : myRow[colName]; } }
public object? this[string colName] =>
colName == "id" ? myID : myRow[colName];
#endregion
#region IDictionary Members
public void Add(object key, object value) { throw new Exception("Row is read only."); }
public void Add(object key, object? value) =>
throw new NotSupportedException();
public void Clear() { throw new Exception("Row is read only."); }
public void Clear() =>
throw new NotSupportedException();
public bool Contains(object key) { return myRow.Contains(key); }
@@ -68,9 +71,13 @@ namespace BCG {
}
}
public object this[object key] {
get { return key is string ? this[key as string] : this[(int)key]; }
set { throw new Exception("The method or operation is not implemented."); }
public object? this[object key] {
get => key switch {
string colName => this[colName],
int colIndex => this[colIndex],
_ => throw new ArgumentException("Must be string or int", nameof(key)),
};
set => throw new NotSupportedException();
}
#endregion
@@ -112,18 +119,18 @@ namespace BCG {
myIDMap.Add(tableNo, new Dictionary<int, IBlueRow>());
IBlueRow nr = new MyRow(r, myID++); // BlueDataUtility.getNewReplacedRow(r, myID++, "id");
myRows.addRow(nr);
myIDMap[tableNo].Add((int)r["id"], nr);
myIDMap[tableNo].Add(r.Id(), nr);
return nr;
}
public IBlueRow removeRow(IBlueRow r, int tableNo) {
IBlueRow dr = myIDMap[tableNo][(int)r["id"]];
IBlueRow dr = myIDMap[tableNo][r.Id()];
myRows.deleteRow(dr);
myIDMap[tableNo].Remove((int)r["id"]);
myIDMap[tableNo].Remove(r.Id());
return dr;
}
public IBlueRowCollection removeTable(int tableNo) {
public IBlueRowCollection? removeTable(int tableNo) {
if (!myIDMap.ContainsKey(tableNo))
return null;
else {
@@ -170,7 +177,7 @@ namespace BCG {
#endregion
}
private bool mySubscriptionActive = false;
private MutableAppendTable myTable;
MutableAppendTable? myTable;
private int myMovingIndex;
bool isFirstUpdate = true;
ImmutableDictionary<int, (ViewStream Input, ViewStream Output, IDisposable? Subscription)> myStreamIndex =
@@ -222,7 +229,7 @@ namespace BCG {
if (mySubscriptionActive) {
sub?.Dispose();
if (myTable != null) {
IBlueRowCollection delRows = myTable.removeTable(kvp.Key);
IBlueRowCollection? delRows = myTable.removeTable(kvp.Key);
if (delRows != null)
myFireEvent(new DataPacket(myTable, new BlueTable(myTable.schema), delRows));
}
@@ -99,7 +99,8 @@ namespace BCG
}
}
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
public void Add(object key, object? value) =>
throw new NotSupportedException();
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
@@ -1,4 +1,4 @@
using System;
#nullable enable
using System.Collections;
namespace BCG
@@ -10,14 +10,7 @@ namespace BCG
public class BlueRowDictionaryEnumerator : BlueRowEnumerator, IDictionaryEnumerator
{
public BlueRowDictionaryEnumerator (IBlueRow row) : base (row) {}
public override object Current {get{return Entry;}}
#region IDictionaryEnumerator members
public object Key {get{return myRow.schema[myPos].colName;}}
public object Value {get{return myRow[myPos];}}
public DictionaryEntry Entry {get{return new DictionaryEntry (Key, Value);}}
#endregion
public override object? Current => Entry;
}
}
@@ -1,13 +1,14 @@
using System;
using System.Data;
using System.Collections;
using System.IO;
using System.Reflection;
using System.Collections.Generic;
using System.Linq;
using BCG.Utilities.General;
using Spark.Core.Collections;
using System;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.Linq;
using System.Reflection;
namespace BCG {
/// <summary>
/// The BlueTableSchema is an immutable schema holder. Once created it cannot be modified. It can be created
@@ -117,7 +118,7 @@ namespace BCG {
// }
}
protected Hashtable myIndexByColName = new Hashtable();
protected Dictionary<string, ColumnInfo> myIndexByColName = new();
protected ColumnInfo[] myIndexByColOrder;
private string[] myTableColNames;
protected string myTableName;
@@ -187,18 +188,8 @@ namespace BCG {
/// <summary>
/// The returned Array is sorted according to the Column No
/// </summary>
public string[] columnNames {
get {
if (myTableColNames == null) {
myTableColNames = new string[myIndexByColName.Count];
foreach (DictionaryEntry de in myIndexByColName) {
ColumnInfo temp = (ColumnInfo)de.Value;
myTableColNames[temp.colNo] = temp.colName;
}
}
return myTableColNames;
}
}
public string[] columnNames =>
myTableColNames ??= myIndexByColName.Values.ToIndexedArray(c => (c.colNo, c.colName));
/// <summary>
/// The returned Array is sorted according to the Column No
@@ -1,3 +1,4 @@
#nullable enable
using System;
using System.Text.Json.Serialization;
@@ -6,10 +7,10 @@ namespace BCG {
/// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
/// </summary>
[Serializable]
public class ColumnInfo : IComparable {
public class ColumnInfo : IComparable<ColumnInfo>, IComparable, IEquatable<ColumnInfo> {
//the default column info does not cach values in a lookup table, therefore it is in group 0
public ColumnInfo(string colName, int colNo, Type colType, object colValue = null) {
public ColumnInfo(string colName, int colNo, Type colType, object? colValue = null) {
myColName = colName;
myColNo = colNo;
myColType = colType;
@@ -27,8 +28,14 @@ namespace BCG {
type == typeof(string) ||
type == typeof(DateTime);
public int CompareTo(object obj) {
int otherColNo = ((ColumnInfo)(obj)).colNo;
public int CompareTo(object? obj) =>
CompareTo(obj as ColumnInfo);
public int CompareTo(ColumnInfo? other) {
if (other is null) {
return 1;
}
int otherColNo = other.colNo;
if (myColNo < otherColNo)
return -1;
else if (myColNo > otherColNo)
@@ -36,18 +43,21 @@ namespace BCG {
return 0;
}
public override bool Equals(object obj) {
if (obj is ColumnInfo) {
ColumnInfo other = obj as ColumnInfo;
return (colNo == other.colNo) && (colName == other.colName) && (colType == other.colType);
}
return false;
}
public override bool Equals(object? obj) =>
Equals(obj as ColumnInfo);
public bool Equals(ColumnInfo? other) =>
other is not null
&& colNo == other.colNo
&& colName == other.colName
&& colType == other.colType;
public override string ToString() {
return colName + " " + colNo + " " + colType;
}
public override int GetHashCode() =>
HashCode.Combine(colNo, colName, colType);
// for backwards compatibility to allow build
public bool isConstant {get{return myColValue != null;}}
@@ -55,7 +65,7 @@ namespace BCG {
private int myColNo;
private string myColName;
private Type myColType;
private object myColValue;
private object? myColValue;
[JsonPropertyName("colNo")]
public int colNo { get { return myColNo; } }
@@ -67,6 +77,6 @@ namespace BCG {
public Type colType { get { return myColType; } }
[JsonIgnore]
public object colValue {get{return myColValue;}}
public object? colValue {get{return myColValue;}}
}
}
@@ -1,24 +1,39 @@
using System;
#nullable enable
using BCG;
using System.Collections;
using Spark.Core.Collections;
using System;
using System.IO;
// common place for all serialization interfaces
namespace bts {
// this is the data that is sent across the write for stream IO
public class StreamIOMessage {
public class StreamIOMessage : IEquatable<StreamIOMessage> {
public enum StreamOp {
CREATE = 0,
// removes and adds multiple rows
// other operations can be expressed with this
MULTIPLE_REMOVE_ADD = 1,
}
public class CreateMessage {
public class CreateMessage : IEquatable<CreateMessage> {
public IBlueRowCollection rows;
public CreateMessage(IBlueRowCollection rows_) {
rows = rows_;
}
public bool Equals(CreateMessage? other) =>
other is not null
&& BlueDataUtility.compute_equality(rows, other.rows);
public override bool Equals(object? obj) =>
Equals(obj as CreateMessage);
public override int GetHashCode() =>
rows.Count;
public override string ToString() {
string rv = "Create message ";
if (rows == null) {
@@ -30,21 +45,27 @@ namespace bts {
}
return rv;
}
public override bool Equals(object obj) {
if (obj != null && obj is CreateMessage) {
var other = obj as CreateMessage;
return BlueDataUtility.compute_equality(rows, other.rows);
}
return false;
}
}
public class MultipleRemoveAddMessage {
public class MultipleRemoveAddMessage : IEquatable<MultipleRemoveAddMessage> {
public IBlueRowCollection removed_rows;
public IBlueRowCollection added_rows;
public MultipleRemoveAddMessage(IBlueRowCollection removed_rows_, IBlueRowCollection added_rows_) {
removed_rows = removed_rows_;
added_rows = added_rows_;
}
public bool Equals(MultipleRemoveAddMessage? other) =>
other is not null
&& BlueDataUtility.compute_equality(removed_rows, other.removed_rows)
&& BlueDataUtility.compute_equality(added_rows, other.added_rows);
public override bool Equals(object? obj) =>
Equals(obj as MultipleRemoveAddMessage);
public override int GetHashCode() =>
HashCode.Combine(removed_rows.Count, added_rows.Count);
public override string ToString() {
string rv = "MultipleRemoveAdd removed: ";
foreach (IBlueRow r in removed_rows) {
@@ -56,56 +77,46 @@ namespace bts {
}
return rv;
}
public override bool Equals(object obj) {
if (obj != null && obj is MultipleRemoveAddMessage) {
MultipleRemoveAddMessage other = obj as MultipleRemoveAddMessage;
return BlueDataUtility.compute_equality(removed_rows, other.removed_rows) &&
BlueDataUtility.compute_equality(added_rows, other.added_rows);
}
return false;
}
}
public StreamOp op;
public readonly CreateMessage create_message = null;
public readonly MultipleRemoveAddMessage multiple_remove_add_message = null;
public readonly CreateMessage? create_message;
public readonly MultipleRemoveAddMessage? multiple_remove_add_message;
public StreamIOMessage(CreateMessage create_message_) {
op = StreamOp.CREATE;
create_message = create_message_;
multiple_remove_add_message = null;
}
public StreamIOMessage(MultipleRemoveAddMessage multiple_remove_add_message_) {
op = StreamOp.MULTIPLE_REMOVE_ADD;
create_message = null;
multiple_remove_add_message = multiple_remove_add_message_;
}
public bool Equals(StreamIOMessage? other) =>
other is not null
&& op == other.op
&& op switch {
StreamOp.CREATE => NullCheckEqualityComparer.Equals(create_message, other.create_message),
StreamOp.MULTIPLE_REMOVE_ADD => NullCheckEqualityComparer.Equals(multiple_remove_add_message, other.multiple_remove_add_message),
_ => false,
};
public override bool Equals(object? obj) =>
Equals(obj as StreamIOMessage);
public override int GetHashCode() =>
HashCode.Combine(op, create_message, multiple_remove_add_message);
public override string ToString() {
switch (op) {
case StreamIOMessage.StreamOp.CREATE:
return create_message.ToString();
return create_message?.ToString() ?? string.Empty;
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD:
return multiple_remove_add_message.ToString();
return multiple_remove_add_message?.ToString() ?? string.Empty;
}
bts.Log.assert(false);
return String.Empty;
}
public override bool Equals(object obj) {
if (obj != null && obj is StreamIOMessage) {
StreamIOMessage other = obj as StreamIOMessage;
if (op == other.op) {
switch (op) {
case StreamOp.CREATE:
return create_message.Equals(other.create_message);
case StreamOp.MULTIPLE_REMOVE_ADD:
return multiple_remove_add_message.Equals(other.multiple_remove_add_message);
}
}
}
return false;
}
}
#region serialization
// This helps putting stream callbacks (op + data) on the wire, to send it to the pubsubserver.
@@ -134,4 +145,4 @@ namespace bts {
}
#endregion
}
}
@@ -1,12 +1,10 @@
#nullable enable
namespace bts {
using BCG;
using System;
using System.Collections.Generic;
using System.IO;
using System.Reactive.Subjects;
using System.Reflection.Metadata;
using System.Runtime.Serialization.Formatters.Binary;
using System.Text;
public static class StreamSerializationUtils {
@@ -122,17 +120,24 @@ namespace bts {
}
public static object deserialize_column(ref BinaryReader br, Type t) {
Func<BinaryReader, object> reader_func;
Log.assert(_col_deser_map.TryGetValue(t, out reader_func));
return reader_func(br);
if (_col_deser_map.TryGetValue(t, out var deserialize)) {
return deserialize(br);
}
var msg = $"Deserialization function not defined for type '{t}'";
Log.assert(false, msg);
throw new ArgumentException(msg, nameof(t));
}
public static void serialize_row_column(BinaryWriter bw, IBlueRow r, int col_no) {
var ci = r.schema.columnInfo[col_no];
var type = ci.colType;
Action<BinaryWriter, IBlueRow, int> ser_action;
Log.assert(_col_ser_map.TryGetValue(type, out ser_action));
ser_action(bw, r, col_no);
if (_col_ser_map.TryGetValue(type, out var serialize)) {
serialize(bw, r, col_no);
return;
}
var msg = $"Serialization function not defined for type '{type}' (failed on column '{ci.colName}')";
Log.assert(false, msg);
throw new ArgumentException(msg, nameof(r));
}
public static ColumnInfo deserialize_column_info(ref BinaryReader br) {
@@ -226,4 +231,4 @@ namespace bts {
}
}
}
}
@@ -1,6 +1,4 @@
using NUnit.Framework;
using System.Reactive.Disposables;
using Spark.Core.Diagnostics;
namespace bts {
[TestFixture]
@@ -41,7 +39,7 @@ namespace bts {
sub_msg_desered.construct_deserialize(str);
Assert.That(sub_msg_desered.connection, Is.EqualTo(sub_msg.connection));
Assert.That(sub_msg_desered.subscribing_app, Is.EqualTo(sub_msg.subscribing_app));
compare_arrays(sub_msg.tags, sub_msg_desered.tags);
compare_arrays(sub_msg.tags, ExceptionAssert.NotNull(sub_msg_desered.tags));
Assert.That(sub_msg_desered, Is.EqualTo(sub_msg));
}
@@ -60,7 +58,7 @@ namespace bts {
sub_msg_desered.construct_deserialize(str);
Assert.That(sub_msg_desered.connection, Is.EqualTo(unsub_msg.connection));
Assert.That(sub_msg_desered.subscribing_app, Is.EqualTo(unsub_msg.subscribing_app));
compare_arrays<string>(unsub_msg.tags, sub_msg_desered.tags);
compare_arrays<string>(unsub_msg.tags, ExceptionAssert.NotNull(sub_msg_desered.tags));
}
[Test]
@@ -87,7 +85,7 @@ namespace bts {
Assert.That(pub_msg_desered.tag_type, Is.EqualTo(pub_msg.tag_type));
Assert.That(pub_msg_desered.is_compressed, Is.EqualTo(pub_msg.is_compressed));
Assert.That(pub_msg_desered.publish_time_epoch_ms, Is.EqualTo(pub_msg.publish_time_epoch_ms));
Assert.That(pub_msg_desered.data_to_publish.Length, Is.EqualTo(pub_msg.data_to_publish.Length));
Assert.That(ExceptionAssert.NotNull(pub_msg_desered.data_to_publish).Length, Is.EqualTo(pub_msg.data_to_publish.Length));
compare_arrays(pub_msg.data_to_publish, pub_msg_desered.data_to_publish);
}
@@ -105,7 +103,7 @@ namespace bts {
ps_server_msg_desered.construct_deserialize(str);
Assert.That(ps_server_msg_desered.tag, Is.EqualTo(ps_server_msg.tag));
Assert.That(ps_server_msg_desered.tag_type, Is.EqualTo(ps_server_msg.tag_type));
compare_arrays(ps_server_msg.data, ps_server_msg_desered.data);
compare_arrays(ps_server_msg.data, ExceptionAssert.NotNull(ps_server_msg_desered.data));
}
[Test]
@@ -20,8 +20,7 @@ namespace BCG {
}
public IPubSubServerConnection get_connection(RemoteConnectionInfo server_rci) {
IPubSubServerConnection rv = null;
if (!_conn_map.TryGetValue(server_rci, out rv)) {
if (!_conn_map.TryGetValue(server_rci, out var rv)) {
rv = _build_connection(server_rci);
_conn_map.Add(server_rci, rv);
}
@@ -1,9 +1,11 @@
using bts;
using mbts;
using System;
using System.Collections.Generic;
using System.Text;
using System.Diagnostics.CodeAnalysis;
using System.IO;
//using BCG.Utilities;
using mbts;
using System.Linq;
namespace BCG {
@@ -14,8 +16,8 @@ namespace BCG {
/// </summary>
public class TagPubSubInfo {
public string tag;
public IDictionary<string, string> subscribers,
publishers;
public IDictionary<string, string> subscribers;
public IDictionary<string, string> publishers;
public ICollection<Pair<string, string>> dead_publishers;
public string pubsub_server;
@@ -58,6 +60,8 @@ namespace BCG {
}
bw.Write(dead_publishers.Count);
foreach (Pair<string, string> p in dead_publishers) {
Log.notnull(p.first);
Log.notnull(p.second);
bw.Write(p.first);
bw.Write(p.second);
}
@@ -91,6 +95,7 @@ namespace BCG {
/// constructs itself from the stream
/// </summary>
/// <param name="s"></param>
[MemberNotNull(nameof(tag), nameof(subscribers), nameof(publishers), nameof(dead_publishers), nameof(pubsub_server))]
private void _read_from(Stream s) {
BinaryReader br = new BinaryReader(s);
tag = br.ReadString();
@@ -1,11 +1,10 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
// using BCG.Utilities;
using BCG.Utilities.General;
using BCG.Utilities.General;
using mbts;
using System;
using System.Threading;
namespace BCG
{
/// <summary>
@@ -62,7 +61,7 @@ namespace BCG
// w.ToString(), myWorkers.Count), PS3LogLevels.connection);
myWorkers.Remove(w);
}
foreach (KeyValuePair<IWorker, Pair<TimeSpan, DateTime>> p in myJustAdded)
foreach (var p in myJustAdded)
{
// BlueEventLog.logEvent(String.Format("ThreadDrivenMasterWorker : adding worker {0} , worker count before adding this = {1}",
// p.Key.ToString(), myWorkers.Count), PS3LogLevels.connection);
@@ -78,7 +77,7 @@ namespace BCG
{
// BlueEventLog.logEvent(ex);
if (ex is OutOfMemoryException)
throw ex;
throw;
}
}
}
@@ -1,11 +1,10 @@
using System;
using System.IO;
namespace BCG
{
public interface IGenericFormatter
{
T Deserialize<T> (Stream serializationStream);
void Serialize<T> (Stream serializationStream, T graph);
T Deserialize<T> (Stream serializationStream) where T : notnull;
void Serialize<T> (Stream serializationStream, T graph) where T : notnull;
}
}
@@ -2,17 +2,20 @@ namespace bts;
using BCG;
using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Net;
using System.Net.Sockets;
// opens a TCP port and starts listening to connections
public class AsyncTCPConnectionAcceptor_APM {
private readonly int _port;
private Socket _accept_socket;
private ITCPConnectionHandler _connection_handler;
private HashSet<string> _key_map = new HashSet<string>();
Socket? _accept_socket;
readonly ITCPConnectionHandler _connection_handler;
readonly HashSet<string> _key_map = new();
public AsyncTCPConnectionAcceptor_APM(int port, ITCPConnectionHandler connection_handler) {
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM. cons started on port: {port}");
@@ -25,6 +28,7 @@ public class AsyncTCPConnectionAcceptor_APM {
_begin_accepting();
}
[MemberNotNull(nameof(_accept_socket))]
void _init() {
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._init started on port: {_port}");
_accept_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
@@ -34,7 +38,9 @@ public class AsyncTCPConnectionAcceptor_APM {
_accept_socket.Listen(250);
}
[MemberNotNull(nameof(_accept_socket))]
void _begin_accepting() {
ArgumentNullException.ThrowIfNull(_accept_socket);
var accept_key = Guid.NewGuid().ToString();
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._begin_accepting port: {_port} accept_key: {accept_key}");
var res = _accept_socket.BeginAccept(new AsyncCallback(_on_accept), accept_key);
@@ -46,14 +52,14 @@ public class AsyncTCPConnectionAcceptor_APM {
var log_msg = $"AsyncTCPConnectionAcceptor_APM._on_accept state: {res.AsyncState}";
Log.Gen.info(log_msg);
Console.WriteLine(log_msg);
string key = res.AsyncState.ToString();
var key = ExceptionAssert.NotNull(res.AsyncState).ToString() ?? string.Empty;
if (_key_map.Contains(key)) {
return;
}
_key_map.Add(key);
Socket new_sock;
try {
new_sock = _accept_socket.EndAccept(res);
new_sock = ExceptionAssert.NotNull(_accept_socket).EndAccept(res);
} catch (Exception ex) {
bts.Log.Gen.info("AsyncTCPConnectionAcceptor_APM._on_accept exception thrown by EndAccept ex: {ex}");
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(250));
@@ -5,6 +5,8 @@ using BCG;
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using System.Diagnostics.CodeAnalysis;
using Spark.Core.Diagnostics;
namespace bts {
@@ -13,10 +15,10 @@ namespace bts {
// it uses TcpListener, instead of raw sockets.
public class AsyncTCPConnectionAcceptor_AsyncWrapper {
private readonly int _accept_port;
private Socket _accept_socket;
// private Socket _accept_socket;
private ITCPConnectionHandler _connection_handler;
private TcpListener _server;
private Action<ITCPConnection> _cbs = (c) => { };
TcpListener? _server;
// private Action<ITCPConnection> _cbs = (c) => { };
public AsyncTCPConnectionAcceptor_AsyncWrapper(int accept_port, ITCPConnectionHandler connection_handler) {
Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper.cons started on port: {accept_port}");
@@ -25,9 +27,10 @@ namespace bts {
}
public void resgiter_callback(Action<ITCPConnection> cb) {
_cbs += cb;
// _cbs += cb;
}
[MemberNotNull(nameof(_server))]
public void start() {
Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._init started on port: {_accept_port}");
_server = new TcpListener(IPAddress.Any, _accept_port);
@@ -36,6 +39,7 @@ namespace bts {
}
private void _accept_connections_forever() {
ExceptionAssert.MemberNotNull(_server);
Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port}");
while (true) {
// this is a blocking call
@@ -55,4 +59,4 @@ namespace bts {
}
}
}
}
@@ -2,13 +2,13 @@ namespace bts
{
using BCG;
using BCG.Utilities.General;
using Spark.Core.Diagnostics;
using System;
using System.IO;
using System.Net.Sockets;
using System.Threading.Tasks;
using System.Threading;
using System.Net;
using System.Text;
// this ITCPConnection uses the TCPClient class instead of a raw socket (in the hope, that it would configure the socket correctly)
@@ -34,7 +34,7 @@ namespace bts
private const int HB_MSG_SIGNAL = 1;
private TcpClient _tcp_client;
private NetworkStream _nw_stream;
private ITCPMessageHandler _msg_handler;
ITCPMessageHandler? _msg_handler;
private bool _is_alive = true;
ConnectionID _connection_id;
@@ -191,7 +191,7 @@ namespace bts
private class OneRecvMessage
{
public int msg_type;
public byte[] data;
public byte[]? data;
public OneRecvMessage(int msg_type_, byte[] data_)
{
msg_type = msg_type_;
@@ -211,7 +211,7 @@ namespace bts
public override string ToString()
{
string rv = $"OneMessage msg_type: {msg_type} data_len: {data.Length}";
string rv = $"OneMessage msg_type: {msg_type} data_len: {data?.Length ?? 0}";
return rv;
}
}
@@ -237,6 +237,7 @@ namespace bts
private void _read_msgs_forever()
{
ExceptionAssert.MemberNotNull(_msg_handler);
while (_is_alive)
{
try
@@ -270,7 +271,7 @@ namespace bts
public class ErrorMessage
{
public readonly SocketError? socket_error = null;
public readonly Exception exception = null;
public readonly Exception? exception = null;
public readonly bool is_exception = false;
public readonly string context = String.Empty;
public ErrorMessage(SocketError socket_error_, string context_)
@@ -313,9 +314,9 @@ namespace bts
{
String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket");
if (ex is OutOfMemoryException)
throw ex;
throw;
}
_msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _connection_id.peer));
_msg_handler?.onData(new RemoteMessage(null, RemoteMessageType.Disconnection, _connection_id.peer));
}
}
}
@@ -325,8 +326,10 @@ namespace bts
lock (this)
{
Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
if (err_msg.oom)
if (err_msg.oom) {
Log.notnull(err_msg.exception);
throw err_msg.exception;
}
Dispose();
}
}
@@ -336,4 +339,4 @@ namespace bts
return _connection_id.ToString();
}
}
}
}
@@ -1,10 +1,10 @@
using System.Net.Sockets;
using System.Net;
using BCG;
using System;
using System.Collections.Generic;
using Spark.Core.Diagnostics;
using System;
using System.Net;
using System.Net.Sockets;
namespace bts {
// Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests
@@ -42,7 +42,7 @@ namespace bts {
private static void _on_connect(IAsyncResult res) {
Log.Gen.info($"AsyncTCPConnector_APM._on_connect res: {res.AsyncState}");
SocketConnectionArgs sc_args = res.AsyncState as SocketConnectionArgs;
var sc_args = ExceptionAssert.Cast(res.AsyncState).To<SocketConnectionArgs>();
try {
sc_args.sock.EndConnect(res);
} catch (SocketException socket_ex) {
@@ -1,12 +1,11 @@
using bts;
using Bts.Net;
using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using bts.utils;
using System.Net.NetworkInformation;
using bts;
using Bts.Net;
using System.Linq;
using System.Net;
namespace BCG {
public class RemoteConnectionInfo : IEquatable<RemoteConnectionInfo> {
@@ -106,13 +105,7 @@ namespace BCG {
return ip.GetHashCode() ^ port.GetHashCode();
}
public override bool Equals(object? obj) {
if (obj == null) {
return false;
}
RemoteConnectionInfo other = obj as RemoteConnectionInfo;
return other != null && other.ip == ip && other.port == port;
}
public override bool Equals(object? obj) =>
obj is RemoteConnectionInfo other && other.ip == ip && other.port == port;
}
}
@@ -1,14 +1,14 @@
using bts;
using Spark.Core.Diagnostics;
using System;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using bts;
using System.Threading;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
namespace BCG {
// utility functions for TCP socket async communications. Does the following
@@ -50,7 +50,9 @@ namespace BCG {
Marshal.Copy(ptr, inOptionValues, 0, size);
Marshal.FreeHGlobal(ptr);
server_socket.IOControl(IOControlCode.KeepAliveValues, inOptionValues, null);
if (OperatingSystem.IsWindows()) {
server_socket.IOControl(IOControlCode.KeepAliveValues, inOptionValues, null);
}
}
// Set other socket options as needed
@@ -67,14 +69,14 @@ namespace BCG {
}
public static RemoteConnectionInfo get_peer_address(Socket sock) {
IPEndPoint ip_ep = sock.RemoteEndPoint as IPEndPoint;
var ip_ep = ExceptionAssert.Cast(sock.RemoteEndPoint).To<IPEndPoint>();
int port = ip_ep.Port;
string ip = ip_ep.Address.ToString();
return new RemoteConnectionInfo(ip, port);
}
public static RemoteConnectionInfo get_self_address(Socket socket) {
IPEndPoint ip_ep = socket.LocalEndPoint as IPEndPoint;
var ip_ep = ExceptionAssert.Cast(socket.LocalEndPoint).To<IPEndPoint>();
int port = ip_ep.Port;
string ip = ip_ep.Address.ToString();
return new RemoteConnectionInfo(ip, port);
@@ -101,7 +103,9 @@ namespace BCG {
Marshal.Copy(ptr, inOptionValues, 0, size);
Marshal.FreeHGlobal(ptr);
client_socket.IOControl(IOControlCode.KeepAliveValues, inOptionValues, null);
if (OperatingSystem.IsWindows()) {
client_socket.IOControl(IOControlCode.KeepAliveValues, inOptionValues, null);
}
}
// Set other socket options as needed
@@ -126,7 +130,7 @@ namespace BCG {
Socket listener = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
listener.Bind(ip_end_point);
listener.Listen(10);
Socket client_socket = null;
Socket? client_socket;
while (true) {
try {
// keeps on accepting connections
@@ -135,16 +139,16 @@ namespace BCG {
handler.onConnection(new_conn);
} catch (Exception ex) {
bts.Log.Gen.error($"TCPConnection.accept ex: {ex} trying again");
Task.Delay(wait_after_error);
await Task.Delay(wait_after_error);
}
}
}
// connects to the server and ona successful connection, calls con__handler.on_connection_async with an ITCPConnection.
// In case of failures to connect it waits for a given amount of time before retrying
public static async Task<Socket> connect_to_server_async(string ip, int server_port, ITCPConnectionHandler conn_handler,
public static async Task<Socket?> connect_to_server_async(string ip, int server_port, ITCPConnectionHandler conn_handler,
TimeSpan wait_after_failed_attempt, int n_max_attempts = 1024) {
Socket client_socket = null;
Socket? client_socket = null;
for (int i = 0; i < n_max_attempts; i++) {
try {
string log_conn = $"{ip}:{server_port}";
@@ -158,7 +162,7 @@ namespace BCG {
break;
} catch (Exception ex) {
bts.Log.Gen.error($"AsyncTCPConnection2.connect_to_server threw ex: {ex}, will reattempt now");
Task.Delay(wait_after_failed_attempt);
await Task.Delay(wait_after_failed_attempt);
}
}
return client_socket;
@@ -281,4 +285,4 @@ namespace BCG {
#endregion
}
}
@@ -2,9 +2,6 @@ using System;
using System.Collections;
using System.Threading;
using BCG;
using BCG.Utilities;
namespace BCG
{
[Serializable]
@@ -36,13 +33,10 @@ namespace BCG
}
}
public object dequeue()
{
return dequeue(Timeout.Infinite);
}
public object? dequeue() =>
dequeue(Timeout.Infinite);
public object dequeue (int millisecondsToWait)
{
public object? dequeue (int millisecondsToWait) {
for (;;)
lock (this)
{
@@ -60,13 +54,13 @@ namespace BCG
}
}
public object[] dequeueMultipleElements(int millisecondsToWait, int maxElements)
public object?[] dequeueMultipleElements(int millisecondsToWait, int maxElements)
{
ArrayList rv = new ArrayList();
int n = getCount();
for (int i = 0; i < n; i++)
{
object o = dequeue(0);
var o = dequeue(0);
if (o != null)
{
rv.Add(o);
@@ -78,7 +72,7 @@ namespace BCG
int timeRemaining = millisecondsToWait;
while (true)
{
object o = dequeue(timeRemaining);
var o = dequeue(timeRemaining);
if (o != null)
rv.Add(o);
timeRemaining = millisecondsToWait - Convert.ToInt32 (((TimeSpan)(DateTime.Now - startTime)).TotalMilliseconds);
@@ -87,10 +81,8 @@ namespace BCG
}
}
public object[] dequeueMultipleElements(TimeSpan ts, int maxElements)
{
return dequeueMultipleElements(Convert.ToInt32 (ts.TotalMilliseconds), maxElements);
}
public object?[] dequeueMultipleElements(TimeSpan ts, int maxElements) =>
dequeueMultipleElements(Convert.ToInt32 (ts.TotalMilliseconds), maxElements);
public int getCount()
{
@@ -1,8 +1,5 @@
using System;
using System.Text;
using BCG.Utilities.General;
namespace BCG
{
@@ -43,7 +40,7 @@ namespace BCG
set { myStartTime = value; }
}
private string myProcessString;
string? myProcessString;
public string processString
{
get
@@ -84,7 +81,7 @@ namespace BCG
myStartTime = System.Diagnostics.Process.GetCurrentProcess().StartTime;
}
private static ClientInfo myLocalClient;
static ClientInfo? myLocalClient;
private static object myLock = new object();
public static ClientInfo thisClient
@@ -135,4 +132,4 @@ namespace BCG
return builder.ToString();
}
}
}
}
@@ -33,8 +33,8 @@
// exception statement from your version.
using System;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Text;
namespace BCG.Utilities.SharpZipLib.Tar {
@@ -55,10 +55,10 @@ namespace BCG.Utilities.SharpZipLib.Tar {
public static readonly int DEFAULT_RCDSIZE = 512;
public static readonly int DEFAULT_BLKSIZE = DEFAULT_RCDSIZE * 20;
Stream inputStream;
Stream outputStream;
Stream? inputStream;
Stream? outputStream;
byte[] blockBuffer;
byte[]? blockBuffer;
int currBlkIdx;
int currRecIdx;
int blockSize;
@@ -110,6 +110,7 @@ namespace BCG.Utilities.SharpZipLib.Tar {
/// <summary>
/// Initialization common to all constructors.
/// </summary>
[MemberNotNull(nameof(blockBuffer))]
void Initialize(int blockSize, int recordSize)
{
this.debug = false;
@@ -197,7 +198,7 @@ namespace BCG.Utilities.SharpZipLib.Tar {
/// <returns>
/// The record data.
/// </returns>
public byte[] ReadRecord()
public byte[]? ReadRecord()
{
if (this.debug) {
Console.Error.WriteLine( "ReadRecord: recIdx = " + this.currRecIdx + " blkIdx = " + this.currBlkIdx );
@@ -214,7 +215,7 @@ namespace BCG.Utilities.SharpZipLib.Tar {
}
byte[] result = new byte[this.recordSize];
requireBlockBuffer();
Array.Copy(this.blockBuffer, (this.currRecIdx * this.recordSize), result, 0, this.recordSize );
this.currRecIdx++;
return result;
@@ -238,34 +239,37 @@ namespace BCG.Utilities.SharpZipLib.Tar {
int offset = 0;
int bytesNeeded = this.blockSize;
for (; bytesNeeded > 0 ;) {
long numBytes = this.inputStream.Read(this.blockBuffer, offset, bytesNeeded);
//
// NOTE
// We have fit EOF, and the block is not full!
//
// This is a broken archive. It does not follow the standard
// blocking algorithm. However, because we are generous, and
// it requires little effort, we will simply ignore the error
// and continue as if the entire block were read. This does
// not appear to break anything upstream. We used to return
// false in this case.
//
// Thanks to 'Yohann.Roussel@alcatel.fr' for this fix.
//
if (numBytes <= 0) {
break;
}
offset += (int)numBytes;
bytesNeeded -= (int)numBytes;
if (numBytes != this.blockSize) {
if (this.debug) {
Console.Error.WriteLine("ReadBlock: INCOMPLETE READ " + numBytes + " of " + this.blockSize + " bytes read.");
}
}
}
if (bytesNeeded > 0) {
requireBlockBuffer();
do {
long numBytes = this.inputStream.Read(this.blockBuffer, offset, bytesNeeded);
//
// NOTE
// We have fit EOF, and the block is not full!
//
// This is a broken archive. It does not follow the standard
// blocking algorithm. However, because we are generous, and
// it requires little effort, we will simply ignore the error
// and continue as if the entire block were read. This does
// not appear to break anything upstream. We used to return
// false in this case.
//
// Thanks to 'Yohann.Roussel@alcatel.fr' for this fix.
//
if (numBytes <= 0) {
break;
}
offset += (int)numBytes;
bytesNeeded -= (int)numBytes;
if (numBytes != this.blockSize) {
if (this.debug) {
Console.Error.WriteLine("ReadBlock: INCOMPLETE READ " + numBytes + " of " + this.blockSize + " bytes read.");
}
}
} while(bytesNeeded > 0);
}
this.currBlkIdx++;
return true;
@@ -318,6 +322,7 @@ namespace BCG.Utilities.SharpZipLib.Tar {
if (this.currRecIdx >= this.recsPerBlock) {
this.WriteBlock();
}
requireBlockBuffer();
Array.Copy(record, 0, this.blockBuffer, (this.currRecIdx * this.recordSize), this.recordSize );
this.currRecIdx++;
}
@@ -350,7 +355,7 @@ namespace BCG.Utilities.SharpZipLib.Tar {
if (this.currRecIdx >= this.recsPerBlock) {
this.WriteBlock();
}
requireBlockBuffer();
Array.Copy(buf, offset, this.blockBuffer, (this.currRecIdx * this.recordSize), this.recordSize );
this.currRecIdx++;
@@ -368,7 +373,7 @@ namespace BCG.Utilities.SharpZipLib.Tar {
if (this.outputStream == null) {
throw new System.IO.IOException("no output stream defined");
}
requireBlockBuffer();
this.outputStream.Write(this.blockBuffer, 0, this.blockSize);
this.outputStream.Flush();
@@ -418,6 +423,13 @@ namespace BCG.Utilities.SharpZipLib.Tar {
inputStream = null;
}
}
[MemberNotNull(nameof(blockBuffer))]
void requireBlockBuffer() {
if (blockBuffer is null) {
throw new InvalidOperationException("blockBuffer is null");
}
}
}
}
/* The original Java file had this header:
@@ -32,7 +32,6 @@
// obligated to do so. If you do not wish to do so, delete this
// exception statement from your version.
#nullable disable
using System;
using System.IO;
using System.Text;
@@ -53,11 +52,11 @@ namespace BCG.Utilities.SharpZipLib.Tar {
protected int entrySize;
protected int entryOffset;
protected byte[] readBuf;
protected byte[]? readBuf;
protected TarBuffer buffer;
protected TarEntry currEntry;
protected IEntryFactory eFactory;
protected TarEntry? currEntry;
protected IEntryFactory? eFactory;
Stream inputStream;
/// <summary>
@@ -308,7 +307,7 @@ namespace BCG.Utilities.SharpZipLib.Tar {
this.readBuf = null;
}
byte[] headerBuf = this.buffer.ReadRecord();
byte[]? headerBuf = this.buffer.ReadRecord();
if (headerBuf == null) {
if (this.debug) {
@@ -327,6 +326,9 @@ namespace BCG.Utilities.SharpZipLib.Tar {
if (this.hasHitEOF) {
this.currEntry = null;
} else {
if (headerBuf is null) {
throw new InvalidOperationException("headerBuf is null");
}
try {
if (this.eFactory == null) {
this.currEntry = new TarEntry(headerBuf);
@@ -422,13 +424,9 @@ namespace BCG.Utilities.SharpZipLib.Tar {
}
while (numToRead > 0) {
byte[] rec = this.buffer.ReadRecord();
if (rec == null) {
// Unexpected EOF!
throw new IOException("unexpected EOF with " + numToRead + " bytes unread");
}
int sz = numToRead;
byte[]? rec = buffer.ReadRecord()
?? throw new IOException("unexpected EOF with " + numToRead + " bytes unread");
int sz = numToRead;
int recLen = rec.Length;
if (recLen > sz) {
@@ -44,6 +44,8 @@ using BCG.Utilities.SharpZipLib.Checksums;
using BCG.Utilities.SharpZipLib.Zip.Compression;
using BCG.Utilities.SharpZipLib.Zip.Compression.Streams;
using Spark.Core.Diagnostics;
namespace BCG.Utilities.SharpZipLib.Zip {
/// <summary>
@@ -96,9 +98,9 @@ namespace BCG.Utilities.SharpZipLib.Zip {
/// </example>
public class ZipOutputStream : DeflaterOutputStream
{
private ArrayList entries = new ArrayList();
private Crc32 crc = new Crc32();
private ZipEntry curEntry = null;
private ArrayList? entries = new();
private Crc32 crc = new();
private ZipEntry? curEntry = null;
private CompressionMethod curMethod;
private int size;
@@ -308,10 +310,7 @@ namespace BCG.Utilities.SharpZipLib.Zip {
if (name.Length > 0xFFFF) {
throw new ZipException("Name too long.");
}
byte[] extra = entry.ExtraData;
if (extra == null) {
extra = new byte[0];
}
byte[] extra = entry.ExtraData ?? Array.Empty<byte>();
if (extra.Length > 0xFFFF) {
throw new ZipException("Extra data too long.");
}
@@ -397,7 +396,7 @@ namespace BCG.Utilities.SharpZipLib.Zip {
}
}
entries.Add(curEntry);
ExceptionAssert.MemberNotNull(entries).Add(curEntry);
curEntry = null;
}
@@ -468,10 +467,7 @@ namespace BCG.Utilities.SharpZipLib.Zip {
if (name.Length > 0xffff) {
throw new ZipException("Name too long.");
}
byte[] extra = entry.ExtraData;
if (extra == null) {
extra = new byte[0];
}
byte[] extra = entry.ExtraData ?? Array.Empty<byte>();
string strComment = entry.Comment;
byte[] comment = strComment != null ? ZipConstants.ConvertToArray(strComment) : new byte[0];
if (comment.Length > 0xffff) {
@@ -1,4 +1,3 @@
using System;
using System.IO;
using System.Runtime.Serialization;
using System.Runtime.Serialization.Formatters.Binary;
@@ -10,14 +9,11 @@ namespace BCG
{
IFormatter myFormatter = new F ();
public T Deserialize<T> (Stream serializationStream)
{
return (T) myFormatter.Deserialize (serializationStream);
}
public void Serialize<T> (Stream serializationStream, T graph)
{
public T Deserialize<T> (Stream serializationStream) where T : notnull =>
(T)myFormatter.Deserialize (serializationStream);
public void Serialize<T> (Stream serializationStream, T graph) where T : notnull =>
myFormatter.Serialize (serializationStream, graph);
}
}
public class GenericBinaryFormatter : GenericFormatter<BinaryFormatter> { }
@@ -17,12 +17,13 @@ namespace BCG {
myLastLogTime = DateTime.MinValue;
protected TimeSpan myPeriod = new TimeSpan(0, 0, 0, 0, 100),
myLogPeriod = new TimeSpan(0, 1, 0);
protected Dictionary<IWorker, Pair<TimeSpan, DateTime>> myWorkers = new Dictionary<IWorker, Pair<TimeSpan, DateTime>>(),
myJustAdded = new Dictionary<IWorker, Pair<TimeSpan, DateTime>>(),
myJustRemoved = new Dictionary<IWorker, Pair<TimeSpan, DateTime>>();
protected readonly Dictionary<IWorker, Pair<TimeSpan, DateTime>?> myWorkers = new();
protected readonly Dictionary<IWorker, Pair<TimeSpan, DateTime>?> myJustAdded = new();
protected readonly Dictionary<IWorker, Pair<TimeSpan, DateTime>?> myJustRemoved = new();
readonly Stopwatch timer = new();
private static MasterWorker myThreadDriven = null,
myDoWorkDriven = null;
static MasterWorker? myThreadDriven;
static MasterWorker? myDoWorkDriven;
internal MasterWorker() {
myInit();
}
@@ -35,7 +36,6 @@ namespace BCG {
// TODO
myThreadDriven = myThreadDriven ?? new ThreadDrivenMasterWorker();
return myThreadDriven;
return null;
} else {
myDoWorkDriven = myDoWorkDriven ?? new MasterWorker();
return myDoWorkDriven;
@@ -61,10 +61,10 @@ namespace BCG {
public virtual void doWork() {
foreach (KeyValuePair<IWorker, Pair<TimeSpan, DateTime>> p in myJustAdded) {
foreach (var p in myJustAdded) {
myWorkers.Add(p.Key, p.Value);
}
foreach (KeyValuePair<IWorker, Pair<TimeSpan, DateTime>> p in myJustRemoved) {
foreach (var p in myJustRemoved) {
myWorkers.Remove(p.Key);
}
myJustAdded.Clear();
+1 -8
View File
@@ -1,12 +1,5 @@
//using ProtoBuf;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
//using System.Windows;
//using System.Windows.Media;
namespace mbts
{
@@ -18,7 +11,7 @@ namespace mbts
public S? second;
public Pair() { }
public Pair(F f, S s)
public Pair(F? f, S? s)
{
first = f;
second = s;
@@ -1,13 +1,11 @@
using System;
using System.Collections.Generic;
using System.Text;
using BCG.Utilities;
using bts;
using mbts;
using Spark.Core.Diagnostics;
using System.Collections.Generic;
namespace BCG {
/// <summary>
/// SingleTagPubSubMap consists of all the state w.r.t to
@@ -79,6 +77,7 @@ namespace BCG {
public bool handle_sub(SubscribeMsg sub_msg) {
Log.Gen.info($"SingleTagConnectionData.handle_sub sub_msg: {sub_msg}");
ExceptionAssert.NotNull(sub_msg.connection);
if (_sub_conn_to_app_map.ContainsKey(sub_msg.connection)) {
bts.Log.Gen.info($"tag: {_tag} has already been subscribed by connection: {sub_msg.connection}");
return false;
@@ -89,6 +88,7 @@ namespace BCG {
}
public bool handle_unsub(UnsubscribeMsg unsub_msg) {
ExceptionAssert.NotNull(unsub_msg.connection);
if (_sub_conn_to_app_map.ContainsKey(unsub_msg.connection)) {
_sub_conn_to_app_map.Remove(unsub_msg.connection);
return true;
@@ -123,6 +123,7 @@ namespace BCG {
}
public void handle_pub(PublishMsg pub_msg, out bool new_publisher) {
ExceptionAssert.NotNull(pub_msg.connection);
if (_active_pub_conn_to_app_map.ContainsKey(pub_msg.connection)) {
new_publisher = false;
} else {
@@ -131,4 +132,4 @@ namespace BCG {
}
}
}
}
}
@@ -1,6 +1,9 @@
using bts;
using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic;
using bts;
using System.Threading;
namespace BCG {
@@ -40,15 +43,13 @@ namespace BCG {
return;
}
RemoteConnectionInfo rci = msg.remoteConnectionInfo;
ITCPConnection this_conn;
bool found = _conn_map.TryGetValue(rci, out this_conn);
if (!found) {
if (!_conn_map.TryGetValue(rci, out var this_conn)) {
Log.Gen.error($"Could not find connection: {rci}");
return;
}
Log.Gen.info($"\n\n\nSimpleTCPServerAsync.onData msgs_received: {_n_msgs_received} msg: {msg} ci: {rci} NOW echoing it back\n\n\n");
if (msg.messageType == RemoteMessageType.Data) {
this_conn.send(msg.message);
this_conn.send(ExceptionAssert.NotNull(msg.message));
} else {
if (msg.messageType == RemoteMessageType.Disconnection) {
var ci = msg.remoteConnectionInfo;
@@ -1,103 +1,102 @@
using bts.SilverPine;
namespace bts.BlueOrb {
using bts;
//using bts.api;
//using bts.spark;
using System;
using ZstdSharp.Unsafe;
using System.Diagnostics.CodeAnalysis;
public static class ContractUtils {
public class FutureTradableContract {
public string symbol;
public DateTime expiration;
public Exchange exchange;
public string contract_text;
}
public record FutureTradableContract(
string Symbol,
DateTime Expiration,
Exchange Exchange,
string? ContractText);
public class FutureSpreadContract {
public List<Tuple<double, FutureTradableContract>> legs;
public string symbol {
public List<Tuple<double, FutureTradableContract>> legs = new(2);
public string Symbol {
get {
return legs == null || legs.Count == 0 ? string.Empty : legs.ElementAt(0).Item2.symbol;
return legs == null || legs.Count == 0 ? string.Empty : legs.ElementAt(0).Item2.Symbol;
}
}
public string contract_text;
}
public class FutureContinousContract {
public string symbol;
public Exchange exchange;
public string? ContractText;
}
public record FutureContinousContract(string Symbol, Exchange Exchange);
// this is for all future conytracts belonging to a symbol
public class FutureWildcardContract {
public string symbol;
public Exchange exchange;
}
public record FutureWildcardContract(string Symbol, Exchange Exchange);
public class FutureContract {
public FutureWildcardContract wildcard_contract = null;
public FutureTradableContract future_tradable_contract = null;
public FutureSpreadContract spread_tradable_contract = null;
public FutureContinousContract continous_constract = null;
public FutureWildcardContract? wildcard_contract;
public FutureTradableContract? future_tradable_contract;
public FutureSpreadContract? spread_tradable_contract;
public FutureContinousContract? continous_constract;
public FutureContract(string txt) {
wildcard_contract = parse_wildcard_contract(txt);
if (wildcard_contract != null)
return;
FutureTradableContract fc = parse_future_contract(txt);
var fc = parse_future_contract(txt);
if (fc != null) {
future_tradable_contract = fc;
return;
}
FutureSpreadContract sc = parse_future_spread_contract(txt);
var sc = parse_future_spread_contract(txt);
if (sc != null) {
spread_tradable_contract = sc;
return;
}
// this is not totally true, since ParseFutureSpreadContract does not
// parse ICS spreads
continous_constract = new FutureContinousContract() { symbol = txt };
continous_constract = new(txt, Exchange.CME);
}
[MemberNotNullWhen(true, nameof(continous_constract))]
public bool is_continous_contract => continous_constract != null;
[MemberNotNullWhen(true, nameof(wildcard_contract))]
public bool is_wildcard_contract => wildcard_contract != null;
[MemberNotNullWhen(true, nameof(future_tradable_contract))]
public bool is_future_tradable_contract => future_tradable_contract != null;
[MemberNotNullWhen(true, nameof(spread_tradable_contract))]
public bool is_spread_tradable_contract => spread_tradable_contract != null;
public string symbol() {
if (wildcard_contract != null)
return wildcard_contract.symbol;
return wildcard_contract.Symbol;
if (continous_constract != null) {
return continous_constract.symbol;
return continous_constract.Symbol;
} else if (future_tradable_contract != null) {
return future_tradable_contract.symbol;
return future_tradable_contract.Symbol;
} else if (spread_tradable_contract != null) {
return spread_tradable_contract.symbol;
return spread_tradable_contract.Symbol;
}
Log.assert(false);
return string.Empty;
}
public string contract_text {
public string? contract_text {
get {
if (future_tradable_contract != null)
return future_tradable_contract.contract_text;
return future_tradable_contract.ContractText;
if (spread_tradable_contract != null)
return spread_tradable_contract.contract_text;
return spread_tradable_contract.ContractText;
return null;
}
}
}
#region Parsing
public static FutureWildcardContract parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) {
public static FutureWildcardContract? parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) {
if (contract_txt.EndsWith("*")) {
contract_txt = contract_txt.Replace("*", string.Empty);
return new FutureWildcardContract() {
symbol = contract_txt,
exchange = exchange
};
return new FutureWildcardContract(contract_txt, exchange);
}
return null;
}
public static FutureTradableContract parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) {
public static FutureTradableContract? parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) {
// Validate the future string length
if (!contract_txt.All(c => char.IsLetterOrDigit(c)))
return null;
@@ -143,17 +142,10 @@ namespace bts.BlueOrb {
// Create the FutureContract instance and set its properties
DateTime expiration = new DateTime(year, month, 1);
var futureContract = new FutureTradableContract {
symbol = symbol,
expiration = expiration,
exchange = exchange,
contract_text = contract_txt
};
return futureContract;
return new(symbol, expiration, exchange, contract_txt);
}
// Method to parse a futures spread, e.g. "CLX4-CLZ4"
public static FutureSpreadContract parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) {
public static FutureSpreadContract? parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) {
// Split the string by '-' to handle each leg
var spreadParts = contract_txt.Split('-');
if (spreadParts.Length < 2) {
@@ -201,11 +193,11 @@ namespace bts.BlueOrb {
#endregion
public static ContractType get_type(string contract_txt) {
FutureTradableContract fc = parse_future_contract(contract_txt);
var fc = parse_future_contract(contract_txt);
if (fc != null) {
return ContractType.Future;
}
FutureSpreadContract sc = parse_future_spread_contract(contract_txt);
var sc = parse_future_spread_contract(contract_txt);
if (sc != null) {
return ContractType.FutureSpread;
}
@@ -220,13 +212,13 @@ namespace bts.BlueOrb {
}
public static Tuple<ContractCategory, string> get_symbol_category(string symbol) {
FutureTradableContract fc = parse_future_contract(symbol);
var fc = parse_future_contract(symbol);
if (fc != null) {
return Tuple.Create(ContractCategory.FutureString, fc.symbol);
return Tuple.Create(ContractCategory.FutureString, fc.Symbol);
}
var sc = parse_future_spread_contract(symbol);
if (sc != null) {
return Tuple.Create(ContractCategory.FutureString, sc.symbol);
return Tuple.Create(ContractCategory.FutureString, sc.Symbol);
}
return Tuple.Create(ContractCategory.Symbol, symbol);
}
@@ -2,6 +2,7 @@ namespace bts.BlueOrb {
using bts.SilverPine;
using BCG;
using Bts.DataStreamExpressions;
public class DateTimeRange {
public readonly DateTime range_beg;
@@ -18,7 +19,7 @@ namespace bts.BlueOrb {
}
public class FutureBarsLoader {
public static string get_db_table_name(BarSize bar_size, string symbol, bool is_continous) {
public static string? get_db_table_name(BarSize bar_size, string symbol, bool is_continous) {
switch (bar_size) {
case BarSize.ONE_DAY: {
if (is_continous) {
@@ -62,7 +63,8 @@ namespace bts.BlueOrb {
beg = beg.Date;
end = end.Date.Add(new TimeSpan(23, 59, 59));
// this returns rows from beg and end inclusive
Func<string, SQLiteStream> get_sqlite_str = (table_name) => {
Func<string?, SQLiteStream> get_sqlite_str = (table_name) => {
ArgumentException.ThrowIfNullOrEmpty(table_name);
string range_beg_iso8601 = beg.ToString("yyyy-MM-ddTHH:mm:ss"),
range_end_iso8601 = end.ToString("yyyy-MM-ddTHH:mm:ss");
string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'";
@@ -70,27 +72,27 @@ namespace bts.BlueOrb {
};
string symbol = future_contract.symbol();
if (future_contract.wildcard_contract != null) {
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
ViewStream rs = get_sqlite_str(table_name);
//rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
mySetRealSource(rs);
return;
}
if (future_contract.wildcard_contract != null) {
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
ViewStream rs = get_sqlite_str(table_name);
//rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
mySetRealSource(rs);
return;
}
if (future_contract.is_continous_contract) {
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true);
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true);
ViewStream rs = get_sqlite_str(table_name);
//rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
mySetRealSource(rs);
return;
} else {
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false);
ViewStream rs = get_sqlite_str(table_name);
//rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime);
//TODO: filter it
@@ -115,7 +117,8 @@ namespace bts.BlueOrb {
// it gets the raw table with all contracts for that symbol
public FuturesHistorical1MinuteBarsStream(ContractUtils.FutureContract future_contract,
DateTimeRange dt_range) {
Func<string, SQLiteStream> get_sqlite_str = (table_name) => {
Func<string?, SQLiteStream> get_sqlite_str = (table_name) => {
ArgumentException.ThrowIfNullOrEmpty(table_name);
string range_beg_iso8601 = dt_range.range_beg.ToString("yyyy-MM-ddTHH:mm:ss"),
range_end_iso8601 = dt_range.range_end.ToString("yyyy-MM-ddTHH:mm:ss");
//string sql_str = $"select * from {table_name} where DATE(ts_event) >= '{range_beg_iso8601}' AND DATE(ts_event) <= '{range_end_iso8601}'";
@@ -124,18 +127,18 @@ namespace bts.BlueOrb {
};
string symbol = future_contract.symbol();
if (future_contract.wildcard_contract != null) {
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
ViewStream rs = get_sqlite_str(table_name);
mySetRealSource(rs);
return;
}
if (future_contract.is_continous_contract) {
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true);
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true);
ViewStream rs = get_sqlite_str(table_name);
mySetRealSource(rs);
return;
} else {
string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false);
ViewStream rs = get_sqlite_str(table_name);
// TODO: filter
rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'");
@@ -1,6 +1,4 @@
namespace bts.BlueOrb {
using System;
using BCG;
using bts.api;
@@ -24,23 +22,23 @@
// wild card
if (fc.is_wildcard_contract) {
var wc_contr = fc.wildcard_contract;
ViewStream rs = new bts.BlueOrb.FilterStream(parent_str, $"symbol='{wc_contr.symbol}'");
ViewStream rs = new FilterStream(parent_str, $"symbol='{wc_contr.Symbol}'");
rs = new bts.BlueOrb.FilterStream(parent_str, $"sec_type='{SecurityType.Future}' or sec_type = '{SecurityType.Spread}'");
mySetRealSource(rs);
return;
} else if (fc.is_future_tradable_contract) {
var fut_contr = fc.future_tradable_contract;
ViewStream rs = new bts.BlueOrb.FilterStream(parent_str, $"sec='{fut_contr.contract_text}'");
ViewStream rs = new FilterStream(parent_str, $"sec='{fut_contr.ContractText}'");
mySetRealSource(rs);
return;
} else if (fc.is_spread_tradable_contract) {
var spr_contr = fc.future_tradable_contract;
ViewStream rs = new bts.BlueOrb.FilterStream(parent_str, $"sec='{spr_contr.contract_text}'");
ViewStream rs = new FilterStream(parent_str, $"sec='{spr_contr.ContractText}'");
mySetRealSource(rs);
return;
} else if (fc.is_continous_contract) {
var cont_contr = fc.continous_constract;
ViewStream rs = new bts.BlueOrb.FilterStream(parent_str, $"symbol='{cont_contr.symbol}' and sec_type='{SecurityType.Future}'");
ViewStream rs = new FilterStream(parent_str, $"symbol='{cont_contr.Symbol}' and sec_type='{SecurityType.Future}'");
rs = new bts.BlueOrb.GroupByStream(rs, () => { return new string[] { "symbol" }; },
(group_rows) => {
IBlueRow r = group_rows.ToList().OrderBy(r => r.get_datetime("expiration")).ElementAt(0);
@@ -1,6 +1,4 @@
namespace bts.BlueOrb {
using System;
using BCG;
using bts.api;
@@ -10,7 +8,7 @@
public LiveTradeStream(string target)
: this(SecurityType.Future, target) { }
public LiveTradeStream(SecurityType sec_type, string target) {
ViewStream rs = null;
ViewStream? rs = null;
switch (sec_type) {
case SecurityType.Future:
rs = new LiveFuturesTradeStream(target);
@@ -1,11 +1,11 @@
namespace bts.BlueOrb {
using BCG;
using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BCG;
[DataStreamUserRespresentation("ReplayFutureLiveBBOStream")]
public class ReplayFutureLiveBBOStream : ViewStream, BCG.Utilities.General.IWorker{
@@ -14,13 +14,13 @@
public DateTime last_firetime_clock = DateTime.MinValue;
}
private class RowsWithPointer {
public List<IBlueRow> rows;
public List<IBlueRow>? rows;
public int pointer;
}
private readonly ViewStreamDatabase _vsd;
private readonly ContractUtils.FutureContinousContract _fcc;
readonly ViewStreamDatabase? _vsd;
readonly ContractUtils.FutureContinousContract? _fcc;
private readonly LastFireTime _last_fire_time = new LastFireTime();
private RowsWithPointer _rows = new RowsWithPointer();
@@ -62,20 +62,22 @@
}
protected override void myOnFirstSubscriber() {
var file_path = Path.Combine(SQLiteHelper.location, $"mbp1_{_fcc.symbol}_small.csv".ToLower());
Log.notnull(_fcc);
var file_path = Path.Combine(SQLiteHelper.location, $"mbp1_{_fcc.Symbol}_small.csv".ToLower());
var rows = BCG.CSVReaderStream.LoadFile(file_path);
_rows.rows = new(rows.Count);
_rows.rows.AddRange(rows.OfType<IBlueRow>().OrderBy(r => r.get<string>("ts_event")));
_rows.pointer = 0;
_vsd.addWorker(this);
ExceptionAssert.NotNull(_vsd).addWorker(this);
}
protected override void myOnLastSubscriber() {
_vsd.removeWorker(this);
ExceptionAssert.NotNull(_vsd).removeWorker(this);
}
public void doWork() {
IBlueRow cand_row = _rows.rows[_rows.pointer];
var rows = ExceptionAssert.NotNull(_rows.rows);
IBlueRow cand_row = rows[_rows.pointer];
string cand_ts_event = cand_row.get<string>("ts_event");
DateTime cand_ts_event_dt = DateTime.Parse(cand_ts_event, null, System.Globalization.DateTimeStyles.RoundtripKind);
TimeSpan cand_ts_span_file = cand_ts_event_dt - _last_fire_time.last_fireitme_file;
@@ -1,11 +1,11 @@
namespace bts.BlueOrb {
using BCG;
using Spark.Core.Diagnostics;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BCG;
[DataStreamUserRespresentation("ReplayFutureLiveTradeStream")]
public class ReplayFutureLiveTradeStream : ViewStream, BCG.Utilities.General.IWorker {
@@ -14,15 +14,15 @@
public DateTime last_firetime_clock = DateTime.MinValue;
}
private class RowsWithPointer {
public List<IBlueRow> rows;
public List<IBlueRow>? rows;
public int pointer;
}
private readonly ViewStreamDatabase _vsd;
private readonly ContractUtils.FutureContinousContract _fcc;
readonly ViewStreamDatabase? _vsd;
readonly ContractUtils.FutureContinousContract? _fcc;
private readonly LastFireTime _last_fire_time = new LastFireTime();
private RowsWithPointer _rows = new RowsWithPointer();
readonly RowsWithPointer _rows = new();
public ReplayFutureLiveTradeStream(string continous_contract) {
Func<string> iso_8601_now = () => {
@@ -61,21 +61,24 @@
}
protected override void myOnFirstSubscriber() {
var file_path = Path.Combine(SQLiteHelper.location, $"trades_{_fcc.symbol}.csv");
Log.notnull(_vsd);
Log.notnull(_fcc);
var file_path = Path.Combine(SQLiteHelper.location, $"trades_{_fcc.Symbol}.csv");
var csv_rd = new CSVReaderStream(file_path, false);
var rows = BCG.CSVReaderStream.LoadFile(file_path);
_rows.rows = new(rows.Count);
_rows.rows.AddRange(rows.OfType<IBlueRow>().OrderBy(r => r.get<string>("ts_event")));
_rows.pointer = 0;
_vsd.addWorker(this);
ExceptionAssert.NotNull(_vsd).addWorker(this);
}
protected override void myOnLastSubscriber() {
_vsd.removeWorker(this);
ExceptionAssert.NotNull(_vsd).removeWorker(this);
}
public void doWork() {
IBlueRow cand_row = _rows.rows[_rows.pointer];
var rows = ExceptionAssert.NotNull(_rows.rows);
IBlueRow cand_row = rows[_rows.pointer];
string cand_ts_event = cand_row.get<string>("ts_event");
DateTime cand_ts_event_dt = DateTime.Parse(cand_ts_event, null, System.Globalization.DateTimeStyles.RoundtripKind);
TimeSpan cand_ts_span_file = cand_ts_event_dt - _last_fire_time.last_fireitme_file;
@@ -8,7 +8,7 @@ namespace bts {
using System.Text.Json.Nodes;
[DataStreamUserRespresentation("benzingaNews")]
public class BenzingaNewsStream : ViewStream {
private IBlueTableSchema _schema = null;
IBlueTableSchema? _schema;
private int _id = 0;
@@ -63,14 +63,14 @@ namespace bts {
}
foreach (JsonNode jnode in json_objs) {
if (jnode is JsonObject json_object) {
string symbol = jnode["symbol"] == null ? String.Empty : jnode["symbol"].ToString();
string exchange = jnode["exchange"] == null ? String.Empty : jnode["exchange"].ToString();
string symbol = jnode["symbol"]?.ToString() ?? string.Empty;
string exchange = jnode["exchange"]?.ToString() ?? string.Empty;
MutableBlueRow nr = new MutableBlueRow(_schema, r);
nr.setValue("id", _id++);
nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
nr.setValue("doc_id_internal", $"{r["doc_id"]}_{symbol}");
nr.setValue("symbol", symbol);
nr.setValue("exchange", exchange);
nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
nr.setValue("is_primary_exchange", jnode["primary"]?.GetValue<bool>() ?? false);
expanded_rows.Add(nr);
}
}
@@ -84,4 +84,4 @@ namespace bts {
mySetRealSource(output_stream);
}
}
}
}
Loaded 100 of 118 files, more files were not shown because too many files have changed in this diff. Show more