cleanup: Fix a ton of build warnings (cs)

All these warnings have been bothering me for a while. The work isn't done, but
this commit chops them down from ~1000 to ~600.

* AppendCollection.cs
* AppendStreamModified.cs
* BenzingaNewsStream.cs
* BenzingaNewsSymbolStateStream.cs
* BookStream.cs
* CSVSaverStream.cs
* ChartsUtils.cs
* ColumnInfo.cs
* ContractUtils.cs
* ContractUtils.cs
* DiffStream.cs
* FuturesActiveContractStream.cs
* FuzzyJoinOverTimeStream.cs
* HistoricalBarsStream.cs
* HistoricalBarsStream.cs
* InstrumentsStream.cs
* JoinOperandRowIndex.cs
* JoinRow.cs
* JoinStream.cs
* JoinTable.cs
* JoinTableSchema.cs
* LatencyStream.cs
* LeftInternalJoinStream.cs
* LightForeignJoinRow.cs
* LightForeignJoinRowList.cs
* LightLeftForeignJoinRow.cs
* LiteralStream.cs
* NLStream.cs
* NLToDSLStream.cs
* NLToIndexedFewShotPromptStream.cs
* NonIncGroupByStream.cs
* NonIncJoinBookkeeper.cs
* NonIncJoinSpec.cs
* OHLCVStream.cs
* OpenAIPromptStream.cs
* PriceStatsStream.cs
* PriceStatsStream.cs
* ProjectHelper.cs
* ProjectStream.cs
* PubSubServerStateStream.cs
* PubsubHeartbeatStream.cs
* SimpleNode.cs
* SortRow.cs
* SpatialArbStatsStream.cs
* SpatialFuturePairsStream.cs
* SpreadPerformanceEvaluator.cs
* SpreadPerformanceStream.cs
* StreamIOCommon.cs
* SuperJoinStream.cs
* TableMetaDataStream.cs
* TextWriterStream.cs
* TimeAverageStream.cs
This commit is contained in:
John Colvin committed 2025-11-19 13:57:47 -05:00
1 parent 7418f5b5a1
commit a4efaa0089
48 files changed
+610 -590

No files matched your search

@@ -1,5 +1,4 @@
using System;
using System.Collections;
using System.Collections.Generic;
using mbts;
@@ -12,30 +11,21 @@ namespace BCG
{
ViewStream myLStream, myRStream;
DataPacket myLastR = null, myLastL = null;
DataPacket? myLastR = null, myLastL = null;
BlueTable myLastFired = null;
BlueTable? myLastFired = null;
double myThreshold = .00005;
string[] myColumnsToMatch;
public DiffStream(ViewStream v1, ViewStream v2) : this(null, v1, v2, "id") { }
public DiffStream(ViewStream v1, ViewStream v2, params string[] ColumnsToMatch) : this(null, v1, v2, ColumnsToMatch) { }
public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) :
this(null, v1, v2, thresh, ColumnsToMatch) { }
public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
{
public DiffStream(ViewStream v1, ViewStream v2) : this(v1, v2, "id") { }
public DiffStream(ViewStream v1, ViewStream v2, params string[] columnsToMatch) : this(v1, v2, .00005, columnsToMatch) { }
public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch) {
myThreshold = thresh;
myLStream = v1;
myRStream = v2;
myColumnsToMatch = columnsToMatch;
}
public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
: this(db,v1,v2,.00005,columnsToMatch)
{
}
protected override void myOnFirstSubscriber()
{
@@ -52,8 +42,14 @@ namespace BCG
struct ListComparer : IEqualityComparer<List<object>>
{
public bool Equals(List<object> x, List<object> y)
public bool Equals(List<object>? x, List<object>? y)
{
if (x is null) {
return y is null;
}
if (y is null) {
return false;
}
if (x.Count != y.Count)
return false;
@@ -74,21 +70,20 @@ namespace BCG
void myHandleCallbacks()
{
DataPacket packet = null;
DataPacket? packet;
if (myLastL == null || myLastR == null)
{
packet = myLastL == null ? myLastR : myLastL;
if (packet != null)
if (packet != null) {
myFireEvent(packet);
myLastFired = new BlueTable(packet.newRows);
myLastFired = new BlueTable(packet.newRows);
}
return;
}
BlueTable table = new BlueTable(myLastL.newRows.schema);
Dictionary<List<object>, Pair<BlueRow, BlueRow>> rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(new ListComparer());
Dictionary<List<object>, Pair<BlueRow?, BlueRow?>> rowPairs = new(new ListComparer());
foreach(BlueRow row in myLastL.newRows)
{
@@ -104,7 +99,7 @@ namespace BCG
}
}
rowPairs[hash] = new Pair<BlueRow, BlueRow>(row, null);
rowPairs[hash] = new(row, null);
}
foreach (BlueRow row in myLastR.newRows)
@@ -119,12 +114,11 @@ namespace BCG
catch (Exception) { }
}
Pair<BlueRow, BlueRow> pair;
rowPairs.TryGetValue(hash, out pair);
rowPairs.TryGetValue(hash, out var pair);
if (pair == null)
pair = new Pair<BlueRow, BlueRow>(null, row);
pair = new(null, row);
else
pair.second = row;
@@ -132,13 +126,11 @@ namespace BCG
}
int id = 0;
foreach (KeyValuePair<List<object>, Pair<BlueRow, BlueRow>> hashpair in rowPairs)
foreach (var (hashVals, pair) in rowPairs)
{
Pair<BlueRow, BlueRow> pair = hashpair.Value;
if (pair.first == null || pair.second == null)
{
BlueRow row = pair.first == null ? pair.second : pair.first;
var row = pair.first ?? pair.second ?? throw new InvalidOperationException("Both rows in pair are null");
Dictionary<string, object> newRow = new Dictionary<string, object>();
foreach (ColumnInfo info in row.schema.columnInfo)
@@ -211,4 +203,4 @@ namespace BCG
}
}
}
@@ -1,4 +1,8 @@
using System;
using bts;
using Bts.DataStreamCore;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
@@ -40,12 +44,12 @@ namespace BCG
private IComparer<IBlueRow> myRightCompare;
protected FuzzyRowComparer myFuzzyCompare;
protected TimeSpan myTimeWindow;
protected IBlueTableSchema myLeftSchema;
protected IBlueTableSchema myRightSchema;
protected IBlueTableSchema myOutputSchema;
protected IBlueTableSchema? myLeftSchema;
protected IBlueTableSchema? myRightSchema;
protected IBlueTableSchema? myOutputSchema;
private int myID = 0;
protected TimeWindowTable myStateRows;
protected TimeWindowTable? myStateRows;
private DateTime myNextCull;
const long MinCullIntervalMS = 1000;
@@ -65,7 +69,7 @@ namespace BCG
this.myFuzzyCompare = fuzzyCompare;
}
public virtual IBlueTableSchema OutputSchema
public virtual IBlueTableSchema? OutputSchema
{
get
{
@@ -150,7 +154,7 @@ namespace BCG
int count;
for (count = 0; count < list.Count; count++)
{
if ((DateTime)list[count][timeColumn] > expiry)
if (list[count].GetValue<DateTime>(timeColumn) > expiry)
{
// Stop when we find the first row that we can keep. Since this is sorted all others are also kept.
break;
@@ -172,14 +176,17 @@ namespace BCG
this.myFitness = fitness;
}
public IBlueRow myRightRow { get; private set; }
public IBlueRow myLeftRow { get; set; }
public IBlueRow? myLeftRow { get; set; }
public double myFitness { get; private set; }
public IBlueRow myMergedRow { get; set; }
public IBlueRow? myMergedRow { get; set; }
#region IComparable<RowMatch> Members
public int CompareTo(RowMatch other)
public int CompareTo(RowMatch? other)
{
if (other is null) {
return 1;
}
return myFitness.CompareTo(other.myFitness);
}
@@ -262,6 +269,10 @@ namespace BCG
private void HandleCorrection(List<RowMatch> corrections, BlueTable adds, BlueTable dels)
{
if (corrections.Count == 0) {
return;
}
Log.notnull(myStateRows);
// We already emitted correction.MergedRow so we need to redact that and
// then emit correction.left as an add since it is no longer matched
foreach (RowMatch correction in corrections.Distinct())
@@ -285,7 +296,7 @@ namespace BCG
int r = 0;
for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; )
{
if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn])
if (leftExpired[l].GetValue<DateTime>(myLeftTimeColumn) <= rightExpired[r].GetValue<DateTime>(myRightTimeColumn))
{
// Advance Left
IEnumerable<RowMatch> rm = rightMatches[l];
@@ -332,7 +343,7 @@ namespace BCG
private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable<RowMatch> rightMatches)
{
IBlueRow merged = null;
IBlueRow? merged = null;
if (rightMatches != null)
{
foreach (RowMatch match in rightMatches)
@@ -366,9 +377,9 @@ namespace BCG
/// Override this function to provide a custom row merge.
/// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting.
/// </summary>
protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right)
protected virtual IBlueRow MergeRows(IBlueRow? left, IBlueRow? right)
{
Dictionary<string, object> newRow = new Dictionary<string, object>();
Dictionary<string, object?> newRow = new();
if (left == null && right == null) throw new ArgumentException("oh come on.");
@@ -393,14 +404,14 @@ namespace BCG
}
}
newRow["id"] = this.myID++;
newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn];
newRow[NEW_TIME_COLUMN] = (left != null) ? left[myLeftTimeColumn] : right![myRightTimeColumn];
return new BlueRow(this.OutputSchema, newRow);
}
private IEnumerable<RowMatch> FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
private IEnumerable<RowMatch>? FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
{
List<RowMatch> matches = null;
List<RowMatch>? matches = null;
// Search through all rows in list and compare with the fuzzy comparer
for (int x = 0; x < list.Count; x++)
@@ -421,21 +432,15 @@ namespace BCG
matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch);
// Mark the best match for this particular right hand match
RowMatch oldMatch;
if (this.myBestMatch.TryGetValue(match, out oldMatch))
{
if (currentMatch.myFitness < oldMatch.myFitness)
{
if (myBestMatch.TryGetValue(match, out var oldMatch)) {
if (currentMatch.myFitness < oldMatch.myFitness) {
this.myBestMatch[match] = currentMatch;
if (oldMatch.myMergedRow != null)
{
if (oldMatch.myMergedRow != null) {
// We already emitted oldMatch.MergedRow so we need to fix it
corrections.Add(oldMatch);
}
}
}
else
{
} else {
this.myBestMatch[match] = currentMatch;
}
}
@@ -7,6 +7,8 @@ using System.Text.Json;
using bts;
using bts.SilverPine;
using mbts.utils;
namespace BCG {
//[DataStreamUserRespresentation("AggregateStream")]
public class NonIncGroupByStream : ViewStream {
@@ -80,7 +82,7 @@ namespace BCG {
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
private void _main_stream_onchange(object nv) {
DataPacket dp = nv as DataPacket;
var dp = ExceptionAssert.Argument.Is<DataPacket>(nv);
///grouping together
_timer.Restart();
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
@@ -109,7 +111,12 @@ namespace BCG {
}
// TODO,
if (total_state != null)
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
myFireEvent(new DataPacket(
total_state,
total_state,
myLastValue == null
? new BlueTable(total_state.schema)
: ExceptionAssert.of_type<DataPacket>(myLastValue).newRows));
}
}
}
@@ -1,6 +1,5 @@
using System;
using System.Collections;
using BCG.Utilities;
using System.Collections.Generic;
namespace BCG
{
@@ -11,7 +10,8 @@ namespace BCG
/// </summary>
internal class JoinOperandRowIndex
{
private IDictionary myJoinColDict = new Hashtable (), myIdDict = new Hashtable ();
readonly Dictionary<object, object> myJoinColDict = new();
readonly Dictionary<object, object> myIdDict = new();
private int myJoinColOrdinal, myIdColOrdinal;
/// <summary>
@@ -35,18 +35,18 @@ namespace BCG
public void add (IBlueRow row)
{
RowList list;
if (myJoinColDict[row[myJoinColOrdinal]] == null)
myJoinColDict[row[myJoinColOrdinal]] = list = new RowList ();
if (myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value] == null)
myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value] = list = new RowList ();
else
list = (RowList)myJoinColDict[row[myJoinColOrdinal]];
myIdDict[row[myIdColOrdinal]] = list.add (row);
list = (RowList)myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value];
myIdDict[row[myIdColOrdinal] ?? DBNull.Value] = list.add (row);
}
public void remove (IBlueRow row)
{
RowNode node = (RowNode)myIdDict[row[myIdColOrdinal]];
((RowList)myJoinColDict[row[myJoinColOrdinal]]).remove (node);
myIdDict.Remove (row[myIdColOrdinal]);
RowNode node = (RowNode)myIdDict[row[myIdColOrdinal] ?? DBNull.Value];
((RowList)myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value]).remove (node);
myIdDict.Remove (row[myIdColOrdinal] ?? DBNull.Value);
}
public RowList getMatchingRows (object key)
@@ -1,18 +1,20 @@
using System;
using BCG.Utilities;
using System.Collections;
using bts;
using Bts.DataStreamCore;
namespace BCG
{
internal class JoinRow : IBlueRow
{
private RowNode myLeftNode, myRightNode;
protected IBlueRow myLeftRow, myRightRow;
RowNode? myLeftNode;
RowNode? myRightNode;
protected IBlueRow myLeftRow;
protected IBlueRow? myRightRow;
protected int myId;
protected JoinTableSchema mySchema;
private object[] myValues;
private object?[] myValues;
#region Constructor(s)
/// <summary>
@@ -21,8 +23,8 @@ namespace BCG
/// <param name="leftRow">left row of the join</param>
/// <param name="rightRow">right row of the join</param>
/// <param name="schema">schema of the joined row</param>
public JoinRow (IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
: this ((int)leftRow["id"], leftRow, rightRow, schema) {}
public JoinRow(IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
: this(leftRow.Id(), leftRow, rightRow, schema) { }
/// <summary>
/// Constructs a row assigning it the ID provioded as the first argument
@@ -31,7 +33,7 @@ namespace BCG
/// <param name="leftRow">left row of the join</param>
/// <param name="rightRow">right row of the join</param>
/// <param name="schema">schema of the joined row</param>
public JoinRow (int id, IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
public JoinRow (int id, IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema)
{
myId = id;
myLeftRow = leftRow;
@@ -48,13 +50,13 @@ namespace BCG
public bool isPartialJoin{get{return myRightRow == null;}}
public RowNode leftNode
public RowNode? leftNode
{
get{return myLeftNode;}
set{myLeftNode = value;}
}
public RowNode rightNode
public RowNode? rightNode
{
get{return myRightNode;}
set{myRightNode = value;}
@@ -85,7 +87,7 @@ namespace BCG
/// The key can either be the string corrsponding to the column name or
/// the order of the column
/// </summary>
public virtual object this [object key]
public virtual object? this [object key]
{
get
{
@@ -96,7 +98,7 @@ namespace BCG
set {throw new NotSupportedException ("The IBlueRow is read only");}
}
public object this [int index]
public object? this [int index]
{
get
{
@@ -119,7 +121,7 @@ namespace BCG
}
}
public object this [string colName] {get{return this[mySchema[colName].colNo];}}
public object? this [string colName] {get{return this[mySchema[colName].colNo];}}
public ICollection Keys{get{return mySchema.columnNames;}}
@@ -133,9 +135,11 @@ namespace BCG
}
}
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
public void Add(object key, object? value) =>
throw new NotSupportedException("The IBlueRow is readonly");
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
public void Clear() =>
throw new NotSupportedException ("The IBlueRow is readonly");
public bool Contains (object key){return mySchema[(string)key] != null;}
@@ -1,11 +1,10 @@
using System;
using System.Data;
using System.Collections;
using System.Text.RegularExpressions;
using BCG.Utilities;
using BCG.Utilities.General;
using bts;
using mbts.utils;
using System;
using System.Collections;
namespace BCG
{
public class JoinStream : ViewStream
@@ -111,8 +110,15 @@ namespace BCG
{
if (myPs == null)
{
JoinNameResolver resolver = new JoinNameResolver ((JoinTableSchema)myIjs.schema, myIjs.getSchema (true), myLeftCm.columnExpression,
myIjs.getSchema (false), myRightCm.columnExpression);
ArgumentNullException.ThrowIfNull(myIjs);
ArgumentNullException.ThrowIfNull(myLeftCm);
ArgumentNullException.ThrowIfNull(myRightCm);
var resolver = new JoinNameResolver(
ExceptionAssert.of_type<JoinTableSchema>(myIjs.schema),
myIjs.getSchema(true),
myLeftCm.columnExpression,
myIjs.getSchema(false),
myRightCm.columnExpression);
IDictionary map = resolver.getMap ();
map["id"] = "id";
myPs = new ProjectStream (myIjs, map);
@@ -123,26 +129,27 @@ namespace BCG
Log.assert(false, "This call back should not have been called");
}
else
myFireEvent ((o as ICloneable).Clone());
myFireEvent(ExceptionAssert.of_type<ICloneable>(o).Clone());
}
private void myProjectChangeHandler (object o)
{
ArgumentNullException.ThrowIfNull(myIjs);
if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1)
myIjs.onChange -= myInternalJoinChangeHandler;
myFireEvent ((o as ICloneable).Clone());
myFireEvent(ExceptionAssert.of_type<ICloneable>(o).Clone());
}
protected override void myOnFirstSubscriber ()
{
myIjs.onChange += myInternalJoinChangeHandler;
ExceptionAssert.not_null(myIjs).onChange += myInternalJoinChangeHandler;
}
protected override void myOnLastSubscriber ()
{
// how does this relate to the -= in myProjectChangeHandler?
myIjs.onChange -= myInternalJoinChangeHandler;
ExceptionAssert.not_null(myIjs).onChange -= myInternalJoinChangeHandler;
// if there is a project after the ijs we need to
// unsubscribe to it otherwise references will be kept and
@@ -159,7 +166,9 @@ namespace BCG
/// </summary>
/// <param name="leftPacket"></param>
/// <param name="rightPacket"></param>
public void test (DataPacket dp, bool isLeft) {myIjs.test (dp, isLeft);}
public void test (DataPacket dp, bool isLeft) =>
ExceptionAssert.not_null(myIjs).test(dp, isLeft);
/***********************************************************************************************/
}
}
@@ -1,7 +1,7 @@
using Bts.DataStreamCore;
using System;
using System.Collections;
using System.IO;
using BCG.Utilities.General;
namespace BCG
{
@@ -19,20 +19,13 @@ namespace BCG
myRightIndex = new JoinDeletionIndex (mySchema.getRenamedName ("id", false), false);
}
public IBlueRow this [int id]
{
get
{
if (myIdIndex[id] == null)
throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
return (IBlueRow)myIdIndex[id];
}
}
public IBlueRow this [int id] =>
myIdIndex[id] as IBlueRow ?? throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
#region JoinTable members
public virtual void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp)
public virtual void addRow (IBlueRow leftRow, IBlueRow? rightRow, DataPacket dp)
{
JoinRow newRow = new JoinRow (myNewRowId, leftRow, rightRow, mySchema);
var newRow = new JoinRow(myNewRowId, leftRow, rightRow, mySchema);
myIdIndex[myNewRowId ++] = newRow;
((BlueTable)(dp.addedRows)).addRow (newRow);
myLeftIndex.add (newRow);
@@ -48,17 +41,17 @@ namespace BCG
thisIndex = myLeftIndex;
otherIndex = myRightIndex;
}
RowList matchingNodes = thisIndex.getMatchingRows (deletedRow["id"]);
RowList matchingNodes = thisIndex.getMatchingRows (deletedRow.Id());
foreach (RowNode node in matchingNodes)
{
JoinRow row = (JoinRow)node.row;
RowNode correspondingNode = isLeft ? row.rightNode : row.leftNode;
var correspondingNode = isLeft ? row.rightNode : row.leftNode;
if (correspondingNode != null)
otherIndex.remove (correspondingNode); //in case of partial left joinss
myIdIndex.Remove (row["id"]);
myIdIndex.Remove (row.Id());
((BlueTable)dp.deletedRows).addRow (row);
}
thisIndex.removeAll (deletedRow["id"]);
thisIndex.removeAll(deletedRow.Id());
}
#endregion
#region ICollection members
@@ -1,9 +1,10 @@
using System;
using System.Collections;
using BCG.Utilities;
using System.Collections.Generic;
using System.Linq;
using bts;
using mbts.utils;
namespace BCG
{
/// <summary>
@@ -13,10 +14,11 @@ namespace BCG
public class JoinTableSchema : BlueTableSchema
{
private string myLeftJc = "", myRightJc = "", myLeftId = "", myRightId = "";
private IDictionary myLeftNameToAlias = new Hashtable ();
private IDictionary myRightNameToAlias = new Hashtable ();
private IDictionary myAliasToLeftName = new Hashtable(),
myAliasToRightName = new Hashtable();
readonly Dictionary<string, string> myLeftNameToAlias = new();
readonly Dictionary<string, string> myRightNameToAlias = new();
readonly Dictionary<string, string> myAliasToLeftName = new();
readonly Dictionary<string, string> myAliasToRightName = new();
/// <summary>
/// Constructs the schema of the Jointable with the schema of the left and right tables. Incase of
/// a name being there in both the shemas it prefixes the name with "L." or "R." accordingly. Column "id"
@@ -85,9 +87,9 @@ namespace BCG
renamed = false;
}
myIndexByColName["id"] = new ColumnInfo("id", myIndexByColName.Count, typeof(int));
myIndexByColOrder[myIndexByColOrder.Length - 1] = (ColumnInfo)myIndexByColName["id"];
myAliasToLeftName = ProjectHelper.invertDic(myLeftNameToAlias);
myAliasToRightName = ProjectHelper.invertDic(myRightNameToAlias);
myIndexByColOrder[myIndexByColOrder.Length - 1] = ExceptionAssert.of_type<ColumnInfo>(myIndexByColName["id"]);
myAliasToLeftName = myLeftNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key);
myAliasToRightName = myRightNameToAlias.ToDictionary(kv => kv.Key, kv => kv.Value);
}
public string leftJoinCol {get{return myLeftJc;}}
@@ -97,8 +99,8 @@ namespace BCG
public string getRenamedName (string origName, bool isLeft)
{
if (isLeft)
return (string)myLeftNameToAlias[origName];
return (string)myRightNameToAlias[origName];
return myLeftNameToAlias[origName] as string ?? string.Empty;
return myRightNameToAlias[origName] as string ?? string.Empty;
}
public string getOriginalName(string rename, bool isLeft)
@@ -2,16 +2,15 @@
using bts.collections;
using mbts;
using mbts.utils;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Security.Cryptography;
namespace BCG {
public class NonIncJoinBookkeeper : IDisposable {
public Action on_new_data;
public Action? on_new_data;
private readonly NonIncJoinSpec _join_spec;
private readonly int _join_position;
private readonly int _start_index_in_output;
@@ -20,16 +19,19 @@ namespace BCG {
private readonly bool _is_tag_along;
private int _n_total_fires = 0, _n_expensive_fires = 0;
private IBlueTableSchema _schema = null;
private IBlueTableSchema? _schema;
private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
private IBlueRow _solitary_row = null;
private IBlueRow? _solitary_row;
private int[] _join_col_indices = null,
_data_col_indices = null;
int[]? _join_col_indices;
int[]? _data_col_indices;
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
_join_spec = the_join_spec;
public NonIncJoinBookkeeper(NonIncJoinSpec join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
if (join_spec.join_col_names is null) {
throw new ArgumentException("join_col_names is null", nameof(join_spec));
}
_join_spec = join_spec;
_join_position = join_position;
_start_index_in_output = start_index_in_output;
_row_hasher = row_hasher;
@@ -41,7 +43,7 @@ namespace BCG {
private void _fill_missing_data(DataPacket dp) {
if (_filled_missing_data)
return;
Func<string[], bool> null_or_empty = (str_arr) => {
bool null_or_empty(string[]? str_arr) {
if (str_arr == null || str_arr.Length == 0)
return true;
foreach (string s in str_arr) {
@@ -50,13 +52,16 @@ namespace BCG {
}
}
return true;
};
Func<string[], string[]> remove_bad_cols = (col_names) => {
}
string[] remove_bad_cols(string[]? col_names) {
if (col_names is null) {
return Array.Empty<string>();
}
HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
List<string> cols_list = new List<string>(col_names);
cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
return cols_list.ToArray();
};
}
if (null_or_empty(_join_spec.data_col_names)) {
_join_spec.data_col_names = dp.schema.columnNames;
}
@@ -70,7 +75,7 @@ namespace BCG {
}
public void init() {
_join_spec.stream.onChange += _on_stream_cb;
ExceptionAssert.not_null(_join_spec.stream).onChange += _on_stream_cb;
}
public void _on_stream_cb(object dp_obj) {
@@ -111,14 +116,19 @@ namespace BCG {
public bool is_ready { get { return _schema != null; } }
public void constribute_cols_to_schema(List<ColumnInfo> cis) {
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
ArgumentNullException.ThrowIfNull(_join_spec.data_col_names);
if (_join_spec.data_col_names.Length > 0) {
ArgumentNullException.ThrowIfNull(_join_spec.projected_col_names);
ArgumentNullException.ThrowIfNull(_schema);
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
}
}
public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
public void copy_master_row_to_output_row(int row_index, object[] output_row) {
IBlueRow this_row = _row_list[row_index].second;
IBlueRow? this_row = _row_list[row_index].second;
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
}
@@ -129,8 +139,7 @@ namespace BCG {
else
copy_dbnull_to_output_row(output_row);
} else {
IBlueRow this_row = null;
if (_row_map.TryGetValue(row_hash, out this_row))
if (_row_map.TryGetValue(row_hash, out var this_row))
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
else
copy_dbnull_to_output_row(output_row);
@@ -138,13 +147,16 @@ namespace BCG {
}
private void copy_dbnull_to_output_row(object[] output_row) {
ArgumentNullException.ThrowIfNull(_data_col_indices);
for (int i = 0; i < _data_col_indices.Length; i++)
output_row[i + _start_index_in_output] = DBNull.Value;
}
#region IDisposable
public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
public void Dispose() {
ExceptionAssert.not_null(_join_spec.stream).onChange -= _on_stream_cb;
}
#endregion
}
@@ -1,20 +1,18 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using mbts;
using mbts.utils;
namespace BCG {
// this a collection of data, theres no joining logic in here
public class NonIncJoinSpec {
public ViewStream stream;
public ViewStream? stream;
// join col names are the names on which the streams are joined (for equality),
// data col names are the names in the original stream
// projected col names are their new names, ie if its x:l_x, then x is in the
// data_col_names and l_x in projected cols
public string[] join_col_names, data_col_names, projected_col_names;
public string[]? join_col_names, data_col_names, projected_col_names;
public NonIncJoinSpec() { }
@@ -31,8 +29,8 @@ namespace BCG {
public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
foreach (Triplet<ViewStream, string, string> r in join_specs) {
string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
string[] visible_cols = r.third.Split(' ');
string[] keys = (r.second ?? string.Empty).Split(' ');
string[] visible_cols = ExceptionAssert.not_null(r.third).Split(' ');
List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
foreach (string s in visible_cols) {
string[] toks = s.Split(':');
@@ -44,22 +42,15 @@ namespace BCG {
mapped_col_names.Add(toks[1]);
}
}
rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
rv.Add(new NonIncJoinSpec(ExceptionAssert.not_null(r.first), keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
}
return rv.ToArray();
}
public override string ToString() {
Func<string[], string> arr_txt = (a) => {
string rv = "";
foreach (string s in a) {
rv += "{s} ";
}
rv = rv.Trim();
return rv;
};
string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
return rv;
}
public override string ToString() =>
$"stream: {stream} "
+ $"join_col_names: { string.Join(" ", join_col_names ?? Array.Empty<string>())} "
+ $"data_col_names: { string.Join(" ", data_col_names ?? Array.Empty<string>())} "
+ $"projected_col_names: { string.Join(" ", projected_col_names ?? Array.Empty<string>())}";
}
}
@@ -1,7 +1,14 @@
using System;
using System.Collections;
using System.Collections.Generic;
using bts;
using Bts.DataStreamCore;
using mbts;
using mbts.utils;
using Spark.Core.Collections;
namespace BCG
@@ -178,10 +185,11 @@ namespace BCG
public class SuperJoinRow : IBlueRow
{
private JoinProjectHelper myHelper;
private IBlueRow myLeftRow, myRightRow;
private IBlueRow myLeftRow;
private IBlueRow? myRightRow;
int myID;
public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow rightRow, JoinProjectHelper joinProjectHelper)
public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow? rightRow, JoinProjectHelper joinProjectHelper)
{
myID = id;
myLeftRow = leftRow;
@@ -189,21 +197,21 @@ namespace BCG
myHelper = joinProjectHelper;
}
public int leftID { get { return (int)myLeftRow["id"]; } }
public int leftID => myLeftRow.Id();
public int rightID { get { return (int)myRightRow["id"]; } }
public int rightID => ExceptionAssert.not_null(myRightRow).Id();
public IBlueRow leftRow { get { return myLeftRow; } }
public IBlueRow rightRow { get { return myRightRow; } }
public IBlueRow? rightRow => myRightRow;
#region IBlueRow Members
public IBlueTableSchema schema { get { return myHelper.schema; } }
public object this[int index] { get { return this[schema[index].colName]; } }
public object? this[int index] { get { return this[schema[index].colName]; } }
public object this[string colName]
public object? this[string colName]
{
get
{
@@ -211,8 +219,7 @@ namespace BCG
return myID;
else
{
bool isLeft;
string col = myHelper.getOriginalCol(colName, out isLeft);
string col = myHelper.getOriginalCol(colName, out bool isLeft);
return isLeft ? myLeftRow[col] : (myRightRow == null ? (object)DBNull.Value : myRightRow[col]);
}
}
@@ -229,7 +236,7 @@ namespace BCG
public bool Contains(object key)
{
return myLeftRow.Contains(key) || myRightRow.Contains(key);
return myLeftRow.Contains(key) || (myRightRow?.Contains(key) ?? false);
}
public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
@@ -246,7 +253,7 @@ namespace BCG
{
get
{
List<object> rv = new List<object>();
var rv = new List<object?>();
foreach (string col in Keys)
rv.Add(this[col]);
return rv;
@@ -291,9 +298,10 @@ namespace BCG
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
private bool myManageUnmappedLeftRows = false;
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
protected IBlueTableSchema myLeftSchema, myRightSchema;
protected IBlueTableSchema? myLeftSchema;
protected IBlueTableSchema? myRightSchema;
protected JoinProjectHelper myJoinProjectHelper;
protected BlueTable myState;
protected BlueTable? myState;
protected int myID;
protected ViewStream myLeftStream, myRightStream;
protected string myLeftKeyCol, myRightKeyCol;
@@ -346,25 +354,25 @@ namespace BCG
private void myHandleLeftStreamChanges(object newValue)
{
myHandleStreamChanges(newValue as DataPacket, true);
myHandleStreamChanges(ExceptionAssert.Argument.Is<DataPacket>(newValue), true);
}
private void myHandleRightStreamChanges(object newValue)
{
myHandleStreamChanges(newValue as DataPacket, false);
myHandleStreamChanges(ExceptionAssert.Argument.Is<DataPacket>(newValue), false);
}
private void myHandleStreamChanges(DataPacket dp, bool isLeft)
{
myInitSchema(dp.schema, isLeft);
BlueTable dels = null, adds = null;
BlueTable? dels = null, adds = null;
if (myState != null)
{
dels = new BlueTable(myState.schema);
adds = new BlueTable(myState.schema);
}
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
myAddRows(dp.addedRows, isLeft, myState, adds);
myDeleteRows(dp.deletedRows, isLeft, ExceptionAssert.not_null(myState), ExceptionAssert.not_null(dels));
myAddRows(dp.addedRows, isLeft, myState, ExceptionAssert.not_null(adds));
if (myState != null)
{
myFireEvent(new DataPacket(myState, adds, dels));
@@ -410,8 +418,8 @@ namespace BCG
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
foreach (IBlueRow r in addedRows)
{
object currentKey = r[thisKeyCol];
int currentID = (int)r["id"];
var currentKey = r[thisKeyCol] ?? DBNull.Value;
int currentID = r.Id();
///add it to 1.
if (!thisKeyToRows.ContainsKey(currentKey))
thisKeyToRows.Add(currentKey, new Dictionary<int, IBlueRow>());
@@ -422,14 +430,14 @@ namespace BCG
ICollection<IBlueRow> matchingRows = otherSideKeyToRows[currentKey].Values;
foreach (IBlueRow mr in matchingRows)
{
int mrID = (int)mr["id"];
int mrID = mr.Id();
SuperJoinRow rr = new SuperJoinRow(myID++, isLeft ? r : mr, isLeft ? mr : r, myJoinProjectHelper);
state.addRow(rr);
addedRowsOut.addRow(rr);
///add to 3.
if (!thisIDToResultRows.ContainsKey(currentID))
thisIDToResultRows.Add(currentID, new Dictionary<int, SuperJoinRow>());
thisIDToResultRows[currentID].Add((int)rr["id"], rr);
thisIDToResultRows[currentID].Add(rr.Id(), rr);
///add to 4.
if (!otherIDToResultRows.ContainsKey(mrID))
{
@@ -437,7 +445,7 @@ namespace BCG
if (myManageUnmappedLeftRows && !isLeft)
myTakeCareOfUnmatachedLeftRows(mr, false);
}
otherIDToResultRows[mrID].Add((int)rr["id"], rr);
otherIDToResultRows[mrID].Add(rr.Id(), rr);
}
}
else
@@ -458,22 +466,21 @@ namespace BCG
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
foreach (IBlueRow r in deletedRows)
{
int id = (int)r["id"];
object key = r[thisKeyCol];
int id = r.Id();
object key = ExceptionAssert.not_null(r[thisKeyCol]);
///delete it from 1.
thisKeyToRows[key].Remove(id);
if (thisKeyToRows[key].Count == 0)
thisKeyToRows.Remove(key);
///form return rows and delete from 3.
if (thisIDToResultRows.ContainsKey(id))
if (thisIDToResultRows.TryGetValue(id, out IDictionary<int, SuperJoinRow>? removedResultRows))
{
ICollection removedResultRows = thisIDToResultRows[id].Values as ICollection;
thisIDToResultRows.Remove(id);
foreach (SuperJoinRow rr in removedResultRows)
foreach (var rr in removedResultRows.Values)
{
///delete from 4.
int otherId = isLeft ? rr.rightID : rr.leftID;
otherIDToResultRows[otherId].Remove((int)rr["id"]);
otherIDToResultRows[otherId].Remove(rr.Id());
if (otherIDToResultRows[otherId].Count == 0)
otherIDToResultRows.Remove(otherId);
delRowsOut.addRow(rr);
@@ -493,15 +500,11 @@ namespace BCG
private void myTakeCareOfUnmatachedLeftRows(IBlueRow leftRow, bool add)
{
int lid = (int)leftRow["id"];
if (add)
int lid = leftRow.Id();
if (add) {
myLeftUnmatchedRowsAdds.Add(lid, leftRow);
else
{
if (myLeftUnmatchedRowsAdds.ContainsKey(lid))
myLeftUnmatchedRowsAdds.Remove(lid);
else
myLeftUnmatchedRowsDels.Add((int)leftRow["id"], leftRow);
} else if (!myLeftUnmatchedRowsAdds.Remove(lid)) {
myLeftUnmatchedRowsDels.Add(lid, leftRow);
}
}
}
@@ -522,44 +525,48 @@ namespace BCG
protected override void myFireEvent(object o)
{
myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, o as DataPacket);
var dp = ExceptionAssert.Argument.Is<DataPacket>(o);
myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, dp);
}
private void myHandleJoinResults(Dictionary<int, IBlueRow> unmatchedLeftRowsDels,
Dictionary<int, IBlueRow> unmatchedLeftRowAdds, DataPacket resultPacket)
{
BlueTable dels = resultPacket.deletedRows as BlueTable,
adds = resultPacket.addedRows as BlueTable;
foreach (IBlueRow lr in unmatchedLeftRowsDels.Values)
{
///an lr will be here if its actually deleted or
///a right row is added, so now it has a partner
///and is no longer unmatched.
///in both cases the half row that this lr
///created needs to be removed from this
///class's state and added to dels
///it means a right is added
int lid = (int)lr["id"];
SuperJoinRow rr = myLeftIDToHalfRows[lid];
myLeftIDToHalfRows.Remove(lid);
myState.deleteRow(rr);
dels.addRow(rr);
var dels = ExceptionAssert.of_type<BlueTable>(resultPacket.deletedRows);
var adds = ExceptionAssert.of_type<BlueTable>(resultPacket.addedRows);
if (unmatchedLeftRowsDels.Count > 0) {
Log.notnull(myState);
foreach (IBlueRow lr in unmatchedLeftRowsDels.Values) {
///an lr will be here if its actually deleted or
///a right row is added, so now it has a partner
///and is no longer unmatched.
///in both cases the half row that this lr
///created needs to be removed from this
///class's state and added to dels
///it means a right is added
int lid = lr.Id();
SuperJoinRow rr = myLeftIDToHalfRows[lid];
myLeftIDToHalfRows.Remove(lid);
myState.deleteRow(rr);
dels.addRow(rr);
}
}
foreach (IBlueRow lr in unmatchedLeftRowAdds.Values)
{
///an lr will be here if its added with no
///matching right, or a right matching row
///is gone so it is unmatched now.
///in both cases a half row needs to be
///added to this class's state and added
///to adds
int lid = (int)lr["id"];
SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
myLeftIDToHalfRows.Add(lid, rr);
myState.addRow(rr);
adds.addRow(rr);
if (unmatchedLeftRowAdds.Count > 0) {
Log.notnull(myState);
foreach (IBlueRow lr in unmatchedLeftRowAdds.Values) {
///an lr will be here if its added with no
///matching right, or a right matching row
///is gone so it is unmatched now.
///in both cases a half row needs to be
///added to this class's state and added
///to adds
int lid = lr.Id();
var rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
myLeftIDToHalfRows.Add(lid, rr);
myState.addRow(rr);
adds.addRow(rr);
}
}
base.myFireEvent(new DataPacket(myState, adds, dels));
}
@@ -1,6 +1,9 @@
using bts;
using Bts.DataStreamCore;
using System;
using System.Collections;
using BCG.Utilities;
namespace BCG
{
@@ -11,9 +14,10 @@ namespace BCG
[DataStreamUserRespresentation (false)]
public class LeftInternalJoinStream : InternalJoinStream
{
protected DataPacket myLeftDataPacket, myRightDataPacket;
protected DataPacket? myLeftDataPacket, myRightDataPacket;
//Container to contain all the left rows that are orphaned during a deletion operation
private IDictionary myLeftOrphanedRows;
IDictionary? myLeftOrphanedRows;
public LeftInternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight) :
base (vsLeft, jcLeft, vsRight, jcRight) {}
@@ -62,10 +66,10 @@ namespace BCG
private void myTestAndFire (bool isLeft)
{
JoinOperand jo = isLeft ? myLeftOperand : myRightOperand;
DataPacket dp = isLeft ? myLeftDataPacket : myRightDataPacket;
var dp = isLeft ? myLeftDataPacket : myRightDataPacket;
//to test whether both have received call backs
if (myLeftOperand.hasReceivedData)
jo.changeHandler (dp);
jo.changeHandler(Log.notnull(dp));
else if (myLeftDataPacket != null && myRightDataPacket != null)
{
myRightOperand.changeHandler (myRightDataPacket);
@@ -82,9 +86,11 @@ namespace BCG
{
foreach (IBlueRow row in rows)
myAddRow (row, isLeft);
if (!isLeft)
if (!isLeft) {
Log.notnull(myLeftOrphanedRows);
foreach (RowNode node in myLeftOrphanedRows.Values)
((JoinTable)myLastFiredPacket.newRows).addRow (node.row, null, myLastFiredPacket);
((JoinTable)myLastFiredPacket.newRows).addRow(node.row, null, myLastFiredPacket);
}
}
/// <summary>
@@ -104,7 +110,7 @@ namespace BCG
if (isLeft)
{
myLeftOperand.addRow (row);
RowList matchedRows = myRightOperand.getMatchingRows (row[myLeftJoinColOrdinal]);
RowList matchedRows = myRightOperand.getMatchingRows (row[myLeftJoinColOrdinal] ?? DBNull.Value);
if (matchedRows.Count == 0)
((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
else
@@ -113,10 +119,10 @@ namespace BCG
}
else
{
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]);
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value);
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count;
myRightOperand.addRow (row);
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count;
if (matchedRows.Count != 0){
foreach (RowNode matchedNode in matchedRows)
{
@@ -125,10 +131,11 @@ namespace BCG
//the jointable as a new row will replace it
if (rightMatchingRowCount == 0 && newRightMatchingRowCount > 0)
{
if (myLeftOrphanedRows[matchedNode.row["id"]] == null)
Log.notnull(myLeftOrphanedRows);
if (myLeftOrphanedRows[matchedNode.row.Id()] == null)
((JoinTable)myLastFiredPacket.newRows).deleteRow (true, matchedNode.row, myLastFiredPacket);
else
myLeftOrphanedRows.Remove (matchedNode.row["id"]);
myLeftOrphanedRows.Remove (matchedNode.row.Id());
}
//they were orphaned by a previous transaction and hence the orphans need to be
//removed from the Jointable and the complete row added
@@ -148,21 +155,23 @@ namespace BCG
}
else
{
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]);
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value);
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count;
myRightOperand.deleteRow (row);
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count;
if (matchedRows.Count != 0)
{
if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0)
if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0) {
//ie all the matchedRows were joined with 'this' right row (method arg), and hence they need to
//be handled specially, ie joined with nulls, if there are no right rows with this
//joincolumn value being inserted.
Log.notnull(myLeftOrphanedRows);
foreach (RowNode matchedNode in matchedRows)
{
// BlueEventLog.logAssert (myLeftOrphanedRows[matchedNode.row["id"]] == null);
myLeftOrphanedRows[matchedNode.row["id"]] = matchedNode;
myLeftOrphanedRows[matchedNode.row.Id()] = matchedNode;
}
}
((JoinTable)myLastFiredPacket.newRows).deleteRow (false, row, myLastFiredPacket);
}
}
@@ -173,9 +182,11 @@ namespace BCG
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
myLastFiredPacket.newRows = new JoinTable (mySchema);
myLastFiredPacket.addedRows = new BlueTable (mySchema);
if (!isLeft)
if (!isLeft) {
Log.notnull(myLeftDataPacket);
foreach (IBlueRow row in myLeftDataPacket.newRows)
((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
((JoinTable)myLastFiredPacket.newRows).addRow(row, null, myLastFiredPacket);
}
}
}
@@ -30,7 +30,7 @@ namespace BCG
/// <summary>
/// Node containing the referencedRow
/// </summary>
IBlueRow referencedRow {get;}
IBlueRow? referencedRow {get;}
/// <summary>
/// removes all referenced or referencing nodes (indicated by the bool), p.s. theres only one referenced row
@@ -1,5 +1,3 @@
using System;
namespace BCG
{
/// <summary>
@@ -15,12 +13,12 @@ namespace BCG
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
IBlueRow getRow (int id);
IBlueRow? getRow (int id);
/// <summary>
/// Gets all the rows in the node
/// </summary>
/// <returns></returns>
IBlueRow[] getRows ();
IBlueRow?[] getRows ();
/// <summary>
/// Replaces the row with the given id, with the row passed as the second arg
/// </summary>
@@ -1,7 +1,7 @@
using Bts.DataStreamCore;
using System;
using System.Collections;
using BCG.Utilities;
using BCG;
namespace BCG
{
@@ -56,18 +56,18 @@ namespace BCG
/// The key can either be the string corrsponding to the column name or
/// the order of the column
/// </summary>
public virtual object this [object key]
public virtual object? this [object key]
{
get {return key is string ? this [key as string] : this [(int)key];}
get => key is string ks ? this[ks] : this[(int)key];
set {throw new NotSupportedException ("The IBlueRow is read only");}
}
public virtual object this [int index]
public virtual object? this [int index]
{
get
{
if (index == mySchema.columnNames.Length - 1)
return myRingGRow["id"];
return myRingGRow.Id();
if (index < myRingGRow.Count)
return myRingGRow[index];
else if (index >= myRingGRow.Count && index < mySchema.columnNames.Length - 1)
@@ -77,7 +77,8 @@ namespace BCG
}
}
public object this [string colName] {get{return this[mySchema[colName].colNo];}}
public object? this [string colName] =>
this[mySchema[colName].colNo];
public ICollection Keys{get{return mySchema.columnNames;}}
@@ -1,6 +1,7 @@
using mbts.utils;
using System;
using System.Collections;
using BCG.Utilities;
namespace BCG
{
@@ -70,7 +71,10 @@ namespace BCG
}
else
{
SimpleNode sn = n as SimpleNode;
var sn = ExceptionAssert.Argument.Is<SimpleNode>(n);
if (sn.prev is null) {
throw new ArgumentException("prev is null", nameof(n));
}
sn.prev.next = sn.next;
if (sn.next != null)
sn.next.prev = sn.prev;
@@ -81,7 +85,7 @@ namespace BCG
public int count {get {return myFirst.listCount;}}
public IBlueRow referencedRow {get {return myFirst.row;}}
public IBlueRow? referencedRow => myFirst.row;
public void removeAll (bool isReD)
{
@@ -110,10 +114,17 @@ namespace BCG
public class LightForeignJoinRowListEnumerator : IEnumerator
{
private ILightForeignJoinRowList myList;
private SimpleNode myPtr, myFirst;
public LightForeignJoinRowListEnumerator (ILightForeignJoinRowList list, SimpleNode first)
{myList = list; myFirst = first;}
public object Current {get{return myPtr.row;}}
private SimpleNode? myPtr;
private SimpleNode myFirst;
public LightForeignJoinRowListEnumerator(ILightForeignJoinRowList list, SimpleNode first) {
myList = list;
myFirst = first;
}
public object? Current =>
myPtr is null ? throw new InvalidOperationException("myPtr is null") : myPtr.row;
public bool MoveNext() {return (myPtr = myPtr == null ? myFirst : myPtr.next) != null;}
public void Reset() {myPtr = null;}
public ILightForeignJoinRowList list {get {return myList;}}
@@ -1,5 +1,6 @@
using Bts.DataStreamCore;
using System;
using BCG.Utilities;
namespace BCG
{
@@ -13,12 +14,12 @@ namespace BCG
: base (reDRow, rinGRow, schema)
{}
public override object this [int index]
public override object? this [int index]
{
get
{
if (index == mySchema.columnNames.Length - 1)
return myRingGRow["id"];
return myRingGRow.Id();
if (index < myRingGRow.Count)
return myRingGRow[index];
else if (index >= myRingGRow.Count && index < mySchema.columnNames.Length - 1)
@@ -1,5 +1,3 @@
using System;
namespace BCG
{
/// <summary>
@@ -9,15 +7,15 @@ namespace BCG
/// </summary>
public class SimpleNode : INode
{
public IBlueRow row;
public SimpleNode next, prev;
public IBlueRow? row;
public SimpleNode? next;
public SimpleNode? prev;
public SimpleNode (IBlueRow r)
: this (r, null, null)
{}
public SimpleNode (IBlueRow r, SimpleNode nx, SimpleNode pv)
{
public SimpleNode(IBlueRow r, SimpleNode? nx, SimpleNode? pv) {
row = r;
next = nx;
prev = pv;
@@ -25,13 +23,15 @@ namespace BCG
#region INode Members
public IBlueRow getRow (int id) {return row;}
public IBlueRow? getRow (int id) =>
row;
public IBlueRow[] getRows() {return new IBlueRow[] {row};}
public IBlueRow?[] getRows() =>
new[] { row };
public void replace (int id, IBlueRow nr) {row = nr;}
#endregion
}
}
}
@@ -1,14 +1,8 @@
using System;
using System.Collections;
using System.Collections.Generic;
using Amazon.Runtime.Internal.Util;
using BCG;
using BCG.Utilities;
using bts;
using System;
using System.Collections.Generic;
namespace BCG {
// Prefixing with "old" to avoid name conflict with the BlueOrb LiteralStream
[DataStreamUserRespresentation("oldLiteral")]
@@ -44,13 +38,14 @@ namespace BCG {
return new BlueRow(map);
};
string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
BlueTable table = null;
BlueTable? table = null;
foreach (string json_string in row_arr) {
IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
get_row(json_string, table == null ? 0 : table.Count);
table = table ?? new BlueTable(r.schema);
table ??= new BlueTable(r.schema);
table.addRow(r);
}
table ??= BlueTable.Empty(table_name);
ViewStream str = new BlueTableStream(table);
str = new SchemaGuesserStream(str);
mySetRealSource(str);
@@ -67,8 +62,11 @@ namespace BCG {
string[] columnAndType = c.Split('/');
if (columnAndType.Length == 1)
ci[n] = new ColumnInfo(c, n, typeof(double));
else
ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
else {
var colType = Type.GetType("System." + columnAndType[1], true)
?? throw new ArgumentException($"Failed to get column type at index {n}", nameof(schema));
ci[n] = new ColumnInfo(columnAndType[0], n, colType);
}
n++;
}
@@ -1,13 +1,14 @@
using bts;
using Bts.ColumnMapExpressions;
using Bts.DataStream;
using mbts.config;
using mbts.utils;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
// using BCG.Evaluator.ColumnMapExpression;
namespace BCG
{
@@ -26,7 +27,7 @@ namespace BCG
/// <param name="isch">original(unprojected) schema</param>
/// <param name="exp">expression defining the mapping</param>
/// <param name="tableName">table name of the new schema</param>
internal ProjectHelper (IBlueTableSchema isch, string exp, string tableName)
internal ProjectHelper (IBlueTableSchema isch, string exp, string? tableName)
: this (isch, myGetDictionary (isch, exp), tableName)
{}
@@ -36,10 +37,10 @@ namespace BCG
/// <param name="origToAlias">dictionary mapping unprojected names to their projected names, dropped cols should not be there</param>
/// <param name="tableName">table name of the new schema</param>
internal ProjectHelper (
IBlueTableSchema isch, IDictionary<string, string> origToAlias, string tableName)
IBlueTableSchema isch, IDictionary<string, string> origToAlias, string? tableName)
{
ArrayList colMapAl = new ArrayList (),
colInfoAl = new ArrayList ();
var colMapAl = new List<int>();
var colInfoAl = new List<ColumnInfo>();
relevantOrigCols = new string[origToAlias.Count-1];
int c = 0;
ArrayList iColNames = new ArrayList (isch.columnNames);
@@ -49,19 +50,15 @@ namespace BCG
if (origToAlias.TryGetValue(colName, out var colAlias))
{
ColumnInfo ci = isch[colName];
colInfoAl.Add (new ColumnInfo (colAlias,
colInfoAl.Count,
ci.colType)
);
colMapAl.Add (ci.colNo);
colInfoAl.Add(new ColumnInfo (colAlias, colInfoAl.Count, ci.colType));
colMapAl.Add(ci.colNo);
if (ci.colName != "id")
relevantOrigCols[c++] = ci.colName;
}
}
schema = new BlueTableSchema (tableName == null ? isch.tableName : tableName,
colInfoAl.ToArray (typeof(ColumnInfo)) as ColumnInfo[]);
colMap = colMapAl.ToArray (typeof(int)) as int[];
schema = new BlueTableSchema(tableName ?? isch.tableName, colInfoAl.ToArray());
colMap = colMapAl.ToArray();
}
/// <summary>
@@ -75,16 +72,11 @@ namespace BCG
{
IDictionary<string, string> origToAlias = new Dictionary<string, string>();
ColumnMapExpression e;
if (Configuration.get<DataStreamConfig>().UseStaticParser)
{
e = ColumnMapExpression.Parse(exp);
}
else
{
var e = Configuration.get<DataStreamConfig>().UseStaticParser
? ColumnMapExpression.Parse(exp)
// TODO (maybe): Actually use non-static parser (unclear what advantages it will have)
e = ColumnMapExpression.Parse(exp);
}
: ColumnMapExpression.Parse(exp);
Log.notnull(e);
if (e.IsInclusive) {
origToAlias = e.Columns.ToDictionary();
origToAlias["id"] = "id";
@@ -104,7 +96,7 @@ namespace BCG
{
IDictionary retVal = new Hashtable (dic.Count);
foreach (DictionaryEntry de in dic)
retVal.Add (de.Value, de.Key);
retVal.Add(ExceptionAssert.not_null(de.Value), de.Key);
return retVal;
}
}
@@ -1,4 +1,8 @@
namespace BCG {
using bts;
using mbts.utils;
using Spark.Core.Collections;
using System.Collections;
@@ -8,10 +12,11 @@ namespace BCG {
[DataStreamUserRespresentation("oldproject")]
public class ProjectStream : ViewStream
{
private ProjectHelper myHelper;
private string myColExp, myTableName;
private ViewStream myVs;
private IDictionary<string, string> myDic;
ProjectHelper? myHelper;
string? myColExp;
string? myTableName;
ViewStream? myVs;
IDictionary<string, string>? myDic;
/// <summary>
/// </summary>
/// <param name="vs">stream to susbsribe to as the input stream</param>
@@ -54,17 +59,17 @@ namespace BCG {
protected override void myOnFirstSubscriber()
{
myVs.onChange += myHandleChange;
Log.notnull(myVs).onChange += myHandleChange;
}
protected override void myOnLastSubscriber()
{
myVs.onChange -= myHandleChange;
Log.notnull(myVs).onChange -= myHandleChange;
}
internal void myHandleChange (object newValue)
{
DataPacket dp = newValue as DataPacket;
var dp = ExceptionAssert.Argument.Is<IDataPacket>(newValue);
if (myHelper != null)
{
IBlueRowCollection dels, adds;
@@ -79,8 +84,8 @@ namespace BCG {
else
{ //first time a callback is received
myHelper = myDic == null ?
new ProjectHelper (dp.schema, myColExp, myTableName) :
new ProjectHelper (dp.schema, myDic, myTableName);
new ProjectHelper (dp.Schema, myColExp ?? string.Empty, myTableName) :
new ProjectHelper (dp.Schema, myDic, myTableName);
IBlueRowCollection temp;
myFireEvent (new DataPacket (temp = new ProjectedTable (dp.newRows, myHelper),
@@ -1,13 +1,10 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using BCG.Utilities;
using System.Diagnostics;
using Microsoft.Data.Analysis;
using Parquet;
using Parquet.Data;
using mbts.utils;
using Microsoft.Data.Analysis;
using System;
using System.Diagnostics;
using System.IO;
namespace BCG {
[DataStreamUserRespresentation("CSVSaver")]
@@ -40,7 +37,7 @@ namespace BCG {
myStream.onChange -= myHandleChanges;
private void myHandleChanges(object newValue) {
IBlueRowCollection totalState = (newValue as DataPacket).newRows;
IBlueRowCollection totalState = ExceptionAssert.Argument.Is<DataPacket>(newValue).newRows;
if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
myNWrites++;
int i = 0;
@@ -62,7 +59,10 @@ namespace BCG {
IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
BlueTable nRows = new BlueTable(myResultSchema);
nRows.addRow(nr);
myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
BlueTable deletedRows = myLastValue is null
? new(myResultSchema)
: new(ExceptionAssert.of_type<DataPacket>(myLastValue).newRows);
myFireEvent(new DataPacket(nRows, nRows, deletedRows));
} else
myFireEvent(newValue);
}
@@ -83,7 +83,7 @@ namespace BCG {
if (col != "id") {
string colText = String.Empty;
if (r[col] != null && r[col] != DBNull.Value)
colText = r[col].ToString();
colText = r[col]?.ToString() ?? string.Empty;
if (r.schema[col].colType == typeof(string))
colText = "\"" + colText + "\"";
rv += (i++ == 0 ? colText : "," + colText);
@@ -136,7 +136,7 @@ namespace BCG {
}
public class CSVReaderStreamMapper : IStreamMapper {
private IBlueTableSchema mySchema;
IBlueTableSchema? mySchema;
private readonly string[] myOutputSchemaStringArray;
public CSVReaderStreamMapper(string outputSchema) {
@@ -162,6 +162,7 @@ namespace BCG {
}
public IBlueRow map(IBlueRow row) {
ArgumentNullException.ThrowIfNull(mySchema);
object[] vals = new object[row.schema.columnInfo.Length];
row.Values.CopyTo(vals, 0);
for (int i = 0; i < mySchema.columnInfo.Length; i++) {
@@ -1,5 +1,7 @@
using Bts.ColumnSortExpressions;
using mbts.utils;
using System;
using System.Collections;
using System.Collections.Generic;
@@ -39,7 +41,7 @@ public class SortRow : IComparable, IBlueRow {
var otherVal = otherRow[col.Name];
int rv = myIsNull(this[col.Name]) ?
(myIsNull(otherVal) ? 0 : -1) :
(myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
(myIsNull(otherVal) ? 1 : ExceptionAssert.of_type<IComparable>(this[col.Name]).CompareTo(otherVal));
rv *= col.IsAscending ? 1 : -1;
return rv;
}
@@ -1,8 +1,7 @@
using System;
using System.Diagnostics;
using System.IO;
using BCG.Utilities.General;
using BCG.Utilities.General;
using System;
using System.IO;
namespace BCG
{
@@ -49,8 +48,7 @@ namespace BCG
#region IWorker
public void doWork()
{
if ((myNextWrite <= DateTime.Now && myGotUpdate))
{
if (myNextWrite <= DateTime.Now && myGotUpdate) {
myGotUpdate = false;
myNextWrite = DateTime.Now.AddSeconds(myWriteFreqSecs);
myWrite(myStream.rows);
@@ -96,7 +94,7 @@ namespace BCG
rowStr += DELIM;
}
rowStr += row[columnName].ToString();
rowStr += row[columnName]?.ToString() ?? string.Empty;
}
writer.WriteLine(rowStr);
@@ -139,7 +137,7 @@ namespace BCG
private int myWriteFreqSecs;
private DateTime myNextWrite;
private bool myGotUpdate;
private string mySchemaStr;
private string? mySchemaStr;
#endregion
}
}
@@ -1,9 +1,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Bts.DataStreamCore;
using System;
using System.Collections;
using System.Collections.Generic;
namespace BCG
{
@@ -11,7 +10,7 @@ namespace BCG
{
private ViewStream _src;
private string _time_stamp_field;
private IBlueTableSchema _schema;
private IBlueTableSchema? _schema;
private string _latency_field;
public LatencyStream(ViewStream input, string time_stamp_field, string latency_field)
@@ -38,10 +37,10 @@ namespace BCG
public IBlueRow map(IBlueRow row)
{
var ht = new Hashtable(row);
if (row[_time_stamp_field].Equals(DBNull.Value))
if (Equals(row[_time_stamp_field], DBNull.Value))
ht[_latency_field] = DBNull.Value;
else
ht[_latency_field] = (DateTime.Now - (DateTime)row[_time_stamp_field]).TotalMilliseconds;
ht[_latency_field] = (DateTime.Now - row.GetValue<DateTime>(_time_stamp_field)).TotalMilliseconds;
return new BlueRow(_schema, ht);
}
@@ -1,7 +1,8 @@
using System;
using Bts.DataStreamCore;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BCG
{
@@ -42,8 +43,8 @@ namespace BCG
BlueTable result = new BlueTable(dp.schema);
foreach (IBlueRow r in dp.newRows)
{
string key = r[myCol_Key].ToString();
DateTime dt = (DateTime) r[myCol_Time];
var key = r[myCol_Key]?.ToString() ?? string.Empty;
var dt = r.GetValue<DateTime>(myCol_Time);
double value = Convert.ToDouble(r[myCol_Value]);
object[] rowValues = new object[r.Count];
@@ -68,9 +69,7 @@ namespace BCG
private double myUpdateValue(string key, DateTime time, double value)
{
SortedList<DateTime, double> seriesValues;
if (!myKeyToTimeToValue.TryGetValue(key, out seriesValues))
{
if (!myKeyToTimeToValue.TryGetValue(key, out SortedList<DateTime, double>? seriesValues)) {
seriesValues = new SortedList<DateTime, double>();
myKeyToTimeToValue[key] = seriesValues;
}
@@ -1,3 +1,7 @@
using Bts.DataStreamCore;
using mbts.utils;
using System;
using System.Collections;
@@ -12,7 +16,7 @@ namespace BCG
private int myId = 0;
private IDictionary myLeftIndex = new Hashtable (),
myRightIndex = new Hashtable ();
private IBlueTableSchema mySchema;
private IBlueTableSchema? mySchema;
public AppendCollection () {}
@@ -24,15 +28,15 @@ namespace BCG
IDictionary index = isLeft ? myLeftIndex : myRightIndex;
foreach (IBlueRow r in dp.deletedRows)
{
IBlueRow dr = index[r["id"]] as IBlueRow;
var dr = ExceptionAssert.of_type<IBlueRow>(index[r.Id()]);
dRows.addRow (dr);
index.Remove (r["id"]);
index.Remove (r.Id());
}
foreach (IBlueRow r in dp.addedRows)
{
IBlueRow ar = new AppendRow (r, myId++, mySchema);
aRows.addRow (ar);
index.Add (r["id"], ar);
index.Add(r.Id(), ar);
}
return new DataPacket (this, aRows, dRows);
}
@@ -43,7 +47,8 @@ namespace BCG
public bool containsID (int id) {throw new ApplicationException ("IBlueRow is readonly");}
public IBlueTableSchema schema {get {return mySchema;}}
public IBlueTableSchema schema =>
ExceptionAssert.not_null(mySchema);
#endregion
@@ -60,15 +65,14 @@ namespace BCG
array.SetValue (r, i++);
}
public object SyncRoot {get {return null;}}
public object SyncRoot => this;
#endregion
#region IEnumerable Members
public System.Collections.IEnumerator GetEnumerator()
{
return new MyEnumerator (myLeftIndex, myRightIndex);
public System.Collections.IEnumerator GetEnumerator() {
return new MyEnumerator(myLeftIndex, myRightIndex);
}
private class MyEnumerator : IEnumerator
@@ -1,11 +1,12 @@
using System;
using mbts.utils;
namespace BCG
{
public class AppendStreamModified : ViewStream
{
private AppendCollection myColl = new AppendCollection ();
private ViewStream myLeftStream, myRightstream;
ViewStream? myLeftStream;
ViewStream? myRightstream;
public AppendStreamModified (ViewStream leftStream, ViewStream rightStream)
{
@@ -19,7 +20,7 @@ namespace BCG
mySetRealSource (vss[0]);
else
{
AppendStreamModified rv = null;
AppendStreamModified? rv = null;
foreach (ViewStream s in vss)
rv = rv == null ? new AppendStreamModified (new ViewStream[] {s}) : new AppendStreamModified (rv, s);
mySetRealSource (rv);
@@ -28,24 +29,26 @@ namespace BCG
protected override void myOnFirstSubscriber()
{
myLeftStream.onChange += myHandleLeftChanges;
myRightstream.onChange += myHandleRightChanges;
ExceptionAssert.not_null(myLeftStream).onChange += myHandleLeftChanges;
ExceptionAssert.not_null(myRightstream).onChange += myHandleRightChanges;
}
protected override void myOnLastSubscriber()
{
myLeftStream.onChange -= myHandleLeftChanges;
myRightstream.onChange -= myHandleRightChanges;
ExceptionAssert.not_null(myLeftStream).onChange -= myHandleLeftChanges;
ExceptionAssert.not_null(myRightstream).onChange -= myHandleRightChanges;
}
private void myHandleLeftChanges (object newValue)
{
myFireEvent (myGetDataPacket (newValue as DataPacket, true));
var dp = ExceptionAssert.Argument.Is<DataPacket>(newValue);
myFireEvent(myGetDataPacket(dp, true));
}
private void myHandleRightChanges (object newValue)
{
myFireEvent (myGetDataPacket (newValue as DataPacket, false));
var dp = ExceptionAssert.Argument.Is<DataPacket>(newValue);
myFireEvent(myGetDataPacket(dp, false));
}
private DataPacket myGetDataPacket (DataPacket dp, bool isLeft)