From a4efaa0089639f1f0835f4fa3b6e061e495c2dff Mon Sep 17 00:00:00 2001 From: John Colvin Date: Fri, 22 Aug 2025 14:50:01 -0400 Subject: [PATCH] cleanup: Fix a ton of build warnings (cs) All these warnings have been bothering me for a while. The work isn't done, but this commit chops them down from ~1000 to ~600. * AppendCollection.cs * AppendStreamModified.cs * BenzingaNewsStream.cs * BenzingaNewsSymbolStateStream.cs * BookStream.cs * CSVSaverStream.cs * ChartsUtils.cs * ColumnInfo.cs * ContractUtils.cs * ContractUtils.cs * DiffStream.cs * FuturesActiveContractStream.cs * FuzzyJoinOverTimeStream.cs * HistoricalBarsStream.cs * HistoricalBarsStream.cs * InstrumentsStream.cs * JoinOperandRowIndex.cs * JoinRow.cs * JoinStream.cs * JoinTable.cs * JoinTableSchema.cs * LatencyStream.cs * LeftInternalJoinStream.cs * LightForeignJoinRow.cs * LightForeignJoinRowList.cs * LightLeftForeignJoinRow.cs * LiteralStream.cs * NLStream.cs * NLToDSLStream.cs * NLToIndexedFewShotPromptStream.cs * NonIncGroupByStream.cs * NonIncJoinBookkeeper.cs * NonIncJoinSpec.cs * OHLCVStream.cs * OpenAIPromptStream.cs * PriceStatsStream.cs * PriceStatsStream.cs * ProjectHelper.cs * ProjectStream.cs * PubSubServerStateStream.cs * PubsubHeartbeatStream.cs * SimpleNode.cs * SortRow.cs * SpatialArbStatsStream.cs * SpatialFuturePairsStream.cs * SpreadPerformanceEvaluator.cs * SpreadPerformanceStream.cs * StreamIOCommon.cs * SuperJoinStream.cs * TableMetaDataStream.cs * TextWriterStream.cs * TimeAverageStream.cs --- .../DataStream/Primitives/Diff/DiffStream.cs | 54 +++---- .../Fuzzy/FuzzyJoinOverTimeStream.cs | 59 ++++---- .../NonIncremental/NonIncGroupByStream.cs | 11 +- .../Primitives/Join/JoinOperandRowIndex.cs | 20 +-- .../DataStream/Primitives/Join/JoinRow.cs | 34 +++-- .../DataStream/Primitives/Join/JoinStream.cs | 35 +++-- .../DataStream/Primitives/Join/JoinTable.cs | 27 ++-- .../Primitives/Join/JoinTableSchema.cs | 26 ++-- .../NonIncremental/NonIncJoinBookkeeper.cs | 52 ++++--- .../Join/NonIncremental/NonIncJoinSpec.cs | 31 ++-- .../Primitives/Join/SuperJoinStream.cs | 143 +++++++++--------- .../LeftJoin/LeftInternalJoinStream.cs | 51 ++++--- .../ILightForeignJoinRowList.cs | 2 +- .../Primitives/LightJoinStreams/INode.cs | 6 +- .../LightJoinStreams/LightForeignJoinRow.cs | 15 +- .../LightForeignJoinRowList.cs | 25 ++- .../LightLeftForeignJoinRow.cs | 7 +- .../Primitives/LightJoinStreams/SimpleNode.cs | 18 +-- .../Primitives/Literal/LiteralStream.cs | 24 ++- .../Primitives/Project/ProjectHelper.cs | 40 ++--- .../Primitives/Project/ProjectStream.cs | 23 +-- .../SideEffectStreams/CSVSaverStream.cs | 27 ++-- .../DataStream/Primitives/Sort/SortRow.cs | 4 +- .../TextFileStreams/TextWriterStream.cs | 14 +- .../Primitives/Time/LatencyStream.cs | 15 +- .../TimeAverage/TimeAverageStream.cs | 13 +- .../Primitives/Union/AppendCollection.cs | 22 +-- .../Primitives/Union/AppendStreamModified.cs | 21 +-- .../DataStreamCore/BlueData/ColumnInfo.cs | 40 ++--- .../Publish/Serialization/StreamIOCommon.cs | 87 ++++++----- .../BlueOrb/MarketData/ContractUtils.cs | 46 +++--- .../Historical/HistoricalBarsStream.cs | 20 ++- .../BenzingaNews/BenzingaNewsStream.cs | 12 +- .../BenzingaNewsSymbolStateStream.cs | 16 +- .../Business/Indicators/ChartsUtils.cs | 7 +- .../Analytics/SpatialArbStatsStream.cs | 6 +- .../Analytics/SpatialFuturePairsStream.cs | 4 +- .../Analytics/SpreadPerformanceEvaluator.cs | 35 +---- .../Analytics/SpreadPerformanceStream.cs | 6 +- .../Business/MarketData/BookStream.cs | 6 +- .../MarketData/FuturesActiveContractStream.cs | 2 +- .../Business/MarketData/InstrumentsStream.cs | 20 +-- .../Business/MarketData/OHLCVStream.cs | 4 +- .../Business/MarketData/PriceStatsStream.cs | 41 ++--- .../SharedStreams/NL/OpenAIPromptStream.cs | 3 +- .../Subscription/PubSubServerStateStream.cs | 6 +- .../PS3/Subscription/PubsubHeartbeatStream.cs | 4 +- .../System/PS3/TableMetaDataStream.cs | 16 +- 48 files changed, 610 insertions(+), 590 deletions(-) diff --git a/src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs b/src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs index b25e2aec0..85f95e9ee 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs @@ -1,5 +1,4 @@ using System; -using System.Collections; using System.Collections.Generic; using mbts; @@ -12,30 +11,21 @@ namespace BCG { ViewStream myLStream, myRStream; - DataPacket myLastR = null, myLastL = null; + DataPacket? myLastR = null, myLastL = null; - BlueTable myLastFired = null; + BlueTable? myLastFired = null; double myThreshold = .00005; string[] myColumnsToMatch; - public DiffStream(ViewStream v1, ViewStream v2) : this(null, v1, v2, "id") { } - public DiffStream(ViewStream v1, ViewStream v2, params string[] ColumnsToMatch) : this(null, v1, v2, ColumnsToMatch) { } - public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) : - this(null, v1, v2, thresh, ColumnsToMatch) { } - - public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch) - { + public DiffStream(ViewStream v1, ViewStream v2) : this(v1, v2, "id") { } + public DiffStream(ViewStream v1, ViewStream v2, params string[] columnsToMatch) : this(v1, v2, .00005, columnsToMatch) { } + public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch) { myThreshold = thresh; myLStream = v1; myRStream = v2; myColumnsToMatch = columnsToMatch; } - public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch) - : this(db,v1,v2,.00005,columnsToMatch) - { - } - protected override void myOnFirstSubscriber() { @@ -52,8 +42,14 @@ namespace BCG struct ListComparer : IEqualityComparer> { - public bool Equals(List x, List y) + public bool Equals(List? x, List? y) { + if (x is null) { + return y is null; + } + if (y is null) { + return false; + } if (x.Count != y.Count) return false; @@ -74,21 +70,20 @@ namespace BCG void myHandleCallbacks() { - DataPacket packet = null; - + DataPacket? packet; if (myLastL == null || myLastR == null) { packet = myLastL == null ? myLastR : myLastL; - if (packet != null) + if (packet != null) { myFireEvent(packet); - - myLastFired = new BlueTable(packet.newRows); + myLastFired = new BlueTable(packet.newRows); + } return; } BlueTable table = new BlueTable(myLastL.newRows.schema); - Dictionary, Pair> rowPairs = new Dictionary, Pair>(new ListComparer()); + Dictionary, Pair> rowPairs = new(new ListComparer()); foreach(BlueRow row in myLastL.newRows) { @@ -104,7 +99,7 @@ namespace BCG } } - rowPairs[hash] = new Pair(row, null); + rowPairs[hash] = new(row, null); } foreach (BlueRow row in myLastR.newRows) @@ -119,12 +114,11 @@ namespace BCG catch (Exception) { } } - Pair pair; - rowPairs.TryGetValue(hash, out pair); + rowPairs.TryGetValue(hash, out var pair); if (pair == null) - pair = new Pair(null, row); + pair = new(null, row); else pair.second = row; @@ -132,13 +126,11 @@ namespace BCG } int id = 0; - foreach (KeyValuePair, Pair> hashpair in rowPairs) + foreach (var (hashVals, pair) in rowPairs) { - Pair pair = hashpair.Value; - if (pair.first == null || pair.second == null) { - BlueRow row = pair.first == null ? pair.second : pair.first; + var row = pair.first ?? pair.second ?? throw new InvalidOperationException("Both rows in pair are null"); Dictionary newRow = new Dictionary(); foreach (ColumnInfo info in row.schema.columnInfo) @@ -211,4 +203,4 @@ namespace BCG } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs b/src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs index 069f662e3..486281275 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs @@ -1,4 +1,8 @@ -using System; +using bts; + +using Bts.DataStreamCore; + +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -40,12 +44,12 @@ namespace BCG private IComparer myRightCompare; protected FuzzyRowComparer myFuzzyCompare; protected TimeSpan myTimeWindow; - protected IBlueTableSchema myLeftSchema; - protected IBlueTableSchema myRightSchema; - protected IBlueTableSchema myOutputSchema; + protected IBlueTableSchema? myLeftSchema; + protected IBlueTableSchema? myRightSchema; + protected IBlueTableSchema? myOutputSchema; private int myID = 0; - protected TimeWindowTable myStateRows; + protected TimeWindowTable? myStateRows; private DateTime myNextCull; const long MinCullIntervalMS = 1000; @@ -65,7 +69,7 @@ namespace BCG this.myFuzzyCompare = fuzzyCompare; } - public virtual IBlueTableSchema OutputSchema + public virtual IBlueTableSchema? OutputSchema { get { @@ -150,7 +154,7 @@ namespace BCG int count; for (count = 0; count < list.Count; count++) { - if ((DateTime)list[count][timeColumn] > expiry) + if (list[count].GetValue(timeColumn) > expiry) { // Stop when we find the first row that we can keep. Since this is sorted all others are also kept. break; @@ -172,14 +176,17 @@ namespace BCG this.myFitness = fitness; } public IBlueRow myRightRow { get; private set; } - public IBlueRow myLeftRow { get; set; } + public IBlueRow? myLeftRow { get; set; } public double myFitness { get; private set; } - public IBlueRow myMergedRow { get; set; } + public IBlueRow? myMergedRow { get; set; } #region IComparable Members - public int CompareTo(RowMatch other) + public int CompareTo(RowMatch? other) { + if (other is null) { + return 1; + } return myFitness.CompareTo(other.myFitness); } @@ -262,6 +269,10 @@ namespace BCG private void HandleCorrection(List corrections, BlueTable adds, BlueTable dels) { + if (corrections.Count == 0) { + return; + } + Log.notnull(myStateRows); // We already emitted correction.MergedRow so we need to redact that and // then emit correction.left as an add since it is no longer matched foreach (RowMatch correction in corrections.Distinct()) @@ -285,7 +296,7 @@ namespace BCG int r = 0; for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; ) { - if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn]) + if (leftExpired[l].GetValue(myLeftTimeColumn) <= rightExpired[r].GetValue(myRightTimeColumn)) { // Advance Left IEnumerable rm = rightMatches[l]; @@ -332,7 +343,7 @@ namespace BCG private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable rightMatches) { - IBlueRow merged = null; + IBlueRow? merged = null; if (rightMatches != null) { foreach (RowMatch match in rightMatches) @@ -366,9 +377,9 @@ namespace BCG /// Override this function to provide a custom row merge. /// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting. /// - protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right) + protected virtual IBlueRow MergeRows(IBlueRow? left, IBlueRow? right) { - Dictionary newRow = new Dictionary(); + Dictionary newRow = new(); if (left == null && right == null) throw new ArgumentException("oh come on."); @@ -393,14 +404,14 @@ namespace BCG } } newRow["id"] = this.myID++; - newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn]; + newRow[NEW_TIME_COLUMN] = (left != null) ? left[myLeftTimeColumn] : right![myRightTimeColumn]; return new BlueRow(this.OutputSchema, newRow); } - private IEnumerable FindMatches(IBlueRow row, List list, List corrections) + private IEnumerable? FindMatches(IBlueRow row, List list, List corrections) { - List matches = null; + List? matches = null; // Search through all rows in list and compare with the fuzzy comparer for (int x = 0; x < list.Count; x++) @@ -421,21 +432,15 @@ namespace BCG matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch); // Mark the best match for this particular right hand match - RowMatch oldMatch; - if (this.myBestMatch.TryGetValue(match, out oldMatch)) - { - if (currentMatch.myFitness < oldMatch.myFitness) - { + if (myBestMatch.TryGetValue(match, out var oldMatch)) { + if (currentMatch.myFitness < oldMatch.myFitness) { this.myBestMatch[match] = currentMatch; - if (oldMatch.myMergedRow != null) - { + if (oldMatch.myMergedRow != null) { // We already emitted oldMatch.MergedRow so we need to fix it corrections.Add(oldMatch); } } - } - else - { + } else { this.myBestMatch[match] = currentMatch; } } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs index fbc8ef6e4..447aaf662 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs @@ -7,6 +7,8 @@ using System.Text.Json; using bts; using bts.SilverPine; +using mbts.utils; + namespace BCG { //[DataStreamUserRespresentation("AggregateStream")] public class NonIncGroupByStream : ViewStream { @@ -80,7 +82,7 @@ namespace BCG { protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; } private void _main_stream_onchange(object nv) { - DataPacket dp = nv as DataPacket; + var dp = ExceptionAssert.Argument.Is(nv); ///grouping together _timer.Restart(); IDictionary> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ? @@ -109,7 +111,12 @@ namespace BCG { } // TODO, if (total_state != null) - myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows)); + myFireEvent(new DataPacket( + total_state, + total_state, + myLastValue == null + ? new BlueTable(total_state.schema) + : ExceptionAssert.of_type(myLastValue).newRows)); } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperandRowIndex.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperandRowIndex.cs index 3c22cd209..64111d784 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperandRowIndex.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinOperandRowIndex.cs @@ -1,6 +1,5 @@ using System; -using System.Collections; -using BCG.Utilities; +using System.Collections.Generic; namespace BCG { @@ -11,7 +10,8 @@ namespace BCG /// internal class JoinOperandRowIndex { - private IDictionary myJoinColDict = new Hashtable (), myIdDict = new Hashtable (); + readonly Dictionary myJoinColDict = new(); + readonly Dictionary myIdDict = new(); private int myJoinColOrdinal, myIdColOrdinal; /// @@ -35,18 +35,18 @@ namespace BCG public void add (IBlueRow row) { RowList list; - if (myJoinColDict[row[myJoinColOrdinal]] == null) - myJoinColDict[row[myJoinColOrdinal]] = list = new RowList (); + if (myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value] == null) + myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value] = list = new RowList (); else - list = (RowList)myJoinColDict[row[myJoinColOrdinal]]; - myIdDict[row[myIdColOrdinal]] = list.add (row); + list = (RowList)myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value]; + myIdDict[row[myIdColOrdinal] ?? DBNull.Value] = list.add (row); } public void remove (IBlueRow row) { - RowNode node = (RowNode)myIdDict[row[myIdColOrdinal]]; - ((RowList)myJoinColDict[row[myJoinColOrdinal]]).remove (node); - myIdDict.Remove (row[myIdColOrdinal]); + RowNode node = (RowNode)myIdDict[row[myIdColOrdinal] ?? DBNull.Value]; + ((RowList)myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value]).remove (node); + myIdDict.Remove (row[myIdColOrdinal] ?? DBNull.Value); } public RowList getMatchingRows (object key) diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinRow.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinRow.cs index e4f8425b5..bd06127a4 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/JoinRow.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinRow.cs @@ -1,18 +1,20 @@ using System; -using BCG.Utilities; using System.Collections; using bts; +using Bts.DataStreamCore; namespace BCG { internal class JoinRow : IBlueRow { - private RowNode myLeftNode, myRightNode; - protected IBlueRow myLeftRow, myRightRow; + RowNode? myLeftNode; + RowNode? myRightNode; + protected IBlueRow myLeftRow; + protected IBlueRow? myRightRow; protected int myId; protected JoinTableSchema mySchema; - private object[] myValues; - + private object?[] myValues; + #region Constructor(s) /// @@ -21,8 +23,8 @@ namespace BCG /// left row of the join /// right row of the join /// schema of the joined row - public JoinRow (IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema) - : this ((int)leftRow["id"], leftRow, rightRow, schema) {} + public JoinRow(IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema) + : this(leftRow.Id(), leftRow, rightRow, schema) { } /// /// Constructs a row assigning it the ID provioded as the first argument @@ -31,7 +33,7 @@ namespace BCG /// left row of the join /// right row of the join /// schema of the joined row - public JoinRow (int id, IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema) + public JoinRow (int id, IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema) { myId = id; myLeftRow = leftRow; @@ -48,13 +50,13 @@ namespace BCG public bool isPartialJoin{get{return myRightRow == null;}} - public RowNode leftNode + public RowNode? leftNode { get{return myLeftNode;} set{myLeftNode = value;} } - public RowNode rightNode + public RowNode? rightNode { get{return myRightNode;} set{myRightNode = value;} @@ -85,7 +87,7 @@ namespace BCG /// The key can either be the string corrsponding to the column name or /// the order of the column /// - public virtual object this [object key] + public virtual object? this [object key] { get { @@ -96,7 +98,7 @@ namespace BCG set {throw new NotSupportedException ("The IBlueRow is read only");} } - public object this [int index] + public object? this [int index] { get { @@ -119,7 +121,7 @@ namespace BCG } } - public object this [string colName] {get{return this[mySchema[colName].colNo];}} + public object? this [string colName] {get{return this[mySchema[colName].colNo];}} public ICollection Keys{get{return mySchema.columnNames;}} @@ -133,9 +135,11 @@ namespace BCG } } - public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");} + public void Add(object key, object? value) => + throw new NotSupportedException("The IBlueRow is readonly"); - public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");} + public void Clear() => + throw new NotSupportedException ("The IBlueRow is readonly"); public bool Contains (object key){return mySchema[(string)key] != null;} diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs index db494e8b7..23e098b8c 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinStream.cs @@ -1,11 +1,10 @@ -using System; -using System.Data; -using System.Collections; -using System.Text.RegularExpressions; -using BCG.Utilities; -using BCG.Utilities.General; using bts; +using mbts.utils; + +using System; +using System.Collections; + namespace BCG { public class JoinStream : ViewStream @@ -111,8 +110,15 @@ namespace BCG { if (myPs == null) { - JoinNameResolver resolver = new JoinNameResolver ((JoinTableSchema)myIjs.schema, myIjs.getSchema (true), myLeftCm.columnExpression, - myIjs.getSchema (false), myRightCm.columnExpression); + ArgumentNullException.ThrowIfNull(myIjs); + ArgumentNullException.ThrowIfNull(myLeftCm); + ArgumentNullException.ThrowIfNull(myRightCm); + var resolver = new JoinNameResolver( + ExceptionAssert.of_type(myIjs.schema), + myIjs.getSchema(true), + myLeftCm.columnExpression, + myIjs.getSchema(false), + myRightCm.columnExpression); IDictionary map = resolver.getMap (); map["id"] = "id"; myPs = new ProjectStream (myIjs, map); @@ -123,26 +129,27 @@ namespace BCG Log.assert(false, "This call back should not have been called"); } else - myFireEvent ((o as ICloneable).Clone()); + myFireEvent(ExceptionAssert.of_type(o).Clone()); } private void myProjectChangeHandler (object o) { + ArgumentNullException.ThrowIfNull(myIjs); if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1) myIjs.onChange -= myInternalJoinChangeHandler; - myFireEvent ((o as ICloneable).Clone()); + myFireEvent(ExceptionAssert.of_type(o).Clone()); } protected override void myOnFirstSubscriber () { - myIjs.onChange += myInternalJoinChangeHandler; + ExceptionAssert.not_null(myIjs).onChange += myInternalJoinChangeHandler; } protected override void myOnLastSubscriber () { // how does this relate to the -= in myProjectChangeHandler? - myIjs.onChange -= myInternalJoinChangeHandler; + ExceptionAssert.not_null(myIjs).onChange -= myInternalJoinChangeHandler; // if there is a project after the ijs we need to // unsubscribe to it otherwise references will be kept and @@ -159,7 +166,9 @@ namespace BCG /// /// /// - public void test (DataPacket dp, bool isLeft) {myIjs.test (dp, isLeft);} + public void test (DataPacket dp, bool isLeft) => + ExceptionAssert.not_null(myIjs).test(dp, isLeft); + /***********************************************************************************************/ } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs index a2d88c1ea..154a992dd 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinTable.cs @@ -1,7 +1,7 @@ +using Bts.DataStreamCore; + using System; using System.Collections; -using System.IO; -using BCG.Utilities.General; namespace BCG { @@ -19,20 +19,13 @@ namespace BCG myRightIndex = new JoinDeletionIndex (mySchema.getRenamedName ("id", false), false); } - public IBlueRow this [int id] - { - get - { - if (myIdIndex[id] == null) - throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName); - return (IBlueRow)myIdIndex[id]; - } - } + public IBlueRow this [int id] => + myIdIndex[id] as IBlueRow ?? throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName); #region JoinTable members - public virtual void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp) + public virtual void addRow (IBlueRow leftRow, IBlueRow? rightRow, DataPacket dp) { - JoinRow newRow = new JoinRow (myNewRowId, leftRow, rightRow, mySchema); + var newRow = new JoinRow(myNewRowId, leftRow, rightRow, mySchema); myIdIndex[myNewRowId ++] = newRow; ((BlueTable)(dp.addedRows)).addRow (newRow); myLeftIndex.add (newRow); @@ -48,17 +41,17 @@ namespace BCG thisIndex = myLeftIndex; otherIndex = myRightIndex; } - RowList matchingNodes = thisIndex.getMatchingRows (deletedRow["id"]); + RowList matchingNodes = thisIndex.getMatchingRows (deletedRow.Id()); foreach (RowNode node in matchingNodes) { JoinRow row = (JoinRow)node.row; - RowNode correspondingNode = isLeft ? row.rightNode : row.leftNode; + var correspondingNode = isLeft ? row.rightNode : row.leftNode; if (correspondingNode != null) otherIndex.remove (correspondingNode); //in case of partial left joinss - myIdIndex.Remove (row["id"]); + myIdIndex.Remove (row.Id()); ((BlueTable)dp.deletedRows).addRow (row); } - thisIndex.removeAll (deletedRow["id"]); + thisIndex.removeAll(deletedRow.Id()); } #endregion #region ICollection members diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/JoinTableSchema.cs b/src/DataStreamSuite/DataStream/Primitives/Join/JoinTableSchema.cs index 74ae14431..00be130f7 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/JoinTableSchema.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/JoinTableSchema.cs @@ -1,9 +1,10 @@ -using System; -using System.Collections; -using BCG.Utilities; +using System.Collections.Generic; +using System.Linq; using bts; +using mbts.utils; + namespace BCG { /// @@ -13,10 +14,11 @@ namespace BCG public class JoinTableSchema : BlueTableSchema { private string myLeftJc = "", myRightJc = "", myLeftId = "", myRightId = ""; - private IDictionary myLeftNameToAlias = new Hashtable (); - private IDictionary myRightNameToAlias = new Hashtable (); - private IDictionary myAliasToLeftName = new Hashtable(), - myAliasToRightName = new Hashtable(); + readonly Dictionary myLeftNameToAlias = new(); + readonly Dictionary myRightNameToAlias = new(); + readonly Dictionary myAliasToLeftName = new(); + readonly Dictionary myAliasToRightName = new(); + /// /// Constructs the schema of the Jointable with the schema of the left and right tables. Incase of /// a name being there in both the shemas it prefixes the name with "L." or "R." accordingly. Column "id" @@ -85,9 +87,9 @@ namespace BCG renamed = false; } myIndexByColName["id"] = new ColumnInfo("id", myIndexByColName.Count, typeof(int)); - myIndexByColOrder[myIndexByColOrder.Length - 1] = (ColumnInfo)myIndexByColName["id"]; - myAliasToLeftName = ProjectHelper.invertDic(myLeftNameToAlias); - myAliasToRightName = ProjectHelper.invertDic(myRightNameToAlias); + myIndexByColOrder[myIndexByColOrder.Length - 1] = ExceptionAssert.of_type(myIndexByColName["id"]); + myAliasToLeftName = myLeftNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key); + myAliasToRightName = myRightNameToAlias.ToDictionary(kv => kv.Key, kv => kv.Value); } public string leftJoinCol {get{return myLeftJc;}} @@ -97,8 +99,8 @@ namespace BCG public string getRenamedName (string origName, bool isLeft) { if (isLeft) - return (string)myLeftNameToAlias[origName]; - return (string)myRightNameToAlias[origName]; + return myLeftNameToAlias[origName] as string ?? string.Empty; + return myRightNameToAlias[origName] as string ?? string.Empty; } public string getOriginalName(string rename, bool isLeft) diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs index 03305d575..695f70347 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs @@ -2,16 +2,15 @@ using bts.collections; using mbts; +using mbts.utils; using System; using System.Collections.Generic; using System.Diagnostics; -using System.Linq; -using System.Security.Cryptography; namespace BCG { public class NonIncJoinBookkeeper : IDisposable { - public Action on_new_data; + public Action? on_new_data; private readonly NonIncJoinSpec _join_spec; private readonly int _join_position; private readonly int _start_index_in_output; @@ -20,16 +19,19 @@ namespace BCG { private readonly bool _is_tag_along; private int _n_total_fires = 0, _n_expensive_fires = 0; - private IBlueTableSchema _schema = null; + private IBlueTableSchema? _schema; private IDictionary _row_map = new Dictionary(); private List> _row_list = new List>(); - private IBlueRow _solitary_row = null; + private IBlueRow? _solitary_row; - private int[] _join_col_indices = null, - _data_col_indices = null; + int[]? _join_col_indices; + int[]? _data_col_indices; - public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) { - _join_spec = the_join_spec; + public NonIncJoinBookkeeper(NonIncJoinSpec join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) { + if (join_spec.join_col_names is null) { + throw new ArgumentException("join_col_names is null", nameof(join_spec)); + } + _join_spec = join_spec; _join_position = join_position; _start_index_in_output = start_index_in_output; _row_hasher = row_hasher; @@ -41,7 +43,7 @@ namespace BCG { private void _fill_missing_data(DataPacket dp) { if (_filled_missing_data) return; - Func null_or_empty = (str_arr) => { + bool null_or_empty(string[]? str_arr) { if (str_arr == null || str_arr.Length == 0) return true; foreach (string s in str_arr) { @@ -50,13 +52,16 @@ namespace BCG { } } return true; - }; - Func remove_bad_cols = (col_names) => { + } + string[] remove_bad_cols(string[]? col_names) { + if (col_names is null) { + return Array.Empty(); + } HashSet schema_col_map = new HashSet(dp.schema.columnNames); List cols_list = new List(col_names); cols_list.RemoveWhere(c => !schema_col_map.Contains(c)); return cols_list.ToArray(); - }; + } if (null_or_empty(_join_spec.data_col_names)) { _join_spec.data_col_names = dp.schema.columnNames; } @@ -70,7 +75,7 @@ namespace BCG { } public void init() { - _join_spec.stream.onChange += _on_stream_cb; + ExceptionAssert.not_null(_join_spec.stream).onChange += _on_stream_cb; } public void _on_stream_cb(object dp_obj) { @@ -111,14 +116,19 @@ namespace BCG { public bool is_ready { get { return _schema != null; } } public void constribute_cols_to_schema(List cis) { - for (int i = 0; i < _join_spec.data_col_names.Length; i++) - cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType)); + ArgumentNullException.ThrowIfNull(_join_spec.data_col_names); + if (_join_spec.data_col_names.Length > 0) { + ArgumentNullException.ThrowIfNull(_join_spec.projected_col_names); + ArgumentNullException.ThrowIfNull(_schema); + for (int i = 0; i < _join_spec.data_col_names.Length; i++) + cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType)); + } } public List> get_list() { return _row_list; } public void copy_master_row_to_output_row(int row_index, object[] output_row) { - IBlueRow this_row = _row_list[row_index].second; + IBlueRow? this_row = _row_list[row_index].second; BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output); } @@ -129,8 +139,7 @@ namespace BCG { else copy_dbnull_to_output_row(output_row); } else { - IBlueRow this_row = null; - if (_row_map.TryGetValue(row_hash, out this_row)) + if (_row_map.TryGetValue(row_hash, out var this_row)) BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output); else copy_dbnull_to_output_row(output_row); @@ -138,13 +147,16 @@ namespace BCG { } private void copy_dbnull_to_output_row(object[] output_row) { + ArgumentNullException.ThrowIfNull(_data_col_indices); for (int i = 0; i < _data_col_indices.Length; i++) output_row[i + _start_index_in_output] = DBNull.Value; } #region IDisposable - public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; } + public void Dispose() { + ExceptionAssert.not_null(_join_spec.stream).onChange -= _on_stream_cb; + } #endregion } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs index 432dbd539..470f2dc51 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinSpec.cs @@ -1,20 +1,18 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using mbts; +using mbts.utils; namespace BCG { // this a collection of data, theres no joining logic in here public class NonIncJoinSpec { - public ViewStream stream; + public ViewStream? stream; // join col names are the names on which the streams are joined (for equality), // data col names are the names in the original stream // projected col names are their new names, ie if its x:l_x, then x is in the // data_col_names and l_x in projected cols - public string[] join_col_names, data_col_names, projected_col_names; + public string[]? join_col_names, data_col_names, projected_col_names; public NonIncJoinSpec() { } @@ -31,8 +29,8 @@ namespace BCG { public static NonIncJoinSpec[] from_triplet(params Triplet[] join_specs) { List rv = new List(); foreach (Triplet r in join_specs) { - string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' '); - string[] visible_cols = r.third.Split(' '); + string[] keys = (r.second ?? string.Empty).Split(' '); + string[] visible_cols = ExceptionAssert.not_null(r.third).Split(' '); List data_col_names = new List(), mapped_col_names = new List(); foreach (string s in visible_cols) { string[] toks = s.Split(':'); @@ -44,22 +42,15 @@ namespace BCG { mapped_col_names.Add(toks[1]); } } - rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray())); + rv.Add(new NonIncJoinSpec(ExceptionAssert.not_null(r.first), keys, data_col_names.ToArray(), mapped_col_names.ToArray())); } return rv.ToArray(); } - public override string ToString() { - Func arr_txt = (a) => { - string rv = ""; - foreach (string s in a) { - rv += "{s} "; - } - rv = rv.Trim(); - return rv; - }; - string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}"; - return rv; - } + public override string ToString() => + $"stream: {stream} " + + $"join_col_names: { string.Join(" ", join_col_names ?? Array.Empty())} " + + $"data_col_names: { string.Join(" ", data_col_names ?? Array.Empty())} " + + $"projected_col_names: { string.Join(" ", projected_col_names ?? Array.Empty())}"; } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs index 52edd07bd..b29cf1cbe 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs @@ -1,7 +1,14 @@ using System; using System.Collections; using System.Collections.Generic; + +using bts; + +using Bts.DataStreamCore; + using mbts; +using mbts.utils; + using Spark.Core.Collections; namespace BCG @@ -178,10 +185,11 @@ namespace BCG public class SuperJoinRow : IBlueRow { private JoinProjectHelper myHelper; - private IBlueRow myLeftRow, myRightRow; + private IBlueRow myLeftRow; + private IBlueRow? myRightRow; int myID; - public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow rightRow, JoinProjectHelper joinProjectHelper) + public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow? rightRow, JoinProjectHelper joinProjectHelper) { myID = id; myLeftRow = leftRow; @@ -189,21 +197,21 @@ namespace BCG myHelper = joinProjectHelper; } - public int leftID { get { return (int)myLeftRow["id"]; } } + public int leftID => myLeftRow.Id(); - public int rightID { get { return (int)myRightRow["id"]; } } + public int rightID => ExceptionAssert.not_null(myRightRow).Id(); public IBlueRow leftRow { get { return myLeftRow; } } - public IBlueRow rightRow { get { return myRightRow; } } + public IBlueRow? rightRow => myRightRow; #region IBlueRow Members public IBlueTableSchema schema { get { return myHelper.schema; } } - public object this[int index] { get { return this[schema[index].colName]; } } + public object? this[int index] { get { return this[schema[index].colName]; } } - public object this[string colName] + public object? this[string colName] { get { @@ -211,8 +219,7 @@ namespace BCG return myID; else { - bool isLeft; - string col = myHelper.getOriginalCol(colName, out isLeft); + string col = myHelper.getOriginalCol(colName, out bool isLeft); return isLeft ? myLeftRow[col] : (myRightRow == null ? (object)DBNull.Value : myRightRow[col]); } } @@ -229,7 +236,7 @@ namespace BCG public bool Contains(object key) { - return myLeftRow.Contains(key) || myRightRow.Contains(key); + return myLeftRow.Contains(key) || (myRightRow?.Contains(key) ?? false); } public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } @@ -246,7 +253,7 @@ namespace BCG { get { - List rv = new List(); + var rv = new List(); foreach (string col in Keys) rv.Add(this[col]); return rv; @@ -291,9 +298,10 @@ namespace BCG myLeftUnmatchedRowsDels = new Dictionary(); private bool myManageUnmappedLeftRows = false; protected string myLeftProj = String.Empty, myRightProj = String.Empty; - protected IBlueTableSchema myLeftSchema, myRightSchema; + protected IBlueTableSchema? myLeftSchema; + protected IBlueTableSchema? myRightSchema; protected JoinProjectHelper myJoinProjectHelper; - protected BlueTable myState; + protected BlueTable? myState; protected int myID; protected ViewStream myLeftStream, myRightStream; protected string myLeftKeyCol, myRightKeyCol; @@ -346,25 +354,25 @@ namespace BCG private void myHandleLeftStreamChanges(object newValue) { - myHandleStreamChanges(newValue as DataPacket, true); + myHandleStreamChanges(ExceptionAssert.Argument.Is(newValue), true); } private void myHandleRightStreamChanges(object newValue) { - myHandleStreamChanges(newValue as DataPacket, false); + myHandleStreamChanges(ExceptionAssert.Argument.Is(newValue), false); } private void myHandleStreamChanges(DataPacket dp, bool isLeft) { myInitSchema(dp.schema, isLeft); - BlueTable dels = null, adds = null; + BlueTable? dels = null, adds = null; if (myState != null) { dels = new BlueTable(myState.schema); adds = new BlueTable(myState.schema); } - myDeleteRows(dp.deletedRows, isLeft, myState, dels); - myAddRows(dp.addedRows, isLeft, myState, adds); + myDeleteRows(dp.deletedRows, isLeft, ExceptionAssert.not_null(myState), ExceptionAssert.not_null(dels)); + myAddRows(dp.addedRows, isLeft, myState, ExceptionAssert.not_null(adds)); if (myState != null) { myFireEvent(new DataPacket(myState, adds, dels)); @@ -410,8 +418,8 @@ namespace BCG otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows; foreach (IBlueRow r in addedRows) { - object currentKey = r[thisKeyCol]; - int currentID = (int)r["id"]; + var currentKey = r[thisKeyCol] ?? DBNull.Value; + int currentID = r.Id(); ///add it to 1. if (!thisKeyToRows.ContainsKey(currentKey)) thisKeyToRows.Add(currentKey, new Dictionary()); @@ -422,14 +430,14 @@ namespace BCG ICollection matchingRows = otherSideKeyToRows[currentKey].Values; foreach (IBlueRow mr in matchingRows) { - int mrID = (int)mr["id"]; + int mrID = mr.Id(); SuperJoinRow rr = new SuperJoinRow(myID++, isLeft ? r : mr, isLeft ? mr : r, myJoinProjectHelper); state.addRow(rr); addedRowsOut.addRow(rr); ///add to 3. if (!thisIDToResultRows.ContainsKey(currentID)) thisIDToResultRows.Add(currentID, new Dictionary()); - thisIDToResultRows[currentID].Add((int)rr["id"], rr); + thisIDToResultRows[currentID].Add(rr.Id(), rr); ///add to 4. if (!otherIDToResultRows.ContainsKey(mrID)) { @@ -437,7 +445,7 @@ namespace BCG if (myManageUnmappedLeftRows && !isLeft) myTakeCareOfUnmatachedLeftRows(mr, false); } - otherIDToResultRows[mrID].Add((int)rr["id"], rr); + otherIDToResultRows[mrID].Add(rr.Id(), rr); } } else @@ -458,22 +466,21 @@ namespace BCG otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows; foreach (IBlueRow r in deletedRows) { - int id = (int)r["id"]; - object key = r[thisKeyCol]; + int id = r.Id(); + object key = ExceptionAssert.not_null(r[thisKeyCol]); ///delete it from 1. thisKeyToRows[key].Remove(id); if (thisKeyToRows[key].Count == 0) thisKeyToRows.Remove(key); ///form return rows and delete from 3. - if (thisIDToResultRows.ContainsKey(id)) + if (thisIDToResultRows.TryGetValue(id, out IDictionary? removedResultRows)) { - ICollection removedResultRows = thisIDToResultRows[id].Values as ICollection; thisIDToResultRows.Remove(id); - foreach (SuperJoinRow rr in removedResultRows) + foreach (var rr in removedResultRows.Values) { ///delete from 4. int otherId = isLeft ? rr.rightID : rr.leftID; - otherIDToResultRows[otherId].Remove((int)rr["id"]); + otherIDToResultRows[otherId].Remove(rr.Id()); if (otherIDToResultRows[otherId].Count == 0) otherIDToResultRows.Remove(otherId); delRowsOut.addRow(rr); @@ -493,15 +500,11 @@ namespace BCG private void myTakeCareOfUnmatachedLeftRows(IBlueRow leftRow, bool add) { - int lid = (int)leftRow["id"]; - if (add) + int lid = leftRow.Id(); + if (add) { myLeftUnmatchedRowsAdds.Add(lid, leftRow); - else - { - if (myLeftUnmatchedRowsAdds.ContainsKey(lid)) - myLeftUnmatchedRowsAdds.Remove(lid); - else - myLeftUnmatchedRowsDels.Add((int)leftRow["id"], leftRow); + } else if (!myLeftUnmatchedRowsAdds.Remove(lid)) { + myLeftUnmatchedRowsDels.Add(lid, leftRow); } } } @@ -522,44 +525,48 @@ namespace BCG protected override void myFireEvent(object o) { - myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, o as DataPacket); + var dp = ExceptionAssert.Argument.Is(o); + myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, dp); } private void myHandleJoinResults(Dictionary unmatchedLeftRowsDels, Dictionary unmatchedLeftRowAdds, DataPacket resultPacket) { - BlueTable dels = resultPacket.deletedRows as BlueTable, - adds = resultPacket.addedRows as BlueTable; - - foreach (IBlueRow lr in unmatchedLeftRowsDels.Values) - { - ///an lr will be here if its actually deleted or - ///a right row is added, so now it has a partner - ///and is no longer unmatched. - ///in both cases the half row that this lr - ///created needs to be removed from this - ///class's state and added to dels - ///it means a right is added - int lid = (int)lr["id"]; - SuperJoinRow rr = myLeftIDToHalfRows[lid]; - myLeftIDToHalfRows.Remove(lid); - myState.deleteRow(rr); - dels.addRow(rr); + var dels = ExceptionAssert.of_type(resultPacket.deletedRows); + var adds = ExceptionAssert.of_type(resultPacket.addedRows); + if (unmatchedLeftRowsDels.Count > 0) { + Log.notnull(myState); + foreach (IBlueRow lr in unmatchedLeftRowsDels.Values) { + ///an lr will be here if its actually deleted or + ///a right row is added, so now it has a partner + ///and is no longer unmatched. + ///in both cases the half row that this lr + ///created needs to be removed from this + ///class's state and added to dels + ///it means a right is added + int lid = lr.Id(); + SuperJoinRow rr = myLeftIDToHalfRows[lid]; + myLeftIDToHalfRows.Remove(lid); + myState.deleteRow(rr); + dels.addRow(rr); + } } - foreach (IBlueRow lr in unmatchedLeftRowAdds.Values) - { - ///an lr will be here if its added with no - ///matching right, or a right matching row - ///is gone so it is unmatched now. - ///in both cases a half row needs to be - ///added to this class's state and added - ///to adds - int lid = (int)lr["id"]; - SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper); - myLeftIDToHalfRows.Add(lid, rr); - myState.addRow(rr); - adds.addRow(rr); + if (unmatchedLeftRowAdds.Count > 0) { + Log.notnull(myState); + foreach (IBlueRow lr in unmatchedLeftRowAdds.Values) { + ///an lr will be here if its added with no + ///matching right, or a right matching row + ///is gone so it is unmatched now. + ///in both cases a half row needs to be + ///added to this class's state and added + ///to adds + int lid = lr.Id(); + var rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper); + myLeftIDToHalfRows.Add(lid, rr); + myState.addRow(rr); + adds.addRow(rr); + } } base.myFireEvent(new DataPacket(myState, adds, dels)); } diff --git a/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinStream.cs index 08287f70d..ac77cb49b 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LeftJoin/LeftInternalJoinStream.cs @@ -1,6 +1,9 @@ +using bts; + +using Bts.DataStreamCore; + using System; using System.Collections; -using BCG.Utilities; namespace BCG { @@ -11,9 +14,10 @@ namespace BCG [DataStreamUserRespresentation (false)] public class LeftInternalJoinStream : InternalJoinStream { - protected DataPacket myLeftDataPacket, myRightDataPacket; + protected DataPacket? myLeftDataPacket, myRightDataPacket; //Container to contain all the left rows that are orphaned during a deletion operation - private IDictionary myLeftOrphanedRows; + IDictionary? myLeftOrphanedRows; + public LeftInternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight) : base (vsLeft, jcLeft, vsRight, jcRight) {} @@ -62,10 +66,10 @@ namespace BCG private void myTestAndFire (bool isLeft) { JoinOperand jo = isLeft ? myLeftOperand : myRightOperand; - DataPacket dp = isLeft ? myLeftDataPacket : myRightDataPacket; + var dp = isLeft ? myLeftDataPacket : myRightDataPacket; //to test whether both have received call backs if (myLeftOperand.hasReceivedData) - jo.changeHandler (dp); + jo.changeHandler(Log.notnull(dp)); else if (myLeftDataPacket != null && myRightDataPacket != null) { myRightOperand.changeHandler (myRightDataPacket); @@ -82,9 +86,11 @@ namespace BCG { foreach (IBlueRow row in rows) myAddRow (row, isLeft); - if (!isLeft) + if (!isLeft) { + Log.notnull(myLeftOrphanedRows); foreach (RowNode node in myLeftOrphanedRows.Values) - ((JoinTable)myLastFiredPacket.newRows).addRow (node.row, null, myLastFiredPacket); + ((JoinTable)myLastFiredPacket.newRows).addRow(node.row, null, myLastFiredPacket); + } } /// @@ -104,7 +110,7 @@ namespace BCG if (isLeft) { myLeftOperand.addRow (row); - RowList matchedRows = myRightOperand.getMatchingRows (row[myLeftJoinColOrdinal]); + RowList matchedRows = myRightOperand.getMatchingRows (row[myLeftJoinColOrdinal] ?? DBNull.Value); if (matchedRows.Count == 0) ((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket); else @@ -113,10 +119,10 @@ namespace BCG } else { - RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]); - int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count; + RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value); + int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count; myRightOperand.addRow (row); - int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count; + int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count; if (matchedRows.Count != 0){ foreach (RowNode matchedNode in matchedRows) { @@ -125,10 +131,11 @@ namespace BCG //the jointable as a new row will replace it if (rightMatchingRowCount == 0 && newRightMatchingRowCount > 0) { - if (myLeftOrphanedRows[matchedNode.row["id"]] == null) + Log.notnull(myLeftOrphanedRows); + if (myLeftOrphanedRows[matchedNode.row.Id()] == null) ((JoinTable)myLastFiredPacket.newRows).deleteRow (true, matchedNode.row, myLastFiredPacket); else - myLeftOrphanedRows.Remove (matchedNode.row["id"]); + myLeftOrphanedRows.Remove (matchedNode.row.Id()); } //they were orphaned by a previous transaction and hence the orphans need to be //removed from the Jointable and the complete row added @@ -148,21 +155,23 @@ namespace BCG } else { - RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]); - int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count; + RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value); + int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count; myRightOperand.deleteRow (row); - int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count; + int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count; if (matchedRows.Count != 0) { - if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0) + if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0) { //ie all the matchedRows were joined with 'this' right row (method arg), and hence they need to //be handled specially, ie joined with nulls, if there are no right rows with this //joincolumn value being inserted. + Log.notnull(myLeftOrphanedRows); foreach (RowNode matchedNode in matchedRows) { // BlueEventLog.logAssert (myLeftOrphanedRows[matchedNode.row["id"]] == null); - myLeftOrphanedRows[matchedNode.row["id"]] = matchedNode; + myLeftOrphanedRows[matchedNode.row.Id()] = matchedNode; } + } ((JoinTable)myLastFiredPacket.newRows).deleteRow (false, row, myLastFiredPacket); } } @@ -173,9 +182,11 @@ namespace BCG myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; myLastFiredPacket.newRows = new JoinTable (mySchema); myLastFiredPacket.addedRows = new BlueTable (mySchema); - if (!isLeft) + if (!isLeft) { + Log.notnull(myLeftDataPacket); foreach (IBlueRow row in myLeftDataPacket.newRows) - ((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket); + ((JoinTable)myLastFiredPacket.newRows).addRow(row, null, myLastFiredPacket); + } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/ILightForeignJoinRowList.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/ILightForeignJoinRowList.cs index 5766330e6..1db20f64d 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/ILightForeignJoinRowList.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/ILightForeignJoinRowList.cs @@ -30,7 +30,7 @@ namespace BCG /// /// Node containing the referencedRow /// - IBlueRow referencedRow {get;} + IBlueRow? referencedRow {get;} /// /// removes all referenced or referencing nodes (indicated by the bool), p.s. theres only one referenced row diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/INode.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/INode.cs index 7adab3364..a352f9d87 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/INode.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/INode.cs @@ -1,5 +1,3 @@ -using System; - namespace BCG { /// @@ -15,12 +13,12 @@ namespace BCG /// /// /// - IBlueRow getRow (int id); + IBlueRow? getRow (int id); /// /// Gets all the rows in the node /// /// - IBlueRow[] getRows (); + IBlueRow?[] getRows (); /// /// Replaces the row with the given id, with the row passed as the second arg /// diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRow.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRow.cs index 7e02ff7ec..8bda10fdf 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRow.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRow.cs @@ -1,7 +1,7 @@ +using Bts.DataStreamCore; + using System; using System.Collections; -using BCG.Utilities; -using BCG; namespace BCG { @@ -56,18 +56,18 @@ namespace BCG /// The key can either be the string corrsponding to the column name or /// the order of the column /// - public virtual object this [object key] + public virtual object? this [object key] { - get {return key is string ? this [key as string] : this [(int)key];} + get => key is string ks ? this[ks] : this[(int)key]; set {throw new NotSupportedException ("The IBlueRow is read only");} } - public virtual object this [int index] + public virtual object? this [int index] { get { if (index == mySchema.columnNames.Length - 1) - return myRingGRow["id"]; + return myRingGRow.Id(); if (index < myRingGRow.Count) return myRingGRow[index]; else if (index >= myRingGRow.Count && index < mySchema.columnNames.Length - 1) @@ -77,7 +77,8 @@ namespace BCG } } - public object this [string colName] {get{return this[mySchema[colName].colNo];}} + public object? this [string colName] => + this[mySchema[colName].colNo]; public ICollection Keys{get{return mySchema.columnNames;}} diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRowList.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRowList.cs index fb88d8ebf..349405b6b 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRowList.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightForeignJoinRowList.cs @@ -1,6 +1,7 @@ +using mbts.utils; + using System; using System.Collections; -using BCG.Utilities; namespace BCG { @@ -70,7 +71,10 @@ namespace BCG } else { - SimpleNode sn = n as SimpleNode; + var sn = ExceptionAssert.Argument.Is(n); + if (sn.prev is null) { + throw new ArgumentException("prev is null", nameof(n)); + } sn.prev.next = sn.next; if (sn.next != null) sn.next.prev = sn.prev; @@ -81,7 +85,7 @@ namespace BCG public int count {get {return myFirst.listCount;}} - public IBlueRow referencedRow {get {return myFirst.row;}} + public IBlueRow? referencedRow => myFirst.row; public void removeAll (bool isReD) { @@ -110,10 +114,17 @@ namespace BCG public class LightForeignJoinRowListEnumerator : IEnumerator { private ILightForeignJoinRowList myList; - private SimpleNode myPtr, myFirst; - public LightForeignJoinRowListEnumerator (ILightForeignJoinRowList list, SimpleNode first) - {myList = list; myFirst = first;} - public object Current {get{return myPtr.row;}} + private SimpleNode? myPtr; + private SimpleNode myFirst; + + public LightForeignJoinRowListEnumerator(ILightForeignJoinRowList list, SimpleNode first) { + myList = list; + myFirst = first; + } + + public object? Current => + myPtr is null ? throw new InvalidOperationException("myPtr is null") : myPtr.row; + public bool MoveNext() {return (myPtr = myPtr == null ? myFirst : myPtr.next) != null;} public void Reset() {myPtr = null;} public ILightForeignJoinRowList list {get {return myList;}} diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinRow.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinRow.cs index 27d4ff0df..e9066cbef 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinRow.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/LightLeftForeignJoinRow.cs @@ -1,5 +1,6 @@ +using Bts.DataStreamCore; + using System; -using BCG.Utilities; namespace BCG { @@ -13,12 +14,12 @@ namespace BCG : base (reDRow, rinGRow, schema) {} - public override object this [int index] + public override object? this [int index] { get { if (index == mySchema.columnNames.Length - 1) - return myRingGRow["id"]; + return myRingGRow.Id(); if (index < myRingGRow.Count) return myRingGRow[index]; else if (index >= myRingGRow.Count && index < mySchema.columnNames.Length - 1) diff --git a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/SimpleNode.cs b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/SimpleNode.cs index 5166c1243..d81acd18e 100644 --- a/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/SimpleNode.cs +++ b/src/DataStreamSuite/DataStream/Primitives/LightJoinStreams/SimpleNode.cs @@ -1,5 +1,3 @@ -using System; - namespace BCG { /// @@ -9,15 +7,15 @@ namespace BCG /// public class SimpleNode : INode { - public IBlueRow row; - public SimpleNode next, prev; + public IBlueRow? row; + public SimpleNode? next; + public SimpleNode? prev; public SimpleNode (IBlueRow r) : this (r, null, null) {} - public SimpleNode (IBlueRow r, SimpleNode nx, SimpleNode pv) - { + public SimpleNode(IBlueRow r, SimpleNode? nx, SimpleNode? pv) { row = r; next = nx; prev = pv; @@ -25,13 +23,15 @@ namespace BCG #region INode Members - public IBlueRow getRow (int id) {return row;} + public IBlueRow? getRow (int id) => + row; - public IBlueRow[] getRows() {return new IBlueRow[] {row};} + public IBlueRow?[] getRows() => + new[] { row }; public void replace (int id, IBlueRow nr) {row = nr;} #endregion } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs b/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs index db6c6dbf0..0104b44e6 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Literal/LiteralStream.cs @@ -1,14 +1,8 @@ -using System; -using System.Collections; -using System.Collections.Generic; - -using Amazon.Runtime.Internal.Util; - -using BCG; -using BCG.Utilities; - using bts; +using System; +using System.Collections.Generic; + namespace BCG { // Prefixing with "old" to avoid name conflict with the BlueOrb LiteralStream [DataStreamUserRespresentation("oldLiteral")] @@ -44,13 +38,14 @@ namespace BCG { return new BlueRow(map); }; string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries); - BlueTable table = null; + BlueTable? table = null; foreach (string json_string in row_arr) { IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count); get_row(json_string, table == null ? 0 : table.Count); - table = table ?? new BlueTable(r.schema); + table ??= new BlueTable(r.schema); table.addRow(r); } + table ??= BlueTable.Empty(table_name); ViewStream str = new BlueTableStream(table); str = new SchemaGuesserStream(str); mySetRealSource(str); @@ -67,8 +62,11 @@ namespace BCG { string[] columnAndType = c.Split('/'); if (columnAndType.Length == 1) ci[n] = new ColumnInfo(c, n, typeof(double)); - else - ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true)); + else { + var colType = Type.GetType("System." + columnAndType[1], true) + ?? throw new ArgumentException($"Failed to get column type at index {n}", nameof(schema)); + ci[n] = new ColumnInfo(columnAndType[0], n, colType); + } n++; } diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectHelper.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectHelper.cs index 82dca1783..c145df85a 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectHelper.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectHelper.cs @@ -1,13 +1,14 @@ +using bts; + using Bts.ColumnMapExpressions; using Bts.DataStream; using mbts.config; +using mbts.utils; -using System; using System.Collections; using System.Collections.Generic; using System.Linq; -// using BCG.Evaluator.ColumnMapExpression; namespace BCG { @@ -26,7 +27,7 @@ namespace BCG /// original(unprojected) schema /// expression defining the mapping /// table name of the new schema - internal ProjectHelper (IBlueTableSchema isch, string exp, string tableName) + internal ProjectHelper (IBlueTableSchema isch, string exp, string? tableName) : this (isch, myGetDictionary (isch, exp), tableName) {} @@ -36,10 +37,10 @@ namespace BCG /// dictionary mapping unprojected names to their projected names, dropped cols should not be there /// table name of the new schema internal ProjectHelper ( - IBlueTableSchema isch, IDictionary origToAlias, string tableName) + IBlueTableSchema isch, IDictionary origToAlias, string? tableName) { - ArrayList colMapAl = new ArrayList (), - colInfoAl = new ArrayList (); + var colMapAl = new List(); + var colInfoAl = new List(); relevantOrigCols = new string[origToAlias.Count-1]; int c = 0; ArrayList iColNames = new ArrayList (isch.columnNames); @@ -49,19 +50,15 @@ namespace BCG if (origToAlias.TryGetValue(colName, out var colAlias)) { ColumnInfo ci = isch[colName]; - colInfoAl.Add (new ColumnInfo (colAlias, - colInfoAl.Count, - ci.colType) - ); - colMapAl.Add (ci.colNo); + colInfoAl.Add(new ColumnInfo (colAlias, colInfoAl.Count, ci.colType)); + colMapAl.Add(ci.colNo); if (ci.colName != "id") relevantOrigCols[c++] = ci.colName; } } - schema = new BlueTableSchema (tableName == null ? isch.tableName : tableName, - colInfoAl.ToArray (typeof(ColumnInfo)) as ColumnInfo[]); - colMap = colMapAl.ToArray (typeof(int)) as int[]; + schema = new BlueTableSchema(tableName ?? isch.tableName, colInfoAl.ToArray()); + colMap = colMapAl.ToArray(); } /// @@ -75,16 +72,11 @@ namespace BCG { IDictionary origToAlias = new Dictionary(); - ColumnMapExpression e; - if (Configuration.get().UseStaticParser) - { - e = ColumnMapExpression.Parse(exp); - } - else - { + var e = Configuration.get().UseStaticParser + ? ColumnMapExpression.Parse(exp) // TODO (maybe): Actually use non-static parser (unclear what advantages it will have) - e = ColumnMapExpression.Parse(exp); - } + : ColumnMapExpression.Parse(exp); + Log.notnull(e); if (e.IsInclusive) { origToAlias = e.Columns.ToDictionary(); origToAlias["id"] = "id"; @@ -104,7 +96,7 @@ namespace BCG { IDictionary retVal = new Hashtable (dic.Count); foreach (DictionaryEntry de in dic) - retVal.Add (de.Value, de.Key); + retVal.Add(ExceptionAssert.not_null(de.Value), de.Key); return retVal; } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs index fab2e9956..954b83eb3 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs @@ -1,4 +1,8 @@ namespace BCG { + using bts; + + using mbts.utils; + using Spark.Core.Collections; using System.Collections; @@ -8,10 +12,11 @@ namespace BCG { [DataStreamUserRespresentation("oldproject")] public class ProjectStream : ViewStream { - private ProjectHelper myHelper; - private string myColExp, myTableName; - private ViewStream myVs; - private IDictionary myDic; + ProjectHelper? myHelper; + string? myColExp; + string? myTableName; + ViewStream? myVs; + IDictionary? myDic; /// /// /// stream to susbsribe to as the input stream @@ -54,17 +59,17 @@ namespace BCG { protected override void myOnFirstSubscriber() { - myVs.onChange += myHandleChange; + Log.notnull(myVs).onChange += myHandleChange; } protected override void myOnLastSubscriber() { - myVs.onChange -= myHandleChange; + Log.notnull(myVs).onChange -= myHandleChange; } internal void myHandleChange (object newValue) { - DataPacket dp = newValue as DataPacket; + var dp = ExceptionAssert.Argument.Is(newValue); if (myHelper != null) { IBlueRowCollection dels, adds; @@ -79,8 +84,8 @@ namespace BCG { else { //first time a callback is received myHelper = myDic == null ? - new ProjectHelper (dp.schema, myColExp, myTableName) : - new ProjectHelper (dp.schema, myDic, myTableName); + new ProjectHelper (dp.Schema, myColExp ?? string.Empty, myTableName) : + new ProjectHelper (dp.Schema, myDic, myTableName); IBlueRowCollection temp; myFireEvent (new DataPacket (temp = new ProjectedTable (dp.newRows, myHelper), diff --git a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs index 2fadb400a..8f8656bf5 100644 --- a/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/CSVSaverStream.cs @@ -1,13 +1,10 @@ -using System; -using System.Collections.Generic; -using System.Text; -using System.IO; -using BCG.Utilities; -using System.Diagnostics; -using Microsoft.Data.Analysis; -using Parquet; -using Parquet.Data; +using mbts.utils; +using Microsoft.Data.Analysis; + +using System; +using System.Diagnostics; +using System.IO; namespace BCG { [DataStreamUserRespresentation("CSVSaver")] @@ -40,7 +37,7 @@ namespace BCG { myStream.onChange -= myHandleChanges; private void myHandleChanges(object newValue) { - IBlueRowCollection totalState = (newValue as DataPacket).newRows; + IBlueRowCollection totalState = ExceptionAssert.Argument.Is(newValue).newRows; if (totalState.Count > 0 && myNWrites < myNMaxWrites) { myNWrites++; int i = 0; @@ -62,7 +59,10 @@ namespace BCG { IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count); BlueTable nRows = new BlueTable(myResultSchema); nRows.addRow(nr); - myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows))); + BlueTable deletedRows = myLastValue is null + ? new(myResultSchema) + : new(ExceptionAssert.of_type(myLastValue).newRows); + myFireEvent(new DataPacket(nRows, nRows, deletedRows)); } else myFireEvent(newValue); } @@ -83,7 +83,7 @@ namespace BCG { if (col != "id") { string colText = String.Empty; if (r[col] != null && r[col] != DBNull.Value) - colText = r[col].ToString(); + colText = r[col]?.ToString() ?? string.Empty; if (r.schema[col].colType == typeof(string)) colText = "\"" + colText + "\""; rv += (i++ == 0 ? colText : "," + colText); @@ -136,7 +136,7 @@ namespace BCG { } public class CSVReaderStreamMapper : IStreamMapper { - private IBlueTableSchema mySchema; + IBlueTableSchema? mySchema; private readonly string[] myOutputSchemaStringArray; public CSVReaderStreamMapper(string outputSchema) { @@ -162,6 +162,7 @@ namespace BCG { } public IBlueRow map(IBlueRow row) { + ArgumentNullException.ThrowIfNull(mySchema); object[] vals = new object[row.schema.columnInfo.Length]; row.Values.CopyTo(vals, 0); for (int i = 0; i < mySchema.columnInfo.Length; i++) { diff --git a/src/DataStreamSuite/DataStream/Primitives/Sort/SortRow.cs b/src/DataStreamSuite/DataStream/Primitives/Sort/SortRow.cs index ef778eecb..28d1af7f1 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Sort/SortRow.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Sort/SortRow.cs @@ -1,5 +1,7 @@ using Bts.ColumnSortExpressions; +using mbts.utils; + using System; using System.Collections; using System.Collections.Generic; @@ -39,7 +41,7 @@ public class SortRow : IComparable, IBlueRow { var otherVal = otherRow[col.Name]; int rv = myIsNull(this[col.Name]) ? (myIsNull(otherVal) ? 0 : -1) : - (myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal)); + (myIsNull(otherVal) ? 1 : ExceptionAssert.of_type(this[col.Name]).CompareTo(otherVal)); rv *= col.IsAscending ? 1 : -1; return rv; } diff --git a/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextWriterStream.cs b/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextWriterStream.cs index bd6719a47..dbcbbe9a2 100644 --- a/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextWriterStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/TextFileStreams/TextWriterStream.cs @@ -1,8 +1,7 @@ -using System; -using System.Diagnostics; -using System.IO; +using BCG.Utilities.General; -using BCG.Utilities.General; +using System; +using System.IO; namespace BCG { @@ -49,8 +48,7 @@ namespace BCG #region IWorker public void doWork() { - if ((myNextWrite <= DateTime.Now && myGotUpdate)) - { + if (myNextWrite <= DateTime.Now && myGotUpdate) { myGotUpdate = false; myNextWrite = DateTime.Now.AddSeconds(myWriteFreqSecs); myWrite(myStream.rows); @@ -96,7 +94,7 @@ namespace BCG rowStr += DELIM; } - rowStr += row[columnName].ToString(); + rowStr += row[columnName]?.ToString() ?? string.Empty; } writer.WriteLine(rowStr); @@ -139,7 +137,7 @@ namespace BCG private int myWriteFreqSecs; private DateTime myNextWrite; private bool myGotUpdate; - private string mySchemaStr; + private string? mySchemaStr; #endregion } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Time/LatencyStream.cs b/src/DataStreamSuite/DataStream/Primitives/Time/LatencyStream.cs index 226408212..97375e537 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Time/LatencyStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Time/LatencyStream.cs @@ -1,9 +1,8 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using Bts.DataStreamCore; + +using System; using System.Collections; +using System.Collections.Generic; namespace BCG { @@ -11,7 +10,7 @@ namespace BCG { private ViewStream _src; private string _time_stamp_field; - private IBlueTableSchema _schema; + private IBlueTableSchema? _schema; private string _latency_field; public LatencyStream(ViewStream input, string time_stamp_field, string latency_field) @@ -38,10 +37,10 @@ namespace BCG public IBlueRow map(IBlueRow row) { var ht = new Hashtable(row); - if (row[_time_stamp_field].Equals(DBNull.Value)) + if (Equals(row[_time_stamp_field], DBNull.Value)) ht[_latency_field] = DBNull.Value; else - ht[_latency_field] = (DateTime.Now - (DateTime)row[_time_stamp_field]).TotalMilliseconds; + ht[_latency_field] = (DateTime.Now - row.GetValue(_time_stamp_field)).TotalMilliseconds; return new BlueRow(_schema, ht); } diff --git a/src/DataStreamSuite/DataStream/Primitives/TimeAverage/TimeAverageStream.cs b/src/DataStreamSuite/DataStream/Primitives/TimeAverage/TimeAverageStream.cs index cf05ebf64..774b13469 100644 --- a/src/DataStreamSuite/DataStream/Primitives/TimeAverage/TimeAverageStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/TimeAverage/TimeAverageStream.cs @@ -1,7 +1,8 @@ -using System; +using Bts.DataStreamCore; + +using System; using System.Collections.Generic; using System.Linq; -using System.Text; namespace BCG { @@ -42,8 +43,8 @@ namespace BCG BlueTable result = new BlueTable(dp.schema); foreach (IBlueRow r in dp.newRows) { - string key = r[myCol_Key].ToString(); - DateTime dt = (DateTime) r[myCol_Time]; + var key = r[myCol_Key]?.ToString() ?? string.Empty; + var dt = r.GetValue(myCol_Time); double value = Convert.ToDouble(r[myCol_Value]); object[] rowValues = new object[r.Count]; @@ -68,9 +69,7 @@ namespace BCG private double myUpdateValue(string key, DateTime time, double value) { - SortedList seriesValues; - if (!myKeyToTimeToValue.TryGetValue(key, out seriesValues)) - { + if (!myKeyToTimeToValue.TryGetValue(key, out SortedList? seriesValues)) { seriesValues = new SortedList(); myKeyToTimeToValue[key] = seriesValues; } diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs b/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs index 86e782c7a..ca74f321f 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Union/AppendCollection.cs @@ -1,3 +1,7 @@ +using Bts.DataStreamCore; + +using mbts.utils; + using System; using System.Collections; @@ -12,7 +16,7 @@ namespace BCG private int myId = 0; private IDictionary myLeftIndex = new Hashtable (), myRightIndex = new Hashtable (); - private IBlueTableSchema mySchema; + private IBlueTableSchema? mySchema; public AppendCollection () {} @@ -24,15 +28,15 @@ namespace BCG IDictionary index = isLeft ? myLeftIndex : myRightIndex; foreach (IBlueRow r in dp.deletedRows) { - IBlueRow dr = index[r["id"]] as IBlueRow; + var dr = ExceptionAssert.of_type(index[r.Id()]); dRows.addRow (dr); - index.Remove (r["id"]); + index.Remove (r.Id()); } foreach (IBlueRow r in dp.addedRows) { IBlueRow ar = new AppendRow (r, myId++, mySchema); aRows.addRow (ar); - index.Add (r["id"], ar); + index.Add(r.Id(), ar); } return new DataPacket (this, aRows, dRows); } @@ -43,7 +47,8 @@ namespace BCG public bool containsID (int id) {throw new ApplicationException ("IBlueRow is readonly");} - public IBlueTableSchema schema {get {return mySchema;}} + public IBlueTableSchema schema => + ExceptionAssert.not_null(mySchema); #endregion @@ -60,15 +65,14 @@ namespace BCG array.SetValue (r, i++); } - public object SyncRoot {get {return null;}} + public object SyncRoot => this; #endregion #region IEnumerable Members - public System.Collections.IEnumerator GetEnumerator() - { - return new MyEnumerator (myLeftIndex, myRightIndex); + public System.Collections.IEnumerator GetEnumerator() { + return new MyEnumerator(myLeftIndex, myRightIndex); } private class MyEnumerator : IEnumerator diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/AppendStreamModified.cs b/src/DataStreamSuite/DataStream/Primitives/Union/AppendStreamModified.cs index b95956dea..296072351 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Union/AppendStreamModified.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Union/AppendStreamModified.cs @@ -1,11 +1,12 @@ -using System; +using mbts.utils; namespace BCG { public class AppendStreamModified : ViewStream { private AppendCollection myColl = new AppendCollection (); - private ViewStream myLeftStream, myRightstream; + ViewStream? myLeftStream; + ViewStream? myRightstream; public AppendStreamModified (ViewStream leftStream, ViewStream rightStream) { @@ -19,7 +20,7 @@ namespace BCG mySetRealSource (vss[0]); else { - AppendStreamModified rv = null; + AppendStreamModified? rv = null; foreach (ViewStream s in vss) rv = rv == null ? new AppendStreamModified (new ViewStream[] {s}) : new AppendStreamModified (rv, s); mySetRealSource (rv); @@ -28,24 +29,26 @@ namespace BCG protected override void myOnFirstSubscriber() { - myLeftStream.onChange += myHandleLeftChanges; - myRightstream.onChange += myHandleRightChanges; + ExceptionAssert.not_null(myLeftStream).onChange += myHandleLeftChanges; + ExceptionAssert.not_null(myRightstream).onChange += myHandleRightChanges; } protected override void myOnLastSubscriber() { - myLeftStream.onChange -= myHandleLeftChanges; - myRightstream.onChange -= myHandleRightChanges; + ExceptionAssert.not_null(myLeftStream).onChange -= myHandleLeftChanges; + ExceptionAssert.not_null(myRightstream).onChange -= myHandleRightChanges; } private void myHandleLeftChanges (object newValue) { - myFireEvent (myGetDataPacket (newValue as DataPacket, true)); + var dp = ExceptionAssert.Argument.Is(newValue); + myFireEvent(myGetDataPacket(dp, true)); } private void myHandleRightChanges (object newValue) { - myFireEvent (myGetDataPacket (newValue as DataPacket, false)); + var dp = ExceptionAssert.Argument.Is(newValue); + myFireEvent(myGetDataPacket(dp, false)); } private DataPacket myGetDataPacket (DataPacket dp, bool isLeft) diff --git a/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs index 62e9b4f17..089612243 100644 --- a/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs +++ b/src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs @@ -1,3 +1,4 @@ +#nullable enable using System; using System.Text.Json.Serialization; @@ -6,10 +7,10 @@ namespace BCG { /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable /// [Serializable] - public class ColumnInfo : IComparable { + public class ColumnInfo : IEquatable, IComparable, IComparable { //the default column info does not cach values in a lookup table, therefore it is in group 0 - public ColumnInfo(string colName, int colNo, Type colType, object colValue = null) { + public ColumnInfo(string colName, int colNo, Type colType, object? colValue = null) { myColName = colName; myColNo = colNo; myColType = colType; @@ -27,22 +28,23 @@ namespace BCG { type == typeof(string) || type == typeof(DateTime); - public int CompareTo(object obj) { - int otherColNo = ((ColumnInfo)(obj)).colNo; - if (myColNo < otherColNo) - return -1; - else if (myColNo > otherColNo) - return 1; - return 0; - } + public int CompareTo(ColumnInfo? other) => + other is null ? 1 : colNo.CompareTo(other.colNo); - public override bool Equals(object obj) { - if (obj is ColumnInfo) { - ColumnInfo other = obj as ColumnInfo; - return (colNo == other.colNo) && (colName == other.colName) && (colType == other.colType); - } - return false; - } + public int CompareTo(object? obj) => + CompareTo(obj as ColumnInfo); + + public bool Equals(ColumnInfo? other) => + other is not null + && colNo == other.colNo + && colType == other.colType + && colName == other.colName; + + public override bool Equals(object? obj) => + Equals(obj as ColumnInfo); + + public override int GetHashCode() => + HashCode.Combine(colNo, colType, colName); public override string ToString() { return colName + " " + colNo + " " + colType; @@ -55,7 +57,7 @@ namespace BCG { private int myColNo; private string myColName; private Type myColType; - private object myColValue; + private object? myColValue; [JsonPropertyName("colNo")] public int colNo { get { return myColNo; } } @@ -67,6 +69,6 @@ namespace BCG { public Type colType { get { return myColType; } } [JsonIgnore] - public object colValue {get{return myColValue;}} + public object? colValue {get{return myColValue;}} } } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamIOCommon.cs b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamIOCommon.cs index 6f93ea702..5d35f6350 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamIOCommon.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/v2/Publish/Serialization/StreamIOCommon.cs @@ -1,6 +1,7 @@ -using System; +#nullable enable using BCG; -using System.Collections; + +using System; using System.IO; // common place for all serialization interfaces @@ -14,7 +15,7 @@ namespace bts { // other operations can be expressed with this MULTIPLE_REMOVE_ADD = 1, } - public class CreateMessage { + public class CreateMessage : IEquatable { public IBlueRowCollection rows; public CreateMessage(IBlueRowCollection rows_) { rows = rows_; @@ -30,15 +31,19 @@ namespace bts { } return rv; } - public override bool Equals(object obj) { - if (obj != null && obj is CreateMessage) { - var other = obj as CreateMessage; - return BlueDataUtility.compute_equality(rows, other.rows); - } - return false; - } + + public bool Equals(CreateMessage? other) => + other is not null && BlueDataUtility.compute_equality(rows, other.rows); + + public override bool Equals(object? obj) => + Equals(obj as CreateMessage); + + public override int GetHashCode() => + // Not the most elegant hash, but it works with the Equals fn. CreateMessages that are + // 'Equal' are guaranteed to have the same hash. + HashCode.Combine(rows.Count, rows.schema.columnInfo.Length); } - public class MultipleRemoveAddMessage { + public class MultipleRemoveAddMessage : IEquatable { public IBlueRowCollection removed_rows; public IBlueRowCollection added_rows; public MultipleRemoveAddMessage(IBlueRowCollection removed_rows_, IBlueRowCollection added_rows_) { @@ -56,18 +61,23 @@ namespace bts { } return rv; } - public override bool Equals(object obj) { - if (obj != null && obj is MultipleRemoveAddMessage) { - MultipleRemoveAddMessage other = obj as MultipleRemoveAddMessage; - return BlueDataUtility.compute_equality(removed_rows, other.removed_rows) && - BlueDataUtility.compute_equality(added_rows, other.added_rows); - } - return false; - } + public bool Equals(MultipleRemoveAddMessage? other) => + other is not null + && BlueDataUtility.compute_equality(removed_rows, other.removed_rows) + && BlueDataUtility.compute_equality(added_rows, other.added_rows); + + public override bool Equals(object? obj) => + Equals(obj as MultipleRemoveAddMessage); + + public override int GetHashCode() => + // Not the most elegant hash, but it works with the Equals fn. CreateMessages that are + // 'Equal' are guaranteed to have the same hash. + HashCode.Combine( + removed_rows.Count, removed_rows.schema.columnNames.Length, added_rows.Count, added_rows.schema.columnInfo.Length); } public StreamOp op; - public readonly CreateMessage create_message = null; - public readonly MultipleRemoveAddMessage multiple_remove_add_message = null; + public readonly CreateMessage? create_message = null; + public readonly MultipleRemoveAddMessage? multiple_remove_add_message = null; public StreamIOMessage(CreateMessage create_message_) { op = StreamOp.CREATE; @@ -83,29 +93,26 @@ namespace bts { public override string ToString() { switch (op) { - case StreamIOMessage.StreamOp.CREATE: - return create_message.ToString(); - case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: - return multiple_remove_add_message.ToString(); + case StreamOp.CREATE: + return create_message!.ToString(); + case StreamOp.MULTIPLE_REMOVE_ADD: + return multiple_remove_add_message!.ToString(); } bts.Log.assert(false); return String.Empty; } - public override bool Equals(object obj) { - if (obj != null && obj is StreamIOMessage) { - StreamIOMessage other = obj as StreamIOMessage; - if (op == other.op) { - switch (op) { - case StreamOp.CREATE: - return create_message.Equals(other.create_message); - case StreamOp.MULTIPLE_REMOVE_ADD: - return multiple_remove_add_message.Equals(other.multiple_remove_add_message); - } - } - } - return false; - } + public bool Equals(StreamIOMessage? other) => + op == other?.op && op switch { + StreamOp.CREATE => create_message!.Equals(other.create_message), + StreamOp.MULTIPLE_REMOVE_ADD => multiple_remove_add_message!.Equals(other.multiple_remove_add_message), + _ => false, + }; + public override bool Equals(object? obj) => + Equals(obj as StreamIOMessage); + + public override int GetHashCode() => + HashCode.Combine((int)op, create_message?.GetHashCode() ?? multiple_remove_add_message?.GetHashCode() ?? 0); } #region serialization // This helps putting stream callbacks (op + data) on the wire, to send it to the pubsubserver. @@ -134,4 +141,4 @@ namespace bts { } #endregion -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/ContractUtils.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/ContractUtils.cs index cdb35395f..0d677f57e 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/ContractUtils.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/ContractUtils.cs @@ -11,45 +11,45 @@ namespace bts.BlueOrb { public static class ContractUtils { public class FutureTradableContract { - public string symbol; + public string? symbol; public DateTime expiration; public Exchange exchange; - public string contract_text; + public string? contract_text; } public class FutureSpreadContract { - public List> legs; - public string symbol { + public List>? legs; + public string? symbol { get { return legs == null || legs.Count == 0 ? string.Empty : legs.ElementAt(0).Item2.symbol; } } - public string contract_text; + public string? contract_text; } public class FutureContinousContract { - public string symbol; + public string? symbol; public Exchange exchange; } // this is for all future conytracts belonging to a symbol public class FutureWildcardContract { - public string symbol; + public string? symbol; public Exchange exchange; } public class FutureContract { - public FutureWildcardContract wildcard_contract = null; - public FutureTradableContract future_tradable_contract = null; - public FutureSpreadContract spread_tradable_contract = null; - public FutureContinousContract continous_constract = null; + public FutureWildcardContract? wildcard_contract = null; + public FutureTradableContract? future_tradable_contract = null; + public FutureSpreadContract? spread_tradable_contract = null; + public FutureContinousContract? continous_constract = null; public FutureContract(string txt) { wildcard_contract = parse_wildcard_contract(txt); if (wildcard_contract != null) return; - FutureTradableContract fc = parse_future_contract(txt); + var fc = parse_future_contract(txt); if (fc != null) { future_tradable_contract = fc; return; } - FutureSpreadContract sc = parse_future_spread_contract(txt); + var sc = parse_future_spread_contract(txt); if (sc != null) { spread_tradable_contract = sc; return; @@ -63,7 +63,7 @@ namespace bts.BlueOrb { public bool is_future_tradable_contract => future_tradable_contract != null; public bool is_spread_tradable_contract => spread_tradable_contract != null; - public string symbol() { + public string? symbol() { if (wildcard_contract != null) return wildcard_contract.symbol; if (continous_constract != null) { @@ -76,7 +76,7 @@ namespace bts.BlueOrb { Log.assert(false); return string.Empty; } - public string contract_text { + public string? contract_text { get { if (future_tradable_contract != null) return future_tradable_contract.contract_text; @@ -87,7 +87,7 @@ namespace bts.BlueOrb { } } #region Parsing - public static FutureWildcardContract parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) { + public static FutureWildcardContract? parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME) { if (contract_txt.EndsWith("*")) { contract_txt = contract_txt.Replace("*", string.Empty); return new FutureWildcardContract() { @@ -97,7 +97,7 @@ namespace bts.BlueOrb { } return null; } - public static FutureTradableContract parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) { + public static FutureTradableContract? parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME) { // Validate the future string length if (!contract_txt.All(c => char.IsLetterOrDigit(c))) return null; @@ -153,7 +153,7 @@ namespace bts.BlueOrb { return futureContract; } // Method to parse a futures spread, e.g. "CLX4-CLZ4" - public static FutureSpreadContract parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) { + public static FutureSpreadContract? parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME) { // Split the string by '-' to handle each leg var spreadParts = contract_txt.Split('-'); if (spreadParts.Length < 2) { @@ -201,11 +201,11 @@ namespace bts.BlueOrb { #endregion public static ContractType get_type(string contract_txt) { - FutureTradableContract fc = parse_future_contract(contract_txt); + var fc = parse_future_contract(contract_txt); if (fc != null) { return ContractType.Future; } - FutureSpreadContract sc = parse_future_spread_contract(contract_txt); + var sc = parse_future_spread_contract(contract_txt); if (sc != null) { return ContractType.FutureSpread; } @@ -220,13 +220,13 @@ namespace bts.BlueOrb { } public static Tuple get_symbol_category(string symbol) { - FutureTradableContract fc = parse_future_contract(symbol); + var fc = parse_future_contract(symbol); if (fc != null) { - return Tuple.Create(ContractCategory.FutureString, fc.symbol); + return Tuple.Create(ContractCategory.FutureString, fc.symbol ?? string.Empty); } var sc = parse_future_spread_contract(symbol); if (sc != null) { - return Tuple.Create(ContractCategory.FutureString, sc.symbol); + return Tuple.Create(ContractCategory.FutureString, sc.symbol ?? string.Empty); } return Tuple.Create(ContractCategory.Symbol, symbol); } diff --git a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricalBarsStream.cs b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricalBarsStream.cs index 9be7a9941..363991970 100644 --- a/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricalBarsStream.cs +++ b/src/DataStreamSuite/SharedStreams/BlueOrb/MarketData/Historical/HistoricalBarsStream.cs @@ -1,8 +1,6 @@ namespace bts.BlueOrb { - using bts.SilverPine; using BCG; - public class DateTimeRange { public readonly DateTime range_beg; public readonly DateTime range_end; @@ -18,7 +16,7 @@ namespace bts.BlueOrb { } public class FutureBarsLoader { - public static string get_db_table_name(BarSize bar_size, string symbol, bool is_continous) { + public static string? get_db_table_name(BarSize bar_size, string symbol, bool is_continous) { switch (bar_size) { case BarSize.ONE_DAY: { if (is_continous) { @@ -68,22 +66,22 @@ namespace bts.BlueOrb { string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'"; return new SQLiteStream(SQLiteHelper.db_name, sql_str); }; - string symbol = future_contract.symbol(); + var symbol = future_contract.symbol() ?? string.Empty; if (future_contract.wildcard_contract != null) { - string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false); + var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, false) ?? string.Empty; ViewStream rs = get_sqlite_str(table_name); //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime); mySetRealSource(rs); return; } if (future_contract.is_continous_contract) { - string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true); + var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, true) ?? string.Empty; ViewStream rs = get_sqlite_str(table_name); //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime); mySetRealSource(rs); return; } else { - string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, future_contract.symbol(), false); + var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_DAY, symbol, false) ?? string.Empty; ViewStream rs = get_sqlite_str(table_name); //rs = new DateStandardizationStream(rs, "ts_event", DateConversion.ChopTime); //TODO: filter it @@ -115,20 +113,20 @@ namespace bts.BlueOrb { string sql_str = $"select * from {table_name} where ts_event >= '{range_beg_iso8601}' AND ts_event <= '{range_end_iso8601}'"; return new SQLiteStream(SQLiteHelper.db_name, sql_str); }; - string symbol = future_contract.symbol(); + var symbol = future_contract.symbol() ?? string.Empty; if (future_contract.wildcard_contract != null) { - string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false); + var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, false) ?? string.Empty; ViewStream rs = get_sqlite_str(table_name); mySetRealSource(rs); return; } if (future_contract.is_continous_contract) { - string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true); + var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, true) ?? string.Empty; ViewStream rs = get_sqlite_str(table_name); mySetRealSource(rs); return; } else { - string table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, future_contract.symbol(), false); + var table_name = FutureBarsLoader.get_db_table_name(BarSize.ONE_MIN, symbol, false) ?? string.Empty; ViewStream rs = get_sqlite_str(table_name); // TODO: filter rs = new FilterStream(rs, $"symbol = '{future_contract.contract_text}'"); diff --git a/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsStream.cs b/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsStream.cs index 2b0868bef..7f16a3aea 100644 --- a/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsStream.cs @@ -1,6 +1,8 @@ namespace bts { using BCG; + using mbts.utils; + using System; using System.Collections.Generic; using System.Net; @@ -8,7 +10,7 @@ namespace bts { using System.Text.Json.Nodes; [DataStreamUserRespresentation("benzingaNews")] public class BenzingaNewsStream : ViewStream { - private IBlueTableSchema _schema = null; + private IBlueTableSchema? _schema; private int _id = 0; @@ -63,14 +65,14 @@ namespace bts { } foreach (JsonNode jnode in json_objs) { if (jnode is JsonObject json_object) { - string symbol = jnode["symbol"] == null ? String.Empty : jnode["symbol"].ToString(); - string exchange = jnode["exchange"] == null ? String.Empty : jnode["exchange"].ToString(); + string symbol = jnode["symbol"]?.ToString() ?? string.Empty; + string exchange = jnode["exchange"]?.ToString() ?? string.Empty; MutableBlueRow nr = new MutableBlueRow(_schema, r); nr.setValue("id", _id++); nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}"); nr.setValue("symbol", symbol); nr.setValue("exchange", exchange); - nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString())); + nr.setValue("is_primary_exchange", bool.Parse(ExceptionAssert.not_null(jnode["primary"]).ToString())); expanded_rows.Add(nr); } } @@ -84,4 +86,4 @@ namespace bts { mySetRealSource(output_stream); } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsSymbolStateStream.cs b/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsSymbolStateStream.cs index 944c9b73a..2a0ce1de7 100644 --- a/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsSymbolStateStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsSymbolStateStream.cs @@ -2,8 +2,6 @@ namespace bts { using BCG; using System.Collections.Generic; - using System.Linq; - using System.Security.Cryptography; //[DataStreamUserRespresentation("benzingaNewsSymbols")] public class BenzingaNewsSymbolStateStream : ViewStream, INonIncGroupByEvaluator { @@ -11,23 +9,23 @@ namespace bts { ViewStream r = new NonIncGroupByStream(news_str, this); mySetRealSource(r); } - public BenzingaNewsSymbolStateStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl) : - this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) { - } + + public BenzingaNewsSymbolStateStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl) + : this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) { } public IBlueRowCollection evaluate(IDictionary> grouped_rows) { - BlueTable rv = null; + BlueTable? rv = null; foreach (var kv in grouped_rows) { var symbol_rows = kv.Value; IBlueRow state_row = BlueDataUtility.find_max(symbol_rows, "created_at"); - rv = rv == null ? new BlueTable(state_row.schema) : rv; + rv ??= new(state_row.schema); rv.addRow(state_row); } - return rv; + return rv ?? BlueTable.Empty("BenzingaNewsSymbolState"); } public string[] get_group_by_cols() { return new string[] { "symbol" }; } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/SharedStreams/Business/Indicators/ChartsUtils.cs b/src/DataStreamSuite/SharedStreams/Business/Indicators/ChartsUtils.cs index 904e4529a..8dd1abbad 100644 --- a/src/DataStreamSuite/SharedStreams/Business/Indicators/ChartsUtils.cs +++ b/src/DataStreamSuite/SharedStreams/Business/Indicators/ChartsUtils.cs @@ -3,6 +3,7 @@ using bts.config; using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; namespace BCG { @@ -21,7 +22,9 @@ namespace BCG { } public class RenkoCalculator { public readonly double brick_size; - public RenkoBrick current_brick = null; + public RenkoBrick? current_brick = null; + + [MemberNotNull(nameof(current_brick))] private bool _add_price(double price) { if (current_brick == null) { current_brick = new RenkoBrick { @@ -309,4 +312,4 @@ namespace BCG { return wma_values; } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs index c06b8ef25..3c760c15e 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialArbStatsStream.cs @@ -3,6 +3,8 @@ using bts.md; + using mbts.utils; + using System; using System.Collections.Generic; @@ -13,7 +15,7 @@ public class SpatialArbStatsStream : ViewStream{ private ViewStream _spatial_pairs_stream; private ViewStreamDatabase _vsd; - private ViewStream _tot_str = null; + private ViewStream? _tot_str = null; private HashSet _key_map = new HashSet(); private ViewStream _bbo_stream; private readonly TimeSpan _window_sz, _sampling_period_sz; @@ -89,7 +91,7 @@ //_tot_str = new SortStream(_tot_str, "-n_trade_ticks"); _tot_str.onChange += _handle_tot_stream; - DataPacket dp = dp_obj as DataPacket; + var dp = ExceptionAssert.Argument.Is(dp_obj); foreach (IBlueRow r in dp.newRows) { string s1 = r.get("s1"); string s2 = r.get("s2"); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs index 084e92631..15391c54d 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpatialFuturePairsStream.cs @@ -4,6 +4,8 @@ using bts.api; using bts.md; + using mbts.utils; + using System; using System.Collections.Generic; @@ -67,7 +69,7 @@ private void _main_stream_onchange(object dp_obj) { _key_map.Clear(); - DataPacket dp = dp_obj as DataPacket; + var dp = ExceptionAssert.Argument.Is(dp_obj); BlueTable table = new BlueTable(_schema); var rows = dp.newRows.to_list(); foreach (var r1 in rows) { diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceEvaluator.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceEvaluator.cs index 606ae305e..ef41eebb5 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceEvaluator.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceEvaluator.cs @@ -1,30 +1,7 @@ -using System; -using System.Collections.Generic; -using System.Linq; using MathNet.Numerics.Statistics; using MathNet.Numerics.LinearRegression; -using System.Collections.Generic; -using System.Linq; -using MathNet.Numerics.Statistics; using MathNet.Numerics.LinearAlgebra; -using Amazon.Runtime; -using System; -using System.Collections.Generic; -using System.Linq; -using MathNet.Numerics.Statistics; -using System; -using System.Collections.Generic; -using System.Linq; -using MathNet.Numerics.Statistics; -using System; -using System.Collections.Generic; -using System.Linq; -using MathNet.Numerics.Statistics; // stats calculator -using System; -using System.Collections.Generic; -using System.Linq; -using MathNet.Numerics.Statistics; /// /// Provides evaluation metrics to assess the quality of a synthetic spread for mean-reversion trading. @@ -35,13 +12,13 @@ public static class SpreadPerformanceEvaluator public static class SpreadStatistics { public class SpreadStatsResult { - public Dictionary bid_zscore_distribution { get; set; } - public Dictionary ask_zscore_distribution { get; set; } - public Dictionary mid_zscore_distribution { get; set; } + public Dictionary bid_zscore_distribution { get; set; } = new(0); + public Dictionary ask_zscore_distribution { get; set; } = new(0); + public Dictionary mid_zscore_distribution { get; set; } = new(0); - public DescriptiveStatistics bid_stats { get; set; } - public DescriptiveStatistics ask_stats { get; set; } - public DescriptiveStatistics mid_stats { get; set; } + public DescriptiveStatistics bid_stats { get; set; } = new(); + public DescriptiveStatistics ask_stats { get; set; } = new(); + public DescriptiveStatistics mid_stats { get; set; } = new(); } private static readonly string[] _zscore_labels = new[] diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs index 76fca12e3..9312fcd05 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/Analytics/SpreadPerformanceStream.cs @@ -1,10 +1,12 @@ namespace bts.BlueOrb { using BCG; + using mbts.utils; + using System.Diagnostics; public class SpreadPerformanceStream : ViewStream { - ViewStream _spread_price_stream = null; + ViewStream _spread_price_stream; private readonly IBlueTableSchema _schema; private readonly int _window_secs; private readonly double _buy_threshold, _sell_threshold, _flat_threshold; @@ -70,7 +72,7 @@ namespace bts.BlueOrb { } private void _handle_spread_prices(object dp_obj) { - DataPacket dp = dp_obj as DataPacket; + var dp = ExceptionAssert.Argument.Is(dp_obj); List> spread_ts_prices = new List>(); DateTime min_time = DateTime.MaxValue; DateTime max_time = DateTime.MinValue; var sw = Stopwatch.StartNew(); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs index 1b25cd42a..c36f3bbbb 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs @@ -5,6 +5,8 @@ namespace bts.BlueOrb{ using bts; using bts.md; + using mbts.utils; + using Spark.Core.Disposables; using Spark.Core.Reactive; @@ -103,11 +105,11 @@ namespace bts.BlueOrb{ new_state.addRow(r); } } - DataPacket dp = null; + DataPacket? dp; if (myLastValue == null) { dp = new DataPacket(new_state, new_state, new BlueTable(_bbo_schema)); } else { - DataPacket old_dp = myLastValue as DataPacket; + var old_dp = ExceptionAssert.of_type(myLastValue); IBlueRowCollection added_rows, del_rows; BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows); dp = new DataPacket(new_state, added_rows, del_rows); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs index 9d23bbfb6..033c7ff33 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/FuturesActiveContractStream.cs @@ -26,7 +26,7 @@ namespace bts.BlueOrb{ public IBlueRowCollection evaluate(IDictionary> grouped_rows) { Func is_future = (c) => ContractUtils.get_category(c) == ContractUtils.ContractCategory.FutureString; - BlueTable rv = null; + BlueTable? rv = null; Func, IBlueRow> reduce = (rows) => { var future_rows = rows.Where(r => is_future(r.get("contract"))); IBlueRow rv = future_rows.ElementAt(0); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs index 6fb5c4a4c..8d528e2f1 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/InstrumentsStream.cs @@ -4,8 +4,12 @@ namespace bts.BlueOrb { using bts; using bts.api; using bts.md; + using BCG; + using mbts.utils; + + using Spark.Core; using Spark.Core.Disposables; using Spark.Core.Reactive; @@ -13,18 +17,14 @@ namespace bts.BlueOrb { using System.Collections.Generic; - using System.Linq; using System.Reactive.Linq; - using SecurityKey = System.UInt32; - [DataStreamUserRespresentation("InstrumentsStream")] public class InstrumentsStream : ViewStream, IWorker { private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500); private DateTime _last_poll_time = DateTime.MinValue; private ViewStreamDatabase _vsd; - private int _id = 0; - IReadOnlyDictionary _cached_state = null; + IReadOnlyDictionary? _cached_state; MarketDataClient _md_client; private bool _dirty = false; private readonly IBlueTableSchema _instrument_schema = BlueDataUtility.form_schema("instruments", @@ -68,7 +68,7 @@ namespace bts.BlueOrb { _dirty = true; //_log_updates(state); // just cacching in here, will populate the rows in _poll_scan and fire the datapacket - bts.Log.Gen.info($"InstrumentsStream._on_instruments state: {state.Count()}"); + bts.Log.Gen.info($"InstrumentsStream._on_instruments state: {state.Count}"); _cached_state = state; } @@ -85,13 +85,13 @@ namespace bts.BlueOrb { return; BlueTable new_state = new BlueTable(_instrument_schema); foreach (var kv in _cached_state) { - SecurityKey skey = kv.Key; + var skey = kv.Key; int id = Convert.ToInt32(skey); SecurityImmutable sec = kv.Value; bts.api.SecurityType sec_type = sec.security_type(); object exp = DBNull.Value; if (sec_type == bts.api.SecurityType.Future || sec_type == bts.api.SecurityType.Option) - exp = sec.security_expiration(); + exp = sec.security_expiration().assert_value(); object strike = DBNull.Value; if (sec_type == bts.api.SecurityType.Option) strike = sec.strike(); @@ -101,11 +101,11 @@ namespace bts.BlueOrb { IBlueRow r = new BlueRow(_instrument_schema, id, skey, sec.security_desc(), sec.security_type().ToString(), sec.display_exchange().ToString(), sec.Symbol, sec.Currency, sec.ContractSpec.CashPerPoint, exp, strike, cp); new_state.addRow(r); } - DataPacket dp = null; + DataPacket? dp; if (myLastValue == null) { dp = new DataPacket(new_state, new_state, new BlueTable(_instrument_schema)); } else { - DataPacket old_dp = myLastValue as DataPacket; + var old_dp = ExceptionAssert.of_type(myLastValue); IBlueRowCollection added_rows, del_rows; BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows); dp = new DataPacket(new_state, added_rows, del_rows); diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs index 4713fe802..a8b7cddf4 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/OHLCVStream.cs @@ -167,8 +167,8 @@ namespace bts.BlueOrb { return String.Empty; } - private Source _sourse = Source.SQL; // get from config - private readonly string _location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:\\BlueOrb_Data\\":Path.Combine(Environment.GetEnvironmentVariable("HOME") , "BlueOrb_Data"); + // private Source _sourse = Source.SQL; // get from config + // private readonly string _location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:\\BlueOrb_Data\\":Path.Combine(Environment.GetEnvironmentVariable("HOME") , "BlueOrb_Data"); private readonly string _db_name = "stream_data.db"; // the symbol is used ot retreive the correct table, the data flows as-is from the table // with some name / type changes diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs index 8857848c8..dc1a80518 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs @@ -1,44 +1,28 @@ namespace BCG { using BCG.Utilities.General; - using bts.api; - using bts.api.domain; - using bts.api.synthetic; + using bts; + using bts.api.core; using bts.md; - using bts.spark; - using bts.utils; - using mbts; - using mbts.co2; - using mbts.config; - using mbts.spark; - using mbts.ui; + using mbts.utils; - using Spark.Core.Data; using Spark.Core.Disposables; using Spark.Core.Reactive; using System; - using System.Collections.Concurrent; using System.Collections.Generic; - using System.Collections.ObjectModel; using System.Linq; using System.Reactive.Linq; - using bts.api.core; using Spark.Api.MarketData; - using SecurityKey = System.UInt32; - using System.Data; - using System.Reflection.Metadata; - using bts; - [DataStreamUserRespresentation("price_stats")] public class PriceStatsStream : ViewStream, IWorker { private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500); private DateTime _last_poll_time = DateTime.MinValue; private ViewStreamDatabase _vsd; private int _id = 0; - IReadOnlyDictionary _cached_state = null; + IReadOnlyDictionary? _cached_state = null; MarketDataClient _md_client; private bool _dirty = false; private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo", @@ -52,14 +36,13 @@ namespace BCG { IDisposable? subscription; // TODO: add cons for symbol, exchange etc - public PriceStatsStream() - : this(ViewStreamDatabase.instance) { - } + public PriceStatsStream(MarketDataClient mdClient) + : this(mdClient, ViewStreamDatabase.instance) { } - public PriceStatsStream(ViewStreamDatabase vsd) { + public PriceStatsStream(MarketDataClient mdClient, ViewStreamDatabase vsd) { bts.Log.Gen.info("PriceStatsStream.cons"); + _md_client = mdClient; _vsd = vsd; - _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5))); } protected override void myOnFirstSubscriber() { @@ -69,7 +52,7 @@ namespace BCG { } private void _log_updates(TableUpdate update) { - int i = 0; + // int i = 0; // foreach (BookTuple bt in update.added) { // bts.Log.Gen.info($"PriceStatsStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}"); // } @@ -99,18 +82,18 @@ namespace BCG { return; BlueTable new_state = new BlueTable(_pstats_scheam); foreach (var kv in _cached_state) { - SecurityKey skey = kv.Key; + var skey = kv.Key; int id = Convert.ToInt32(skey); PriceStats pstats = kv.Value; var today = DateTime.Now.Date; IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), today, pstats.open, pstats.close, pstats.settlement); new_state.addRow(r); } - DataPacket dp = null; + DataPacket? dp; if (myLastValue == null) { dp = new DataPacket(new_state, new_state, new BlueTable(_pstats_scheam)); } else { - DataPacket old_dp = myLastValue as DataPacket; + var old_dp = ExceptionAssert.of_type(myLastValue); IBlueRowCollection added_rows, del_rows; BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows); dp = new DataPacket(new_state, added_rows, del_rows); diff --git a/src/DataStreamSuite/SharedStreams/NL/OpenAIPromptStream.cs b/src/DataStreamSuite/SharedStreams/NL/OpenAIPromptStream.cs index 8033541bb..cb4a21755 100644 --- a/src/DataStreamSuite/SharedStreams/NL/OpenAIPromptStream.cs +++ b/src/DataStreamSuite/SharedStreams/NL/OpenAIPromptStream.cs @@ -85,7 +85,8 @@ namespace bts.BlueOrb { .GetProperty("choices")[0] .GetProperty("message") .GetProperty("content") - .GetString(); + .GetString() + ?? string.Empty; //return gpt_response; DateTime t1 = DateTime.Now; diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs index f395cfbfb..0f5cbb8d3 100644 --- a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs +++ b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubSubServerStateStream.cs @@ -1,5 +1,6 @@ using BCG.Utilities.General; using BCG; +using mbts.utils; namespace bts { /// @@ -10,7 +11,7 @@ namespace bts { public class PubSubServerStateStream : ViewStream, IWorker { private IDictionary _tag_map = new Dictionary(); private readonly string _tag = "System_AllSubscriptions"; - private IBlueTableSchema _schema = null; + private IBlueTableSchema? _schema; private SubscriberCore _sub_core; private ViewStreamDatabase _vsd; @@ -47,12 +48,11 @@ namespace bts { var updates = _sub_core.get_updates(TimeSpan.Zero); //this is where the pubsub updates are converted to rows if (updates.Count() > 0) { - int id = 0; foreach (var u in updates) { if (u.tag != _tag) { continue; } - var tpsi_list = TagPubSubInfo.deserialize(u.data); + var tpsi_list = TagPubSubInfo.deserialize(Log.notnull(u.data)); foreach (var tsi in tpsi_list) { _tag_map[tsi.tag] = tsi; } diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubsubHeartbeatStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubsubHeartbeatStream.cs index 17bda994f..d2c3fc06c 100644 --- a/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubsubHeartbeatStream.cs +++ b/src/DataStreamSuite/SharedStreams/System/PS3/Subscription/PubsubHeartbeatStream.cs @@ -7,8 +7,8 @@ using BCG.Utilities.General; public class PubsubHeartbeatStream : ViewStream, IWorker { private ViewStreamDatabase _vsd; private SubscriberCore _subcore = new SubscriberCore(); - private MutableBlueRow _row = null; - private readonly IBlueTableSchema _schema = null; + private MutableBlueRow? _row; + private readonly IBlueTableSchema? _schema; public PubsubHeartbeatStream(ViewStreamDatabase vsd) { _vsd = vsd; _schema = BlueDataUtility.form_schema("PS_HBB", diff --git a/src/DataStreamSuite/SharedStreams/System/PS3/TableMetaDataStream.cs b/src/DataStreamSuite/SharedStreams/System/PS3/TableMetaDataStream.cs index b4b49051b..d1accbe88 100644 --- a/src/DataStreamSuite/SharedStreams/System/PS3/TableMetaDataStream.cs +++ b/src/DataStreamSuite/SharedStreams/System/PS3/TableMetaDataStream.cs @@ -5,12 +5,14 @@ namespace bts using Bts.DataStreamCore.StreamFactories; + using mbts.utils; + public class TableMetaDataStream : ViewStream, IWorker { private class TableState { public ViewStream table_stream; - public IBlueTableSchema schema = null; + public IBlueTableSchema? schema; public bool processed = false; public TableState(ViewStream table_stream_) { @@ -19,8 +21,7 @@ namespace bts } private void _handle_table_update(object dp_obj) { - DataPacket dp = dp_obj as DataPacket; - schema = dp.schema; + schema = ExceptionAssert.Argument.Is(dp_obj).schema; } public void unsub() { @@ -93,7 +94,7 @@ namespace bts private void _handle_base_stream(object dp_obj) { - DataPacket dp = dp_obj as DataPacket; + var dp = ExceptionAssert.Argument.Is(dp_obj); foreach (IBlueRow r in dp.newRows) { string tag = r.get("tag"); @@ -108,12 +109,7 @@ namespace bts { Log.assert(tag_name.StartsWith("DS_")); string table_name = tag_name.Remove(0, 3); - TableState ts = null; - if (!_table_map.TryGetValue(table_name, out ts)) - { - ts = new TableState(tableStreamFactory.Create(table_name)); - _table_map.Add(table_name, ts); - } + _table_map.GetOrAdd(table_name, _ => new(tableStreamFactory.Create(table_name))); } } }