cleanup: Fix a ton of build warnings (cs)
All these warnings have been bothering me for a while. The work isn't done, but this commit chops them down from ~1000 to ~600. * AppendCollection.cs * AppendStreamModified.cs * BenzingaNewsStream.cs * BenzingaNewsSymbolStateStream.cs * BookStream.cs * CSVSaverStream.cs * ChartsUtils.cs * ColumnInfo.cs * ContractUtils.cs * ContractUtils.cs * DiffStream.cs * FuturesActiveContractStream.cs * FuzzyJoinOverTimeStream.cs * HistoricalBarsStream.cs * HistoricalBarsStream.cs * InstrumentsStream.cs * JoinOperandRowIndex.cs * JoinRow.cs * JoinStream.cs * JoinTable.cs * JoinTableSchema.cs * LatencyStream.cs * LeftInternalJoinStream.cs * LightForeignJoinRow.cs * LightForeignJoinRowList.cs * LightLeftForeignJoinRow.cs * LiteralStream.cs * NLStream.cs * NLToDSLStream.cs * NLToIndexedFewShotPromptStream.cs * NonIncGroupByStream.cs * NonIncJoinBookkeeper.cs * NonIncJoinSpec.cs * OHLCVStream.cs * OpenAIPromptStream.cs * PriceStatsStream.cs * PriceStatsStream.cs * ProjectHelper.cs * ProjectStream.cs * PubSubServerStateStream.cs * PubsubHeartbeatStream.cs * SimpleNode.cs * SortRow.cs * SpatialArbStatsStream.cs * SpatialFuturePairsStream.cs * SpreadPerformanceEvaluator.cs * SpreadPerformanceStream.cs * StreamIOCommon.cs * SuperJoinStream.cs * TableMetaDataStream.cs * TextWriterStream.cs * TimeAverageStream.cs
This commit is contained in:
1 parent
7418f5b5a1
commit
a4efaa0089
48 files changed
+610
-590
No files matched your search
@@ -1,5 +1,4 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using mbts;
|
||||
|
||||
@@ -12,30 +11,21 @@ namespace BCG
|
||||
{
|
||||
ViewStream myLStream, myRStream;
|
||||
|
||||
DataPacket myLastR = null, myLastL = null;
|
||||
DataPacket? myLastR = null, myLastL = null;
|
||||
|
||||
BlueTable myLastFired = null;
|
||||
BlueTable? myLastFired = null;
|
||||
|
||||
double myThreshold = .00005;
|
||||
|
||||
string[] myColumnsToMatch;
|
||||
public DiffStream(ViewStream v1, ViewStream v2) : this(null, v1, v2, "id") { }
|
||||
public DiffStream(ViewStream v1, ViewStream v2, params string[] ColumnsToMatch) : this(null, v1, v2, ColumnsToMatch) { }
|
||||
public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) :
|
||||
this(null, v1, v2, thresh, ColumnsToMatch) { }
|
||||
|
||||
public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
|
||||
{
|
||||
public DiffStream(ViewStream v1, ViewStream v2) : this(v1, v2, "id") { }
|
||||
public DiffStream(ViewStream v1, ViewStream v2, params string[] columnsToMatch) : this(v1, v2, .00005, columnsToMatch) { }
|
||||
public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch) {
|
||||
myThreshold = thresh;
|
||||
myLStream = v1;
|
||||
myRStream = v2;
|
||||
myColumnsToMatch = columnsToMatch;
|
||||
}
|
||||
public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
|
||||
: this(db,v1,v2,.00005,columnsToMatch)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
@@ -52,8 +42,14 @@ namespace BCG
|
||||
|
||||
struct ListComparer : IEqualityComparer<List<object>>
|
||||
{
|
||||
public bool Equals(List<object> x, List<object> y)
|
||||
public bool Equals(List<object>? x, List<object>? y)
|
||||
{
|
||||
if (x is null) {
|
||||
return y is null;
|
||||
}
|
||||
if (y is null) {
|
||||
return false;
|
||||
}
|
||||
if (x.Count != y.Count)
|
||||
return false;
|
||||
|
||||
@@ -74,21 +70,20 @@ namespace BCG
|
||||
|
||||
void myHandleCallbacks()
|
||||
{
|
||||
DataPacket packet = null;
|
||||
|
||||
DataPacket? packet;
|
||||
if (myLastL == null || myLastR == null)
|
||||
{
|
||||
packet = myLastL == null ? myLastR : myLastL;
|
||||
if (packet != null)
|
||||
if (packet != null) {
|
||||
myFireEvent(packet);
|
||||
|
||||
myLastFired = new BlueTable(packet.newRows);
|
||||
myLastFired = new BlueTable(packet.newRows);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
BlueTable table = new BlueTable(myLastL.newRows.schema);
|
||||
|
||||
Dictionary<List<object>, Pair<BlueRow, BlueRow>> rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(new ListComparer());
|
||||
Dictionary<List<object>, Pair<BlueRow?, BlueRow?>> rowPairs = new(new ListComparer());
|
||||
|
||||
foreach(BlueRow row in myLastL.newRows)
|
||||
{
|
||||
@@ -104,7 +99,7 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
rowPairs[hash] = new Pair<BlueRow, BlueRow>(row, null);
|
||||
rowPairs[hash] = new(row, null);
|
||||
}
|
||||
|
||||
foreach (BlueRow row in myLastR.newRows)
|
||||
@@ -119,12 +114,11 @@ namespace BCG
|
||||
catch (Exception) { }
|
||||
}
|
||||
|
||||
Pair<BlueRow, BlueRow> pair;
|
||||
rowPairs.TryGetValue(hash, out pair);
|
||||
rowPairs.TryGetValue(hash, out var pair);
|
||||
|
||||
|
||||
if (pair == null)
|
||||
pair = new Pair<BlueRow, BlueRow>(null, row);
|
||||
pair = new(null, row);
|
||||
else
|
||||
pair.second = row;
|
||||
|
||||
@@ -132,13 +126,11 @@ namespace BCG
|
||||
}
|
||||
|
||||
int id = 0;
|
||||
foreach (KeyValuePair<List<object>, Pair<BlueRow, BlueRow>> hashpair in rowPairs)
|
||||
foreach (var (hashVals, pair) in rowPairs)
|
||||
{
|
||||
Pair<BlueRow, BlueRow> pair = hashpair.Value;
|
||||
|
||||
if (pair.first == null || pair.second == null)
|
||||
{
|
||||
BlueRow row = pair.first == null ? pair.second : pair.first;
|
||||
var row = pair.first ?? pair.second ?? throw new InvalidOperationException("Both rows in pair are null");
|
||||
|
||||
Dictionary<string, object> newRow = new Dictionary<string, object>();
|
||||
foreach (ColumnInfo info in row.schema.columnInfo)
|
||||
@@ -211,4 +203,4 @@ namespace BCG
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,8 @@
|
||||
using System;
|
||||
using bts;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
@@ -40,12 +44,12 @@ namespace BCG
|
||||
private IComparer<IBlueRow> myRightCompare;
|
||||
protected FuzzyRowComparer myFuzzyCompare;
|
||||
protected TimeSpan myTimeWindow;
|
||||
protected IBlueTableSchema myLeftSchema;
|
||||
protected IBlueTableSchema myRightSchema;
|
||||
protected IBlueTableSchema myOutputSchema;
|
||||
protected IBlueTableSchema? myLeftSchema;
|
||||
protected IBlueTableSchema? myRightSchema;
|
||||
protected IBlueTableSchema? myOutputSchema;
|
||||
private int myID = 0;
|
||||
|
||||
protected TimeWindowTable myStateRows;
|
||||
protected TimeWindowTable? myStateRows;
|
||||
|
||||
private DateTime myNextCull;
|
||||
const long MinCullIntervalMS = 1000;
|
||||
@@ -65,7 +69,7 @@ namespace BCG
|
||||
this.myFuzzyCompare = fuzzyCompare;
|
||||
}
|
||||
|
||||
public virtual IBlueTableSchema OutputSchema
|
||||
public virtual IBlueTableSchema? OutputSchema
|
||||
{
|
||||
get
|
||||
{
|
||||
@@ -150,7 +154,7 @@ namespace BCG
|
||||
int count;
|
||||
for (count = 0; count < list.Count; count++)
|
||||
{
|
||||
if ((DateTime)list[count][timeColumn] > expiry)
|
||||
if (list[count].GetValue<DateTime>(timeColumn) > expiry)
|
||||
{
|
||||
// Stop when we find the first row that we can keep. Since this is sorted all others are also kept.
|
||||
break;
|
||||
@@ -172,14 +176,17 @@ namespace BCG
|
||||
this.myFitness = fitness;
|
||||
}
|
||||
public IBlueRow myRightRow { get; private set; }
|
||||
public IBlueRow myLeftRow { get; set; }
|
||||
public IBlueRow? myLeftRow { get; set; }
|
||||
public double myFitness { get; private set; }
|
||||
public IBlueRow myMergedRow { get; set; }
|
||||
public IBlueRow? myMergedRow { get; set; }
|
||||
|
||||
#region IComparable<RowMatch> Members
|
||||
|
||||
public int CompareTo(RowMatch other)
|
||||
public int CompareTo(RowMatch? other)
|
||||
{
|
||||
if (other is null) {
|
||||
return 1;
|
||||
}
|
||||
return myFitness.CompareTo(other.myFitness);
|
||||
}
|
||||
|
||||
@@ -262,6 +269,10 @@ namespace BCG
|
||||
|
||||
private void HandleCorrection(List<RowMatch> corrections, BlueTable adds, BlueTable dels)
|
||||
{
|
||||
if (corrections.Count == 0) {
|
||||
return;
|
||||
}
|
||||
Log.notnull(myStateRows);
|
||||
// We already emitted correction.MergedRow so we need to redact that and
|
||||
// then emit correction.left as an add since it is no longer matched
|
||||
foreach (RowMatch correction in corrections.Distinct())
|
||||
@@ -285,7 +296,7 @@ namespace BCG
|
||||
int r = 0;
|
||||
for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; )
|
||||
{
|
||||
if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn])
|
||||
if (leftExpired[l].GetValue<DateTime>(myLeftTimeColumn) <= rightExpired[r].GetValue<DateTime>(myRightTimeColumn))
|
||||
{
|
||||
// Advance Left
|
||||
IEnumerable<RowMatch> rm = rightMatches[l];
|
||||
@@ -332,7 +343,7 @@ namespace BCG
|
||||
|
||||
private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable<RowMatch> rightMatches)
|
||||
{
|
||||
IBlueRow merged = null;
|
||||
IBlueRow? merged = null;
|
||||
if (rightMatches != null)
|
||||
{
|
||||
foreach (RowMatch match in rightMatches)
|
||||
@@ -366,9 +377,9 @@ namespace BCG
|
||||
/// Override this function to provide a custom row merge.
|
||||
/// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting.
|
||||
/// </summary>
|
||||
protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right)
|
||||
protected virtual IBlueRow MergeRows(IBlueRow? left, IBlueRow? right)
|
||||
{
|
||||
Dictionary<string, object> newRow = new Dictionary<string, object>();
|
||||
Dictionary<string, object?> newRow = new();
|
||||
|
||||
if (left == null && right == null) throw new ArgumentException("oh come on.");
|
||||
|
||||
@@ -393,14 +404,14 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
newRow["id"] = this.myID++;
|
||||
newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn];
|
||||
newRow[NEW_TIME_COLUMN] = (left != null) ? left[myLeftTimeColumn] : right![myRightTimeColumn];
|
||||
|
||||
return new BlueRow(this.OutputSchema, newRow);
|
||||
}
|
||||
|
||||
private IEnumerable<RowMatch> FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
|
||||
private IEnumerable<RowMatch>? FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
|
||||
{
|
||||
List<RowMatch> matches = null;
|
||||
List<RowMatch>? matches = null;
|
||||
|
||||
// Search through all rows in list and compare with the fuzzy comparer
|
||||
for (int x = 0; x < list.Count; x++)
|
||||
@@ -421,21 +432,15 @@ namespace BCG
|
||||
matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch);
|
||||
|
||||
// Mark the best match for this particular right hand match
|
||||
RowMatch oldMatch;
|
||||
if (this.myBestMatch.TryGetValue(match, out oldMatch))
|
||||
{
|
||||
if (currentMatch.myFitness < oldMatch.myFitness)
|
||||
{
|
||||
if (myBestMatch.TryGetValue(match, out var oldMatch)) {
|
||||
if (currentMatch.myFitness < oldMatch.myFitness) {
|
||||
this.myBestMatch[match] = currentMatch;
|
||||
if (oldMatch.myMergedRow != null)
|
||||
{
|
||||
if (oldMatch.myMergedRow != null) {
|
||||
// We already emitted oldMatch.MergedRow so we need to fix it
|
||||
corrections.Add(oldMatch);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
this.myBestMatch[match] = currentMatch;
|
||||
}
|
||||
}
|
||||
|
||||
+9
-2
@@ -7,6 +7,8 @@ using System.Text.Json;
|
||||
using bts;
|
||||
using bts.SilverPine;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
namespace BCG {
|
||||
//[DataStreamUserRespresentation("AggregateStream")]
|
||||
public class NonIncGroupByStream : ViewStream {
|
||||
@@ -80,7 +82,7 @@ namespace BCG {
|
||||
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
|
||||
|
||||
private void _main_stream_onchange(object nv) {
|
||||
DataPacket dp = nv as DataPacket;
|
||||
var dp = ExceptionAssert.Argument.Is<DataPacket>(nv);
|
||||
///grouping together
|
||||
_timer.Restart();
|
||||
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
|
||||
@@ -109,7 +111,12 @@ namespace BCG {
|
||||
}
|
||||
// TODO,
|
||||
if (total_state != null)
|
||||
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
||||
myFireEvent(new DataPacket(
|
||||
total_state,
|
||||
total_state,
|
||||
myLastValue == null
|
||||
? new BlueTable(total_state.schema)
|
||||
: ExceptionAssert.of_type<DataPacket>(myLastValue).newRows));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -11,7 +10,8 @@ namespace BCG
|
||||
/// </summary>
|
||||
internal class JoinOperandRowIndex
|
||||
{
|
||||
private IDictionary myJoinColDict = new Hashtable (), myIdDict = new Hashtable ();
|
||||
readonly Dictionary<object, object> myJoinColDict = new();
|
||||
readonly Dictionary<object, object> myIdDict = new();
|
||||
private int myJoinColOrdinal, myIdColOrdinal;
|
||||
|
||||
/// <summary>
|
||||
@@ -35,18 +35,18 @@ namespace BCG
|
||||
public void add (IBlueRow row)
|
||||
{
|
||||
RowList list;
|
||||
if (myJoinColDict[row[myJoinColOrdinal]] == null)
|
||||
myJoinColDict[row[myJoinColOrdinal]] = list = new RowList ();
|
||||
if (myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value] == null)
|
||||
myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value] = list = new RowList ();
|
||||
else
|
||||
list = (RowList)myJoinColDict[row[myJoinColOrdinal]];
|
||||
myIdDict[row[myIdColOrdinal]] = list.add (row);
|
||||
list = (RowList)myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value];
|
||||
myIdDict[row[myIdColOrdinal] ?? DBNull.Value] = list.add (row);
|
||||
}
|
||||
|
||||
public void remove (IBlueRow row)
|
||||
{
|
||||
RowNode node = (RowNode)myIdDict[row[myIdColOrdinal]];
|
||||
((RowList)myJoinColDict[row[myJoinColOrdinal]]).remove (node);
|
||||
myIdDict.Remove (row[myIdColOrdinal]);
|
||||
RowNode node = (RowNode)myIdDict[row[myIdColOrdinal] ?? DBNull.Value];
|
||||
((RowList)myJoinColDict[row[myJoinColOrdinal] ?? DBNull.Value]).remove (node);
|
||||
myIdDict.Remove (row[myIdColOrdinal] ?? DBNull.Value);
|
||||
}
|
||||
|
||||
public RowList getMatchingRows (object key)
|
||||
|
||||
@@ -1,18 +1,20 @@
|
||||
using System;
|
||||
using BCG.Utilities;
|
||||
using System.Collections;
|
||||
using bts;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
internal class JoinRow : IBlueRow
|
||||
{
|
||||
private RowNode myLeftNode, myRightNode;
|
||||
protected IBlueRow myLeftRow, myRightRow;
|
||||
RowNode? myLeftNode;
|
||||
RowNode? myRightNode;
|
||||
protected IBlueRow myLeftRow;
|
||||
protected IBlueRow? myRightRow;
|
||||
protected int myId;
|
||||
protected JoinTableSchema mySchema;
|
||||
private object[] myValues;
|
||||
|
||||
private object?[] myValues;
|
||||
|
||||
#region Constructor(s)
|
||||
|
||||
/// <summary>
|
||||
@@ -21,8 +23,8 @@ namespace BCG
|
||||
/// <param name="leftRow">left row of the join</param>
|
||||
/// <param name="rightRow">right row of the join</param>
|
||||
/// <param name="schema">schema of the joined row</param>
|
||||
public JoinRow (IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
|
||||
: this ((int)leftRow["id"], leftRow, rightRow, schema) {}
|
||||
public JoinRow(IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
|
||||
: this(leftRow.Id(), leftRow, rightRow, schema) { }
|
||||
|
||||
/// <summary>
|
||||
/// Constructs a row assigning it the ID provioded as the first argument
|
||||
@@ -31,7 +33,7 @@ namespace BCG
|
||||
/// <param name="leftRow">left row of the join</param>
|
||||
/// <param name="rightRow">right row of the join</param>
|
||||
/// <param name="schema">schema of the joined row</param>
|
||||
public JoinRow (int id, IBlueRow leftRow, IBlueRow rightRow, JoinTableSchema schema)
|
||||
public JoinRow (int id, IBlueRow leftRow, IBlueRow? rightRow, JoinTableSchema schema)
|
||||
{
|
||||
myId = id;
|
||||
myLeftRow = leftRow;
|
||||
@@ -48,13 +50,13 @@ namespace BCG
|
||||
|
||||
public bool isPartialJoin{get{return myRightRow == null;}}
|
||||
|
||||
public RowNode leftNode
|
||||
public RowNode? leftNode
|
||||
{
|
||||
get{return myLeftNode;}
|
||||
set{myLeftNode = value;}
|
||||
}
|
||||
|
||||
public RowNode rightNode
|
||||
public RowNode? rightNode
|
||||
{
|
||||
get{return myRightNode;}
|
||||
set{myRightNode = value;}
|
||||
@@ -85,7 +87,7 @@ namespace BCG
|
||||
/// The key can either be the string corrsponding to the column name or
|
||||
/// the order of the column
|
||||
/// </summary>
|
||||
public virtual object this [object key]
|
||||
public virtual object? this [object key]
|
||||
{
|
||||
get
|
||||
{
|
||||
@@ -96,7 +98,7 @@ namespace BCG
|
||||
set {throw new NotSupportedException ("The IBlueRow is read only");}
|
||||
}
|
||||
|
||||
public object this [int index]
|
||||
public object? this [int index]
|
||||
{
|
||||
get
|
||||
{
|
||||
@@ -119,7 +121,7 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public object this [string colName] {get{return this[mySchema[colName].colNo];}}
|
||||
public object? this [string colName] {get{return this[mySchema[colName].colNo];}}
|
||||
|
||||
public ICollection Keys{get{return mySchema.columnNames;}}
|
||||
|
||||
@@ -133,9 +135,11 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||
public void Add(object key, object? value) =>
|
||||
throw new NotSupportedException("The IBlueRow is readonly");
|
||||
|
||||
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
|
||||
public void Clear() =>
|
||||
throw new NotSupportedException ("The IBlueRow is readonly");
|
||||
|
||||
public bool Contains (object key){return mySchema[(string)key] != null;}
|
||||
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
using System;
|
||||
using System.Data;
|
||||
using System.Collections;
|
||||
using System.Text.RegularExpressions;
|
||||
using BCG.Utilities;
|
||||
using BCG.Utilities.General;
|
||||
using bts;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class JoinStream : ViewStream
|
||||
@@ -111,8 +110,15 @@ namespace BCG
|
||||
{
|
||||
if (myPs == null)
|
||||
{
|
||||
JoinNameResolver resolver = new JoinNameResolver ((JoinTableSchema)myIjs.schema, myIjs.getSchema (true), myLeftCm.columnExpression,
|
||||
myIjs.getSchema (false), myRightCm.columnExpression);
|
||||
ArgumentNullException.ThrowIfNull(myIjs);
|
||||
ArgumentNullException.ThrowIfNull(myLeftCm);
|
||||
ArgumentNullException.ThrowIfNull(myRightCm);
|
||||
var resolver = new JoinNameResolver(
|
||||
ExceptionAssert.of_type<JoinTableSchema>(myIjs.schema),
|
||||
myIjs.getSchema(true),
|
||||
myLeftCm.columnExpression,
|
||||
myIjs.getSchema(false),
|
||||
myRightCm.columnExpression);
|
||||
IDictionary map = resolver.getMap ();
|
||||
map["id"] = "id";
|
||||
myPs = new ProjectStream (myIjs, map);
|
||||
@@ -123,26 +129,27 @@ namespace BCG
|
||||
Log.assert(false, "This call back should not have been called");
|
||||
}
|
||||
else
|
||||
myFireEvent ((o as ICloneable).Clone());
|
||||
myFireEvent(ExceptionAssert.of_type<ICloneable>(o).Clone());
|
||||
}
|
||||
|
||||
private void myProjectChangeHandler (object o)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(myIjs);
|
||||
if (myIjs.myOnChangeDelegate.GetInvocationList().Length > 1)
|
||||
myIjs.onChange -= myInternalJoinChangeHandler;
|
||||
myFireEvent ((o as ICloneable).Clone());
|
||||
myFireEvent(ExceptionAssert.of_type<ICloneable>(o).Clone());
|
||||
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber ()
|
||||
{
|
||||
myIjs.onChange += myInternalJoinChangeHandler;
|
||||
ExceptionAssert.not_null(myIjs).onChange += myInternalJoinChangeHandler;
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber ()
|
||||
{
|
||||
// how does this relate to the -= in myProjectChangeHandler?
|
||||
myIjs.onChange -= myInternalJoinChangeHandler;
|
||||
ExceptionAssert.not_null(myIjs).onChange -= myInternalJoinChangeHandler;
|
||||
|
||||
// if there is a project after the ijs we need to
|
||||
// unsubscribe to it otherwise references will be kept and
|
||||
@@ -159,7 +166,9 @@ namespace BCG
|
||||
/// </summary>
|
||||
/// <param name="leftPacket"></param>
|
||||
/// <param name="rightPacket"></param>
|
||||
public void test (DataPacket dp, bool isLeft) {myIjs.test (dp, isLeft);}
|
||||
public void test (DataPacket dp, bool isLeft) =>
|
||||
ExceptionAssert.not_null(myIjs).test(dp, isLeft);
|
||||
|
||||
/***********************************************************************************************/
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.IO;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -19,20 +19,13 @@ namespace BCG
|
||||
myRightIndex = new JoinDeletionIndex (mySchema.getRenamedName ("id", false), false);
|
||||
}
|
||||
|
||||
public IBlueRow this [int id]
|
||||
{
|
||||
get
|
||||
{
|
||||
if (myIdIndex[id] == null)
|
||||
throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
|
||||
return (IBlueRow)myIdIndex[id];
|
||||
}
|
||||
}
|
||||
public IBlueRow this [int id] =>
|
||||
myIdIndex[id] as IBlueRow ?? throw new BlueDataException ("ID " + id.ToString() + " does not exist in table " + mySchema.tableName);
|
||||
|
||||
#region JoinTable members
|
||||
public virtual void addRow (IBlueRow leftRow, IBlueRow rightRow, DataPacket dp)
|
||||
public virtual void addRow (IBlueRow leftRow, IBlueRow? rightRow, DataPacket dp)
|
||||
{
|
||||
JoinRow newRow = new JoinRow (myNewRowId, leftRow, rightRow, mySchema);
|
||||
var newRow = new JoinRow(myNewRowId, leftRow, rightRow, mySchema);
|
||||
myIdIndex[myNewRowId ++] = newRow;
|
||||
((BlueTable)(dp.addedRows)).addRow (newRow);
|
||||
myLeftIndex.add (newRow);
|
||||
@@ -48,17 +41,17 @@ namespace BCG
|
||||
thisIndex = myLeftIndex;
|
||||
otherIndex = myRightIndex;
|
||||
}
|
||||
RowList matchingNodes = thisIndex.getMatchingRows (deletedRow["id"]);
|
||||
RowList matchingNodes = thisIndex.getMatchingRows (deletedRow.Id());
|
||||
foreach (RowNode node in matchingNodes)
|
||||
{
|
||||
JoinRow row = (JoinRow)node.row;
|
||||
RowNode correspondingNode = isLeft ? row.rightNode : row.leftNode;
|
||||
var correspondingNode = isLeft ? row.rightNode : row.leftNode;
|
||||
if (correspondingNode != null)
|
||||
otherIndex.remove (correspondingNode); //in case of partial left joinss
|
||||
myIdIndex.Remove (row["id"]);
|
||||
myIdIndex.Remove (row.Id());
|
||||
((BlueTable)dp.deletedRows).addRow (row);
|
||||
}
|
||||
thisIndex.removeAll (deletedRow["id"]);
|
||||
thisIndex.removeAll(deletedRow.Id());
|
||||
}
|
||||
#endregion
|
||||
#region ICollection members
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
using bts;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
/// <summary>
|
||||
@@ -13,10 +14,11 @@ namespace BCG
|
||||
public class JoinTableSchema : BlueTableSchema
|
||||
{
|
||||
private string myLeftJc = "", myRightJc = "", myLeftId = "", myRightId = "";
|
||||
private IDictionary myLeftNameToAlias = new Hashtable ();
|
||||
private IDictionary myRightNameToAlias = new Hashtable ();
|
||||
private IDictionary myAliasToLeftName = new Hashtable(),
|
||||
myAliasToRightName = new Hashtable();
|
||||
readonly Dictionary<string, string> myLeftNameToAlias = new();
|
||||
readonly Dictionary<string, string> myRightNameToAlias = new();
|
||||
readonly Dictionary<string, string> myAliasToLeftName = new();
|
||||
readonly Dictionary<string, string> myAliasToRightName = new();
|
||||
|
||||
/// <summary>
|
||||
/// Constructs the schema of the Jointable with the schema of the left and right tables. Incase of
|
||||
/// a name being there in both the shemas it prefixes the name with "L." or "R." accordingly. Column "id"
|
||||
@@ -85,9 +87,9 @@ namespace BCG
|
||||
renamed = false;
|
||||
}
|
||||
myIndexByColName["id"] = new ColumnInfo("id", myIndexByColName.Count, typeof(int));
|
||||
myIndexByColOrder[myIndexByColOrder.Length - 1] = (ColumnInfo)myIndexByColName["id"];
|
||||
myAliasToLeftName = ProjectHelper.invertDic(myLeftNameToAlias);
|
||||
myAliasToRightName = ProjectHelper.invertDic(myRightNameToAlias);
|
||||
myIndexByColOrder[myIndexByColOrder.Length - 1] = ExceptionAssert.of_type<ColumnInfo>(myIndexByColName["id"]);
|
||||
myAliasToLeftName = myLeftNameToAlias.ToDictionary(kv => kv.Value, kv => kv.Key);
|
||||
myAliasToRightName = myRightNameToAlias.ToDictionary(kv => kv.Key, kv => kv.Value);
|
||||
}
|
||||
|
||||
public string leftJoinCol {get{return myLeftJc;}}
|
||||
@@ -97,8 +99,8 @@ namespace BCG
|
||||
public string getRenamedName (string origName, bool isLeft)
|
||||
{
|
||||
if (isLeft)
|
||||
return (string)myLeftNameToAlias[origName];
|
||||
return (string)myRightNameToAlias[origName];
|
||||
return myLeftNameToAlias[origName] as string ?? string.Empty;
|
||||
return myRightNameToAlias[origName] as string ?? string.Empty;
|
||||
}
|
||||
|
||||
public string getOriginalName(string rename, bool isLeft)
|
||||
|
||||
+32
-20
@@ -2,16 +2,15 @@
|
||||
using bts.collections;
|
||||
|
||||
using mbts;
|
||||
using mbts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
|
||||
namespace BCG {
|
||||
public class NonIncJoinBookkeeper : IDisposable {
|
||||
public Action on_new_data;
|
||||
public Action? on_new_data;
|
||||
private readonly NonIncJoinSpec _join_spec;
|
||||
private readonly int _join_position;
|
||||
private readonly int _start_index_in_output;
|
||||
@@ -20,16 +19,19 @@ namespace BCG {
|
||||
private readonly bool _is_tag_along;
|
||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||
|
||||
private IBlueTableSchema _schema = null;
|
||||
private IBlueTableSchema? _schema;
|
||||
private IDictionary<int, IBlueRow> _row_map = new Dictionary<int, IBlueRow>();
|
||||
private List<Pair<int, IBlueRow>> _row_list = new List<Pair<int, IBlueRow>>();
|
||||
private IBlueRow _solitary_row = null;
|
||||
private IBlueRow? _solitary_row;
|
||||
|
||||
private int[] _join_col_indices = null,
|
||||
_data_col_indices = null;
|
||||
int[]? _join_col_indices;
|
||||
int[]? _data_col_indices;
|
||||
|
||||
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
|
||||
_join_spec = the_join_spec;
|
||||
public NonIncJoinBookkeeper(NonIncJoinSpec join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
|
||||
if (join_spec.join_col_names is null) {
|
||||
throw new ArgumentException("join_col_names is null", nameof(join_spec));
|
||||
}
|
||||
_join_spec = join_spec;
|
||||
_join_position = join_position;
|
||||
_start_index_in_output = start_index_in_output;
|
||||
_row_hasher = row_hasher;
|
||||
@@ -41,7 +43,7 @@ namespace BCG {
|
||||
private void _fill_missing_data(DataPacket dp) {
|
||||
if (_filled_missing_data)
|
||||
return;
|
||||
Func<string[], bool> null_or_empty = (str_arr) => {
|
||||
bool null_or_empty(string[]? str_arr) {
|
||||
if (str_arr == null || str_arr.Length == 0)
|
||||
return true;
|
||||
foreach (string s in str_arr) {
|
||||
@@ -50,13 +52,16 @@ namespace BCG {
|
||||
}
|
||||
}
|
||||
return true;
|
||||
};
|
||||
Func<string[], string[]> remove_bad_cols = (col_names) => {
|
||||
}
|
||||
string[] remove_bad_cols(string[]? col_names) {
|
||||
if (col_names is null) {
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
|
||||
List<string> cols_list = new List<string>(col_names);
|
||||
cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
|
||||
return cols_list.ToArray();
|
||||
};
|
||||
}
|
||||
if (null_or_empty(_join_spec.data_col_names)) {
|
||||
_join_spec.data_col_names = dp.schema.columnNames;
|
||||
}
|
||||
@@ -70,7 +75,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public void init() {
|
||||
_join_spec.stream.onChange += _on_stream_cb;
|
||||
ExceptionAssert.not_null(_join_spec.stream).onChange += _on_stream_cb;
|
||||
}
|
||||
|
||||
public void _on_stream_cb(object dp_obj) {
|
||||
@@ -111,14 +116,19 @@ namespace BCG {
|
||||
public bool is_ready { get { return _schema != null; } }
|
||||
|
||||
public void constribute_cols_to_schema(List<ColumnInfo> cis) {
|
||||
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
|
||||
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
|
||||
ArgumentNullException.ThrowIfNull(_join_spec.data_col_names);
|
||||
if (_join_spec.data_col_names.Length > 0) {
|
||||
ArgumentNullException.ThrowIfNull(_join_spec.projected_col_names);
|
||||
ArgumentNullException.ThrowIfNull(_schema);
|
||||
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
|
||||
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
|
||||
}
|
||||
}
|
||||
|
||||
public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
|
||||
|
||||
public void copy_master_row_to_output_row(int row_index, object[] output_row) {
|
||||
IBlueRow this_row = _row_list[row_index].second;
|
||||
IBlueRow? this_row = _row_list[row_index].second;
|
||||
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
||||
}
|
||||
|
||||
@@ -129,8 +139,7 @@ namespace BCG {
|
||||
else
|
||||
copy_dbnull_to_output_row(output_row);
|
||||
} else {
|
||||
IBlueRow this_row = null;
|
||||
if (_row_map.TryGetValue(row_hash, out this_row))
|
||||
if (_row_map.TryGetValue(row_hash, out var this_row))
|
||||
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
||||
else
|
||||
copy_dbnull_to_output_row(output_row);
|
||||
@@ -138,13 +147,16 @@ namespace BCG {
|
||||
}
|
||||
|
||||
private void copy_dbnull_to_output_row(object[] output_row) {
|
||||
ArgumentNullException.ThrowIfNull(_data_col_indices);
|
||||
for (int i = 0; i < _data_col_indices.Length; i++)
|
||||
output_row[i + _start_index_in_output] = DBNull.Value;
|
||||
}
|
||||
|
||||
#region IDisposable
|
||||
|
||||
public void Dispose() { _join_spec.stream.onChange -= _on_stream_cb; }
|
||||
public void Dispose() {
|
||||
ExceptionAssert.not_null(_join_spec.stream).onChange -= _on_stream_cb;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
@@ -1,20 +1,18 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using mbts;
|
||||
using mbts.utils;
|
||||
|
||||
namespace BCG {
|
||||
// this a collection of data, theres no joining logic in here
|
||||
public class NonIncJoinSpec {
|
||||
public ViewStream stream;
|
||||
public ViewStream? stream;
|
||||
// join col names are the names on which the streams are joined (for equality),
|
||||
// data col names are the names in the original stream
|
||||
// projected col names are their new names, ie if its x:l_x, then x is in the
|
||||
// data_col_names and l_x in projected cols
|
||||
public string[] join_col_names, data_col_names, projected_col_names;
|
||||
public string[]? join_col_names, data_col_names, projected_col_names;
|
||||
|
||||
public NonIncJoinSpec() { }
|
||||
|
||||
@@ -31,8 +29,8 @@ namespace BCG {
|
||||
public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
|
||||
List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
|
||||
foreach (Triplet<ViewStream, string, string> r in join_specs) {
|
||||
string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
|
||||
string[] visible_cols = r.third.Split(' ');
|
||||
string[] keys = (r.second ?? string.Empty).Split(' ');
|
||||
string[] visible_cols = ExceptionAssert.not_null(r.third).Split(' ');
|
||||
List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
|
||||
foreach (string s in visible_cols) {
|
||||
string[] toks = s.Split(':');
|
||||
@@ -44,22 +42,15 @@ namespace BCG {
|
||||
mapped_col_names.Add(toks[1]);
|
||||
}
|
||||
}
|
||||
rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
|
||||
rv.Add(new NonIncJoinSpec(ExceptionAssert.not_null(r.first), keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
|
||||
}
|
||||
return rv.ToArray();
|
||||
}
|
||||
|
||||
public override string ToString() {
|
||||
Func<string[], string> arr_txt = (a) => {
|
||||
string rv = "";
|
||||
foreach (string s in a) {
|
||||
rv += "{s} ";
|
||||
}
|
||||
rv = rv.Trim();
|
||||
return rv;
|
||||
};
|
||||
string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
|
||||
return rv;
|
||||
}
|
||||
public override string ToString() =>
|
||||
$"stream: {stream} "
|
||||
+ $"join_col_names: { string.Join(" ", join_col_names ?? Array.Empty<string>())} "
|
||||
+ $"data_col_names: { string.Join(" ", data_col_names ?? Array.Empty<string>())} "
|
||||
+ $"projected_col_names: { string.Join(" ", projected_col_names ?? Array.Empty<string>())}";
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,14 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
using bts;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using mbts;
|
||||
using mbts.utils;
|
||||
|
||||
using Spark.Core.Collections;
|
||||
|
||||
namespace BCG
|
||||
@@ -178,10 +185,11 @@ namespace BCG
|
||||
public class SuperJoinRow : IBlueRow
|
||||
{
|
||||
private JoinProjectHelper myHelper;
|
||||
private IBlueRow myLeftRow, myRightRow;
|
||||
private IBlueRow myLeftRow;
|
||||
private IBlueRow? myRightRow;
|
||||
int myID;
|
||||
|
||||
public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow rightRow, JoinProjectHelper joinProjectHelper)
|
||||
public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow? rightRow, JoinProjectHelper joinProjectHelper)
|
||||
{
|
||||
myID = id;
|
||||
myLeftRow = leftRow;
|
||||
@@ -189,21 +197,21 @@ namespace BCG
|
||||
myHelper = joinProjectHelper;
|
||||
}
|
||||
|
||||
public int leftID { get { return (int)myLeftRow["id"]; } }
|
||||
public int leftID => myLeftRow.Id();
|
||||
|
||||
public int rightID { get { return (int)myRightRow["id"]; } }
|
||||
public int rightID => ExceptionAssert.not_null(myRightRow).Id();
|
||||
|
||||
public IBlueRow leftRow { get { return myLeftRow; } }
|
||||
|
||||
public IBlueRow rightRow { get { return myRightRow; } }
|
||||
public IBlueRow? rightRow => myRightRow;
|
||||
|
||||
#region IBlueRow Members
|
||||
|
||||
public IBlueTableSchema schema { get { return myHelper.schema; } }
|
||||
|
||||
public object this[int index] { get { return this[schema[index].colName]; } }
|
||||
public object? this[int index] { get { return this[schema[index].colName]; } }
|
||||
|
||||
public object this[string colName]
|
||||
public object? this[string colName]
|
||||
{
|
||||
get
|
||||
{
|
||||
@@ -211,8 +219,7 @@ namespace BCG
|
||||
return myID;
|
||||
else
|
||||
{
|
||||
bool isLeft;
|
||||
string col = myHelper.getOriginalCol(colName, out isLeft);
|
||||
string col = myHelper.getOriginalCol(colName, out bool isLeft);
|
||||
return isLeft ? myLeftRow[col] : (myRightRow == null ? (object)DBNull.Value : myRightRow[col]);
|
||||
}
|
||||
}
|
||||
@@ -229,7 +236,7 @@ namespace BCG
|
||||
|
||||
public bool Contains(object key)
|
||||
{
|
||||
return myLeftRow.Contains(key) || myRightRow.Contains(key);
|
||||
return myLeftRow.Contains(key) || (myRightRow?.Contains(key) ?? false);
|
||||
}
|
||||
|
||||
public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
|
||||
@@ -246,7 +253,7 @@ namespace BCG
|
||||
{
|
||||
get
|
||||
{
|
||||
List<object> rv = new List<object>();
|
||||
var rv = new List<object?>();
|
||||
foreach (string col in Keys)
|
||||
rv.Add(this[col]);
|
||||
return rv;
|
||||
@@ -291,9 +298,10 @@ namespace BCG
|
||||
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
|
||||
private bool myManageUnmappedLeftRows = false;
|
||||
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
|
||||
protected IBlueTableSchema myLeftSchema, myRightSchema;
|
||||
protected IBlueTableSchema? myLeftSchema;
|
||||
protected IBlueTableSchema? myRightSchema;
|
||||
protected JoinProjectHelper myJoinProjectHelper;
|
||||
protected BlueTable myState;
|
||||
protected BlueTable? myState;
|
||||
protected int myID;
|
||||
protected ViewStream myLeftStream, myRightStream;
|
||||
protected string myLeftKeyCol, myRightKeyCol;
|
||||
@@ -346,25 +354,25 @@ namespace BCG
|
||||
|
||||
private void myHandleLeftStreamChanges(object newValue)
|
||||
{
|
||||
myHandleStreamChanges(newValue as DataPacket, true);
|
||||
myHandleStreamChanges(ExceptionAssert.Argument.Is<DataPacket>(newValue), true);
|
||||
}
|
||||
|
||||
private void myHandleRightStreamChanges(object newValue)
|
||||
{
|
||||
myHandleStreamChanges(newValue as DataPacket, false);
|
||||
myHandleStreamChanges(ExceptionAssert.Argument.Is<DataPacket>(newValue), false);
|
||||
}
|
||||
|
||||
private void myHandleStreamChanges(DataPacket dp, bool isLeft)
|
||||
{
|
||||
myInitSchema(dp.schema, isLeft);
|
||||
BlueTable dels = null, adds = null;
|
||||
BlueTable? dels = null, adds = null;
|
||||
if (myState != null)
|
||||
{
|
||||
dels = new BlueTable(myState.schema);
|
||||
adds = new BlueTable(myState.schema);
|
||||
}
|
||||
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
|
||||
myAddRows(dp.addedRows, isLeft, myState, adds);
|
||||
myDeleteRows(dp.deletedRows, isLeft, ExceptionAssert.not_null(myState), ExceptionAssert.not_null(dels));
|
||||
myAddRows(dp.addedRows, isLeft, myState, ExceptionAssert.not_null(adds));
|
||||
if (myState != null)
|
||||
{
|
||||
myFireEvent(new DataPacket(myState, adds, dels));
|
||||
@@ -410,8 +418,8 @@ namespace BCG
|
||||
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
|
||||
foreach (IBlueRow r in addedRows)
|
||||
{
|
||||
object currentKey = r[thisKeyCol];
|
||||
int currentID = (int)r["id"];
|
||||
var currentKey = r[thisKeyCol] ?? DBNull.Value;
|
||||
int currentID = r.Id();
|
||||
///add it to 1.
|
||||
if (!thisKeyToRows.ContainsKey(currentKey))
|
||||
thisKeyToRows.Add(currentKey, new Dictionary<int, IBlueRow>());
|
||||
@@ -422,14 +430,14 @@ namespace BCG
|
||||
ICollection<IBlueRow> matchingRows = otherSideKeyToRows[currentKey].Values;
|
||||
foreach (IBlueRow mr in matchingRows)
|
||||
{
|
||||
int mrID = (int)mr["id"];
|
||||
int mrID = mr.Id();
|
||||
SuperJoinRow rr = new SuperJoinRow(myID++, isLeft ? r : mr, isLeft ? mr : r, myJoinProjectHelper);
|
||||
state.addRow(rr);
|
||||
addedRowsOut.addRow(rr);
|
||||
///add to 3.
|
||||
if (!thisIDToResultRows.ContainsKey(currentID))
|
||||
thisIDToResultRows.Add(currentID, new Dictionary<int, SuperJoinRow>());
|
||||
thisIDToResultRows[currentID].Add((int)rr["id"], rr);
|
||||
thisIDToResultRows[currentID].Add(rr.Id(), rr);
|
||||
///add to 4.
|
||||
if (!otherIDToResultRows.ContainsKey(mrID))
|
||||
{
|
||||
@@ -437,7 +445,7 @@ namespace BCG
|
||||
if (myManageUnmappedLeftRows && !isLeft)
|
||||
myTakeCareOfUnmatachedLeftRows(mr, false);
|
||||
}
|
||||
otherIDToResultRows[mrID].Add((int)rr["id"], rr);
|
||||
otherIDToResultRows[mrID].Add(rr.Id(), rr);
|
||||
}
|
||||
}
|
||||
else
|
||||
@@ -458,22 +466,21 @@ namespace BCG
|
||||
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
|
||||
foreach (IBlueRow r in deletedRows)
|
||||
{
|
||||
int id = (int)r["id"];
|
||||
object key = r[thisKeyCol];
|
||||
int id = r.Id();
|
||||
object key = ExceptionAssert.not_null(r[thisKeyCol]);
|
||||
///delete it from 1.
|
||||
thisKeyToRows[key].Remove(id);
|
||||
if (thisKeyToRows[key].Count == 0)
|
||||
thisKeyToRows.Remove(key);
|
||||
///form return rows and delete from 3.
|
||||
if (thisIDToResultRows.ContainsKey(id))
|
||||
if (thisIDToResultRows.TryGetValue(id, out IDictionary<int, SuperJoinRow>? removedResultRows))
|
||||
{
|
||||
ICollection removedResultRows = thisIDToResultRows[id].Values as ICollection;
|
||||
thisIDToResultRows.Remove(id);
|
||||
foreach (SuperJoinRow rr in removedResultRows)
|
||||
foreach (var rr in removedResultRows.Values)
|
||||
{
|
||||
///delete from 4.
|
||||
int otherId = isLeft ? rr.rightID : rr.leftID;
|
||||
otherIDToResultRows[otherId].Remove((int)rr["id"]);
|
||||
otherIDToResultRows[otherId].Remove(rr.Id());
|
||||
if (otherIDToResultRows[otherId].Count == 0)
|
||||
otherIDToResultRows.Remove(otherId);
|
||||
delRowsOut.addRow(rr);
|
||||
@@ -493,15 +500,11 @@ namespace BCG
|
||||
|
||||
private void myTakeCareOfUnmatachedLeftRows(IBlueRow leftRow, bool add)
|
||||
{
|
||||
int lid = (int)leftRow["id"];
|
||||
if (add)
|
||||
int lid = leftRow.Id();
|
||||
if (add) {
|
||||
myLeftUnmatchedRowsAdds.Add(lid, leftRow);
|
||||
else
|
||||
{
|
||||
if (myLeftUnmatchedRowsAdds.ContainsKey(lid))
|
||||
myLeftUnmatchedRowsAdds.Remove(lid);
|
||||
else
|
||||
myLeftUnmatchedRowsDels.Add((int)leftRow["id"], leftRow);
|
||||
} else if (!myLeftUnmatchedRowsAdds.Remove(lid)) {
|
||||
myLeftUnmatchedRowsDels.Add(lid, leftRow);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -522,44 +525,48 @@ namespace BCG
|
||||
|
||||
protected override void myFireEvent(object o)
|
||||
{
|
||||
myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, o as DataPacket);
|
||||
var dp = ExceptionAssert.Argument.Is<DataPacket>(o);
|
||||
myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, dp);
|
||||
}
|
||||
|
||||
private void myHandleJoinResults(Dictionary<int, IBlueRow> unmatchedLeftRowsDels,
|
||||
Dictionary<int, IBlueRow> unmatchedLeftRowAdds, DataPacket resultPacket)
|
||||
{
|
||||
BlueTable dels = resultPacket.deletedRows as BlueTable,
|
||||
adds = resultPacket.addedRows as BlueTable;
|
||||
|
||||
foreach (IBlueRow lr in unmatchedLeftRowsDels.Values)
|
||||
{
|
||||
///an lr will be here if its actually deleted or
|
||||
///a right row is added, so now it has a partner
|
||||
///and is no longer unmatched.
|
||||
///in both cases the half row that this lr
|
||||
///created needs to be removed from this
|
||||
///class's state and added to dels
|
||||
///it means a right is added
|
||||
int lid = (int)lr["id"];
|
||||
SuperJoinRow rr = myLeftIDToHalfRows[lid];
|
||||
myLeftIDToHalfRows.Remove(lid);
|
||||
myState.deleteRow(rr);
|
||||
dels.addRow(rr);
|
||||
var dels = ExceptionAssert.of_type<BlueTable>(resultPacket.deletedRows);
|
||||
var adds = ExceptionAssert.of_type<BlueTable>(resultPacket.addedRows);
|
||||
if (unmatchedLeftRowsDels.Count > 0) {
|
||||
Log.notnull(myState);
|
||||
foreach (IBlueRow lr in unmatchedLeftRowsDels.Values) {
|
||||
///an lr will be here if its actually deleted or
|
||||
///a right row is added, so now it has a partner
|
||||
///and is no longer unmatched.
|
||||
///in both cases the half row that this lr
|
||||
///created needs to be removed from this
|
||||
///class's state and added to dels
|
||||
///it means a right is added
|
||||
int lid = lr.Id();
|
||||
SuperJoinRow rr = myLeftIDToHalfRows[lid];
|
||||
myLeftIDToHalfRows.Remove(lid);
|
||||
myState.deleteRow(rr);
|
||||
dels.addRow(rr);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (IBlueRow lr in unmatchedLeftRowAdds.Values)
|
||||
{
|
||||
///an lr will be here if its added with no
|
||||
///matching right, or a right matching row
|
||||
///is gone so it is unmatched now.
|
||||
///in both cases a half row needs to be
|
||||
///added to this class's state and added
|
||||
///to adds
|
||||
int lid = (int)lr["id"];
|
||||
SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
|
||||
myLeftIDToHalfRows.Add(lid, rr);
|
||||
myState.addRow(rr);
|
||||
adds.addRow(rr);
|
||||
if (unmatchedLeftRowAdds.Count > 0) {
|
||||
Log.notnull(myState);
|
||||
foreach (IBlueRow lr in unmatchedLeftRowAdds.Values) {
|
||||
///an lr will be here if its added with no
|
||||
///matching right, or a right matching row
|
||||
///is gone so it is unmatched now.
|
||||
///in both cases a half row needs to be
|
||||
///added to this class's state and added
|
||||
///to adds
|
||||
int lid = lr.Id();
|
||||
var rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
|
||||
myLeftIDToHalfRows.Add(lid, rr);
|
||||
myState.addRow(rr);
|
||||
adds.addRow(rr);
|
||||
}
|
||||
}
|
||||
base.myFireEvent(new DataPacket(myState, adds, dels));
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
using bts;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -11,9 +14,10 @@ namespace BCG
|
||||
[DataStreamUserRespresentation (false)]
|
||||
public class LeftInternalJoinStream : InternalJoinStream
|
||||
{
|
||||
protected DataPacket myLeftDataPacket, myRightDataPacket;
|
||||
protected DataPacket? myLeftDataPacket, myRightDataPacket;
|
||||
//Container to contain all the left rows that are orphaned during a deletion operation
|
||||
private IDictionary myLeftOrphanedRows;
|
||||
IDictionary? myLeftOrphanedRows;
|
||||
|
||||
public LeftInternalJoinStream (ViewStream vsLeft, string jcLeft, ViewStream vsRight, string jcRight) :
|
||||
base (vsLeft, jcLeft, vsRight, jcRight) {}
|
||||
|
||||
@@ -62,10 +66,10 @@ namespace BCG
|
||||
private void myTestAndFire (bool isLeft)
|
||||
{
|
||||
JoinOperand jo = isLeft ? myLeftOperand : myRightOperand;
|
||||
DataPacket dp = isLeft ? myLeftDataPacket : myRightDataPacket;
|
||||
var dp = isLeft ? myLeftDataPacket : myRightDataPacket;
|
||||
//to test whether both have received call backs
|
||||
if (myLeftOperand.hasReceivedData)
|
||||
jo.changeHandler (dp);
|
||||
jo.changeHandler(Log.notnull(dp));
|
||||
else if (myLeftDataPacket != null && myRightDataPacket != null)
|
||||
{
|
||||
myRightOperand.changeHandler (myRightDataPacket);
|
||||
@@ -82,9 +86,11 @@ namespace BCG
|
||||
{
|
||||
foreach (IBlueRow row in rows)
|
||||
myAddRow (row, isLeft);
|
||||
if (!isLeft)
|
||||
if (!isLeft) {
|
||||
Log.notnull(myLeftOrphanedRows);
|
||||
foreach (RowNode node in myLeftOrphanedRows.Values)
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow (node.row, null, myLastFiredPacket);
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow(node.row, null, myLastFiredPacket);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -104,7 +110,7 @@ namespace BCG
|
||||
if (isLeft)
|
||||
{
|
||||
myLeftOperand.addRow (row);
|
||||
RowList matchedRows = myRightOperand.getMatchingRows (row[myLeftJoinColOrdinal]);
|
||||
RowList matchedRows = myRightOperand.getMatchingRows (row[myLeftJoinColOrdinal] ?? DBNull.Value);
|
||||
if (matchedRows.Count == 0)
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
|
||||
else
|
||||
@@ -113,10 +119,10 @@ namespace BCG
|
||||
}
|
||||
else
|
||||
{
|
||||
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]);
|
||||
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
|
||||
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value);
|
||||
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count;
|
||||
myRightOperand.addRow (row);
|
||||
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
|
||||
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count;
|
||||
if (matchedRows.Count != 0){
|
||||
foreach (RowNode matchedNode in matchedRows)
|
||||
{
|
||||
@@ -125,10 +131,11 @@ namespace BCG
|
||||
//the jointable as a new row will replace it
|
||||
if (rightMatchingRowCount == 0 && newRightMatchingRowCount > 0)
|
||||
{
|
||||
if (myLeftOrphanedRows[matchedNode.row["id"]] == null)
|
||||
Log.notnull(myLeftOrphanedRows);
|
||||
if (myLeftOrphanedRows[matchedNode.row.Id()] == null)
|
||||
((JoinTable)myLastFiredPacket.newRows).deleteRow (true, matchedNode.row, myLastFiredPacket);
|
||||
else
|
||||
myLeftOrphanedRows.Remove (matchedNode.row["id"]);
|
||||
myLeftOrphanedRows.Remove (matchedNode.row.Id());
|
||||
}
|
||||
//they were orphaned by a previous transaction and hence the orphans need to be
|
||||
//removed from the Jointable and the complete row added
|
||||
@@ -148,21 +155,23 @@ namespace BCG
|
||||
}
|
||||
else
|
||||
{
|
||||
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal]);
|
||||
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
|
||||
RowList matchedRows = myLeftOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value);
|
||||
int rightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count;
|
||||
myRightOperand.deleteRow (row);
|
||||
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal]).Count;
|
||||
int newRightMatchingRowCount = myRightOperand.getMatchingRows (row[myRightJoinColOrdinal] ?? DBNull.Value).Count;
|
||||
if (matchedRows.Count != 0)
|
||||
{
|
||||
if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0)
|
||||
if (rightMatchingRowCount > 0 && newRightMatchingRowCount == 0) {
|
||||
//ie all the matchedRows were joined with 'this' right row (method arg), and hence they need to
|
||||
//be handled specially, ie joined with nulls, if there are no right rows with this
|
||||
//joincolumn value being inserted.
|
||||
Log.notnull(myLeftOrphanedRows);
|
||||
foreach (RowNode matchedNode in matchedRows)
|
||||
{
|
||||
// BlueEventLog.logAssert (myLeftOrphanedRows[matchedNode.row["id"]] == null);
|
||||
myLeftOrphanedRows[matchedNode.row["id"]] = matchedNode;
|
||||
myLeftOrphanedRows[matchedNode.row.Id()] = matchedNode;
|
||||
}
|
||||
}
|
||||
((JoinTable)myLastFiredPacket.newRows).deleteRow (false, row, myLastFiredPacket);
|
||||
}
|
||||
}
|
||||
@@ -173,9 +182,11 @@ namespace BCG
|
||||
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
||||
myLastFiredPacket.newRows = new JoinTable (mySchema);
|
||||
myLastFiredPacket.addedRows = new BlueTable (mySchema);
|
||||
if (!isLeft)
|
||||
if (!isLeft) {
|
||||
Log.notnull(myLeftDataPacket);
|
||||
foreach (IBlueRow row in myLeftDataPacket.newRows)
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow(row, null, myLastFiredPacket);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ namespace BCG
|
||||
/// <summary>
|
||||
/// Node containing the referencedRow
|
||||
/// </summary>
|
||||
IBlueRow referencedRow {get;}
|
||||
IBlueRow? referencedRow {get;}
|
||||
|
||||
/// <summary>
|
||||
/// removes all referenced or referencing nodes (indicated by the bool), p.s. theres only one referenced row
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
using System;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
/// <summary>
|
||||
@@ -15,12 +13,12 @@ namespace BCG
|
||||
/// </summary>
|
||||
/// <param name="id"></param>
|
||||
/// <returns></returns>
|
||||
IBlueRow getRow (int id);
|
||||
IBlueRow? getRow (int id);
|
||||
/// <summary>
|
||||
/// Gets all the rows in the node
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
IBlueRow[] getRows ();
|
||||
IBlueRow?[] getRows ();
|
||||
/// <summary>
|
||||
/// Replaces the row with the given id, with the row passed as the second arg
|
||||
/// </summary>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
using BCG;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -56,18 +56,18 @@ namespace BCG
|
||||
/// The key can either be the string corrsponding to the column name or
|
||||
/// the order of the column
|
||||
/// </summary>
|
||||
public virtual object this [object key]
|
||||
public virtual object? this [object key]
|
||||
{
|
||||
get {return key is string ? this [key as string] : this [(int)key];}
|
||||
get => key is string ks ? this[ks] : this[(int)key];
|
||||
set {throw new NotSupportedException ("The IBlueRow is read only");}
|
||||
}
|
||||
|
||||
public virtual object this [int index]
|
||||
public virtual object? this [int index]
|
||||
{
|
||||
get
|
||||
{
|
||||
if (index == mySchema.columnNames.Length - 1)
|
||||
return myRingGRow["id"];
|
||||
return myRingGRow.Id();
|
||||
if (index < myRingGRow.Count)
|
||||
return myRingGRow[index];
|
||||
else if (index >= myRingGRow.Count && index < mySchema.columnNames.Length - 1)
|
||||
@@ -77,7 +77,8 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
public object this [string colName] {get{return this[mySchema[colName].colNo];}}
|
||||
public object? this [string colName] =>
|
||||
this[mySchema[colName].colNo];
|
||||
|
||||
public ICollection Keys{get{return mySchema.columnNames;}}
|
||||
|
||||
|
||||
+18
-7
@@ -1,6 +1,7 @@
|
||||
using mbts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -70,7 +71,10 @@ namespace BCG
|
||||
}
|
||||
else
|
||||
{
|
||||
SimpleNode sn = n as SimpleNode;
|
||||
var sn = ExceptionAssert.Argument.Is<SimpleNode>(n);
|
||||
if (sn.prev is null) {
|
||||
throw new ArgumentException("prev is null", nameof(n));
|
||||
}
|
||||
sn.prev.next = sn.next;
|
||||
if (sn.next != null)
|
||||
sn.next.prev = sn.prev;
|
||||
@@ -81,7 +85,7 @@ namespace BCG
|
||||
|
||||
public int count {get {return myFirst.listCount;}}
|
||||
|
||||
public IBlueRow referencedRow {get {return myFirst.row;}}
|
||||
public IBlueRow? referencedRow => myFirst.row;
|
||||
|
||||
public void removeAll (bool isReD)
|
||||
{
|
||||
@@ -110,10 +114,17 @@ namespace BCG
|
||||
public class LightForeignJoinRowListEnumerator : IEnumerator
|
||||
{
|
||||
private ILightForeignJoinRowList myList;
|
||||
private SimpleNode myPtr, myFirst;
|
||||
public LightForeignJoinRowListEnumerator (ILightForeignJoinRowList list, SimpleNode first)
|
||||
{myList = list; myFirst = first;}
|
||||
public object Current {get{return myPtr.row;}}
|
||||
private SimpleNode? myPtr;
|
||||
private SimpleNode myFirst;
|
||||
|
||||
public LightForeignJoinRowListEnumerator(ILightForeignJoinRowList list, SimpleNode first) {
|
||||
myList = list;
|
||||
myFirst = first;
|
||||
}
|
||||
|
||||
public object? Current =>
|
||||
myPtr is null ? throw new InvalidOperationException("myPtr is null") : myPtr.row;
|
||||
|
||||
public bool MoveNext() {return (myPtr = myPtr == null ? myFirst : myPtr.next) != null;}
|
||||
public void Reset() {myPtr = null;}
|
||||
public ILightForeignJoinRowList list {get {return myList;}}
|
||||
|
||||
+4
-3
@@ -1,5 +1,6 @@
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -13,12 +14,12 @@ namespace BCG
|
||||
: base (reDRow, rinGRow, schema)
|
||||
{}
|
||||
|
||||
public override object this [int index]
|
||||
public override object? this [int index]
|
||||
{
|
||||
get
|
||||
{
|
||||
if (index == mySchema.columnNames.Length - 1)
|
||||
return myRingGRow["id"];
|
||||
return myRingGRow.Id();
|
||||
if (index < myRingGRow.Count)
|
||||
return myRingGRow[index];
|
||||
else if (index >= myRingGRow.Count && index < mySchema.columnNames.Length - 1)
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
using System;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
/// <summary>
|
||||
@@ -9,15 +7,15 @@ namespace BCG
|
||||
/// </summary>
|
||||
public class SimpleNode : INode
|
||||
{
|
||||
public IBlueRow row;
|
||||
public SimpleNode next, prev;
|
||||
public IBlueRow? row;
|
||||
public SimpleNode? next;
|
||||
public SimpleNode? prev;
|
||||
|
||||
public SimpleNode (IBlueRow r)
|
||||
: this (r, null, null)
|
||||
{}
|
||||
|
||||
public SimpleNode (IBlueRow r, SimpleNode nx, SimpleNode pv)
|
||||
{
|
||||
public SimpleNode(IBlueRow r, SimpleNode? nx, SimpleNode? pv) {
|
||||
row = r;
|
||||
next = nx;
|
||||
prev = pv;
|
||||
@@ -25,13 +23,15 @@ namespace BCG
|
||||
|
||||
#region INode Members
|
||||
|
||||
public IBlueRow getRow (int id) {return row;}
|
||||
public IBlueRow? getRow (int id) =>
|
||||
row;
|
||||
|
||||
public IBlueRow[] getRows() {return new IBlueRow[] {row};}
|
||||
public IBlueRow?[] getRows() =>
|
||||
new[] { row };
|
||||
|
||||
public void replace (int id, IBlueRow nr) {row = nr;}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,8 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
using Amazon.Runtime.Internal.Util;
|
||||
|
||||
using BCG;
|
||||
using BCG.Utilities;
|
||||
|
||||
using bts;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG {
|
||||
// Prefixing with "old" to avoid name conflict with the BlueOrb LiteralStream
|
||||
[DataStreamUserRespresentation("oldLiteral")]
|
||||
@@ -44,13 +38,14 @@ namespace BCG {
|
||||
return new BlueRow(map);
|
||||
};
|
||||
string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
|
||||
BlueTable table = null;
|
||||
BlueTable? table = null;
|
||||
foreach (string json_string in row_arr) {
|
||||
IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
|
||||
get_row(json_string, table == null ? 0 : table.Count);
|
||||
table = table ?? new BlueTable(r.schema);
|
||||
table ??= new BlueTable(r.schema);
|
||||
table.addRow(r);
|
||||
}
|
||||
table ??= BlueTable.Empty(table_name);
|
||||
ViewStream str = new BlueTableStream(table);
|
||||
str = new SchemaGuesserStream(str);
|
||||
mySetRealSource(str);
|
||||
@@ -67,8 +62,11 @@ namespace BCG {
|
||||
string[] columnAndType = c.Split('/');
|
||||
if (columnAndType.Length == 1)
|
||||
ci[n] = new ColumnInfo(c, n, typeof(double));
|
||||
else
|
||||
ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
|
||||
else {
|
||||
var colType = Type.GetType("System." + columnAndType[1], true)
|
||||
?? throw new ArgumentException($"Failed to get column type at index {n}", nameof(schema));
|
||||
ci[n] = new ColumnInfo(columnAndType[0], n, colType);
|
||||
}
|
||||
n++;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
using bts;
|
||||
|
||||
using Bts.ColumnMapExpressions;
|
||||
using Bts.DataStream;
|
||||
|
||||
using mbts.config;
|
||||
using mbts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
// using BCG.Evaluator.ColumnMapExpression;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -26,7 +27,7 @@ namespace BCG
|
||||
/// <param name="isch">original(unprojected) schema</param>
|
||||
/// <param name="exp">expression defining the mapping</param>
|
||||
/// <param name="tableName">table name of the new schema</param>
|
||||
internal ProjectHelper (IBlueTableSchema isch, string exp, string tableName)
|
||||
internal ProjectHelper (IBlueTableSchema isch, string exp, string? tableName)
|
||||
: this (isch, myGetDictionary (isch, exp), tableName)
|
||||
{}
|
||||
|
||||
@@ -36,10 +37,10 @@ namespace BCG
|
||||
/// <param name="origToAlias">dictionary mapping unprojected names to their projected names, dropped cols should not be there</param>
|
||||
/// <param name="tableName">table name of the new schema</param>
|
||||
internal ProjectHelper (
|
||||
IBlueTableSchema isch, IDictionary<string, string> origToAlias, string tableName)
|
||||
IBlueTableSchema isch, IDictionary<string, string> origToAlias, string? tableName)
|
||||
{
|
||||
ArrayList colMapAl = new ArrayList (),
|
||||
colInfoAl = new ArrayList ();
|
||||
var colMapAl = new List<int>();
|
||||
var colInfoAl = new List<ColumnInfo>();
|
||||
relevantOrigCols = new string[origToAlias.Count-1];
|
||||
int c = 0;
|
||||
ArrayList iColNames = new ArrayList (isch.columnNames);
|
||||
@@ -49,19 +50,15 @@ namespace BCG
|
||||
if (origToAlias.TryGetValue(colName, out var colAlias))
|
||||
{
|
||||
ColumnInfo ci = isch[colName];
|
||||
colInfoAl.Add (new ColumnInfo (colAlias,
|
||||
colInfoAl.Count,
|
||||
ci.colType)
|
||||
);
|
||||
colMapAl.Add (ci.colNo);
|
||||
colInfoAl.Add(new ColumnInfo (colAlias, colInfoAl.Count, ci.colType));
|
||||
colMapAl.Add(ci.colNo);
|
||||
if (ci.colName != "id")
|
||||
relevantOrigCols[c++] = ci.colName;
|
||||
}
|
||||
}
|
||||
|
||||
schema = new BlueTableSchema (tableName == null ? isch.tableName : tableName,
|
||||
colInfoAl.ToArray (typeof(ColumnInfo)) as ColumnInfo[]);
|
||||
colMap = colMapAl.ToArray (typeof(int)) as int[];
|
||||
schema = new BlueTableSchema(tableName ?? isch.tableName, colInfoAl.ToArray());
|
||||
colMap = colMapAl.ToArray();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -75,16 +72,11 @@ namespace BCG
|
||||
{
|
||||
IDictionary<string, string> origToAlias = new Dictionary<string, string>();
|
||||
|
||||
ColumnMapExpression e;
|
||||
if (Configuration.get<DataStreamConfig>().UseStaticParser)
|
||||
{
|
||||
e = ColumnMapExpression.Parse(exp);
|
||||
}
|
||||
else
|
||||
{
|
||||
var e = Configuration.get<DataStreamConfig>().UseStaticParser
|
||||
? ColumnMapExpression.Parse(exp)
|
||||
// TODO (maybe): Actually use non-static parser (unclear what advantages it will have)
|
||||
e = ColumnMapExpression.Parse(exp);
|
||||
}
|
||||
: ColumnMapExpression.Parse(exp);
|
||||
Log.notnull(e);
|
||||
if (e.IsInclusive) {
|
||||
origToAlias = e.Columns.ToDictionary();
|
||||
origToAlias["id"] = "id";
|
||||
@@ -104,7 +96,7 @@ namespace BCG
|
||||
{
|
||||
IDictionary retVal = new Hashtable (dic.Count);
|
||||
foreach (DictionaryEntry de in dic)
|
||||
retVal.Add (de.Value, de.Key);
|
||||
retVal.Add(ExceptionAssert.not_null(de.Value), de.Key);
|
||||
return retVal;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
namespace BCG {
|
||||
using bts;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using Spark.Core.Collections;
|
||||
|
||||
using System.Collections;
|
||||
@@ -8,10 +12,11 @@ namespace BCG {
|
||||
[DataStreamUserRespresentation("oldproject")]
|
||||
public class ProjectStream : ViewStream
|
||||
{
|
||||
private ProjectHelper myHelper;
|
||||
private string myColExp, myTableName;
|
||||
private ViewStream myVs;
|
||||
private IDictionary<string, string> myDic;
|
||||
ProjectHelper? myHelper;
|
||||
string? myColExp;
|
||||
string? myTableName;
|
||||
ViewStream? myVs;
|
||||
IDictionary<string, string>? myDic;
|
||||
/// <summary>
|
||||
/// </summary>
|
||||
/// <param name="vs">stream to susbsribe to as the input stream</param>
|
||||
@@ -54,17 +59,17 @@ namespace BCG {
|
||||
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
myVs.onChange += myHandleChange;
|
||||
Log.notnull(myVs).onChange += myHandleChange;
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
myVs.onChange -= myHandleChange;
|
||||
Log.notnull(myVs).onChange -= myHandleChange;
|
||||
}
|
||||
|
||||
internal void myHandleChange (object newValue)
|
||||
{
|
||||
DataPacket dp = newValue as DataPacket;
|
||||
var dp = ExceptionAssert.Argument.Is<IDataPacket>(newValue);
|
||||
if (myHelper != null)
|
||||
{
|
||||
IBlueRowCollection dels, adds;
|
||||
@@ -79,8 +84,8 @@ namespace BCG {
|
||||
else
|
||||
{ //first time a callback is received
|
||||
myHelper = myDic == null ?
|
||||
new ProjectHelper (dp.schema, myColExp, myTableName) :
|
||||
new ProjectHelper (dp.schema, myDic, myTableName);
|
||||
new ProjectHelper (dp.Schema, myColExp ?? string.Empty, myTableName) :
|
||||
new ProjectHelper (dp.Schema, myDic, myTableName);
|
||||
|
||||
IBlueRowCollection temp;
|
||||
myFireEvent (new DataPacket (temp = new ProjectedTable (dp.newRows, myHelper),
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.IO;
|
||||
using BCG.Utilities;
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Data.Analysis;
|
||||
using Parquet;
|
||||
using Parquet.Data;
|
||||
using mbts.utils;
|
||||
|
||||
using Microsoft.Data.Analysis;
|
||||
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
|
||||
namespace BCG {
|
||||
[DataStreamUserRespresentation("CSVSaver")]
|
||||
@@ -40,7 +37,7 @@ namespace BCG {
|
||||
myStream.onChange -= myHandleChanges;
|
||||
|
||||
private void myHandleChanges(object newValue) {
|
||||
IBlueRowCollection totalState = (newValue as DataPacket).newRows;
|
||||
IBlueRowCollection totalState = ExceptionAssert.Argument.Is<DataPacket>(newValue).newRows;
|
||||
if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
|
||||
myNWrites++;
|
||||
int i = 0;
|
||||
@@ -62,7 +59,10 @@ namespace BCG {
|
||||
IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
|
||||
BlueTable nRows = new BlueTable(myResultSchema);
|
||||
nRows.addRow(nr);
|
||||
myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
|
||||
BlueTable deletedRows = myLastValue is null
|
||||
? new(myResultSchema)
|
||||
: new(ExceptionAssert.of_type<DataPacket>(myLastValue).newRows);
|
||||
myFireEvent(new DataPacket(nRows, nRows, deletedRows));
|
||||
} else
|
||||
myFireEvent(newValue);
|
||||
}
|
||||
@@ -83,7 +83,7 @@ namespace BCG {
|
||||
if (col != "id") {
|
||||
string colText = String.Empty;
|
||||
if (r[col] != null && r[col] != DBNull.Value)
|
||||
colText = r[col].ToString();
|
||||
colText = r[col]?.ToString() ?? string.Empty;
|
||||
if (r.schema[col].colType == typeof(string))
|
||||
colText = "\"" + colText + "\"";
|
||||
rv += (i++ == 0 ? colText : "," + colText);
|
||||
@@ -136,7 +136,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public class CSVReaderStreamMapper : IStreamMapper {
|
||||
private IBlueTableSchema mySchema;
|
||||
IBlueTableSchema? mySchema;
|
||||
private readonly string[] myOutputSchemaStringArray;
|
||||
|
||||
public CSVReaderStreamMapper(string outputSchema) {
|
||||
@@ -162,6 +162,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public IBlueRow map(IBlueRow row) {
|
||||
ArgumentNullException.ThrowIfNull(mySchema);
|
||||
object[] vals = new object[row.schema.columnInfo.Length];
|
||||
row.Values.CopyTo(vals, 0);
|
||||
for (int i = 0; i < mySchema.columnInfo.Length; i++) {
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
using Bts.ColumnSortExpressions;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
@@ -39,7 +41,7 @@ public class SortRow : IComparable, IBlueRow {
|
||||
var otherVal = otherRow[col.Name];
|
||||
int rv = myIsNull(this[col.Name]) ?
|
||||
(myIsNull(otherVal) ? 0 : -1) :
|
||||
(myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
|
||||
(myIsNull(otherVal) ? 1 : ExceptionAssert.of_type<IComparable>(this[col.Name]).CompareTo(otherVal));
|
||||
rv *= col.IsAscending ? 1 : -1;
|
||||
return rv;
|
||||
}
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using BCG.Utilities.General;
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -49,8 +48,7 @@ namespace BCG
|
||||
#region IWorker
|
||||
public void doWork()
|
||||
{
|
||||
if ((myNextWrite <= DateTime.Now && myGotUpdate))
|
||||
{
|
||||
if (myNextWrite <= DateTime.Now && myGotUpdate) {
|
||||
myGotUpdate = false;
|
||||
myNextWrite = DateTime.Now.AddSeconds(myWriteFreqSecs);
|
||||
myWrite(myStream.rows);
|
||||
@@ -96,7 +94,7 @@ namespace BCG
|
||||
rowStr += DELIM;
|
||||
}
|
||||
|
||||
rowStr += row[columnName].ToString();
|
||||
rowStr += row[columnName]?.ToString() ?? string.Empty;
|
||||
}
|
||||
|
||||
writer.WriteLine(rowStr);
|
||||
@@ -139,7 +137,7 @@ namespace BCG
|
||||
private int myWriteFreqSecs;
|
||||
private DateTime myNextWrite;
|
||||
private bool myGotUpdate;
|
||||
private string mySchemaStr;
|
||||
private string? mySchemaStr;
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,8 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -11,7 +10,7 @@ namespace BCG
|
||||
{
|
||||
private ViewStream _src;
|
||||
private string _time_stamp_field;
|
||||
private IBlueTableSchema _schema;
|
||||
private IBlueTableSchema? _schema;
|
||||
private string _latency_field;
|
||||
|
||||
public LatencyStream(ViewStream input, string time_stamp_field, string latency_field)
|
||||
@@ -38,10 +37,10 @@ namespace BCG
|
||||
public IBlueRow map(IBlueRow row)
|
||||
{
|
||||
var ht = new Hashtable(row);
|
||||
if (row[_time_stamp_field].Equals(DBNull.Value))
|
||||
if (Equals(row[_time_stamp_field], DBNull.Value))
|
||||
ht[_latency_field] = DBNull.Value;
|
||||
else
|
||||
ht[_latency_field] = (DateTime.Now - (DateTime)row[_time_stamp_field]).TotalMilliseconds;
|
||||
ht[_latency_field] = (DateTime.Now - row.GetValue<DateTime>(_time_stamp_field)).TotalMilliseconds;
|
||||
return new BlueRow(_schema, ht);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
using System;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -42,8 +43,8 @@ namespace BCG
|
||||
BlueTable result = new BlueTable(dp.schema);
|
||||
foreach (IBlueRow r in dp.newRows)
|
||||
{
|
||||
string key = r[myCol_Key].ToString();
|
||||
DateTime dt = (DateTime) r[myCol_Time];
|
||||
var key = r[myCol_Key]?.ToString() ?? string.Empty;
|
||||
var dt = r.GetValue<DateTime>(myCol_Time);
|
||||
double value = Convert.ToDouble(r[myCol_Value]);
|
||||
|
||||
object[] rowValues = new object[r.Count];
|
||||
@@ -68,9 +69,7 @@ namespace BCG
|
||||
private double myUpdateValue(string key, DateTime time, double value)
|
||||
{
|
||||
|
||||
SortedList<DateTime, double> seriesValues;
|
||||
if (!myKeyToTimeToValue.TryGetValue(key, out seriesValues))
|
||||
{
|
||||
if (!myKeyToTimeToValue.TryGetValue(key, out SortedList<DateTime, double>? seriesValues)) {
|
||||
seriesValues = new SortedList<DateTime, double>();
|
||||
myKeyToTimeToValue[key] = seriesValues;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using mbts.utils;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
@@ -12,7 +16,7 @@ namespace BCG
|
||||
private int myId = 0;
|
||||
private IDictionary myLeftIndex = new Hashtable (),
|
||||
myRightIndex = new Hashtable ();
|
||||
private IBlueTableSchema mySchema;
|
||||
private IBlueTableSchema? mySchema;
|
||||
|
||||
public AppendCollection () {}
|
||||
|
||||
@@ -24,15 +28,15 @@ namespace BCG
|
||||
IDictionary index = isLeft ? myLeftIndex : myRightIndex;
|
||||
foreach (IBlueRow r in dp.deletedRows)
|
||||
{
|
||||
IBlueRow dr = index[r["id"]] as IBlueRow;
|
||||
var dr = ExceptionAssert.of_type<IBlueRow>(index[r.Id()]);
|
||||
dRows.addRow (dr);
|
||||
index.Remove (r["id"]);
|
||||
index.Remove (r.Id());
|
||||
}
|
||||
foreach (IBlueRow r in dp.addedRows)
|
||||
{
|
||||
IBlueRow ar = new AppendRow (r, myId++, mySchema);
|
||||
aRows.addRow (ar);
|
||||
index.Add (r["id"], ar);
|
||||
index.Add(r.Id(), ar);
|
||||
}
|
||||
return new DataPacket (this, aRows, dRows);
|
||||
}
|
||||
@@ -43,7 +47,8 @@ namespace BCG
|
||||
|
||||
public bool containsID (int id) {throw new ApplicationException ("IBlueRow is readonly");}
|
||||
|
||||
public IBlueTableSchema schema {get {return mySchema;}}
|
||||
public IBlueTableSchema schema =>
|
||||
ExceptionAssert.not_null(mySchema);
|
||||
|
||||
#endregion
|
||||
|
||||
@@ -60,15 +65,14 @@ namespace BCG
|
||||
array.SetValue (r, i++);
|
||||
}
|
||||
|
||||
public object SyncRoot {get {return null;}}
|
||||
public object SyncRoot => this;
|
||||
|
||||
#endregion
|
||||
|
||||
#region IEnumerable Members
|
||||
|
||||
public System.Collections.IEnumerator GetEnumerator()
|
||||
{
|
||||
return new MyEnumerator (myLeftIndex, myRightIndex);
|
||||
public System.Collections.IEnumerator GetEnumerator() {
|
||||
return new MyEnumerator(myLeftIndex, myRightIndex);
|
||||
}
|
||||
|
||||
private class MyEnumerator : IEnumerator
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
using System;
|
||||
using mbts.utils;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class AppendStreamModified : ViewStream
|
||||
{
|
||||
private AppendCollection myColl = new AppendCollection ();
|
||||
private ViewStream myLeftStream, myRightstream;
|
||||
ViewStream? myLeftStream;
|
||||
ViewStream? myRightstream;
|
||||
|
||||
public AppendStreamModified (ViewStream leftStream, ViewStream rightStream)
|
||||
{
|
||||
@@ -19,7 +20,7 @@ namespace BCG
|
||||
mySetRealSource (vss[0]);
|
||||
else
|
||||
{
|
||||
AppendStreamModified rv = null;
|
||||
AppendStreamModified? rv = null;
|
||||
foreach (ViewStream s in vss)
|
||||
rv = rv == null ? new AppendStreamModified (new ViewStream[] {s}) : new AppendStreamModified (rv, s);
|
||||
mySetRealSource (rv);
|
||||
@@ -28,24 +29,26 @@ namespace BCG
|
||||
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
myLeftStream.onChange += myHandleLeftChanges;
|
||||
myRightstream.onChange += myHandleRightChanges;
|
||||
ExceptionAssert.not_null(myLeftStream).onChange += myHandleLeftChanges;
|
||||
ExceptionAssert.not_null(myRightstream).onChange += myHandleRightChanges;
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
myLeftStream.onChange -= myHandleLeftChanges;
|
||||
myRightstream.onChange -= myHandleRightChanges;
|
||||
ExceptionAssert.not_null(myLeftStream).onChange -= myHandleLeftChanges;
|
||||
ExceptionAssert.not_null(myRightstream).onChange -= myHandleRightChanges;
|
||||
}
|
||||
|
||||
private void myHandleLeftChanges (object newValue)
|
||||
{
|
||||
myFireEvent (myGetDataPacket (newValue as DataPacket, true));
|
||||
var dp = ExceptionAssert.Argument.Is<DataPacket>(newValue);
|
||||
myFireEvent(myGetDataPacket(dp, true));
|
||||
}
|
||||
|
||||
private void myHandleRightChanges (object newValue)
|
||||
{
|
||||
myFireEvent (myGetDataPacket (newValue as DataPacket, false));
|
||||
var dp = ExceptionAssert.Argument.Is<DataPacket>(newValue);
|
||||
myFireEvent(myGetDataPacket(dp, false));
|
||||
}
|
||||
|
||||
private DataPacket myGetDataPacket (DataPacket dp, bool isLeft)
|
||||
|
||||
Reference in new issue
Block a user