Files
spark/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs
T
John Colvin a4efaa0089 cleanup: Fix a ton of build warnings (cs)
All these warnings have been bothering me for a while. The work isn't done, but
this commit chops them down from ~1000 to ~600.

* AppendCollection.cs
* AppendStreamModified.cs
* BenzingaNewsStream.cs
* BenzingaNewsSymbolStateStream.cs
* BookStream.cs
* CSVSaverStream.cs
* ChartsUtils.cs
* ColumnInfo.cs
* ContractUtils.cs
* ContractUtils.cs
* DiffStream.cs
* FuturesActiveContractStream.cs
* FuzzyJoinOverTimeStream.cs
* HistoricalBarsStream.cs
* HistoricalBarsStream.cs
* InstrumentsStream.cs
* JoinOperandRowIndex.cs
* JoinRow.cs
* JoinStream.cs
* JoinTable.cs
* JoinTableSchema.cs
* LatencyStream.cs
* LeftInternalJoinStream.cs
* LightForeignJoinRow.cs
* LightForeignJoinRowList.cs
* LightLeftForeignJoinRow.cs
* LiteralStream.cs
* NLStream.cs
* NLToDSLStream.cs
* NLToIndexedFewShotPromptStream.cs
* NonIncGroupByStream.cs
* NonIncJoinBookkeeper.cs
* NonIncJoinSpec.cs
* OHLCVStream.cs
* OpenAIPromptStream.cs
* PriceStatsStream.cs
* PriceStatsStream.cs
* ProjectHelper.cs
* ProjectStream.cs
* PubSubServerStateStream.cs
* PubsubHeartbeatStream.cs
* SimpleNode.cs
* SortRow.cs
* SpatialArbStatsStream.cs
* SpatialFuturePairsStream.cs
* SpreadPerformanceEvaluator.cs
* SpreadPerformanceStream.cs
* StreamIOCommon.cs
* SuperJoinStream.cs
* TableMetaDataStream.cs
* TextWriterStream.cs
* TimeAverageStream.cs
2025-11-19 13:57:47 -05:00

575 lines
21 KiB
C#

using System;
using System.Collections;
using System.Collections.Generic;
using bts;
using Bts.DataStreamCore;
using mbts;
using mbts.utils;
using Spark.Core.Collections;
namespace BCG
{
internal class SingleRowJoinDicionary<TKey, TValue> : IDictionary<TKey, TValue>
{
private TKey myKey;
private TValue myValue;
#region IDictionary<TKey,TValue> Members
public void Add(TKey key, TValue value)
{
myKey = key;
myValue = value;
}
public bool ContainsKey(TKey key)
{
return myKey.Equals(key);
}
public ICollection<TKey> Keys
{
get { return new TKey[] { myKey }; }
}
public bool Remove(TKey key)
{
bool rv = myKey.Equals(key);
myKey = default(TKey);
return rv;
}
public bool TryGetValue(TKey key, out TValue value)
{
bool rv = false;
value = (rv = myKey.Equals(key)) ? myValue : default(TValue);
return rv;
}
public ICollection<TValue> Values
{
get { throw new Exception("The method or operation is not implemented."); }
}
public TValue this[TKey key]
{
get
{
throw new Exception("The method or operation is not implemented.");
}
set
{
throw new Exception("The method or operation is not implemented.");
}
}
#endregion
#region ICollection<KeyValuePair<TKey,TValue>> Members
public void Add(KeyValuePair<TKey, TValue> item)
{
throw new Exception("The method or operation is not implemented.");
}
public void Clear()
{
throw new Exception("The method or operation is not implemented.");
}
public bool Contains(KeyValuePair<TKey, TValue> item)
{
throw new Exception("The method or operation is not implemented.");
}
public void CopyTo(KeyValuePair<TKey, TValue>[] array, int arrayIndex)
{
throw new Exception("The method or operation is not implemented.");
}
public int Count
{
get { throw new Exception("The method or operation is not implemented."); }
}
public bool IsReadOnly
{
get { throw new Exception("The method or operation is not implemented."); }
}
public bool Remove(KeyValuePair<TKey, TValue> item)
{
throw new Exception("The method or operation is not implemented.");
}
#endregion
#region IEnumerable<KeyValuePair<TKey,TValue>> Members
public IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator()
{
throw new Exception("The method or operation is not implemented.");
}
#endregion
#region IEnumerable Members
IEnumerator IEnumerable.GetEnumerator()
{
throw new Exception("The method or operation is not implemented.");
}
#endregion
}
#region Helper class to deal with stream projections
public class JoinProjectHelper
{
private IBlueTableSchema mySchema;
private Dictionary<string, Pair<string, bool>> myAliasToOriginal = new Dictionary<string, Pair<string, bool>>();
public JoinProjectHelper(IBlueTableSchema leftSchema,
string leftProj,
string leftJoinCol,
IBlueTableSchema rightSchema,
string rightProj,
string rightJoinCol)
{
JoinTableSchema joinedSchema = new JoinTableSchema(leftSchema, leftJoinCol, rightSchema, rightJoinCol);
JoinNameResolver resolver = new JoinNameResolver(joinedSchema, leftSchema, leftProj, rightSchema, rightProj);
IDictionary map = resolver.getMap();
map["id"] = "id";
ProjectHelper helper = new ProjectHelper(
joinedSchema,
GenericDictionaryWrapper.Create<string, string>(ProjectHelper.invertDic(map)),
String.Format("({0} J {1})", leftSchema.tableName, rightSchema.tableName));
mySchema = helper.schema;
for (int aliasCol = 0; aliasCol < helper.colMap.Length; aliasCol++)
{
string aliasColName = mySchema[aliasCol].colName;
if (aliasColName != "id")
{
int joinedColIndex = helper.colMap[aliasCol];
bool isLeft = joinedColIndex < leftSchema.columnInfo.Length;
string originalCol = joinedSchema.getOriginalName(joinedSchema[joinedColIndex].colName, isLeft);
myAliasToOriginal.Add(aliasColName, new Pair<string, bool>(originalCol, isLeft));
}
}
}
public IBlueTableSchema schema
{
get { return mySchema; }
}
public string getOriginalCol(string col, out bool isLeft)
{
Pair<string, bool> p = myAliasToOriginal[col];
isLeft = p.second;
return p.first;
}
}
#endregion
#region IBlueRow implementation for join and leftJoin
public class SuperJoinRow : IBlueRow
{
private JoinProjectHelper myHelper;
private IBlueRow myLeftRow;
private IBlueRow? myRightRow;
int myID;
public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow? rightRow, JoinProjectHelper joinProjectHelper)
{
myID = id;
myLeftRow = leftRow;
myRightRow = rightRow;
myHelper = joinProjectHelper;
}
public int leftID => myLeftRow.Id();
public int rightID => ExceptionAssert.not_null(myRightRow).Id();
public IBlueRow leftRow { get { return myLeftRow; } }
public IBlueRow? rightRow => myRightRow;
#region IBlueRow Members
public IBlueTableSchema schema { get { return myHelper.schema; } }
public object? this[int index] { get { return this[schema[index].colName]; } }
public object? this[string colName]
{
get
{
if (colName == "id")
return myID;
else
{
string col = myHelper.getOriginalCol(colName, out bool isLeft);
return isLeft ? myLeftRow[col] : (myRightRow == null ? (object)DBNull.Value : myRightRow[col]);
}
}
}
#endregion
#region IDictionary Members
public void Add(object key, object? value) =>
throw new NotSupportedException("IBlueRow is readonly");
public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); }
public bool Contains(object key)
{
return myLeftRow.Contains(key) || (myRightRow?.Contains(key) ?? false);
}
public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
public bool IsFixedSize { get { return true; } }
public bool IsReadOnly { get { return true; } }
public System.Collections.ICollection Keys { get { return schema.columnNames; } }
public void Remove(object key) { throw new NotSupportedException("IBlueRow is readonly"); }
public System.Collections.ICollection Values
{
get
{
var rv = new List<object?>();
foreach (string col in Keys)
rv.Add(this[col]);
return rv;
}
}
public object? this[object key] {
get => key is string str ? this[str] : this[(int)key];
set => throw new NotSupportedException("IBlueRow is readonly");
}
#endregion
#region ICollection Members
public void CopyTo(Array array, int index) { Values.CopyTo(array, index); }
public int Count { get { return schema.columnNames.Length; } }
public bool IsSynchronized { get { return false; } }
public object SyncRoot { get { return this; } }
#endregion
#region IEnumerable Members
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
#endregion
}
#endregion
/// <summary>
/// this class is written 3-4 yrs after the original
/// join, and learning from the past mistakes, is
/// made super fast and mem efficent
/// </summary>
public class SuperJoinStream : ViewStream
{
protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
private bool myManageUnmappedLeftRows = false;
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
protected IBlueTableSchema? myLeftSchema;
protected IBlueTableSchema? myRightSchema;
protected JoinProjectHelper myJoinProjectHelper;
protected BlueTable? myState;
protected int myID;
protected ViewStream myLeftStream, myRightStream;
protected string myLeftKeyCol, myRightKeyCol;
protected Dictionary<object, IDictionary<int, IBlueRow>> myLeftKeyToLeftRows = new Dictionary<object, IDictionary<int, IBlueRow>>(),
myRightKeyToRightRows = new Dictionary<object, IDictionary<int, IBlueRow>>();
protected Dictionary<int, IDictionary<int, SuperJoinRow>> myLeftIDToResultRows = new Dictionary<int, IDictionary<int, SuperJoinRow>>(),
myRightIDToResultRows = new Dictionary<int, IDictionary<int, SuperJoinRow>>();
public SuperJoinStream(ViewStream leftStream, string leftKey, ViewStream rightStream, string rightKey)
: this(leftStream, leftKey, "!", rightStream, rightKey, "!")
{ }
public SuperJoinStream(ViewStream leftStream, string leftKey, string leftProj,
ViewStream rightStream, string rightKey, string rightProj)
: this(leftStream, leftKey, leftProj, rightStream, rightKey, rightProj, false)
{
}
internal SuperJoinStream(ViewStream leftStream, string leftKey, string leftProj,
ViewStream rightStream, string rightKey, string rightProj, bool manageUnmappedLeftRows)
{
myLeftStream = leftStream;
myRightStream = rightStream;
myLeftKeyCol = leftKey;
myRightKeyCol = rightKey;
myLeftProj = leftProj;
myRightProj = rightProj;
myManageUnmappedLeftRows = manageUnmappedLeftRows;
}
protected override void myOnFirstSubscriber()
{
myLeftStream.onChange += myHandleLeftStreamChanges;
myRightStream.onChange += myHandleRightStreamChanges;
}
protected override void myOnLastSubscriber()
{
myLeftStream.onChange -= myHandleLeftStreamChanges;
myRightStream.onChange -= myHandleRightStreamChanges;
myLeftKeyToLeftRows.Clear();
myRightKeyToRightRows.Clear();
myLeftIDToResultRows.Clear();
myRightIDToResultRows.Clear();
myState = null;
myLeftSchema = null;
myRightSchema = null;
}
private void myHandleLeftStreamChanges(object newValue)
{
myHandleStreamChanges(ExceptionAssert.Argument.Is<DataPacket>(newValue), true);
}
private void myHandleRightStreamChanges(object newValue)
{
myHandleStreamChanges(ExceptionAssert.Argument.Is<DataPacket>(newValue), false);
}
private void myHandleStreamChanges(DataPacket dp, bool isLeft)
{
myInitSchema(dp.schema, isLeft);
BlueTable? dels = null, adds = null;
if (myState != null)
{
dels = new BlueTable(myState.schema);
adds = new BlueTable(myState.schema);
}
myDeleteRows(dp.deletedRows, isLeft, ExceptionAssert.not_null(myState), ExceptionAssert.not_null(dels));
myAddRows(dp.addedRows, isLeft, myState, ExceptionAssert.not_null(adds));
if (myState != null)
{
myFireEvent(new DataPacket(myState, adds, dels));
myLeftUnmatchedRowsAdds.Clear();
myLeftUnmatchedRowsDels.Clear();
}
}
private void myInitSchema(IBlueTableSchema schema, bool isLeft)
{
if (myState == null)
{
if (isLeft)
{
myLeftSchema = schema;
if (myRightSchema != null)
{
myJoinProjectHelper = new JoinProjectHelper(myLeftSchema, myLeftProj, myLeftKeyCol,
myRightSchema, myRightProj, myRightKeyCol);
myState = new BlueTable(myJoinProjectHelper.schema);
}
}
else
{
myRightSchema = schema;
if (myLeftSchema != null)
{
myJoinProjectHelper = new JoinProjectHelper(myLeftSchema, myLeftProj, myLeftKeyCol,
myRightSchema, myRightProj, myRightKeyCol);
myState = new BlueTable(myJoinProjectHelper.schema);
}
}
}
}
private void myAddRows(IBlueRowCollection addedRows, bool isLeft, BlueTable state, BlueTable addedRowsOut)
{
object thisKeyCol = isLeft ? myLeftKeyCol : myRightKeyCol,
otherSideKeyCol = isLeft ? myRightKeyCol : myLeftKeyCol;
Dictionary<object, IDictionary<int, IBlueRow>> thisKeyToRows = isLeft ? myLeftKeyToLeftRows : myRightKeyToRightRows,
otherSideKeyToRows = isLeft ? myRightKeyToRightRows : myLeftKeyToLeftRows;
Dictionary<int, IDictionary<int, SuperJoinRow>> thisIDToResultRows = isLeft ? myLeftIDToResultRows : myRightIDToResultRows,
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
foreach (IBlueRow r in addedRows)
{
var currentKey = r[thisKeyCol] ?? DBNull.Value;
int currentID = r.Id();
///add it to 1.
if (!thisKeyToRows.ContainsKey(currentKey))
thisKeyToRows.Add(currentKey, new Dictionary<int, IBlueRow>());
thisKeyToRows[currentKey].Add(currentID, r);
///find matching rows, and form result rows
if (otherSideKeyToRows.ContainsKey(currentKey))
{
ICollection<IBlueRow> matchingRows = otherSideKeyToRows[currentKey].Values;
foreach (IBlueRow mr in matchingRows)
{
int mrID = mr.Id();
SuperJoinRow rr = new SuperJoinRow(myID++, isLeft ? r : mr, isLeft ? mr : r, myJoinProjectHelper);
state.addRow(rr);
addedRowsOut.addRow(rr);
///add to 3.
if (!thisIDToResultRows.ContainsKey(currentID))
thisIDToResultRows.Add(currentID, new Dictionary<int, SuperJoinRow>());
thisIDToResultRows[currentID].Add(rr.Id(), rr);
///add to 4.
if (!otherIDToResultRows.ContainsKey(mrID))
{
otherIDToResultRows.Add(mrID, new Dictionary<int, SuperJoinRow>());
if (myManageUnmappedLeftRows && !isLeft)
myTakeCareOfUnmatachedLeftRows(mr, false);
}
otherIDToResultRows[mrID].Add(rr.Id(), rr);
}
}
else
{
if (myManageUnmappedLeftRows && isLeft)
myTakeCareOfUnmatachedLeftRows(r, true);
}
}
}
private void myDeleteRows(IBlueRowCollection deletedRows, bool isLeft, BlueTable state, BlueTable delRowsOut)
{
string thisKeyCol = isLeft ? myLeftKeyCol : myRightKeyCol,
otherSideKeyCol = isLeft ? myRightKeyCol : myLeftKeyCol;
Dictionary<object, IDictionary<int, IBlueRow>> thisKeyToRows = isLeft ? myLeftKeyToLeftRows : myRightKeyToRightRows,
otherSideKeyToRows = isLeft ? myRightKeyToRightRows : myLeftKeyToLeftRows;
Dictionary<int, IDictionary<int, SuperJoinRow>> thisIDToResultRows = isLeft ? myLeftIDToResultRows : myRightIDToResultRows,
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
foreach (IBlueRow r in deletedRows)
{
int id = r.Id();
object key = ExceptionAssert.not_null(r[thisKeyCol]);
///delete it from 1.
thisKeyToRows[key].Remove(id);
if (thisKeyToRows[key].Count == 0)
thisKeyToRows.Remove(key);
///form return rows and delete from 3.
if (thisIDToResultRows.TryGetValue(id, out IDictionary<int, SuperJoinRow>? removedResultRows))
{
thisIDToResultRows.Remove(id);
foreach (var rr in removedResultRows.Values)
{
///delete from 4.
int otherId = isLeft ? rr.rightID : rr.leftID;
otherIDToResultRows[otherId].Remove(rr.Id());
if (otherIDToResultRows[otherId].Count == 0)
otherIDToResultRows.Remove(otherId);
delRowsOut.addRow(rr);
state.deleteRow(rr);
if (myManageUnmappedLeftRows && !isLeft)
if (!thisKeyToRows.ContainsKey(key))
myTakeCareOfUnmatachedLeftRows(rr.leftRow, true);
}
}
else
{
if (myManageUnmappedLeftRows && isLeft)
myTakeCareOfUnmatachedLeftRows(r, false);
}
}
}
private void myTakeCareOfUnmatachedLeftRows(IBlueRow leftRow, bool add)
{
int lid = leftRow.Id();
if (add) {
myLeftUnmatchedRowsAdds.Add(lid, leftRow);
} else if (!myLeftUnmatchedRowsAdds.Remove(lid)) {
myLeftUnmatchedRowsDels.Add(lid, leftRow);
}
}
}
public class SuperLeftJoinStream : SuperJoinStream
{
private Dictionary<int, SuperJoinRow> myLeftIDToHalfRows = new Dictionary<int, SuperJoinRow>();
public SuperLeftJoinStream(ViewStream leftStream, string leftJoinCol,
ViewStream rightStream, string rightJoinCol)
: this(leftStream, leftJoinCol, "!", rightStream, rightJoinCol, "!")
{ }
public SuperLeftJoinStream(ViewStream leftStream, string leftJoinCol, string leftProjExp,
ViewStream rightStream, string rightJoinCol, string rightProjExp)
: base(leftStream, leftJoinCol, leftProjExp, rightStream, rightJoinCol, rightProjExp, true)
{ }
protected override void myFireEvent(object o)
{
var dp = ExceptionAssert.Argument.Is<DataPacket>(o);
myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, dp);
}
private void myHandleJoinResults(Dictionary<int, IBlueRow> unmatchedLeftRowsDels,
Dictionary<int, IBlueRow> unmatchedLeftRowAdds, DataPacket resultPacket)
{
var dels = ExceptionAssert.of_type<BlueTable>(resultPacket.deletedRows);
var adds = ExceptionAssert.of_type<BlueTable>(resultPacket.addedRows);
if (unmatchedLeftRowsDels.Count > 0) {
Log.notnull(myState);
foreach (IBlueRow lr in unmatchedLeftRowsDels.Values) {
///an lr will be here if its actually deleted or
///a right row is added, so now it has a partner
///and is no longer unmatched.
///in both cases the half row that this lr
///created needs to be removed from this
///class's state and added to dels
///it means a right is added
int lid = lr.Id();
SuperJoinRow rr = myLeftIDToHalfRows[lid];
myLeftIDToHalfRows.Remove(lid);
myState.deleteRow(rr);
dels.addRow(rr);
}
}
if (unmatchedLeftRowAdds.Count > 0) {
Log.notnull(myState);
foreach (IBlueRow lr in unmatchedLeftRowAdds.Values) {
///an lr will be here if its added with no
///matching right, or a right matching row
///is gone so it is unmatched now.
///in both cases a half row needs to be
///added to this class's state and added
///to adds
int lid = lr.Id();
var rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
myLeftIDToHalfRows.Add(lid, rr);
myState.addRow(rr);
adds.addRow(rr);
}
}
base.myFireEvent(new DataPacket(myState, adds, dels));
}
}
}