Files
spark/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs
T
John Colvin 9050318ceb refactor: Replace HighPerformanceTimer w/ Stopwatch
This is needed to build DataStreamSuite on Linux.

'Stopwatch' is a platform independent timer class included with the
System.Runtime library in modern dotnet. 'HighPerformanceTimer' is a BTS
class that predates Stopwatch, and is NOT platform independent as it
relies on Windows kernel32.dll QueryPerformanceCounter/Frequency fns.
2024-06-03 13:46:51 +00:00

568 lines
20 KiB
C#

using System;
using System.Collections;
using System.Collections.Generic;
using mbts;
using Spark.Core.Collections;
namespace BCG
{
internal class SingleRowJoinDicionary<TKey, TValue> : IDictionary<TKey, TValue>
{
private TKey myKey;
private TValue myValue;
#region IDictionary<TKey,TValue> Members
public void Add(TKey key, TValue value)
{
myKey = key;
myValue = value;
}
public bool ContainsKey(TKey key)
{
return myKey.Equals(key);
}
public ICollection<TKey> Keys
{
get { return new TKey[] { myKey }; }
}
public bool Remove(TKey key)
{
bool rv = myKey.Equals(key);
myKey = default(TKey);
return rv;
}
public bool TryGetValue(TKey key, out TValue value)
{
bool rv = false;
value = (rv = myKey.Equals(key)) ? myValue : default(TValue);
return rv;
}
public ICollection<TValue> Values
{
get { throw new Exception("The method or operation is not implemented."); }
}
public TValue this[TKey key]
{
get
{
throw new Exception("The method or operation is not implemented.");
}
set
{
throw new Exception("The method or operation is not implemented.");
}
}
#endregion
#region ICollection<KeyValuePair<TKey,TValue>> Members
public void Add(KeyValuePair<TKey, TValue> item)
{
throw new Exception("The method or operation is not implemented.");
}
public void Clear()
{
throw new Exception("The method or operation is not implemented.");
}
public bool Contains(KeyValuePair<TKey, TValue> item)
{
throw new Exception("The method or operation is not implemented.");
}
public void CopyTo(KeyValuePair<TKey, TValue>[] array, int arrayIndex)
{
throw new Exception("The method or operation is not implemented.");
}
public int Count
{
get { throw new Exception("The method or operation is not implemented."); }
}
public bool IsReadOnly
{
get { throw new Exception("The method or operation is not implemented."); }
}
public bool Remove(KeyValuePair<TKey, TValue> item)
{
throw new Exception("The method or operation is not implemented.");
}
#endregion
#region IEnumerable<KeyValuePair<TKey,TValue>> Members
public IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator()
{
throw new Exception("The method or operation is not implemented.");
}
#endregion
#region IEnumerable Members
IEnumerator IEnumerable.GetEnumerator()
{
throw new Exception("The method or operation is not implemented.");
}
#endregion
}
#region Helper class to deal with stream projections
public class JoinProjectHelper
{
private IBlueTableSchema mySchema;
private Dictionary<string, Pair<string, bool>> myAliasToOriginal = new Dictionary<string, Pair<string, bool>>();
public JoinProjectHelper(IBlueTableSchema leftSchema,
string leftProj,
string leftJoinCol,
IBlueTableSchema rightSchema,
string rightProj,
string rightJoinCol)
{
JoinTableSchema joinedSchema = new JoinTableSchema(leftSchema, leftJoinCol, rightSchema, rightJoinCol);
JoinNameResolver resolver = new JoinNameResolver(joinedSchema, leftSchema, leftProj, rightSchema, rightProj);
IDictionary map = resolver.getMap();
map["id"] = "id";
ProjectHelper helper = new ProjectHelper(
joinedSchema,
GenericDictionaryWrapper.Create<string, string>(ProjectHelper.invertDic(map)),
String.Format("({0} J {1})", leftSchema.tableName, rightSchema.tableName));
mySchema = helper.schema;
for (int aliasCol = 0; aliasCol < helper.colMap.Length; aliasCol++)
{
string aliasColName = mySchema[aliasCol].colName;
if (aliasColName != "id")
{
int joinedColIndex = helper.colMap[aliasCol];
bool isLeft = joinedColIndex < leftSchema.columnInfo.Length;
string originalCol = joinedSchema.getOriginalName(joinedSchema[joinedColIndex].colName, isLeft);
myAliasToOriginal.Add(aliasColName, new Pair<string, bool>(originalCol, isLeft));
}
}
}
public IBlueTableSchema schema
{
get { return mySchema; }
}
public string getOriginalCol(string col, out bool isLeft)
{
Pair<string, bool> p = myAliasToOriginal[col];
isLeft = p.second;
return p.first;
}
}
#endregion
#region IBlueRow implementation for join and leftJoin
public class SuperJoinRow : IBlueRow
{
private JoinProjectHelper myHelper;
private IBlueRow myLeftRow, myRightRow;
int myID;
public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow rightRow, JoinProjectHelper joinProjectHelper)
{
myID = id;
myLeftRow = leftRow;
myRightRow = rightRow;
myHelper = joinProjectHelper;
}
public int leftID { get { return (int)myLeftRow["id"]; } }
public int rightID { get { return (int)myRightRow["id"]; } }
public IBlueRow leftRow { get { return myLeftRow; } }
public IBlueRow rightRow { get { return myRightRow; } }
#region IBlueRow Members
public IBlueTableSchema schema { get { return myHelper.schema; } }
public object this[int index] { get { return this[schema[index].colName]; } }
public object this[string colName]
{
get
{
if (colName == "id")
return myID;
else
{
bool isLeft;
string col = myHelper.getOriginalCol(colName, out isLeft);
return isLeft ? myLeftRow[col] : (myRightRow == null ? (object)DBNull.Value : myRightRow[col]);
}
}
}
#endregion
#region IDictionary Members
public void Add(object key, object? value) =>
throw new NotSupportedException("IBlueRow is readonly");
public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); }
public bool Contains(object key)
{
return myLeftRow.Contains(key) || myRightRow.Contains(key);
}
public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
public bool IsFixedSize { get { return true; } }
public bool IsReadOnly { get { return true; } }
public System.Collections.ICollection Keys { get { return schema.columnNames; } }
public void Remove(object key) { throw new NotSupportedException("IBlueRow is readonly"); }
public System.Collections.ICollection Values
{
get
{
List<object> rv = new List<object>();
foreach (string col in Keys)
rv.Add(this[col]);
return rv;
}
}
public object? this[object key] {
get => key is string str ? this[str] : this[(int)key];
set => throw new NotSupportedException("IBlueRow is readonly");
}
#endregion
#region ICollection Members
public void CopyTo(Array array, int index) { Values.CopyTo(array, index); }
public int Count { get { return schema.columnNames.Length; } }
public bool IsSynchronized { get { return false; } }
public object SyncRoot { get { return this; } }
#endregion
#region IEnumerable Members
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
#endregion
}
#endregion
/// <summary>
/// this class is written 3-4 yrs after the original
/// join, and learning from the past mistakes, is
/// made super fast and mem efficent
/// </summary>
public class SuperJoinStream : ViewStream
{
protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
private bool myManageUnmappedLeftRows = false;
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
protected IBlueTableSchema myLeftSchema, myRightSchema;
protected JoinProjectHelper myJoinProjectHelper;
protected BlueTable myState;
protected int myID;
protected ViewStream myLeftStream, myRightStream;
protected string myLeftKeyCol, myRightKeyCol;
protected Dictionary<object, IDictionary<int, IBlueRow>> myLeftKeyToLeftRows = new Dictionary<object, IDictionary<int, IBlueRow>>(),
myRightKeyToRightRows = new Dictionary<object, IDictionary<int, IBlueRow>>();
protected Dictionary<int, IDictionary<int, SuperJoinRow>> myLeftIDToResultRows = new Dictionary<int, IDictionary<int, SuperJoinRow>>(),
myRightIDToResultRows = new Dictionary<int, IDictionary<int, SuperJoinRow>>();
public SuperJoinStream(ViewStream leftStream, string leftKey, ViewStream rightStream, string rightKey)
: this(leftStream, leftKey, "!", rightStream, rightKey, "!")
{ }
public SuperJoinStream(ViewStream leftStream, string leftKey, string leftProj,
ViewStream rightStream, string rightKey, string rightProj)
: this(leftStream, leftKey, leftProj, rightStream, rightKey, rightProj, false)
{
}
internal SuperJoinStream(ViewStream leftStream, string leftKey, string leftProj,
ViewStream rightStream, string rightKey, string rightProj, bool manageUnmappedLeftRows)
{
myLeftStream = leftStream;
myRightStream = rightStream;
myLeftKeyCol = leftKey;
myRightKeyCol = rightKey;
myLeftProj = leftProj;
myRightProj = rightProj;
myManageUnmappedLeftRows = manageUnmappedLeftRows;
}
protected override void myOnFirstSubscriber()
{
myLeftStream.onChange += new ChangeDelegate(myHandleLeftStreamChanges);
myRightStream.onChange += new ChangeDelegate(myHandleRightStreamChanges);
}
protected override void myOnLastSubscriber()
{
myLeftStream.onChange -= new ChangeDelegate(myHandleLeftStreamChanges);
myRightStream.onChange -= new ChangeDelegate(myHandleRightStreamChanges);
myLeftKeyToLeftRows.Clear();
myRightKeyToRightRows.Clear();
myLeftIDToResultRows.Clear();
myRightIDToResultRows.Clear();
myState = null;
myLeftSchema = null;
myRightSchema = null;
}
private void myHandleLeftStreamChanges(object newValue)
{
myHandleStreamChanges(newValue as DataPacket, true);
}
private void myHandleRightStreamChanges(object newValue)
{
myHandleStreamChanges(newValue as DataPacket, false);
}
private void myHandleStreamChanges(DataPacket dp, bool isLeft)
{
myInitSchema(dp.schema, isLeft);
BlueTable dels = null, adds = null;
if (myState != null)
{
dels = new BlueTable(myState.schema);
adds = new BlueTable(myState.schema);
}
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
myAddRows(dp.addedRows, isLeft, myState, adds);
if (myState != null)
{
myFireEvent(new DataPacket(myState, adds, dels));
myLeftUnmatchedRowsAdds.Clear();
myLeftUnmatchedRowsDels.Clear();
}
}
private void myInitSchema(IBlueTableSchema schema, bool isLeft)
{
if (myState == null)
{
if (isLeft)
{
myLeftSchema = schema;
if (myRightSchema != null)
{
myJoinProjectHelper = new JoinProjectHelper(myLeftSchema, myLeftProj, myLeftKeyCol,
myRightSchema, myRightProj, myRightKeyCol);
myState = new BlueTable(myJoinProjectHelper.schema);
}
}
else
{
myRightSchema = schema;
if (myLeftSchema != null)
{
myJoinProjectHelper = new JoinProjectHelper(myLeftSchema, myLeftProj, myLeftKeyCol,
myRightSchema, myRightProj, myRightKeyCol);
myState = new BlueTable(myJoinProjectHelper.schema);
}
}
}
}
private void myAddRows(IBlueRowCollection addedRows, bool isLeft, BlueTable state, BlueTable addedRowsOut)
{
object thisKeyCol = isLeft ? myLeftKeyCol : myRightKeyCol,
otherSideKeyCol = isLeft ? myRightKeyCol : myLeftKeyCol;
Dictionary<object, IDictionary<int, IBlueRow>> thisKeyToRows = isLeft ? myLeftKeyToLeftRows : myRightKeyToRightRows,
otherSideKeyToRows = isLeft ? myRightKeyToRightRows : myLeftKeyToLeftRows;
Dictionary<int, IDictionary<int, SuperJoinRow>> thisIDToResultRows = isLeft ? myLeftIDToResultRows : myRightIDToResultRows,
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
foreach (IBlueRow r in addedRows)
{
object currentKey = r[thisKeyCol];
int currentID = (int)r["id"];
///add it to 1.
if (!thisKeyToRows.ContainsKey(currentKey))
thisKeyToRows.Add(currentKey, new Dictionary<int, IBlueRow>());
thisKeyToRows[currentKey].Add(currentID, r);
///find matching rows, and form result rows
if (otherSideKeyToRows.ContainsKey(currentKey))
{
ICollection<IBlueRow> matchingRows = otherSideKeyToRows[currentKey].Values;
foreach (IBlueRow mr in matchingRows)
{
int mrID = (int)mr["id"];
SuperJoinRow rr = new SuperJoinRow(myID++, isLeft ? r : mr, isLeft ? mr : r, myJoinProjectHelper);
state.addRow(rr);
addedRowsOut.addRow(rr);
///add to 3.
if (!thisIDToResultRows.ContainsKey(currentID))
thisIDToResultRows.Add(currentID, new Dictionary<int, SuperJoinRow>());
thisIDToResultRows[currentID].Add((int)rr["id"], rr);
///add to 4.
if (!otherIDToResultRows.ContainsKey(mrID))
{
otherIDToResultRows.Add(mrID, new Dictionary<int, SuperJoinRow>());
if (myManageUnmappedLeftRows && !isLeft)
myTakeCareOfUnmatachedLeftRows(mr, false);
}
otherIDToResultRows[mrID].Add((int)rr["id"], rr);
}
}
else
{
if (myManageUnmappedLeftRows && isLeft)
myTakeCareOfUnmatachedLeftRows(r, true);
}
}
}
private void myDeleteRows(IBlueRowCollection deletedRows, bool isLeft, BlueTable state, BlueTable delRowsOut)
{
string thisKeyCol = isLeft ? myLeftKeyCol : myRightKeyCol,
otherSideKeyCol = isLeft ? myRightKeyCol : myLeftKeyCol;
Dictionary<object, IDictionary<int, IBlueRow>> thisKeyToRows = isLeft ? myLeftKeyToLeftRows : myRightKeyToRightRows,
otherSideKeyToRows = isLeft ? myRightKeyToRightRows : myLeftKeyToLeftRows;
Dictionary<int, IDictionary<int, SuperJoinRow>> thisIDToResultRows = isLeft ? myLeftIDToResultRows : myRightIDToResultRows,
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
foreach (IBlueRow r in deletedRows)
{
int id = (int)r["id"];
object key = r[thisKeyCol];
///delete it from 1.
thisKeyToRows[key].Remove(id);
if (thisKeyToRows[key].Count == 0)
thisKeyToRows.Remove(key);
///form return rows and delete from 3.
if (thisIDToResultRows.ContainsKey(id))
{
ICollection removedResultRows = thisIDToResultRows[id].Values as ICollection;
thisIDToResultRows.Remove(id);
foreach (SuperJoinRow rr in removedResultRows)
{
///delete from 4.
int otherId = isLeft ? rr.rightID : rr.leftID;
otherIDToResultRows[otherId].Remove((int)rr["id"]);
if (otherIDToResultRows[otherId].Count == 0)
otherIDToResultRows.Remove(otherId);
delRowsOut.addRow(rr);
state.deleteRow(rr);
if (myManageUnmappedLeftRows && !isLeft)
if (!thisKeyToRows.ContainsKey(key))
myTakeCareOfUnmatachedLeftRows(rr.leftRow, true);
}
}
else
{
if (myManageUnmappedLeftRows && isLeft)
myTakeCareOfUnmatachedLeftRows(r, false);
}
}
}
private void myTakeCareOfUnmatachedLeftRows(IBlueRow leftRow, bool add)
{
int lid = (int)leftRow["id"];
if (add)
myLeftUnmatchedRowsAdds.Add(lid, leftRow);
else
{
if (myLeftUnmatchedRowsAdds.ContainsKey(lid))
myLeftUnmatchedRowsAdds.Remove(lid);
else
myLeftUnmatchedRowsDels.Add((int)leftRow["id"], leftRow);
}
}
}
public class SuperLeftJoinStream : SuperJoinStream
{
private Dictionary<int, SuperJoinRow> myLeftIDToHalfRows = new Dictionary<int, SuperJoinRow>();
public SuperLeftJoinStream(ViewStream leftStream, string leftJoinCol,
ViewStream rightStream, string rightJoinCol)
: this(leftStream, leftJoinCol, "!", rightStream, rightJoinCol, "!")
{ }
public SuperLeftJoinStream(ViewStream leftStream, string leftJoinCol, string leftProjExp,
ViewStream rightStream, string rightJoinCol, string rightProjExp)
: base(leftStream, leftJoinCol, leftProjExp, rightStream, rightJoinCol, rightProjExp, true)
{ }
protected override void myFireEvent(object o)
{
myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, o as DataPacket);
}
private void myHandleJoinResults(Dictionary<int, IBlueRow> unmatchedLeftRowsDels,
Dictionary<int, IBlueRow> unmatchedLeftRowAdds, DataPacket resultPacket)
{
BlueTable dels = resultPacket.deletedRows as BlueTable,
adds = resultPacket.addedRows as BlueTable;
foreach (IBlueRow lr in unmatchedLeftRowsDels.Values)
{
///an lr will be here if its actually deleted or
///a right row is added, so now it has a partner
///and is no longer unmatched.
///in both cases the half row that this lr
///created needs to be removed from this
///class's state and added to dels
///it means a right is added
int lid = (int)lr["id"];
SuperJoinRow rr = myLeftIDToHalfRows[lid];
myLeftIDToHalfRows.Remove(lid);
myState.deleteRow(rr);
dels.addRow(rr);
}
foreach (IBlueRow lr in unmatchedLeftRowAdds.Values)
{
///an lr will be here if its added with no
///matching right, or a right matching row
///is gone so it is unmatched now.
///in both cases a half row needs to be
///added to this class's state and added
///to adds
int lid = (int)lr["id"];
SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
myLeftIDToHalfRows.Add(lid, rr);
myState.addRow(rr);
adds.addRow(rr);
}
base.myFireEvent(new DataPacket(myState, adds, dels));
}
}
}