This is needed to build DataStreamSuite on Linux. 'Stopwatch' is a platform independent timer class included with the System.Runtime library in modern dotnet. 'HighPerformanceTimer' is a BTS class that predates Stopwatch, and is NOT platform independent as it relies on Windows kernel32.dll QueryPerformanceCounter/Frequency fns.
568 lines
20 KiB
C#
568 lines
20 KiB
C#
using System;
|
|
using System.Collections;
|
|
using System.Collections.Generic;
|
|
using mbts;
|
|
using Spark.Core.Collections;
|
|
|
|
namespace BCG
|
|
{
|
|
internal class SingleRowJoinDicionary<TKey, TValue> : IDictionary<TKey, TValue>
|
|
{
|
|
private TKey myKey;
|
|
private TValue myValue;
|
|
|
|
#region IDictionary<TKey,TValue> Members
|
|
|
|
public void Add(TKey key, TValue value)
|
|
{
|
|
myKey = key;
|
|
myValue = value;
|
|
}
|
|
|
|
public bool ContainsKey(TKey key)
|
|
{
|
|
return myKey.Equals(key);
|
|
}
|
|
|
|
public ICollection<TKey> Keys
|
|
{
|
|
get { return new TKey[] { myKey }; }
|
|
}
|
|
|
|
public bool Remove(TKey key)
|
|
{
|
|
bool rv = myKey.Equals(key);
|
|
myKey = default(TKey);
|
|
return rv;
|
|
}
|
|
|
|
public bool TryGetValue(TKey key, out TValue value)
|
|
{
|
|
bool rv = false;
|
|
value = (rv = myKey.Equals(key)) ? myValue : default(TValue);
|
|
return rv;
|
|
}
|
|
|
|
public ICollection<TValue> Values
|
|
{
|
|
get { throw new Exception("The method or operation is not implemented."); }
|
|
}
|
|
|
|
public TValue this[TKey key]
|
|
{
|
|
get
|
|
{
|
|
throw new Exception("The method or operation is not implemented.");
|
|
}
|
|
set
|
|
{
|
|
throw new Exception("The method or operation is not implemented.");
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region ICollection<KeyValuePair<TKey,TValue>> Members
|
|
|
|
public void Add(KeyValuePair<TKey, TValue> item)
|
|
{
|
|
throw new Exception("The method or operation is not implemented.");
|
|
}
|
|
|
|
public void Clear()
|
|
{
|
|
throw new Exception("The method or operation is not implemented.");
|
|
}
|
|
|
|
public bool Contains(KeyValuePair<TKey, TValue> item)
|
|
{
|
|
throw new Exception("The method or operation is not implemented.");
|
|
}
|
|
|
|
public void CopyTo(KeyValuePair<TKey, TValue>[] array, int arrayIndex)
|
|
{
|
|
throw new Exception("The method or operation is not implemented.");
|
|
}
|
|
|
|
public int Count
|
|
{
|
|
get { throw new Exception("The method or operation is not implemented."); }
|
|
}
|
|
|
|
public bool IsReadOnly
|
|
{
|
|
get { throw new Exception("The method or operation is not implemented."); }
|
|
}
|
|
|
|
public bool Remove(KeyValuePair<TKey, TValue> item)
|
|
{
|
|
throw new Exception("The method or operation is not implemented.");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region IEnumerable<KeyValuePair<TKey,TValue>> Members
|
|
|
|
public IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator()
|
|
{
|
|
throw new Exception("The method or operation is not implemented.");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region IEnumerable Members
|
|
|
|
IEnumerator IEnumerable.GetEnumerator()
|
|
{
|
|
throw new Exception("The method or operation is not implemented.");
|
|
}
|
|
|
|
#endregion
|
|
}
|
|
|
|
#region Helper class to deal with stream projections
|
|
|
|
public class JoinProjectHelper
|
|
{
|
|
private IBlueTableSchema mySchema;
|
|
private Dictionary<string, Pair<string, bool>> myAliasToOriginal = new Dictionary<string, Pair<string, bool>>();
|
|
|
|
public JoinProjectHelper(IBlueTableSchema leftSchema,
|
|
string leftProj,
|
|
string leftJoinCol,
|
|
IBlueTableSchema rightSchema,
|
|
string rightProj,
|
|
string rightJoinCol)
|
|
{
|
|
|
|
JoinTableSchema joinedSchema = new JoinTableSchema(leftSchema, leftJoinCol, rightSchema, rightJoinCol);
|
|
JoinNameResolver resolver = new JoinNameResolver(joinedSchema, leftSchema, leftProj, rightSchema, rightProj);
|
|
IDictionary map = resolver.getMap();
|
|
map["id"] = "id";
|
|
ProjectHelper helper = new ProjectHelper(
|
|
joinedSchema,
|
|
GenericDictionaryWrapper.Create<string, string>(ProjectHelper.invertDic(map)),
|
|
String.Format("({0} J {1})", leftSchema.tableName, rightSchema.tableName));
|
|
|
|
mySchema = helper.schema;
|
|
for (int aliasCol = 0; aliasCol < helper.colMap.Length; aliasCol++)
|
|
{
|
|
string aliasColName = mySchema[aliasCol].colName;
|
|
if (aliasColName != "id")
|
|
{
|
|
int joinedColIndex = helper.colMap[aliasCol];
|
|
bool isLeft = joinedColIndex < leftSchema.columnInfo.Length;
|
|
string originalCol = joinedSchema.getOriginalName(joinedSchema[joinedColIndex].colName, isLeft);
|
|
myAliasToOriginal.Add(aliasColName, new Pair<string, bool>(originalCol, isLeft));
|
|
}
|
|
}
|
|
}
|
|
|
|
public IBlueTableSchema schema
|
|
{
|
|
get { return mySchema; }
|
|
}
|
|
|
|
public string getOriginalCol(string col, out bool isLeft)
|
|
{
|
|
Pair<string, bool> p = myAliasToOriginal[col];
|
|
isLeft = p.second;
|
|
return p.first;
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region IBlueRow implementation for join and leftJoin
|
|
|
|
public class SuperJoinRow : IBlueRow
|
|
{
|
|
private JoinProjectHelper myHelper;
|
|
private IBlueRow myLeftRow, myRightRow;
|
|
int myID;
|
|
|
|
public SuperJoinRow(int id, IBlueRow leftRow, IBlueRow rightRow, JoinProjectHelper joinProjectHelper)
|
|
{
|
|
myID = id;
|
|
myLeftRow = leftRow;
|
|
myRightRow = rightRow;
|
|
myHelper = joinProjectHelper;
|
|
}
|
|
|
|
public int leftID { get { return (int)myLeftRow["id"]; } }
|
|
|
|
public int rightID { get { return (int)myRightRow["id"]; } }
|
|
|
|
public IBlueRow leftRow { get { return myLeftRow; } }
|
|
|
|
public IBlueRow rightRow { get { return myRightRow; } }
|
|
|
|
#region IBlueRow Members
|
|
|
|
public IBlueTableSchema schema { get { return myHelper.schema; } }
|
|
|
|
public object this[int index] { get { return this[schema[index].colName]; } }
|
|
|
|
public object this[string colName]
|
|
{
|
|
get
|
|
{
|
|
if (colName == "id")
|
|
return myID;
|
|
else
|
|
{
|
|
bool isLeft;
|
|
string col = myHelper.getOriginalCol(colName, out isLeft);
|
|
return isLeft ? myLeftRow[col] : (myRightRow == null ? (object)DBNull.Value : myRightRow[col]);
|
|
}
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region IDictionary Members
|
|
|
|
public void Add(object key, object? value) =>
|
|
throw new NotSupportedException("IBlueRow is readonly");
|
|
|
|
public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); }
|
|
|
|
public bool Contains(object key)
|
|
{
|
|
return myLeftRow.Contains(key) || myRightRow.Contains(key);
|
|
}
|
|
|
|
public System.Collections.IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
|
|
|
|
public bool IsFixedSize { get { return true; } }
|
|
|
|
public bool IsReadOnly { get { return true; } }
|
|
|
|
public System.Collections.ICollection Keys { get { return schema.columnNames; } }
|
|
|
|
public void Remove(object key) { throw new NotSupportedException("IBlueRow is readonly"); }
|
|
|
|
public System.Collections.ICollection Values
|
|
{
|
|
get
|
|
{
|
|
List<object> rv = new List<object>();
|
|
foreach (string col in Keys)
|
|
rv.Add(this[col]);
|
|
return rv;
|
|
}
|
|
}
|
|
|
|
public object? this[object key] {
|
|
get => key is string str ? this[str] : this[(int)key];
|
|
set => throw new NotSupportedException("IBlueRow is readonly");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region ICollection Members
|
|
|
|
public void CopyTo(Array array, int index) { Values.CopyTo(array, index); }
|
|
|
|
public int Count { get { return schema.columnNames.Length; } }
|
|
|
|
public bool IsSynchronized { get { return false; } }
|
|
|
|
public object SyncRoot { get { return this; } }
|
|
|
|
#endregion
|
|
|
|
#region IEnumerable Members
|
|
|
|
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
|
|
|
|
#endregion
|
|
}
|
|
|
|
#endregion
|
|
/// <summary>
|
|
/// this class is written 3-4 yrs after the original
|
|
/// join, and learning from the past mistakes, is
|
|
/// made super fast and mem efficent
|
|
/// </summary>
|
|
public class SuperJoinStream : ViewStream
|
|
{
|
|
protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
|
|
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
|
|
private bool myManageUnmappedLeftRows = false;
|
|
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
|
|
protected IBlueTableSchema myLeftSchema, myRightSchema;
|
|
protected JoinProjectHelper myJoinProjectHelper;
|
|
protected BlueTable myState;
|
|
protected int myID;
|
|
protected ViewStream myLeftStream, myRightStream;
|
|
protected string myLeftKeyCol, myRightKeyCol;
|
|
protected Dictionary<object, IDictionary<int, IBlueRow>> myLeftKeyToLeftRows = new Dictionary<object, IDictionary<int, IBlueRow>>(),
|
|
myRightKeyToRightRows = new Dictionary<object, IDictionary<int, IBlueRow>>();
|
|
|
|
protected Dictionary<int, IDictionary<int, SuperJoinRow>> myLeftIDToResultRows = new Dictionary<int, IDictionary<int, SuperJoinRow>>(),
|
|
myRightIDToResultRows = new Dictionary<int, IDictionary<int, SuperJoinRow>>();
|
|
|
|
public SuperJoinStream(ViewStream leftStream, string leftKey, ViewStream rightStream, string rightKey)
|
|
: this(leftStream, leftKey, "!", rightStream, rightKey, "!")
|
|
{ }
|
|
|
|
public SuperJoinStream(ViewStream leftStream, string leftKey, string leftProj,
|
|
ViewStream rightStream, string rightKey, string rightProj)
|
|
: this(leftStream, leftKey, leftProj, rightStream, rightKey, rightProj, false)
|
|
{
|
|
}
|
|
|
|
internal SuperJoinStream(ViewStream leftStream, string leftKey, string leftProj,
|
|
ViewStream rightStream, string rightKey, string rightProj, bool manageUnmappedLeftRows)
|
|
{
|
|
myLeftStream = leftStream;
|
|
myRightStream = rightStream;
|
|
myLeftKeyCol = leftKey;
|
|
myRightKeyCol = rightKey;
|
|
myLeftProj = leftProj;
|
|
myRightProj = rightProj;
|
|
myManageUnmappedLeftRows = manageUnmappedLeftRows;
|
|
}
|
|
|
|
protected override void myOnFirstSubscriber()
|
|
{
|
|
myLeftStream.onChange += new ChangeDelegate(myHandleLeftStreamChanges);
|
|
myRightStream.onChange += new ChangeDelegate(myHandleRightStreamChanges);
|
|
}
|
|
|
|
protected override void myOnLastSubscriber()
|
|
{
|
|
myLeftStream.onChange -= new ChangeDelegate(myHandleLeftStreamChanges);
|
|
myRightStream.onChange -= new ChangeDelegate(myHandleRightStreamChanges);
|
|
myLeftKeyToLeftRows.Clear();
|
|
myRightKeyToRightRows.Clear();
|
|
myLeftIDToResultRows.Clear();
|
|
myRightIDToResultRows.Clear();
|
|
myState = null;
|
|
myLeftSchema = null;
|
|
myRightSchema = null;
|
|
}
|
|
|
|
private void myHandleLeftStreamChanges(object newValue)
|
|
{
|
|
myHandleStreamChanges(newValue as DataPacket, true);
|
|
}
|
|
|
|
private void myHandleRightStreamChanges(object newValue)
|
|
{
|
|
myHandleStreamChanges(newValue as DataPacket, false);
|
|
}
|
|
|
|
private void myHandleStreamChanges(DataPacket dp, bool isLeft)
|
|
{
|
|
myInitSchema(dp.schema, isLeft);
|
|
BlueTable dels = null, adds = null;
|
|
if (myState != null)
|
|
{
|
|
dels = new BlueTable(myState.schema);
|
|
adds = new BlueTable(myState.schema);
|
|
}
|
|
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
|
|
myAddRows(dp.addedRows, isLeft, myState, adds);
|
|
if (myState != null)
|
|
{
|
|
myFireEvent(new DataPacket(myState, adds, dels));
|
|
myLeftUnmatchedRowsAdds.Clear();
|
|
myLeftUnmatchedRowsDels.Clear();
|
|
}
|
|
}
|
|
|
|
private void myInitSchema(IBlueTableSchema schema, bool isLeft)
|
|
{
|
|
if (myState == null)
|
|
{
|
|
if (isLeft)
|
|
{
|
|
myLeftSchema = schema;
|
|
if (myRightSchema != null)
|
|
{
|
|
myJoinProjectHelper = new JoinProjectHelper(myLeftSchema, myLeftProj, myLeftKeyCol,
|
|
myRightSchema, myRightProj, myRightKeyCol);
|
|
myState = new BlueTable(myJoinProjectHelper.schema);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
myRightSchema = schema;
|
|
if (myLeftSchema != null)
|
|
{
|
|
myJoinProjectHelper = new JoinProjectHelper(myLeftSchema, myLeftProj, myLeftKeyCol,
|
|
myRightSchema, myRightProj, myRightKeyCol);
|
|
myState = new BlueTable(myJoinProjectHelper.schema);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private void myAddRows(IBlueRowCollection addedRows, bool isLeft, BlueTable state, BlueTable addedRowsOut)
|
|
{
|
|
object thisKeyCol = isLeft ? myLeftKeyCol : myRightKeyCol,
|
|
otherSideKeyCol = isLeft ? myRightKeyCol : myLeftKeyCol;
|
|
Dictionary<object, IDictionary<int, IBlueRow>> thisKeyToRows = isLeft ? myLeftKeyToLeftRows : myRightKeyToRightRows,
|
|
otherSideKeyToRows = isLeft ? myRightKeyToRightRows : myLeftKeyToLeftRows;
|
|
Dictionary<int, IDictionary<int, SuperJoinRow>> thisIDToResultRows = isLeft ? myLeftIDToResultRows : myRightIDToResultRows,
|
|
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
|
|
foreach (IBlueRow r in addedRows)
|
|
{
|
|
object currentKey = r[thisKeyCol];
|
|
int currentID = (int)r["id"];
|
|
///add it to 1.
|
|
if (!thisKeyToRows.ContainsKey(currentKey))
|
|
thisKeyToRows.Add(currentKey, new Dictionary<int, IBlueRow>());
|
|
thisKeyToRows[currentKey].Add(currentID, r);
|
|
///find matching rows, and form result rows
|
|
if (otherSideKeyToRows.ContainsKey(currentKey))
|
|
{
|
|
ICollection<IBlueRow> matchingRows = otherSideKeyToRows[currentKey].Values;
|
|
foreach (IBlueRow mr in matchingRows)
|
|
{
|
|
int mrID = (int)mr["id"];
|
|
SuperJoinRow rr = new SuperJoinRow(myID++, isLeft ? r : mr, isLeft ? mr : r, myJoinProjectHelper);
|
|
state.addRow(rr);
|
|
addedRowsOut.addRow(rr);
|
|
///add to 3.
|
|
if (!thisIDToResultRows.ContainsKey(currentID))
|
|
thisIDToResultRows.Add(currentID, new Dictionary<int, SuperJoinRow>());
|
|
thisIDToResultRows[currentID].Add((int)rr["id"], rr);
|
|
///add to 4.
|
|
if (!otherIDToResultRows.ContainsKey(mrID))
|
|
{
|
|
otherIDToResultRows.Add(mrID, new Dictionary<int, SuperJoinRow>());
|
|
if (myManageUnmappedLeftRows && !isLeft)
|
|
myTakeCareOfUnmatachedLeftRows(mr, false);
|
|
}
|
|
otherIDToResultRows[mrID].Add((int)rr["id"], rr);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (myManageUnmappedLeftRows && isLeft)
|
|
myTakeCareOfUnmatachedLeftRows(r, true);
|
|
}
|
|
}
|
|
}
|
|
|
|
private void myDeleteRows(IBlueRowCollection deletedRows, bool isLeft, BlueTable state, BlueTable delRowsOut)
|
|
{
|
|
string thisKeyCol = isLeft ? myLeftKeyCol : myRightKeyCol,
|
|
otherSideKeyCol = isLeft ? myRightKeyCol : myLeftKeyCol;
|
|
Dictionary<object, IDictionary<int, IBlueRow>> thisKeyToRows = isLeft ? myLeftKeyToLeftRows : myRightKeyToRightRows,
|
|
otherSideKeyToRows = isLeft ? myRightKeyToRightRows : myLeftKeyToLeftRows;
|
|
Dictionary<int, IDictionary<int, SuperJoinRow>> thisIDToResultRows = isLeft ? myLeftIDToResultRows : myRightIDToResultRows,
|
|
otherIDToResultRows = isLeft ? myRightIDToResultRows : myLeftIDToResultRows;
|
|
foreach (IBlueRow r in deletedRows)
|
|
{
|
|
int id = (int)r["id"];
|
|
object key = r[thisKeyCol];
|
|
///delete it from 1.
|
|
thisKeyToRows[key].Remove(id);
|
|
if (thisKeyToRows[key].Count == 0)
|
|
thisKeyToRows.Remove(key);
|
|
///form return rows and delete from 3.
|
|
if (thisIDToResultRows.ContainsKey(id))
|
|
{
|
|
ICollection removedResultRows = thisIDToResultRows[id].Values as ICollection;
|
|
thisIDToResultRows.Remove(id);
|
|
foreach (SuperJoinRow rr in removedResultRows)
|
|
{
|
|
///delete from 4.
|
|
int otherId = isLeft ? rr.rightID : rr.leftID;
|
|
otherIDToResultRows[otherId].Remove((int)rr["id"]);
|
|
if (otherIDToResultRows[otherId].Count == 0)
|
|
otherIDToResultRows.Remove(otherId);
|
|
delRowsOut.addRow(rr);
|
|
state.deleteRow(rr);
|
|
if (myManageUnmappedLeftRows && !isLeft)
|
|
if (!thisKeyToRows.ContainsKey(key))
|
|
myTakeCareOfUnmatachedLeftRows(rr.leftRow, true);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (myManageUnmappedLeftRows && isLeft)
|
|
myTakeCareOfUnmatachedLeftRows(r, false);
|
|
}
|
|
}
|
|
}
|
|
|
|
private void myTakeCareOfUnmatachedLeftRows(IBlueRow leftRow, bool add)
|
|
{
|
|
int lid = (int)leftRow["id"];
|
|
if (add)
|
|
myLeftUnmatchedRowsAdds.Add(lid, leftRow);
|
|
else
|
|
{
|
|
if (myLeftUnmatchedRowsAdds.ContainsKey(lid))
|
|
myLeftUnmatchedRowsAdds.Remove(lid);
|
|
else
|
|
myLeftUnmatchedRowsDels.Add((int)leftRow["id"], leftRow);
|
|
}
|
|
}
|
|
}
|
|
|
|
public class SuperLeftJoinStream : SuperJoinStream
|
|
{
|
|
private Dictionary<int, SuperJoinRow> myLeftIDToHalfRows = new Dictionary<int, SuperJoinRow>();
|
|
|
|
public SuperLeftJoinStream(ViewStream leftStream, string leftJoinCol,
|
|
ViewStream rightStream, string rightJoinCol)
|
|
: this(leftStream, leftJoinCol, "!", rightStream, rightJoinCol, "!")
|
|
{ }
|
|
|
|
public SuperLeftJoinStream(ViewStream leftStream, string leftJoinCol, string leftProjExp,
|
|
ViewStream rightStream, string rightJoinCol, string rightProjExp)
|
|
: base(leftStream, leftJoinCol, leftProjExp, rightStream, rightJoinCol, rightProjExp, true)
|
|
{ }
|
|
|
|
protected override void myFireEvent(object o)
|
|
{
|
|
myHandleJoinResults(myLeftUnmatchedRowsDels, myLeftUnmatchedRowsAdds, o as DataPacket);
|
|
}
|
|
|
|
private void myHandleJoinResults(Dictionary<int, IBlueRow> unmatchedLeftRowsDels,
|
|
Dictionary<int, IBlueRow> unmatchedLeftRowAdds, DataPacket resultPacket)
|
|
{
|
|
BlueTable dels = resultPacket.deletedRows as BlueTable,
|
|
adds = resultPacket.addedRows as BlueTable;
|
|
|
|
foreach (IBlueRow lr in unmatchedLeftRowsDels.Values)
|
|
{
|
|
///an lr will be here if its actually deleted or
|
|
///a right row is added, so now it has a partner
|
|
///and is no longer unmatched.
|
|
///in both cases the half row that this lr
|
|
///created needs to be removed from this
|
|
///class's state and added to dels
|
|
///it means a right is added
|
|
int lid = (int)lr["id"];
|
|
SuperJoinRow rr = myLeftIDToHalfRows[lid];
|
|
myLeftIDToHalfRows.Remove(lid);
|
|
myState.deleteRow(rr);
|
|
dels.addRow(rr);
|
|
}
|
|
|
|
foreach (IBlueRow lr in unmatchedLeftRowAdds.Values)
|
|
{
|
|
///an lr will be here if its added with no
|
|
///matching right, or a right matching row
|
|
///is gone so it is unmatched now.
|
|
///in both cases a half row needs to be
|
|
///added to this class's state and added
|
|
///to adds
|
|
int lid = (int)lr["id"];
|
|
SuperJoinRow rr = new SuperJoinRow(myID++, lr, null, myJoinProjectHelper);
|
|
myLeftIDToHalfRows.Add(lid, rr);
|
|
myState.addRow(rr);
|
|
adds.addRow(rr);
|
|
}
|
|
base.myFireEvent(new DataPacket(myState, adds, dels));
|
|
}
|
|
}
|
|
}
|