Files
spark/src/DataStreamSuite/DataStream/Primitives/Union/AppendStream.cs
T
John Colvin 9311c27f47 feat: Resurrect DataStream primitives
They were originally commented out because the expression parsing utils
weren't ready (now they are).
2023-09-29 13:10:50 -04:00

119 lines
4.3 KiB
C#

using System;
using System.Data;
using System.Collections;
using BCG.Utilities;
using BCG.Utilities.General;
namespace BCG
{
/// <summary>
/// AppendStream appends 2 tables with each other, with a generated id having no bearing to the original id.
/// The column ordering the 2 tables need not be the same
/// </summary>
public class AppendStream : UnionStream
{
private bool myUseNewAppend = DataStreamLoadFileConfig.instance.useNewAppend;
public AppendStream(ViewStream v1, ViewStream v2)
{
if (myUseNewAppend)
mySetRealSource(new MutableAppendStream(v1, v2));
else
{
myLeftVs = v1;
myRightVs = v2;
}
}
public AppendStream (params ViewStream[] vss)
{
if (myUseNewAppend)
mySetRealSource(new MutableAppendStream(vss));
else
{
if (vss.Length == 1)
mySetRealSource(vss[0]);
else if (vss.Length > 1)
{
AppendStream rv = null;
foreach (ViewStream s in vss)
rv = rv == null ? new AppendStream(new ViewStream[] { s }) : new AppendStream(rv, s);
mySetRealSource(rv);
}
else
//don't do anything if they didnt' supply any viewstreams
mySetRealSource(null);
}
}
public override void add (IBlueRowCollection rows, bool isLeft)
{
IDictionary index = isLeft ? myLeftIndex : myRightIndex;
int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
foreach (IBlueRow row in rows)
{
UnionRow newRow = new UnionRow (myLastFiredPacket.newRows.schema, row,
myIdCounter ++, myIdIndex, indexToIndexMap);
((BlueTable)myLastFiredPacket.newRows).addRow (newRow);
index[row["id"]] = newRow;
((BlueTable)(myLastFiredPacket.addedRows)).addRow (newRow);
}
}
public override void delete (IBlueRowCollection rows, bool isLeft)
{
IDictionary index = isLeft ? myLeftIndex : myRightIndex;
foreach (IBlueRow axedRow in rows)
{
int rowId = (int)axedRow["id"];
IBlueRow row = (IBlueRow)index[rowId];
index.Remove (rowId);
((BlueTable)myLastFiredPacket.newRows).deleteRow (row);
((BlueTable)(myLastFiredPacket.deletedRows)).addRow (row);
}
}
public override void mergeChanges (IBlueRowCollection dels, IBlueRowCollection adds, bool isLeft)
{
IDictionary temp = new Hashtable ();
IDictionary index = isLeft ? myLeftIndex : myRightIndex;
int[] indexToIndexMap = isLeft ? myLeftIndexToIndexMap : myRightIndexToIndexMap;
IBlueRow corrRow;
foreach (IBlueRow dRow in dels)
{
corrRow = index[dRow["id"]] as IBlueRow;
(myLastFiredPacket.newRows as BlueTable).deleteRow (corrRow);
(myLastFiredPacket.deletedRows as BlueTable).addRow (corrRow);
index.Remove (dRow["id"]);
if (adds.containsID ((int)dRow["id"]))
{
IBlueRow r = adds[(int)dRow["id"]];
corrRow = new UnionRow (myLastFiredPacket.schema, r, (int)corrRow["id"], myIdIndex, indexToIndexMap);
(myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
(myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
index.Add (r["id"], corrRow);
temp.Add (r["id"], true);
}
}
foreach (IBlueRow aRow in adds)
{
if (!temp.Contains (aRow["id"]))
{
corrRow = new UnionRow (myLastFiredPacket.schema, aRow, myIdCounter++, myIdIndex, indexToIndexMap);
(myLastFiredPacket.newRows as BlueTable).addRow (corrRow);
(myLastFiredPacket.addedRows as BlueTable).addRow (corrRow);
index.Add (aRow["id"], corrRow);
}
}
}
}
public class MergeStream : ViewStream
{
public MergeStream (ViewStream v1, ViewStream v2, string[] mergeCols)
{
mySetRealSource (new DistinctRowStream (new AppendStream (v1, v2), mergeCols));
}
}
}