Files
spark/src/DataStreamSuite/DataStream/Primitives/Diff/DiffStream.cs
T
2023-09-04 16:25:40 -04:00

214 lines
5.9 KiB
C#

using System;
using System.Collections;
using System.Collections.Generic;
using mbts;
namespace BCG
{
[DataStreamUserRespresentation("diff")]
public class DiffStream : ViewStream
{
ViewStream myLStream, myRStream;
DataPacket myLastR = null, myLastL = null;
BlueTable myLastFired = null;
double myThreshold = .00005;
string[] myColumnsToMatch;
public DiffStream(ViewStream v1, ViewStream v2) : this(null, v1, v2, "id") { }
public DiffStream(ViewStream v1, ViewStream v2, params string[] ColumnsToMatch) : this(null, v1, v2, ColumnsToMatch) { }
public DiffStream(ViewStream v1, ViewStream v2, double thresh, params string[] ColumnsToMatch) :
this(null, v1, v2, thresh, ColumnsToMatch) { }
public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, double thresh, params string[] columnsToMatch)
{
myThreshold = thresh;
myLStream = v1;
myRStream = v2;
myColumnsToMatch = columnsToMatch;
}
public DiffStream(ViewStreamDatabase db, ViewStream v1, ViewStream v2, params string[] columnsToMatch)
: this(db,v1,v2,.00005,columnsToMatch)
{
}
protected override void myOnFirstSubscriber()
{
myLStream.onChange += myLChange;
myRStream.onChange += myRChange;
}
protected override void myOnLastSubscriber()
{
myLStream.onChange -= myLChange;
myRStream.onChange -= myRChange;
}
struct ListComparer : IEqualityComparer<List<object>>
{
public bool Equals(List<object> x, List<object> y)
{
if (x.Count != y.Count)
return false;
for (int i = 0; i < x.Count; ++i)
if (!x[i].Equals(y[i]))
return false;
return true;
}
public int GetHashCode(List<object> obj)
{
int ret = 0;
for(int i = 0; i < obj.Count; ++i)
ret ^= obj[i].GetHashCode();
return ret;
}
}
void myHandleCallbacks()
{
DataPacket packet = null;
if (myLastL == null || myLastR == null)
{
packet = myLastL == null ? myLastR : myLastL;
if (packet != null)
myFireEvent(packet);
myLastFired = new BlueTable(packet.newRows);
return;
}
BlueTable table = new BlueTable(myLastL.newRows.schema);
Dictionary<List<object>, Pair<BlueRow, BlueRow>> rowPairs = new Dictionary<List<object>, Pair<BlueRow, BlueRow>>(new ListComparer());
foreach(BlueRow row in myLastL.newRows)
{
List<object> hash = new List<object>();
foreach (string column in myColumnsToMatch)
{
try
{
hash.Add(row[column]);
}
catch (Exception)
{
}
}
rowPairs[hash] = new Pair<BlueRow, BlueRow>(row, null);
}
foreach (BlueRow row in myLastR.newRows)
{
List<object> hash = new List<object>();
foreach (string column in myColumnsToMatch)
{
try
{
hash.Add(row[column]);
}
catch (Exception) { }
}
Pair<BlueRow, BlueRow> pair;
rowPairs.TryGetValue(hash, out pair);
if (pair == null)
pair = new Pair<BlueRow, BlueRow>(null, row);
else
pair.second = row;
rowPairs[hash] = pair;
}
int id = 0;
foreach (KeyValuePair<List<object>, Pair<BlueRow, BlueRow>> hashpair in rowPairs)
{
Pair<BlueRow, BlueRow> pair = hashpair.Value;
if (pair.first == null || pair.second == null)
{
BlueRow row = pair.first == null ? pair.second : pair.first;
Dictionary<string, object> newRow = new Dictionary<string, object>();
foreach (ColumnInfo info in row.schema.columnInfo)
{
newRow[info.colName] = row[info.colName];
}
newRow["id"] = ++id;
table.addRow(new BlueRow(newRow));
}
else
{
bool allZeros = true;
Dictionary<string, object> newRow = new Dictionary<string, object>();
foreach (ColumnInfo info in pair.first.schema.columnInfo)
{
if (info.colType != typeof(double))
{
newRow[info.colName] = pair.first[info.colNo];
continue;
}
double l = (double)pair.first[info.colNo];
double r = (double)pair.second[info.colNo];
double value = Math.Abs(l - r);
if (value != 0)
{
double denom = Math.Max(Math.Abs(l), Math.Abs(r));
value /= denom;
if (value > myThreshold)
{
allZeros = false;
}
else value = 0;
}
newRow[info.colName] = value;
}
if (allZeros)
continue;
newRow["id"] = ++id;
table.addRow(new BlueRow(newRow));
}
}
myFireEvent(new DataPacket(table, table, myLastFired == null ? new BlueTable(table.schema) : myLastFired));
myLastFired = table;
}
void myRChange(object newValue)
{
myLastR = (DataPacket)newValue;
myHandleCallbacks();
}
void myLChange(object newValue)
{
myLastL = (DataPacket)newValue;
myHandleCallbacks();
}
}
}