Files
spark/src/DataStreamSuite/DataStream/Primitives/Fuzzy/FuzzyJoinOverTimeStream.cs
T
2023-09-04 16:25:40 -04:00

448 lines
17 KiB
C#

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using BCG.Utilities.General;
using FuzzyRowComparer = System.Func<BCG.IBlueRow, BCG.IBlueRow, double>;
using System.Diagnostics;
namespace BCG
{
/// <summary>
/// This Implements a fuzzy join over two streams across a time window. This works by keeping a buffer
/// specified of a few seconds. When a row expires from one buffer
/// (after timeMS) it is compared to all of the entries currently in the other buffer using the supplied
/// fuzzy compare function. The best (closest to 0) non-infinity match is chosen and the two rows are combined into one
/// and fired. Any rows that expire without a match are still fired but with no value in the other columns.
///
/// Requirements:
/// * Both streams have a date/time column for sorting and expiring from the match buffer
/// * FuzzyCompare must be implemented so that rows that match have a value close to 0.
///
/// 1/10/2011 - Changed the code to ensure that the right hand side is only matched once.
/// This was a fairly significant logical re-architecture and has caused some pollution (side-effects) in some of the matching and merging functions. :(
/// Probably need to go back and refactor some of the helper methods to make the code easier to read.
/// </summary>
public class FuzzyJoinOverTimeStream : ViewStream, IWorker
{
private const string NEW_TIME_COLUMN = "FuzzyJoinTimeColumn";
protected ViewStream myLeftSource;
protected ViewStream myRightSource;
protected ViewStreamDatabase myVsd;
protected List<IBlueRow> myLeftRows = new List<IBlueRow>();
protected List<IBlueRow> myRightRows = new List<IBlueRow>();
private Dictionary<IBlueRow, RowMatch> myBestMatch = new Dictionary<IBlueRow, RowMatch>();
private string myLeftTimeColumn;
private string myRightTimeColumn;
private IComparer<IBlueRow> myLeftCompare;
private IComparer<IBlueRow> myRightCompare;
protected FuzzyRowComparer myFuzzyCompare;
protected TimeSpan myTimeWindow;
protected IBlueTableSchema myLeftSchema;
protected IBlueTableSchema myRightSchema;
protected IBlueTableSchema myOutputSchema;
private int myID = 0;
protected TimeWindowTable myStateRows;
private DateTime myNextCull;
const long MinCullIntervalMS = 1000;
// #constructor
public FuzzyJoinOverTimeStream(ViewStream left, string leftTimeColumn, ViewStream right, string rightTimeColumn, int timeMS, FuzzyRowComparer fuzzyCompare, ViewStreamDatabase vsd)
{
this.myLeftSource = left;
this.myRightSource = right;
this.myLeftTimeColumn = leftTimeColumn;
this.myRightTimeColumn = rightTimeColumn;
this.myLeftCompare = new BlueRowComparer(true, leftTimeColumn, "id");
this.myRightCompare = new BlueRowComparer(true, rightTimeColumn, "id");
this.myNextCull = DateTime.Now.AddMilliseconds(MinCullIntervalMS);
this.myTimeWindow = TimeSpan.FromMilliseconds(timeMS);
this.myVsd = vsd;
this.myFuzzyCompare = fuzzyCompare;
}
public virtual IBlueTableSchema OutputSchema
{
get
{
if (this.myOutputSchema == null)
{
if (this.myLeftSchema != null && this.myRightSchema != null)
{
ColumnInfo[] leftColumns = this.myLeftSchema.columnInfo;
ColumnInfo[] rightColumns = this.myRightSchema.columnInfo;
ColumnInfo[] addColumns = new ColumnInfo[] { new ColumnInfo(NEW_TIME_COLUMN, 0, typeof(DateTime)) };
IEnumerable<ColumnInfo> newColumns = leftColumns.Concat(rightColumns).GroupBy(c => c.colName).Select(s => s.First());
newColumns = newColumns.Concat(addColumns);
// BlueTableSchema constructor assumes that each column has a valid non overlapping column number
ColumnInfo[] newColumnsWithIndex = new ColumnInfo[newColumns.Count()];
int x = 0;
foreach (ColumnInfo ci in newColumns)
{
newColumnsWithIndex[x] = new ColumnInfo(ci.colName, x, ci.colType);
x++;
}
this.myOutputSchema = new BlueTableSchema(newColumnsWithIndex, "FuzzyJoin");
}
}
return this.myOutputSchema;
}
}
protected override void myOnFirstSubscriber()
{
this.myLeftSource.onChange += new ChangeDelegate(myLeft_onChange);
this.myRightSource.onChange += new ChangeDelegate(myRight_onChange);
this.myVsd.addWorker(this);
}
protected override void myOnLastSubscriber()
{
this.myLeftSource.onChange -= new ChangeDelegate(myLeft_onChange);
this.myRightSource.onChange -= new ChangeDelegate(myRight_onChange);
this.myVsd.removeWorker(this);
}
void myRight_onChange(object newValue)
{
// Collect all right hand columns
DataPacket dp = (DataPacket)newValue;
AddNewRows(dp.addedRows, this.myRightRows, this.myRightCompare);
this.myRightSchema = dp.schema;
}
void myLeft_onChange(object newValue)
{
DataPacket dp = (DataPacket)newValue;
AddNewRows(dp.addedRows, this.myLeftRows, this.myLeftCompare);
this.myLeftSchema = dp.schema;
}
void AddNewRows(IBlueRowCollection rows, List<IBlueRow> list, IComparer<IBlueRow> compare)
{
// Rows tend to come in in reverse order since they were added as they arrived
// To avoid unnecessary memcpy in array resize add them to our collection backwards
IBlueRow[] rowsArray = new IBlueRow[rows.Count];
rows.CopyTo(rowsArray, 0);
for (int x = rowsArray.Length - 1; x >= 0; x--)
{
SortedInsert(rowsArray[x], list, compare);
}
}
private void SortedInsert(IBlueRow row, List<IBlueRow> list, IComparer<IBlueRow> compare)
{
int index = list.BinarySearch(row, compare);
if (index < 0)
{
list.Insert(~index, row);
}
}
private IBlueRow[] PruneExpiredRows(List<IBlueRow> list, string timeColumn, DateTime expiry)
{
// Linear search is fine here b/c we don't expect the data to be very large
int count;
for (count = 0; count < list.Count; count++)
{
if ((DateTime)list[count][timeColumn] > expiry)
{
// Stop when we find the first row that we can keep. Since this is sorted all others are also kept.
break;
}
}
IBlueRow[] expiredRows = new IBlueRow[count];
list.CopyTo(0, expiredRows, 0, count);
list.RemoveRange(0, count);
return expiredRows;
}
class RowMatch : IComparable<RowMatch>
{
public RowMatch(IBlueRow row, double fitness)
{
this.myRightRow = row;
this.myFitness = fitness;
}
public IBlueRow myRightRow { get; private set; }
public IBlueRow myLeftRow { get; set; }
public double myFitness { get; private set; }
public IBlueRow myMergedRow { get; set; }
#region IComparable<RowMatch> Members
public int CompareTo(RowMatch other)
{
return myFitness.CompareTo(other.myFitness);
}
#endregion
}
public void doWork()
{
DateTime now = DateTime.Now;
if (now < this.myNextCull) return;
this.myNextCull = now.AddMilliseconds(MinCullIntervalMS);
// Create a BlueTable for sending out. Determine schema
DateTime expirationCutoff = now.Subtract(this.myTimeWindow);
// If we haven't heard from one or both of the input streams then we can't formulate a valid output row
if (this.OutputSchema == null) return;
// Get the removed rows from left
IBlueRow[] leftExpired = PruneExpiredRows(this.myLeftRows, this.myLeftTimeColumn, expirationCutoff);
IEnumerable<RowMatch>[] rightMatches = new IEnumerable<RowMatch>[leftExpired.Length];
List<RowMatch> corrections = new List<RowMatch>();
/* Try to find the best match without making this an NP problem
* 1. Find a list of best matches for each expiring trade
* 2. Mark the right row with the 'best' match that was found for it
* 3. When we emit a trade only include the right hand side if we are the best match
* 4. If later a better match is found !! emit a remove row for the merged row and then a new row for the unmatched left hand side
*/
// Search for matches in right
for (int x = 0; x < leftExpired.Length; x++)
{
IBlueRow row = leftExpired[x];
var matches = FindMatches(row, this.myRightRows, corrections);
if (matches != null)
{
rightMatches[x] = matches;
}
}
// get the removed rows from right --
IBlueRow[] rightExpired = PruneExpiredRows(this.myRightRows, this.myRightTimeColumn, expirationCutoff.Subtract(this.myTimeWindow));
// If there's nothing to do go home early
// note: This test has to happen after searching for matches or myRightRows will not be complete
if (rightExpired.Length == 0 && leftExpired.Length == 0) return;
// Lazy initialize of myStateRows now that we know the full schema
if (this.myStateRows == null)
{
this.myStateRows = new TimeWindowTable(this.OutputSchema, (int)TimeSpan.FromHours(12).TotalMilliseconds, NEW_TIME_COLUMN);
}
BlueTable adds;
MergeExpiredRows(leftExpired, rightMatches, rightExpired, out adds);
// Add adds to state
foreach (IBlueRow row in adds)
{
this.myStateRows.addRow(row);
}
BlueTable dels = (BlueTable)this.myStateRows.moveWindow();
HandleCorrection(corrections, adds, dels);
// Fire event
this.myFireEvent(new DataPacket(this.myStateRows, adds, dels));
// remove matched flags for expired rows
foreach (IBlueRow row in rightExpired)
{
this.myBestMatch.Remove(row);
}
}
private void HandleCorrection(List<RowMatch> corrections, BlueTable adds, BlueTable dels)
{
// We already emitted correction.MergedRow so we need to redact that and
// then emit correction.left as an add since it is no longer matched
foreach (RowMatch correction in corrections.Distinct())
{
Debug.Assert(correction.myMergedRow != null);
this.myStateRows.RemoveRow(correction.myMergedRow);
adds.addRow(this.MergeRows(correction.myLeftRow, null));
dels.addRow(correction.myMergedRow);
}
}
private BlueTable MergeExpiredRows(IBlueRow[] leftExpired, IEnumerable<RowMatch>[] rightMatches, IBlueRow[] rightExpired, out BlueTable adds)
{
adds = new BlueTable(this.OutputSchema);
//
// Loop over both left and right expired rows and emit them in order.
// (If they're going to get shuffled on the wire anyway why bother?)
//
int l = 0;
int r = 0;
for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; )
{
if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn])
{
// Advance Left
IEnumerable<RowMatch> rm = rightMatches[l];
IBlueRow leftRow = leftExpired[l];
IBlueRow merged = MergeLeftWithBestRight(leftRow, rm);
// Increment and fire
l++;
adds.addRow(merged);
}
else
{
IBlueRow rightRow = rightExpired[r];
// If a match was found this row will already be emitted with it's left hand counterpart
if (!this.myBestMatch.ContainsKey(rightRow))
{
IBlueRow merged = this.MergeRows(null, rightRow);
adds.addRow(merged);
}
r++;
}
}
// handle remaining rows. Only one of the next two loops will execute
for (; l < leftExpired.Length; l++)
{
IBlueRow merged = this.MergeLeftWithBestRight(leftExpired[l], rightMatches[l]);
adds.addRow(merged);
}
for (; r < rightExpired.Length; r++)
{
if (!this.myBestMatch.ContainsKey(rightExpired[r]))
{
IBlueRow merged = this.MergeRows(null, rightExpired[r]);
adds.addRow(merged);
}
}
return adds;
}
private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable<RowMatch> rightMatches)
{
IBlueRow merged = null;
if (rightMatches != null)
{
foreach (RowMatch match in rightMatches)
{
RowMatch bestMatch = this.myBestMatch[match.myRightRow];
//if (match.myFitness == bestMatch.myFitness && bestMatch.myLeftRow == null)
if (match == bestMatch)
{
merged = this.MergeRows(leftRow, match.myRightRow);
// store these for later corrections
match.myMergedRow = merged;
match.myLeftRow = leftRow;
// Make sure that we're updating this.myBestMatch[match.myRightRow]
Debug.Assert(match == this.myBestMatch[match.myRightRow]);
break;
}
}
}
// If none of the matches was 'best' emit with empty right hand side
if (merged == null)
{
merged = this.MergeRows(leftRow, null);
}
return merged;
}
/// <summary>
/// Override this function to provide a custom row merge.
/// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting.
/// </summary>
protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right)
{
Dictionary<string, object> newRow = new Dictionary<string, object>();
if (left == null && right == null) throw new ArgumentException("oh come on.");
if (left != null)
{
foreach (string columnName in left.schema.columnNames)
{
if (columnName == "id") continue;
newRow[columnName] = left[columnName];
}
}
if (right != null)
{
foreach (string columnName in right.schema.columnNames)
{
if (columnName == "id") continue;
if (!newRow.ContainsKey(columnName))
{
newRow[columnName] = right[columnName];
}
}
}
newRow["id"] = this.myID++;
newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn];
return new BlueRow(this.OutputSchema, newRow);
}
private IEnumerable<RowMatch> FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
{
List<RowMatch> matches = null;
// Search through all rows in list and compare with the fuzzy comparer
for (int x = 0; x < list.Count; x++)
{
IBlueRow match = list[x];
double fitness = Math.Abs(this.myFuzzyCompare(row, match));
if (!double.IsInfinity(fitness))
{
// Lazy allocate list - not having any matches will be fairly common
if (matches == null)
{
matches = new List<RowMatch>();
}
RowMatch currentMatch = new RowMatch(match, fitness);
int insertIndex = matches.BinarySearch(currentMatch);
matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch);
// Mark the best match for this particular right hand match
RowMatch oldMatch;
if (this.myBestMatch.TryGetValue(match, out oldMatch))
{
if (currentMatch.myFitness < oldMatch.myFitness)
{
this.myBestMatch[match] = currentMatch;
if (oldMatch.myMergedRow != null)
{
// We already emitted oldMatch.MergedRow so we need to fix it
corrections.Add(oldMatch);
}
}
}
else
{
this.myBestMatch[match] = currentMatch;
}
}
}
return matches;
}
}
}