448 lines
17 KiB
C#
448 lines
17 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Text;
|
|
using BCG.Utilities.General;
|
|
|
|
using FuzzyRowComparer = System.Func<BCG.IBlueRow, BCG.IBlueRow, double>;
|
|
using System.Diagnostics;
|
|
|
|
namespace BCG
|
|
{
|
|
/// <summary>
|
|
/// This Implements a fuzzy join over two streams across a time window. This works by keeping a buffer
|
|
/// specified of a few seconds. When a row expires from one buffer
|
|
/// (after timeMS) it is compared to all of the entries currently in the other buffer using the supplied
|
|
/// fuzzy compare function. The best (closest to 0) non-infinity match is chosen and the two rows are combined into one
|
|
/// and fired. Any rows that expire without a match are still fired but with no value in the other columns.
|
|
///
|
|
/// Requirements:
|
|
/// * Both streams have a date/time column for sorting and expiring from the match buffer
|
|
/// * FuzzyCompare must be implemented so that rows that match have a value close to 0.
|
|
///
|
|
/// 1/10/2011 - Changed the code to ensure that the right hand side is only matched once.
|
|
/// This was a fairly significant logical re-architecture and has caused some pollution (side-effects) in some of the matching and merging functions. :(
|
|
/// Probably need to go back and refactor some of the helper methods to make the code easier to read.
|
|
/// </summary>
|
|
public class FuzzyJoinOverTimeStream : ViewStream, IWorker
|
|
{
|
|
private const string NEW_TIME_COLUMN = "FuzzyJoinTimeColumn";
|
|
protected ViewStream myLeftSource;
|
|
protected ViewStream myRightSource;
|
|
protected ViewStreamDatabase myVsd;
|
|
|
|
protected List<IBlueRow> myLeftRows = new List<IBlueRow>();
|
|
protected List<IBlueRow> myRightRows = new List<IBlueRow>();
|
|
private Dictionary<IBlueRow, RowMatch> myBestMatch = new Dictionary<IBlueRow, RowMatch>();
|
|
private string myLeftTimeColumn;
|
|
private string myRightTimeColumn;
|
|
private IComparer<IBlueRow> myLeftCompare;
|
|
private IComparer<IBlueRow> myRightCompare;
|
|
protected FuzzyRowComparer myFuzzyCompare;
|
|
protected TimeSpan myTimeWindow;
|
|
protected IBlueTableSchema myLeftSchema;
|
|
protected IBlueTableSchema myRightSchema;
|
|
protected IBlueTableSchema myOutputSchema;
|
|
private int myID = 0;
|
|
|
|
protected TimeWindowTable myStateRows;
|
|
|
|
private DateTime myNextCull;
|
|
const long MinCullIntervalMS = 1000;
|
|
|
|
// #constructor
|
|
public FuzzyJoinOverTimeStream(ViewStream left, string leftTimeColumn, ViewStream right, string rightTimeColumn, int timeMS, FuzzyRowComparer fuzzyCompare, ViewStreamDatabase vsd)
|
|
{
|
|
this.myLeftSource = left;
|
|
this.myRightSource = right;
|
|
this.myLeftTimeColumn = leftTimeColumn;
|
|
this.myRightTimeColumn = rightTimeColumn;
|
|
this.myLeftCompare = new BlueRowComparer(true, leftTimeColumn, "id");
|
|
this.myRightCompare = new BlueRowComparer(true, rightTimeColumn, "id");
|
|
this.myNextCull = DateTime.Now.AddMilliseconds(MinCullIntervalMS);
|
|
this.myTimeWindow = TimeSpan.FromMilliseconds(timeMS);
|
|
this.myVsd = vsd;
|
|
this.myFuzzyCompare = fuzzyCompare;
|
|
}
|
|
|
|
public virtual IBlueTableSchema OutputSchema
|
|
{
|
|
get
|
|
{
|
|
if (this.myOutputSchema == null)
|
|
{
|
|
if (this.myLeftSchema != null && this.myRightSchema != null)
|
|
{
|
|
ColumnInfo[] leftColumns = this.myLeftSchema.columnInfo;
|
|
ColumnInfo[] rightColumns = this.myRightSchema.columnInfo;
|
|
ColumnInfo[] addColumns = new ColumnInfo[] { new ColumnInfo(NEW_TIME_COLUMN, 0, typeof(DateTime)) };
|
|
IEnumerable<ColumnInfo> newColumns = leftColumns.Concat(rightColumns).GroupBy(c => c.colName).Select(s => s.First());
|
|
newColumns = newColumns.Concat(addColumns);
|
|
|
|
// BlueTableSchema constructor assumes that each column has a valid non overlapping column number
|
|
ColumnInfo[] newColumnsWithIndex = new ColumnInfo[newColumns.Count()];
|
|
int x = 0;
|
|
foreach (ColumnInfo ci in newColumns)
|
|
{
|
|
newColumnsWithIndex[x] = new ColumnInfo(ci.colName, x, ci.colType);
|
|
x++;
|
|
}
|
|
this.myOutputSchema = new BlueTableSchema(newColumnsWithIndex, "FuzzyJoin");
|
|
}
|
|
}
|
|
return this.myOutputSchema;
|
|
}
|
|
}
|
|
|
|
protected override void myOnFirstSubscriber()
|
|
{
|
|
this.myLeftSource.onChange += new ChangeDelegate(myLeft_onChange);
|
|
this.myRightSource.onChange += new ChangeDelegate(myRight_onChange);
|
|
this.myVsd.addWorker(this);
|
|
}
|
|
|
|
protected override void myOnLastSubscriber()
|
|
{
|
|
this.myLeftSource.onChange -= new ChangeDelegate(myLeft_onChange);
|
|
this.myRightSource.onChange -= new ChangeDelegate(myRight_onChange);
|
|
this.myVsd.removeWorker(this);
|
|
}
|
|
|
|
void myRight_onChange(object newValue)
|
|
{
|
|
// Collect all right hand columns
|
|
DataPacket dp = (DataPacket)newValue;
|
|
AddNewRows(dp.addedRows, this.myRightRows, this.myRightCompare);
|
|
this.myRightSchema = dp.schema;
|
|
}
|
|
|
|
void myLeft_onChange(object newValue)
|
|
{
|
|
DataPacket dp = (DataPacket)newValue;
|
|
AddNewRows(dp.addedRows, this.myLeftRows, this.myLeftCompare);
|
|
this.myLeftSchema = dp.schema;
|
|
}
|
|
|
|
void AddNewRows(IBlueRowCollection rows, List<IBlueRow> list, IComparer<IBlueRow> compare)
|
|
{
|
|
// Rows tend to come in in reverse order since they were added as they arrived
|
|
// To avoid unnecessary memcpy in array resize add them to our collection backwards
|
|
IBlueRow[] rowsArray = new IBlueRow[rows.Count];
|
|
rows.CopyTo(rowsArray, 0);
|
|
for (int x = rowsArray.Length - 1; x >= 0; x--)
|
|
{
|
|
SortedInsert(rowsArray[x], list, compare);
|
|
}
|
|
}
|
|
|
|
private void SortedInsert(IBlueRow row, List<IBlueRow> list, IComparer<IBlueRow> compare)
|
|
{
|
|
int index = list.BinarySearch(row, compare);
|
|
if (index < 0)
|
|
{
|
|
list.Insert(~index, row);
|
|
}
|
|
}
|
|
|
|
private IBlueRow[] PruneExpiredRows(List<IBlueRow> list, string timeColumn, DateTime expiry)
|
|
{
|
|
// Linear search is fine here b/c we don't expect the data to be very large
|
|
int count;
|
|
for (count = 0; count < list.Count; count++)
|
|
{
|
|
if ((DateTime)list[count][timeColumn] > expiry)
|
|
{
|
|
// Stop when we find the first row that we can keep. Since this is sorted all others are also kept.
|
|
break;
|
|
}
|
|
}
|
|
|
|
IBlueRow[] expiredRows = new IBlueRow[count];
|
|
list.CopyTo(0, expiredRows, 0, count);
|
|
list.RemoveRange(0, count);
|
|
|
|
return expiredRows;
|
|
}
|
|
|
|
class RowMatch : IComparable<RowMatch>
|
|
{
|
|
public RowMatch(IBlueRow row, double fitness)
|
|
{
|
|
this.myRightRow = row;
|
|
this.myFitness = fitness;
|
|
}
|
|
public IBlueRow myRightRow { get; private set; }
|
|
public IBlueRow myLeftRow { get; set; }
|
|
public double myFitness { get; private set; }
|
|
public IBlueRow myMergedRow { get; set; }
|
|
|
|
#region IComparable<RowMatch> Members
|
|
|
|
public int CompareTo(RowMatch other)
|
|
{
|
|
return myFitness.CompareTo(other.myFitness);
|
|
}
|
|
|
|
#endregion
|
|
}
|
|
|
|
public void doWork()
|
|
{
|
|
DateTime now = DateTime.Now;
|
|
if (now < this.myNextCull) return;
|
|
|
|
this.myNextCull = now.AddMilliseconds(MinCullIntervalMS);
|
|
|
|
// Create a BlueTable for sending out. Determine schema
|
|
|
|
DateTime expirationCutoff = now.Subtract(this.myTimeWindow);
|
|
|
|
// If we haven't heard from one or both of the input streams then we can't formulate a valid output row
|
|
if (this.OutputSchema == null) return;
|
|
|
|
// Get the removed rows from left
|
|
IBlueRow[] leftExpired = PruneExpiredRows(this.myLeftRows, this.myLeftTimeColumn, expirationCutoff);
|
|
IEnumerable<RowMatch>[] rightMatches = new IEnumerable<RowMatch>[leftExpired.Length];
|
|
|
|
|
|
List<RowMatch> corrections = new List<RowMatch>();
|
|
|
|
/* Try to find the best match without making this an NP problem
|
|
* 1. Find a list of best matches for each expiring trade
|
|
* 2. Mark the right row with the 'best' match that was found for it
|
|
* 3. When we emit a trade only include the right hand side if we are the best match
|
|
* 4. If later a better match is found !! emit a remove row for the merged row and then a new row for the unmatched left hand side
|
|
*/
|
|
|
|
// Search for matches in right
|
|
for (int x = 0; x < leftExpired.Length; x++)
|
|
{
|
|
IBlueRow row = leftExpired[x];
|
|
var matches = FindMatches(row, this.myRightRows, corrections);
|
|
if (matches != null)
|
|
{
|
|
rightMatches[x] = matches;
|
|
}
|
|
}
|
|
|
|
// get the removed rows from right --
|
|
IBlueRow[] rightExpired = PruneExpiredRows(this.myRightRows, this.myRightTimeColumn, expirationCutoff.Subtract(this.myTimeWindow));
|
|
|
|
// If there's nothing to do go home early
|
|
// note: This test has to happen after searching for matches or myRightRows will not be complete
|
|
if (rightExpired.Length == 0 && leftExpired.Length == 0) return;
|
|
|
|
// Lazy initialize of myStateRows now that we know the full schema
|
|
if (this.myStateRows == null)
|
|
{
|
|
this.myStateRows = new TimeWindowTable(this.OutputSchema, (int)TimeSpan.FromHours(12).TotalMilliseconds, NEW_TIME_COLUMN);
|
|
}
|
|
|
|
BlueTable adds;
|
|
MergeExpiredRows(leftExpired, rightMatches, rightExpired, out adds);
|
|
|
|
// Add adds to state
|
|
foreach (IBlueRow row in adds)
|
|
{
|
|
this.myStateRows.addRow(row);
|
|
}
|
|
BlueTable dels = (BlueTable)this.myStateRows.moveWindow();
|
|
|
|
HandleCorrection(corrections, adds, dels);
|
|
|
|
// Fire event
|
|
this.myFireEvent(new DataPacket(this.myStateRows, adds, dels));
|
|
|
|
// remove matched flags for expired rows
|
|
foreach (IBlueRow row in rightExpired)
|
|
{
|
|
this.myBestMatch.Remove(row);
|
|
}
|
|
}
|
|
|
|
private void HandleCorrection(List<RowMatch> corrections, BlueTable adds, BlueTable dels)
|
|
{
|
|
// We already emitted correction.MergedRow so we need to redact that and
|
|
// then emit correction.left as an add since it is no longer matched
|
|
foreach (RowMatch correction in corrections.Distinct())
|
|
{
|
|
Debug.Assert(correction.myMergedRow != null);
|
|
this.myStateRows.RemoveRow(correction.myMergedRow);
|
|
adds.addRow(this.MergeRows(correction.myLeftRow, null));
|
|
dels.addRow(correction.myMergedRow);
|
|
}
|
|
}
|
|
|
|
private BlueTable MergeExpiredRows(IBlueRow[] leftExpired, IEnumerable<RowMatch>[] rightMatches, IBlueRow[] rightExpired, out BlueTable adds)
|
|
{
|
|
adds = new BlueTable(this.OutputSchema);
|
|
|
|
//
|
|
// Loop over both left and right expired rows and emit them in order.
|
|
// (If they're going to get shuffled on the wire anyway why bother?)
|
|
//
|
|
int l = 0;
|
|
int r = 0;
|
|
for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; )
|
|
{
|
|
if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn])
|
|
{
|
|
// Advance Left
|
|
IEnumerable<RowMatch> rm = rightMatches[l];
|
|
IBlueRow leftRow = leftExpired[l];
|
|
|
|
IBlueRow merged = MergeLeftWithBestRight(leftRow, rm);
|
|
|
|
// Increment and fire
|
|
l++;
|
|
adds.addRow(merged);
|
|
}
|
|
else
|
|
{
|
|
IBlueRow rightRow = rightExpired[r];
|
|
|
|
// If a match was found this row will already be emitted with it's left hand counterpart
|
|
if (!this.myBestMatch.ContainsKey(rightRow))
|
|
{
|
|
IBlueRow merged = this.MergeRows(null, rightRow);
|
|
adds.addRow(merged);
|
|
}
|
|
r++;
|
|
}
|
|
}
|
|
|
|
// handle remaining rows. Only one of the next two loops will execute
|
|
for (; l < leftExpired.Length; l++)
|
|
{
|
|
IBlueRow merged = this.MergeLeftWithBestRight(leftExpired[l], rightMatches[l]);
|
|
adds.addRow(merged);
|
|
}
|
|
|
|
for (; r < rightExpired.Length; r++)
|
|
{
|
|
if (!this.myBestMatch.ContainsKey(rightExpired[r]))
|
|
{
|
|
IBlueRow merged = this.MergeRows(null, rightExpired[r]);
|
|
adds.addRow(merged);
|
|
}
|
|
}
|
|
|
|
return adds;
|
|
}
|
|
|
|
private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable<RowMatch> rightMatches)
|
|
{
|
|
IBlueRow merged = null;
|
|
if (rightMatches != null)
|
|
{
|
|
foreach (RowMatch match in rightMatches)
|
|
{
|
|
RowMatch bestMatch = this.myBestMatch[match.myRightRow];
|
|
//if (match.myFitness == bestMatch.myFitness && bestMatch.myLeftRow == null)
|
|
if (match == bestMatch)
|
|
{
|
|
merged = this.MergeRows(leftRow, match.myRightRow);
|
|
|
|
// store these for later corrections
|
|
match.myMergedRow = merged;
|
|
match.myLeftRow = leftRow;
|
|
|
|
// Make sure that we're updating this.myBestMatch[match.myRightRow]
|
|
Debug.Assert(match == this.myBestMatch[match.myRightRow]);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// If none of the matches was 'best' emit with empty right hand side
|
|
if (merged == null)
|
|
{
|
|
merged = this.MergeRows(leftRow, null);
|
|
}
|
|
return merged;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Override this function to provide a custom row merge.
|
|
/// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting.
|
|
/// </summary>
|
|
protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right)
|
|
{
|
|
Dictionary<string, object> newRow = new Dictionary<string, object>();
|
|
|
|
if (left == null && right == null) throw new ArgumentException("oh come on.");
|
|
|
|
if (left != null)
|
|
{
|
|
foreach (string columnName in left.schema.columnNames)
|
|
{
|
|
if (columnName == "id") continue;
|
|
newRow[columnName] = left[columnName];
|
|
}
|
|
}
|
|
|
|
if (right != null)
|
|
{
|
|
foreach (string columnName in right.schema.columnNames)
|
|
{
|
|
if (columnName == "id") continue;
|
|
if (!newRow.ContainsKey(columnName))
|
|
{
|
|
newRow[columnName] = right[columnName];
|
|
}
|
|
}
|
|
}
|
|
newRow["id"] = this.myID++;
|
|
newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn];
|
|
|
|
return new BlueRow(this.OutputSchema, newRow);
|
|
}
|
|
|
|
private IEnumerable<RowMatch> FindMatches(IBlueRow row, List<IBlueRow> list, List<RowMatch> corrections)
|
|
{
|
|
List<RowMatch> matches = null;
|
|
|
|
// Search through all rows in list and compare with the fuzzy comparer
|
|
for (int x = 0; x < list.Count; x++)
|
|
{
|
|
IBlueRow match = list[x];
|
|
double fitness = Math.Abs(this.myFuzzyCompare(row, match));
|
|
|
|
if (!double.IsInfinity(fitness))
|
|
{
|
|
// Lazy allocate list - not having any matches will be fairly common
|
|
if (matches == null)
|
|
{
|
|
matches = new List<RowMatch>();
|
|
}
|
|
|
|
RowMatch currentMatch = new RowMatch(match, fitness);
|
|
int insertIndex = matches.BinarySearch(currentMatch);
|
|
matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch);
|
|
|
|
// Mark the best match for this particular right hand match
|
|
RowMatch oldMatch;
|
|
if (this.myBestMatch.TryGetValue(match, out oldMatch))
|
|
{
|
|
if (currentMatch.myFitness < oldMatch.myFitness)
|
|
{
|
|
this.myBestMatch[match] = currentMatch;
|
|
if (oldMatch.myMergedRow != null)
|
|
{
|
|
// We already emitted oldMatch.MergedRow so we need to fix it
|
|
corrections.Add(oldMatch);
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
this.myBestMatch[match] = currentMatch;
|
|
}
|
|
}
|
|
}
|
|
|
|
return matches;
|
|
}
|
|
}
|
|
}
|