using System; using System.Collections.Generic; using System.Linq; using System.Text; using BCG.Utilities.General; using FuzzyRowComparer = System.Func; using System.Diagnostics; namespace BCG { /// /// This Implements a fuzzy join over two streams across a time window. This works by keeping a buffer /// specified of a few seconds. When a row expires from one buffer /// (after timeMS) it is compared to all of the entries currently in the other buffer using the supplied /// fuzzy compare function. The best (closest to 0) non-infinity match is chosen and the two rows are combined into one /// and fired. Any rows that expire without a match are still fired but with no value in the other columns. /// /// Requirements: /// * Both streams have a date/time column for sorting and expiring from the match buffer /// * FuzzyCompare must be implemented so that rows that match have a value close to 0. /// /// 1/10/2011 - Changed the code to ensure that the right hand side is only matched once. /// This was a fairly significant logical re-architecture and has caused some pollution (side-effects) in some of the matching and merging functions. :( /// Probably need to go back and refactor some of the helper methods to make the code easier to read. /// public class FuzzyJoinOverTimeStream : ViewStream, IWorker { private const string NEW_TIME_COLUMN = "FuzzyJoinTimeColumn"; protected ViewStream myLeftSource; protected ViewStream myRightSource; protected ViewStreamDatabase myVsd; protected List myLeftRows = new List(); protected List myRightRows = new List(); private Dictionary myBestMatch = new Dictionary(); private string myLeftTimeColumn; private string myRightTimeColumn; private IComparer myLeftCompare; private IComparer myRightCompare; protected FuzzyRowComparer myFuzzyCompare; protected TimeSpan myTimeWindow; protected IBlueTableSchema myLeftSchema; protected IBlueTableSchema myRightSchema; protected IBlueTableSchema myOutputSchema; private int myID = 0; protected TimeWindowTable myStateRows; private DateTime myNextCull; const long MinCullIntervalMS = 1000; // #constructor public FuzzyJoinOverTimeStream(ViewStream left, string leftTimeColumn, ViewStream right, string rightTimeColumn, int timeMS, FuzzyRowComparer fuzzyCompare, ViewStreamDatabase vsd) { this.myLeftSource = left; this.myRightSource = right; this.myLeftTimeColumn = leftTimeColumn; this.myRightTimeColumn = rightTimeColumn; this.myLeftCompare = new BlueRowComparer(true, leftTimeColumn, "id"); this.myRightCompare = new BlueRowComparer(true, rightTimeColumn, "id"); this.myNextCull = DateTime.Now.AddMilliseconds(MinCullIntervalMS); this.myTimeWindow = TimeSpan.FromMilliseconds(timeMS); this.myVsd = vsd; this.myFuzzyCompare = fuzzyCompare; } public virtual IBlueTableSchema OutputSchema { get { if (this.myOutputSchema == null) { if (this.myLeftSchema != null && this.myRightSchema != null) { ColumnInfo[] leftColumns = this.myLeftSchema.columnInfo; ColumnInfo[] rightColumns = this.myRightSchema.columnInfo; ColumnInfo[] addColumns = new ColumnInfo[] { new ColumnInfo(NEW_TIME_COLUMN, 0, typeof(DateTime)) }; IEnumerable newColumns = leftColumns.Concat(rightColumns).GroupBy(c => c.colName).Select(s => s.First()); newColumns = newColumns.Concat(addColumns); // BlueTableSchema constructor assumes that each column has a valid non overlapping column number ColumnInfo[] newColumnsWithIndex = new ColumnInfo[newColumns.Count()]; int x = 0; foreach (ColumnInfo ci in newColumns) { newColumnsWithIndex[x] = new ColumnInfo(ci.colName, x, ci.colType); x++; } this.myOutputSchema = new BlueTableSchema(newColumnsWithIndex, "FuzzyJoin"); } } return this.myOutputSchema; } } protected override void myOnFirstSubscriber() { this.myLeftSource.onChange += new ChangeDelegate(myLeft_onChange); this.myRightSource.onChange += new ChangeDelegate(myRight_onChange); this.myVsd.addWorker(this); } protected override void myOnLastSubscriber() { this.myLeftSource.onChange -= new ChangeDelegate(myLeft_onChange); this.myRightSource.onChange -= new ChangeDelegate(myRight_onChange); this.myVsd.removeWorker(this); } void myRight_onChange(object newValue) { // Collect all right hand columns DataPacket dp = (DataPacket)newValue; AddNewRows(dp.addedRows, this.myRightRows, this.myRightCompare); this.myRightSchema = dp.schema; } void myLeft_onChange(object newValue) { DataPacket dp = (DataPacket)newValue; AddNewRows(dp.addedRows, this.myLeftRows, this.myLeftCompare); this.myLeftSchema = dp.schema; } void AddNewRows(IBlueRowCollection rows, List list, IComparer compare) { // Rows tend to come in in reverse order since they were added as they arrived // To avoid unnecessary memcpy in array resize add them to our collection backwards IBlueRow[] rowsArray = new IBlueRow[rows.Count]; rows.CopyTo(rowsArray, 0); for (int x = rowsArray.Length - 1; x >= 0; x--) { SortedInsert(rowsArray[x], list, compare); } } private void SortedInsert(IBlueRow row, List list, IComparer compare) { int index = list.BinarySearch(row, compare); if (index < 0) { list.Insert(~index, row); } } private IBlueRow[] PruneExpiredRows(List list, string timeColumn, DateTime expiry) { // Linear search is fine here b/c we don't expect the data to be very large int count; for (count = 0; count < list.Count; count++) { if ((DateTime)list[count][timeColumn] > expiry) { // Stop when we find the first row that we can keep. Since this is sorted all others are also kept. break; } } IBlueRow[] expiredRows = new IBlueRow[count]; list.CopyTo(0, expiredRows, 0, count); list.RemoveRange(0, count); return expiredRows; } class RowMatch : IComparable { public RowMatch(IBlueRow row, double fitness) { this.myRightRow = row; this.myFitness = fitness; } public IBlueRow myRightRow { get; private set; } public IBlueRow myLeftRow { get; set; } public double myFitness { get; private set; } public IBlueRow myMergedRow { get; set; } #region IComparable Members public int CompareTo(RowMatch other) { return myFitness.CompareTo(other.myFitness); } #endregion } public void doWork() { DateTime now = DateTime.Now; if (now < this.myNextCull) return; this.myNextCull = now.AddMilliseconds(MinCullIntervalMS); // Create a BlueTable for sending out. Determine schema DateTime expirationCutoff = now.Subtract(this.myTimeWindow); // If we haven't heard from one or both of the input streams then we can't formulate a valid output row if (this.OutputSchema == null) return; // Get the removed rows from left IBlueRow[] leftExpired = PruneExpiredRows(this.myLeftRows, this.myLeftTimeColumn, expirationCutoff); IEnumerable[] rightMatches = new IEnumerable[leftExpired.Length]; List corrections = new List(); /* Try to find the best match without making this an NP problem * 1. Find a list of best matches for each expiring trade * 2. Mark the right row with the 'best' match that was found for it * 3. When we emit a trade only include the right hand side if we are the best match * 4. If later a better match is found !! emit a remove row for the merged row and then a new row for the unmatched left hand side */ // Search for matches in right for (int x = 0; x < leftExpired.Length; x++) { IBlueRow row = leftExpired[x]; var matches = FindMatches(row, this.myRightRows, corrections); if (matches != null) { rightMatches[x] = matches; } } // get the removed rows from right -- IBlueRow[] rightExpired = PruneExpiredRows(this.myRightRows, this.myRightTimeColumn, expirationCutoff.Subtract(this.myTimeWindow)); // If there's nothing to do go home early // note: This test has to happen after searching for matches or myRightRows will not be complete if (rightExpired.Length == 0 && leftExpired.Length == 0) return; // Lazy initialize of myStateRows now that we know the full schema if (this.myStateRows == null) { this.myStateRows = new TimeWindowTable(this.OutputSchema, (int)TimeSpan.FromHours(12).TotalMilliseconds, NEW_TIME_COLUMN); } BlueTable adds; MergeExpiredRows(leftExpired, rightMatches, rightExpired, out adds); // Add adds to state foreach (IBlueRow row in adds) { this.myStateRows.addRow(row); } BlueTable dels = (BlueTable)this.myStateRows.moveWindow(); HandleCorrection(corrections, adds, dels); // Fire event this.myFireEvent(new DataPacket(this.myStateRows, adds, dels)); // remove matched flags for expired rows foreach (IBlueRow row in rightExpired) { this.myBestMatch.Remove(row); } } private void HandleCorrection(List corrections, BlueTable adds, BlueTable dels) { // We already emitted correction.MergedRow so we need to redact that and // then emit correction.left as an add since it is no longer matched foreach (RowMatch correction in corrections.Distinct()) { Debug.Assert(correction.myMergedRow != null); this.myStateRows.RemoveRow(correction.myMergedRow); adds.addRow(this.MergeRows(correction.myLeftRow, null)); dels.addRow(correction.myMergedRow); } } private BlueTable MergeExpiredRows(IBlueRow[] leftExpired, IEnumerable[] rightMatches, IBlueRow[] rightExpired, out BlueTable adds) { adds = new BlueTable(this.OutputSchema); // // Loop over both left and right expired rows and emit them in order. // (If they're going to get shuffled on the wire anyway why bother?) // int l = 0; int r = 0; for (l = 0, r = 0; l < leftExpired.Length && r < rightExpired.Length; ) { if ((DateTime)leftExpired[l][this.myLeftTimeColumn] <= (DateTime)rightExpired[r][this.myRightTimeColumn]) { // Advance Left IEnumerable rm = rightMatches[l]; IBlueRow leftRow = leftExpired[l]; IBlueRow merged = MergeLeftWithBestRight(leftRow, rm); // Increment and fire l++; adds.addRow(merged); } else { IBlueRow rightRow = rightExpired[r]; // If a match was found this row will already be emitted with it's left hand counterpart if (!this.myBestMatch.ContainsKey(rightRow)) { IBlueRow merged = this.MergeRows(null, rightRow); adds.addRow(merged); } r++; } } // handle remaining rows. Only one of the next two loops will execute for (; l < leftExpired.Length; l++) { IBlueRow merged = this.MergeLeftWithBestRight(leftExpired[l], rightMatches[l]); adds.addRow(merged); } for (; r < rightExpired.Length; r++) { if (!this.myBestMatch.ContainsKey(rightExpired[r])) { IBlueRow merged = this.MergeRows(null, rightExpired[r]); adds.addRow(merged); } } return adds; } private IBlueRow MergeLeftWithBestRight(IBlueRow leftRow, IEnumerable rightMatches) { IBlueRow merged = null; if (rightMatches != null) { foreach (RowMatch match in rightMatches) { RowMatch bestMatch = this.myBestMatch[match.myRightRow]; //if (match.myFitness == bestMatch.myFitness && bestMatch.myLeftRow == null) if (match == bestMatch) { merged = this.MergeRows(leftRow, match.myRightRow); // store these for later corrections match.myMergedRow = merged; match.myLeftRow = leftRow; // Make sure that we're updating this.myBestMatch[match.myRightRow] Debug.Assert(match == this.myBestMatch[match.myRightRow]); break; } } } // If none of the matches was 'best' emit with empty right hand side if (merged == null) { merged = this.MergeRows(leftRow, null); } return merged; } /// /// Override this function to provide a custom row merge. /// The overrided function must add a new column called FuzzyJoinOverTimeStream.NEW_TIME_COLUMN which will contain a timestamp for sorting. /// protected virtual IBlueRow MergeRows(IBlueRow left, IBlueRow right) { Dictionary newRow = new Dictionary(); if (left == null && right == null) throw new ArgumentException("oh come on."); if (left != null) { foreach (string columnName in left.schema.columnNames) { if (columnName == "id") continue; newRow[columnName] = left[columnName]; } } if (right != null) { foreach (string columnName in right.schema.columnNames) { if (columnName == "id") continue; if (!newRow.ContainsKey(columnName)) { newRow[columnName] = right[columnName]; } } } newRow["id"] = this.myID++; newRow[NEW_TIME_COLUMN] = (left != null) ? left[this.myLeftTimeColumn] : right[this.myRightTimeColumn]; return new BlueRow(this.OutputSchema, newRow); } private IEnumerable FindMatches(IBlueRow row, List list, List corrections) { List matches = null; // Search through all rows in list and compare with the fuzzy comparer for (int x = 0; x < list.Count; x++) { IBlueRow match = list[x]; double fitness = Math.Abs(this.myFuzzyCompare(row, match)); if (!double.IsInfinity(fitness)) { // Lazy allocate list - not having any matches will be fairly common if (matches == null) { matches = new List(); } RowMatch currentMatch = new RowMatch(match, fitness); int insertIndex = matches.BinarySearch(currentMatch); matches.Insert(insertIndex >= 0 ? insertIndex : ~insertIndex, currentMatch); // Mark the best match for this particular right hand match RowMatch oldMatch; if (this.myBestMatch.TryGetValue(match, out oldMatch)) { if (currentMatch.myFitness < oldMatch.myFitness) { this.myBestMatch[match] = currentMatch; if (oldMatch.myMergedRow != null) { // We already emitted oldMatch.MergedRow so we need to fix it corrections.Add(oldMatch); } } } else { this.myBestMatch[match] = currentMatch; } } } return matches; } } }