refactor: Replace HighPerformanceTimer w/ Stopwatch

This is needed to build DataStreamSuite on Linux.

'Stopwatch' is a platform independent timer class included with the
System.Runtime library in modern dotnet. 'HighPerformanceTimer' is a BTS
class that predates Stopwatch, and is NOT platform independent as it
relies on Windows kernel32.dll QueryPerformanceCounter/Frequency fns.
This commit is contained in:
John Colvin committed 2024-06-03 13:46:51 +00:00
1 parent 7ee6933789
commit 9050318ceb
18 files changed
+110 -184

No files matched your search

@@ -151,7 +151,6 @@ namespace BCG
private string myFilterString;
private BCG.StreamWorld.Addition.FilterStream myFilterStream;
private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
private HighPerformanceTimer _timer = new HighPerformanceTimer();
#region constructors
public delegate bool OnFilter(IBlueRow r);
@@ -226,7 +225,6 @@ namespace BCG
private void myHandleCallbacks(object newValue)
{
_timer.start();
DataPacket retVal;
DataPacket arrivedPacket = newValue as DataPacket;
myLastReceivedRows = arrivedPacket.newRows;
@@ -255,7 +253,6 @@ namespace BCG
if (myFilter.filter(r))
myLastFiredRows.addRow(r);
}
_timer.stop();
myFireEvent(retVal);
}
@@ -1,10 +1,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using mbts;
using mbts.spark;
using System.Diagnostics;
using bts;
namespace BCG
{
@@ -14,7 +12,7 @@ namespace BCG
private INonIncGroupByEvaluator _groupby_func;
private BlueRowHasher _row_hasher;
private string[] _groupby_cols;
private HighPerformanceTimer _timer = new HighPerformanceTimer();
private Stopwatch _timer = new();
private int _n_total_fires = 0, _n_expensive_fires = 0;
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
@@ -33,21 +31,27 @@ namespace BCG
{
DataPacket dp = nv as DataPacket;
///grouping together
_timer.start();
_timer.Restart();
IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
_timer.stop();
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
_timer.Stop();
var groupElapsedMs = _timer.ElapsedMilliseconds;
if (groupElapsedMs >= 50) {
Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
}
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
/// reducing
_timer.start();
_timer.Restart();
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
_timer.stop();
_timer.Stop();
var applyElapsedMs = _timer.ElapsedMilliseconds;
_n_total_fires++;
if (_timer.lastIntervalTime > 50)
if (applyElapsedMs >= 50)
{
_n_expensive_fires++;
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to evaluate {0} groups {1} , ratio {2}/{3}", grouped_rows.Count, _timer.lastIntervalTime, _n_expensive_fires, _n_total_fires));
Log.DataFlow.info(
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
}
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
}
}
}
}
@@ -1,8 +1,13 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using bts;
using mbts;
using mbts.spark;
@@ -40,8 +45,7 @@ namespace BCG
public void _on_stream_cb(object dp_obj)
{
HighPerformanceTimer timer = new HighPerformanceTimer();
timer.start();
var timer = Stopwatch.StartNew();
DataPacket dp = (DataPacket)dp_obj;
if (_schema == null)
{
@@ -67,13 +71,12 @@ namespace BCG
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
}
}
timer.stop();
timer.Stop();
_n_total_fires++;
if (timer.lastIntervalTime > 50)
{
if (timer.ElapsedMilliseconds >= 50) {
_n_expensive_fires++;
string log_msg = String.Format("NonIncJoinBookkeepingData._on_stream_cb {0} rows {1} key cols {2} join pos time {3} ratio {4}/{5}", dp.newRows.Count, _join_col_indices.Length, _join_position, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
+ $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
}
if (on_new_data != null)
on_new_data();
@@ -1,8 +1,13 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using bts;
using mbts;
using mbts.spark;
@@ -114,8 +119,7 @@ namespace BCG
if (_is_ready())
{
_construct_schema();
HighPerformanceTimer timer = new HighPerformanceTimer();
timer.start();
var timer = Stopwatch.StartNew();
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
List<object[]> output_rows = new List<object[]>();
@@ -138,13 +142,12 @@ namespace BCG
}
}
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
timer.stop();
timer.Stop();
_n_total_fires++;
if (timer.lastIntervalTime >= 250)
{
if (timer.ElapsedMilliseconds >= 250) {
_n_expensive_fires++;
string log_msg = String.Format("Join and map time for {0} tables {1} ratio {2}/{3}", _book_keeping_data.Count, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+ $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
}
myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
}
@@ -154,15 +157,12 @@ namespace BCG
{
if (_output_mapper != null)
{
HighPerformanceTimer timer = new HighPerformanceTimer();
IBlueRowCollection output_rows;
timer.start();
var timer = Stopwatch.StartNew();
output_rows = _output_mapper.map_rows(unmapped_rows);
timer.stop();
if (timer.lastIntervalTime >= 50)
{
// string log_msg = String.Format("Time to map {0} rows into {1} rows {2}", unmapped_rows.Count, output_rows.Count, timer.lastIntervalTime);
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
timer.Stop();
if (timer.ElapsedMilliseconds >= 50) {
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
}
return output_rows;
}
@@ -1,11 +1,7 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Text;
using BCG.Utilities.General;
using BCG.Utilities;
using mbts;
using mbts.spark;
using Spark.Core.Collections;
namespace BCG
@@ -294,7 +290,6 @@ namespace BCG
protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
private bool myManageUnmappedLeftRows = false;
protected HighPerformanceTimer myTimer = new HighPerformanceTimer();
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
protected IBlueTableSchema myLeftSchema, myRightSchema;
protected JoinProjectHelper myJoinProjectHelper;
@@ -361,7 +356,6 @@ namespace BCG
private void myHandleStreamChanges(DataPacket dp, bool isLeft)
{
myTimer.start();
myInitSchema(dp.schema, isLeft);
BlueTable dels = null, adds = null;
if (myState != null)
@@ -371,7 +365,6 @@ namespace BCG
}
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
myAddRows(dp.addedRows, isLeft, myState, adds);
myTimer.stop();
if (myState != null)
{
myFireEvent(new DataPacket(myState, adds, dels));
@@ -1,11 +1,5 @@
using System;
using System.Data;
using System.Collections;
using Bts.DataStream;
using mbts;
using mbts.spark;
using mbts.config;
namespace BCG
@@ -21,7 +15,6 @@ namespace BCG
[DataStreamUserRespresentation (false)]
public class MapStream : MethodDrivenStream
{
private HighPerformanceTimer t = new HighPerformanceTimer ();
protected IStreamMapper myMapper;
protected IChangeableStreamMapper myChangeableMapper = null;
private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
@@ -7,6 +7,8 @@ using BCG.Utilities.General;
using ZstdSharp.Unsafe;
using System.Runtime.CompilerServices;
using mbts.spark;
using System.Diagnostics;
using bts;
namespace BCG {
@@ -50,8 +52,7 @@ namespace BCG {
}
private void _read_mogodb() {
HighPerformanceTimer timer = new HighPerformanceTimer();
timer.start();
var timer = Stopwatch.StartNew();
// Access the collection
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
@@ -66,12 +67,12 @@ namespace BCG {
}
_add_row(document);
}
timer.stop();
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
timer.start();
timer.Stop();
Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
timer.Restart();
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
timer.stop();
bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
timer.Stop();
Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
return;
myFireEvent(dp);
@@ -146,8 +147,7 @@ namespace BCG {
}
private void _read_mogodb() {
HighPerformanceTimer timer = new HighPerformanceTimer();
timer.start();
var timer = Stopwatch.StartNew();
// Access the collection
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
@@ -162,12 +162,12 @@ namespace BCG {
}
_add_doc(document);
}
timer.stop();
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
timer.start();
timer.Stop();
Log.Gen.info($"MongoDBStream time to read {_collection_name}: {timer.ElapsedMilliseconds}ms");
timer.Restart();
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
timer.stop();
bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
timer.Stop();
Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms");;
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
return;
myFireEvent(dp);
@@ -209,4 +209,4 @@ namespace BCG {
}
}
}
}
@@ -1,9 +1,6 @@
namespace BCG {
using mbts.spark;
using Spark.Core.Collections;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
@@ -14,7 +11,6 @@ namespace BCG {
private string myColExp, myTableName;
private ViewStream myVs;
private IDictionary<string, string> myDic;
private HighPerformanceTimer _timer = new();
/// <summary>
/// </summary>
/// <param name="vs">stream to susbsribe to as the input stream</param>
@@ -70,10 +66,8 @@ namespace BCG {
DataPacket dp = newValue as DataPacket;
if (myHelper != null)
{
_timer.start();
IBlueRowCollection dels, adds;
BlueDataUtility.getTrueDeletesAndAdds (dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols);
_timer.stop();
if (dels.Count + adds.Count > 0)
myFireEvent (new DataPacket (new ProjectedTable (dp.newRows, myHelper),
new ProjectedTable (adds, myHelper),
@@ -1,8 +1,6 @@
using System;
using System.Collections.Generic;
using System.Text;
using mbts;
using mbts.spark;
namespace BCG
{
@@ -183,7 +181,6 @@ namespace BCG
/// </summary>
public class MutableAppendStream : ViewStream
{
HighPerformanceTimer _timer = new HighPerformanceTimer();
private bool mySubscriptionActive = false;
private MutableAppendTable myTable;
private int myMovingIndex;
@@ -272,7 +269,6 @@ namespace BCG
private void myHandleStreamChanges(object newValue)
{
DataPacket dp = newValue as DataPacket;
_timer.start();
myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
BlueTable adds = new BlueTable(myTable.schema),
dels = new BlueTable(myTable.schema);
@@ -287,7 +283,6 @@ namespace BCG
IBlueRow nr = myTable.addRow(r, index);
adds.addRow(nr);
}
_timer.stop();
myFireEvent(new DataPacket(myTable, adds, dels));
}
}