refactor: Replace HighPerformanceTimer w/ Stopwatch
This is needed to build DataStreamSuite on Linux. 'Stopwatch' is a platform independent timer class included with the System.Runtime library in modern dotnet. 'HighPerformanceTimer' is a BTS class that predates Stopwatch, and is NOT platform independent as it relies on Windows kernel32.dll QueryPerformanceCounter/Frequency fns.
This commit is contained in:
1 parent
7ee6933789
commit
9050318ceb
18 files changed
+110
-184
No files matched your search
@@ -151,7 +151,6 @@ namespace BCG
|
||||
private string myFilterString;
|
||||
private BCG.StreamWorld.Addition.FilterStream myFilterStream;
|
||||
private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
|
||||
private HighPerformanceTimer _timer = new HighPerformanceTimer();
|
||||
|
||||
#region constructors
|
||||
public delegate bool OnFilter(IBlueRow r);
|
||||
@@ -226,7 +225,6 @@ namespace BCG
|
||||
|
||||
private void myHandleCallbacks(object newValue)
|
||||
{
|
||||
_timer.start();
|
||||
DataPacket retVal;
|
||||
DataPacket arrivedPacket = newValue as DataPacket;
|
||||
myLastReceivedRows = arrivedPacket.newRows;
|
||||
@@ -255,7 +253,6 @@ namespace BCG
|
||||
if (myFilter.filter(r))
|
||||
myLastFiredRows.addRow(r);
|
||||
}
|
||||
_timer.stop();
|
||||
myFireEvent(retVal);
|
||||
}
|
||||
|
||||
|
||||
+18
-14
@@ -1,10 +1,8 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using mbts;
|
||||
using mbts.spark;
|
||||
using System.Diagnostics;
|
||||
|
||||
using bts;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -14,7 +12,7 @@ namespace BCG
|
||||
private INonIncGroupByEvaluator _groupby_func;
|
||||
private BlueRowHasher _row_hasher;
|
||||
private string[] _groupby_cols;
|
||||
private HighPerformanceTimer _timer = new HighPerformanceTimer();
|
||||
private Stopwatch _timer = new();
|
||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||
|
||||
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
|
||||
@@ -33,21 +31,27 @@ namespace BCG
|
||||
{
|
||||
DataPacket dp = nv as DataPacket;
|
||||
///grouping together
|
||||
_timer.start();
|
||||
_timer.Restart();
|
||||
IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
|
||||
_timer.stop();
|
||||
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
||||
_timer.Stop();
|
||||
var groupElapsedMs = _timer.ElapsedMilliseconds;
|
||||
if (groupElapsedMs >= 50) {
|
||||
Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
|
||||
}
|
||||
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
||||
/// reducing
|
||||
_timer.start();
|
||||
_timer.Restart();
|
||||
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
||||
_timer.stop();
|
||||
_timer.Stop();
|
||||
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
||||
_n_total_fires++;
|
||||
if (_timer.lastIntervalTime > 50)
|
||||
if (applyElapsedMs >= 50)
|
||||
{
|
||||
_n_expensive_fires++;
|
||||
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to evaluate {0} groups {1} , ratio {2}/{3}", grouped_rows.Count, _timer.lastIntervalTime, _n_expensive_fires, _n_total_fires));
|
||||
Log.DataFlow.info(
|
||||
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
|
||||
}
|
||||
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+10
-7
@@ -1,8 +1,13 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using bts;
|
||||
|
||||
|
||||
using mbts;
|
||||
using mbts.spark;
|
||||
|
||||
@@ -40,8 +45,7 @@ namespace BCG
|
||||
|
||||
public void _on_stream_cb(object dp_obj)
|
||||
{
|
||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||
timer.start();
|
||||
var timer = Stopwatch.StartNew();
|
||||
DataPacket dp = (DataPacket)dp_obj;
|
||||
if (_schema == null)
|
||||
{
|
||||
@@ -67,13 +71,12 @@ namespace BCG
|
||||
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
|
||||
}
|
||||
}
|
||||
timer.stop();
|
||||
timer.Stop();
|
||||
_n_total_fires++;
|
||||
if (timer.lastIntervalTime > 50)
|
||||
{
|
||||
if (timer.ElapsedMilliseconds >= 50) {
|
||||
_n_expensive_fires++;
|
||||
string log_msg = String.Format("NonIncJoinBookkeepingData._on_stream_cb {0} rows {1} key cols {2} join pos time {3} ratio {4}/{5}", dp.newRows.Count, _join_col_indices.Length, _join_position, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
|
||||
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
|
||||
Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} "
|
||||
+ $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}");
|
||||
}
|
||||
if (on_new_data != null)
|
||||
on_new_data();
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using bts;
|
||||
|
||||
|
||||
using mbts;
|
||||
using mbts.spark;
|
||||
|
||||
@@ -114,8 +119,7 @@ namespace BCG
|
||||
if (_is_ready())
|
||||
{
|
||||
_construct_schema();
|
||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||
timer.start();
|
||||
var timer = Stopwatch.StartNew();
|
||||
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
|
||||
List<object[]> output_rows = new List<object[]>();
|
||||
|
||||
@@ -138,13 +142,12 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
|
||||
timer.stop();
|
||||
timer.Stop();
|
||||
_n_total_fires++;
|
||||
if (timer.lastIntervalTime >= 250)
|
||||
{
|
||||
if (timer.ElapsedMilliseconds >= 250) {
|
||||
_n_expensive_fires++;
|
||||
string log_msg = String.Format("Join and map time for {0} tables {1} ratio {2}/{3}", _book_keeping_data.Count, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
|
||||
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
|
||||
string log_msg = $"Join and map time for {_book_keeping_data.Count} "
|
||||
+ $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
|
||||
}
|
||||
myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
|
||||
}
|
||||
@@ -154,15 +157,12 @@ namespace BCG
|
||||
{
|
||||
if (_output_mapper != null)
|
||||
{
|
||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||
IBlueRowCollection output_rows;
|
||||
timer.start();
|
||||
var timer = Stopwatch.StartNew();
|
||||
output_rows = _output_mapper.map_rows(unmapped_rows);
|
||||
timer.stop();
|
||||
if (timer.lastIntervalTime >= 50)
|
||||
{
|
||||
// string log_msg = String.Format("Time to map {0} rows into {1} rows {2}", unmapped_rows.Count, output_rows.Count, timer.lastIntervalTime);
|
||||
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
|
||||
timer.Stop();
|
||||
if (timer.ElapsedMilliseconds >= 50) {
|
||||
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
|
||||
}
|
||||
return output_rows;
|
||||
}
|
||||
|
||||
@@ -1,11 +1,7 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using BCG.Utilities.General;
|
||||
using BCG.Utilities;
|
||||
using mbts;
|
||||
using mbts.spark;
|
||||
using Spark.Core.Collections;
|
||||
|
||||
namespace BCG
|
||||
@@ -294,7 +290,6 @@ namespace BCG
|
||||
protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
|
||||
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
|
||||
private bool myManageUnmappedLeftRows = false;
|
||||
protected HighPerformanceTimer myTimer = new HighPerformanceTimer();
|
||||
protected string myLeftProj = String.Empty, myRightProj = String.Empty;
|
||||
protected IBlueTableSchema myLeftSchema, myRightSchema;
|
||||
protected JoinProjectHelper myJoinProjectHelper;
|
||||
@@ -361,7 +356,6 @@ namespace BCG
|
||||
|
||||
private void myHandleStreamChanges(DataPacket dp, bool isLeft)
|
||||
{
|
||||
myTimer.start();
|
||||
myInitSchema(dp.schema, isLeft);
|
||||
BlueTable dels = null, adds = null;
|
||||
if (myState != null)
|
||||
@@ -371,7 +365,6 @@ namespace BCG
|
||||
}
|
||||
myDeleteRows(dp.deletedRows, isLeft, myState, dels);
|
||||
myAddRows(dp.addedRows, isLeft, myState, adds);
|
||||
myTimer.stop();
|
||||
if (myState != null)
|
||||
{
|
||||
myFireEvent(new DataPacket(myState, adds, dels));
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
using System;
|
||||
using System.Data;
|
||||
using System.Collections;
|
||||
|
||||
using Bts.DataStream;
|
||||
|
||||
using mbts;
|
||||
using mbts.spark;
|
||||
using mbts.config;
|
||||
|
||||
namespace BCG
|
||||
@@ -21,7 +15,6 @@ namespace BCG
|
||||
[DataStreamUserRespresentation (false)]
|
||||
public class MapStream : MethodDrivenStream
|
||||
{
|
||||
private HighPerformanceTimer t = new HighPerformanceTimer ();
|
||||
protected IStreamMapper myMapper;
|
||||
protected IChangeableStreamMapper myChangeableMapper = null;
|
||||
private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
|
||||
|
||||
@@ -7,6 +7,8 @@ using BCG.Utilities.General;
|
||||
using ZstdSharp.Unsafe;
|
||||
using System.Runtime.CompilerServices;
|
||||
using mbts.spark;
|
||||
using System.Diagnostics;
|
||||
using bts;
|
||||
|
||||
|
||||
namespace BCG {
|
||||
@@ -50,8 +52,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
private void _read_mogodb() {
|
||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||
timer.start();
|
||||
var timer = Stopwatch.StartNew();
|
||||
// Access the collection
|
||||
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
||||
|
||||
@@ -66,12 +67,12 @@ namespace BCG {
|
||||
}
|
||||
_add_row(document);
|
||||
}
|
||||
timer.stop();
|
||||
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
|
||||
timer.start();
|
||||
timer.Stop();
|
||||
Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
|
||||
timer.Restart();
|
||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||
timer.stop();
|
||||
bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
|
||||
timer.Stop();
|
||||
Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
|
||||
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||
return;
|
||||
myFireEvent(dp);
|
||||
@@ -146,8 +147,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
private void _read_mogodb() {
|
||||
HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||
timer.start();
|
||||
var timer = Stopwatch.StartNew();
|
||||
// Access the collection
|
||||
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
||||
|
||||
@@ -162,12 +162,12 @@ namespace BCG {
|
||||
}
|
||||
_add_doc(document);
|
||||
}
|
||||
timer.stop();
|
||||
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
|
||||
timer.start();
|
||||
timer.Stop();
|
||||
Log.Gen.info($"MongoDBStream time to read {_collection_name}: {timer.ElapsedMilliseconds}ms");
|
||||
timer.Restart();
|
||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||
timer.stop();
|
||||
bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
|
||||
timer.Stop();
|
||||
Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms");;
|
||||
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||
return;
|
||||
myFireEvent(dp);
|
||||
@@ -209,4 +209,4 @@ namespace BCG {
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,6 @@
|
||||
namespace BCG {
|
||||
using mbts.spark;
|
||||
|
||||
using Spark.Core.Collections;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
@@ -14,7 +11,6 @@ namespace BCG {
|
||||
private string myColExp, myTableName;
|
||||
private ViewStream myVs;
|
||||
private IDictionary<string, string> myDic;
|
||||
private HighPerformanceTimer _timer = new();
|
||||
/// <summary>
|
||||
/// </summary>
|
||||
/// <param name="vs">stream to susbsribe to as the input stream</param>
|
||||
@@ -70,10 +66,8 @@ namespace BCG {
|
||||
DataPacket dp = newValue as DataPacket;
|
||||
if (myHelper != null)
|
||||
{
|
||||
_timer.start();
|
||||
IBlueRowCollection dels, adds;
|
||||
BlueDataUtility.getTrueDeletesAndAdds (dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols);
|
||||
_timer.stop();
|
||||
if (dels.Count + adds.Count > 0)
|
||||
myFireEvent (new DataPacket (new ProjectedTable (dp.newRows, myHelper),
|
||||
new ProjectedTable (adds, myHelper),
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using mbts;
|
||||
using mbts.spark;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -183,7 +181,6 @@ namespace BCG
|
||||
/// </summary>
|
||||
public class MutableAppendStream : ViewStream
|
||||
{
|
||||
HighPerformanceTimer _timer = new HighPerformanceTimer();
|
||||
private bool mySubscriptionActive = false;
|
||||
private MutableAppendTable myTable;
|
||||
private int myMovingIndex;
|
||||
@@ -272,7 +269,6 @@ namespace BCG
|
||||
private void myHandleStreamChanges(object newValue)
|
||||
{
|
||||
DataPacket dp = newValue as DataPacket;
|
||||
_timer.start();
|
||||
myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo));
|
||||
BlueTable adds = new BlueTable(myTable.schema),
|
||||
dels = new BlueTable(myTable.schema);
|
||||
@@ -287,7 +283,6 @@ namespace BCG
|
||||
IBlueRow nr = myTable.addRow(r, index);
|
||||
adds.addRow(nr);
|
||||
}
|
||||
_timer.stop();
|
||||
myFireEvent(new DataPacket(myTable, adds, dels));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user