From 9050318cebcc49161ca6bb536cd002832004d5e8 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Thu, 23 May 2024 18:36:58 +0000 Subject: [PATCH] refactor: Replace HighPerformanceTimer w/ Stopwatch This is needed to build DataStreamSuite on Linux. 'Stopwatch' is a platform independent timer class included with the System.Runtime library in modern dotnet. 'HighPerformanceTimer' is a BTS class that predates Stopwatch, and is NOT platform independent as it relies on Windows kernel32.dll QueryPerformanceCounter/Frequency fns. --- src/DataStreamSuite/BaseUtilities/Crono.cs | 29 ++++++----------- .../Primitives/Filter/FilterStream.cs | 3 -- .../NonIncremental/NonIncGroupByStream.cs | 32 +++++++++++-------- .../NonIncremental/NonIncJoinBookkeeper.cs | 17 ++++++---- .../Join/NonIncremental/NonIncJoinStream.cs | 28 ++++++++-------- .../Primitives/Join/SuperJoinStream.cs | 7 ---- .../DataStream/Primitives/Map/MapStream.cs | 7 ---- .../Primitives/MongoDB/MongoDBStream.cs | 30 ++++++++--------- .../Primitives/Project/ProjectStream.cs | 6 ---- .../Primitives/Union/MutableAppendStream.cs | 5 --- .../DataStreamCore/DataStream.cs | 11 ++++--- .../Subscription/TableAccumulator2.cs | 14 ++------ .../DataStreamCore/ViewStreamDatabase.cs | 26 +++++++-------- .../DataStreamQueryAnalyzer/MainForm.cs | 18 +++++------ .../PubSub/Misc/MasterWorker.cs | 12 ++++--- .../PubSub/Server/PubSubMessageReactor.cs | 8 +---- src/DataStreamSuite/PubSub/SubscriberCore.cs | 13 -------- .../Business/MarketData/BookStream.cs | 28 ++++------------ 18 files changed, 110 insertions(+), 184 deletions(-) diff --git a/src/DataStreamSuite/BaseUtilities/Crono.cs b/src/DataStreamSuite/BaseUtilities/Crono.cs index a2996614a..e6527fa70 100644 --- a/src/DataStreamSuite/BaseUtilities/Crono.cs +++ b/src/DataStreamSuite/BaseUtilities/Crono.cs @@ -1,7 +1,4 @@ -using mbts.spark; - -using System; -using System.Runtime; +using System.Diagnostics; namespace BCG.Utilities.General { @@ -12,30 +9,22 @@ namespace BCG.Utilities.General { public Crono() { - myTimer = new HighPerformanceTimer(); + myTimer = new(); } public void start() - {myTimer.start();} + {myTimer.Start();} public void stop() - {myTimer.stop();} + {myTimer.Stop();} - public TimeSpan time - { - get - { - return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond)); - } - } + public TimeSpan time => + myTimer.Elapsed; - public double miliseconds - { - get - {return myTimer.totalMilliseconds;} - } + public double miliseconds => + myTimer.Elapsed.TotalMilliseconds; - private HighPerformanceTimer myTimer; + private Stopwatch myTimer; } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs b/src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs index 7f9533064..5d0e24e63 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Filter/FilterStream.cs @@ -151,7 +151,6 @@ namespace BCG private string myFilterString; private BCG.StreamWorld.Addition.FilterStream myFilterStream; private bool myUseNewFilter = Configuration.get().UseNewFilter; - private HighPerformanceTimer _timer = new HighPerformanceTimer(); #region constructors public delegate bool OnFilter(IBlueRow r); @@ -226,7 +225,6 @@ namespace BCG private void myHandleCallbacks(object newValue) { - _timer.start(); DataPacket retVal; DataPacket arrivedPacket = newValue as DataPacket; myLastReceivedRows = arrivedPacket.newRows; @@ -255,7 +253,6 @@ namespace BCG if (myFilter.filter(r)) myLastFiredRows.addRow(r); } - _timer.stop(); myFireEvent(retVal); } diff --git a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs index 0d6e3c42d..2e0b28992 100644 --- a/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/GroupBy/NonIncremental/NonIncGroupByStream.cs @@ -1,10 +1,8 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using mbts; -using mbts.spark; +using System.Diagnostics; + +using bts; namespace BCG { @@ -14,7 +12,7 @@ namespace BCG private INonIncGroupByEvaluator _groupby_func; private BlueRowHasher _row_hasher; private string[] _groupby_cols; - private HighPerformanceTimer _timer = new HighPerformanceTimer(); + private Stopwatch _timer = new(); private int _n_total_fires = 0, _n_expensive_fires = 0; public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) @@ -33,21 +31,27 @@ namespace BCG { DataPacket dp = nv as DataPacket; ///grouping together - _timer.start(); + _timer.Restart(); IDictionary> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher); - _timer.stop(); - // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime)); + _timer.Stop(); + var groupElapsedMs = _timer.ElapsedMilliseconds; + if (groupElapsedMs >= 50) { + Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms"); + } + // Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime)); /// reducing - _timer.start(); + _timer.Restart(); IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows); - _timer.stop(); + _timer.Stop(); + var applyElapsedMs = _timer.ElapsedMilliseconds; _n_total_fires++; - if (_timer.lastIntervalTime > 50) + if (applyElapsedMs >= 50) { _n_expensive_fires++; - // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to evaluate {0} groups {1} , ratio {2}/{3}", grouped_rows.Count, _timer.lastIntervalTime, _n_expensive_fires, _n_total_fires)); + Log.DataFlow.info( + $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}"); } myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows)); - } + } } } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs index f585acb5d..ecc72a4d1 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinBookkeeper.cs @@ -1,8 +1,13 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Text; using System.Threading.Tasks; + +using bts; + + using mbts; using mbts.spark; @@ -40,8 +45,7 @@ namespace BCG public void _on_stream_cb(object dp_obj) { - HighPerformanceTimer timer = new HighPerformanceTimer(); - timer.start(); + var timer = Stopwatch.StartNew(); DataPacket dp = (DataPacket)dp_obj; if (_schema == null) { @@ -67,13 +71,12 @@ namespace BCG _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r; } } - timer.stop(); + timer.Stop(); _n_total_fires++; - if (timer.lastIntervalTime > 50) - { + if (timer.ElapsedMilliseconds >= 50) { _n_expensive_fires++; - string log_msg = String.Format("NonIncJoinBookkeepingData._on_stream_cb {0} rows {1} key cols {2} join pos time {3} ratio {4}/{5}", dp.newRows.Count, _join_col_indices.Length, _join_position, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires); - // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg); + Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} " + + $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}"); } if (on_new_data != null) on_new_data(); diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs index 0ba3113a0..e0bd6467e 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/NonIncremental/NonIncJoinStream.cs @@ -1,8 +1,13 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Text; using System.Threading.Tasks; + +using bts; + + using mbts; using mbts.spark; @@ -114,8 +119,7 @@ namespace BCG if (_is_ready()) { _construct_schema(); - HighPerformanceTimer timer = new HighPerformanceTimer(); - timer.start(); + var timer = Stopwatch.StartNew(); IList> left_row_list = _book_keeping_data[0].get_list(); List output_rows = new List(); @@ -138,13 +142,12 @@ namespace BCG } } IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows)); - timer.stop(); + timer.Stop(); _n_total_fires++; - if (timer.lastIntervalTime >= 250) - { + if (timer.ElapsedMilliseconds >= 250) { _n_expensive_fires++; - string log_msg = String.Format("Join and map time for {0} tables {1} ratio {2}/{3}", _book_keeping_data.Count, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires); - // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg); + string log_msg = $"Join and map time for {_book_keeping_data.Count} " + + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}"; } myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows)); } @@ -154,15 +157,12 @@ namespace BCG { if (_output_mapper != null) { - HighPerformanceTimer timer = new HighPerformanceTimer(); IBlueRowCollection output_rows; - timer.start(); + var timer = Stopwatch.StartNew(); output_rows = _output_mapper.map_rows(unmapped_rows); - timer.stop(); - if (timer.lastIntervalTime >= 50) - { - // string log_msg = String.Format("Time to map {0} rows into {1} rows {2}", unmapped_rows.Count, output_rows.Count, timer.lastIntervalTime); - // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg); + timer.Stop(); + if (timer.ElapsedMilliseconds >= 50) { + Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}"); } return output_rows; } diff --git a/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs b/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs index 35690b29a..b8fae4b51 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Join/SuperJoinStream.cs @@ -1,11 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Text; -using BCG.Utilities.General; -using BCG.Utilities; using mbts; -using mbts.spark; using Spark.Core.Collections; namespace BCG @@ -294,7 +290,6 @@ namespace BCG protected Dictionary myLeftUnmatchedRowsAdds = new Dictionary(), myLeftUnmatchedRowsDels = new Dictionary(); private bool myManageUnmappedLeftRows = false; - protected HighPerformanceTimer myTimer = new HighPerformanceTimer(); protected string myLeftProj = String.Empty, myRightProj = String.Empty; protected IBlueTableSchema myLeftSchema, myRightSchema; protected JoinProjectHelper myJoinProjectHelper; @@ -361,7 +356,6 @@ namespace BCG private void myHandleStreamChanges(DataPacket dp, bool isLeft) { - myTimer.start(); myInitSchema(dp.schema, isLeft); BlueTable dels = null, adds = null; if (myState != null) @@ -371,7 +365,6 @@ namespace BCG } myDeleteRows(dp.deletedRows, isLeft, myState, dels); myAddRows(dp.addedRows, isLeft, myState, adds); - myTimer.stop(); if (myState != null) { myFireEvent(new DataPacket(myState, adds, dels)); diff --git a/src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs b/src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs index 90d7c562d..ec6aaafd4 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Map/MapStream.cs @@ -1,11 +1,5 @@ -using System; -using System.Data; -using System.Collections; - using Bts.DataStream; -using mbts; -using mbts.spark; using mbts.config; namespace BCG @@ -21,7 +15,6 @@ namespace BCG [DataStreamUserRespresentation (false)] public class MapStream : MethodDrivenStream { - private HighPerformanceTimer t = new HighPerformanceTimer (); protected IStreamMapper myMapper; protected IChangeableStreamMapper myChangeableMapper = null; private bool myUseNewMap = Configuration.get().UseNewMap; diff --git a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs index 52015e568..d8854e353 100644 --- a/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs @@ -7,6 +7,8 @@ using BCG.Utilities.General; using ZstdSharp.Unsafe; using System.Runtime.CompilerServices; using mbts.spark; +using System.Diagnostics; +using bts; namespace BCG { @@ -50,8 +52,7 @@ namespace BCG { } private void _read_mogodb() { - HighPerformanceTimer timer = new HighPerformanceTimer(); - timer.start(); + var timer = Stopwatch.StartNew(); // Access the collection IMongoCollection collection = _database.GetCollection(name: _collection_name); @@ -66,12 +67,12 @@ namespace BCG { } _add_row(document); } - timer.stop(); - bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs"); - timer.start(); + timer.Stop(); + Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms"); + timer.Restart(); var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true); - timer.stop(); - bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs"); + timer.Stop(); + Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms"); if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0) return; myFireEvent(dp); @@ -146,8 +147,7 @@ namespace BCG { } private void _read_mogodb() { - HighPerformanceTimer timer = new HighPerformanceTimer(); - timer.start(); + var timer = Stopwatch.StartNew(); // Access the collection IMongoCollection collection = _database.GetCollection(name: _collection_name); @@ -162,12 +162,12 @@ namespace BCG { } _add_doc(document); } - timer.stop(); - bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs"); - timer.start(); + timer.Stop(); + Log.Gen.info($"MongoDBStream time to read {_collection_name}: {timer.ElapsedMilliseconds}ms"); + timer.Restart(); var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true); - timer.stop(); - bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs"); + timer.Stop(); + Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms");; if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0) return; myFireEvent(dp); @@ -209,4 +209,4 @@ namespace BCG { } } -} \ No newline at end of file +} diff --git a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs index 71cdc2c23..8667ca2cf 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Project/ProjectStream.cs @@ -1,9 +1,6 @@ namespace BCG { - using mbts.spark; - using Spark.Core.Collections; - using System; using System.Collections; using System.Collections.Generic; using System.Linq; @@ -14,7 +11,6 @@ namespace BCG { private string myColExp, myTableName; private ViewStream myVs; private IDictionary myDic; - private HighPerformanceTimer _timer = new(); /// /// /// stream to susbsribe to as the input stream @@ -70,10 +66,8 @@ namespace BCG { DataPacket dp = newValue as DataPacket; if (myHelper != null) { - _timer.start(); IBlueRowCollection dels, adds; BlueDataUtility.getTrueDeletesAndAdds (dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols); - _timer.stop(); if (dels.Count + adds.Count > 0) myFireEvent (new DataPacket (new ProjectedTable (dp.newRows, myHelper), new ProjectedTable (adds, myHelper), diff --git a/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs b/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs index 5b563f784..d6ab1cfe8 100644 --- a/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs +++ b/src/DataStreamSuite/DataStream/Primitives/Union/MutableAppendStream.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; -using System.Text; using mbts; -using mbts.spark; namespace BCG { @@ -183,7 +181,6 @@ namespace BCG /// public class MutableAppendStream : ViewStream { - HighPerformanceTimer _timer = new HighPerformanceTimer(); private bool mySubscriptionActive = false; private MutableAppendTable myTable; private int myMovingIndex; @@ -272,7 +269,6 @@ namespace BCG private void myHandleStreamChanges(object newValue) { DataPacket dp = newValue as DataPacket; - _timer.start(); myTable = myTable ?? new MutableAppendTable(new BlueTableSchema("MutableAppend", dp.schema.columnInfo)); BlueTable adds = new BlueTable(myTable.schema), dels = new BlueTable(myTable.schema); @@ -287,7 +283,6 @@ namespace BCG IBlueRow nr = myTable.addRow(r, index); adds.addRow(nr); } - _timer.stop(); myFireEvent(new DataPacket(myTable, adds, dels)); } } diff --git a/src/DataStreamSuite/DataStreamCore/DataStream.cs b/src/DataStreamSuite/DataStreamCore/DataStream.cs index df63d5ce3..071b8b834 100644 --- a/src/DataStreamSuite/DataStreamCore/DataStream.cs +++ b/src/DataStreamSuite/DataStreamCore/DataStream.cs @@ -8,6 +8,8 @@ using System.Collections; using mbts; using mbts.config; using mbts.spark; +using System.Diagnostics; +using bts; namespace BCG { @@ -167,18 +169,17 @@ namespace BCG internal class ChangeDelegateWrapper { - private HighPerformanceTimer myTimer = new HighPerformanceTimer (); + readonly Stopwatch timer = new(); private DataStream.ChangeDelegate myDel; internal ChangeDelegateWrapper (DataStream.ChangeDelegate del) {myDel = del;} internal void wrapperFunction (object o) { - myTimer.start (); + timer.Restart(); myDel (o); - myTimer.stop (); - string minfo = "Type = " + myDel.Method.DeclaringType.FullName + " , method = " + myDel.Method.Name; - // BlueEventLog.logEvent (minfo + " , time = " + myTimer.lastIntervalTime.ToString (), BlueEventLevel.Performance); + timer.Stop(); + Log.Gen.diag($"Type: {myDel.Method.DeclaringType.FullName}, Method = {myDel.Method.Name}"); } } } diff --git a/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs b/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs index 5a97de700..0f44e6c00 100644 --- a/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs +++ b/src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs @@ -1,19 +1,10 @@ using System; using System.IO; -//using BCG.Utilities; -using mbts; -using mbts.spark; -using bts; -using System; -using System.Collections; -using System.IO; -using BCG.Utilities; -using BCG.Utilities.General; using System.Collections.Generic; -using System.Threading; -using mbts; using System.Linq; +using bts; + namespace BCG { // This class is responsible for mantaining the state fo a table. It gets the // bytes from the pubsubserver via the handle_updates call (called by VSD), deserializes @@ -21,7 +12,6 @@ namespace BCG { public class TableAccumulator2 : ITableAccumulator { private readonly string _tag; private readonly ISubscriberCore _subscriber_core; - private HighPerformanceTimer _timer = new HighPerformanceTimer(); IStreamMsgDeserilizer _deser = new StreamIODeserializationController(); private BlueTable _table; private TableUpdateDelegate _on_update_delegate; diff --git a/src/DataStreamSuite/DataStreamCore/ViewStreamDatabase.cs b/src/DataStreamSuite/DataStreamCore/ViewStreamDatabase.cs index de0552cec..a314b308b 100644 --- a/src/DataStreamSuite/DataStreamCore/ViewStreamDatabase.cs +++ b/src/DataStreamSuite/DataStreamCore/ViewStreamDatabase.cs @@ -8,10 +8,9 @@ using System.Collections.Generic; using System.Threading; using mbts; using System.Linq; +using System.Diagnostics; namespace BCG { - using HighPerformanceTimer = mbts.spark.HighPerformanceTimer; - public class ViewStreamDatabase : IWorker, IDisposable { internal delegate void TagFireDelegate(string tag); public delegate void DoWorkDelegate(); @@ -30,9 +29,9 @@ namespace BCG { private bool myIsDisposed = false; private bool _is_in_do_work = false; private static object myTimeInterval; - private HighPerformanceTimer _timer = new HighPerformanceTimer(); - private readonly double myLogTimeThreshold = 10000; - private readonly double _dowork_threshold = 500; + readonly Stopwatch _timer = new(); + readonly TimeSpan myLogTimeThreshold = TimeSpan.FromSeconds(10); + readonly TimeSpan _dowork_threshold = TimeSpan.FromMilliseconds(500); private Thread myThreadOwner = null; // ViewStreamDatabase takes a thread affinity when created. If called from a different thread it will assert. private static readonly object myInstanceLockObject = new object(); @@ -123,16 +122,16 @@ namespace BCG { bts.Log.Gen.warn("Received callback for tag: {tag}"); continue; } - _timer.start(); + _timer.Restart(); if (onTagFireStart != null) onTagFireStart(tag); table_accumulator.handle_updates(tag_updates); if (onTagFireEnd != null) onTagFireEnd(tag); - _timer.stop(); - if (_timer.lastIntervalTime > myLogTimeThreshold) { - bts.Log.Gen.error($"ViewStreamDatabase.doWork tag: {tag} took {_timer.lastIntervalTime} msecs"); + _timer.Stop(); + if (_timer.Elapsed > myLogTimeThreshold) { + bts.Log.Gen.error($"ViewStreamDatabase.doWork tag: {tag} took {_timer.Elapsed}"); } } } @@ -142,11 +141,12 @@ namespace BCG { _is_worker_call_in_progress = true; _pending_worker_ops.Clear(); foreach (IWorker w in _workers) { - _timer.start(); + _timer.Restart(); w.doWork(); - _timer.stop(); - if (_timer.lastIntervalTime > _dowork_threshold) - bts.Log.Gen.warn($"VSD.dowork on worker {w} took {_timer.lastIntervalTime}"); + _timer.Stop(); + if (_timer.Elapsed > _dowork_threshold) { + bts.Log.Gen.warn($"VSD.dowork on worker {w} took {_timer.Elapsed}"); + } } _is_worker_call_in_progress = false; diff --git a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs index cf49929e5..29fa9e0c6 100644 --- a/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs +++ b/src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs @@ -300,10 +300,10 @@ namespace BCG { return Math.Max(header_width, min_width); } } - private HighPerformanceTimer _timer = new HighPerformanceTimer(); + // this is the stream callback private void myCallback(object o) { - _timer.start(); + var timer = Stopwatch.StartNew(); DataPacket dp = o as DataPacket; myCrono.stop(); Log.Ui.diag($"Received callback for {dp.schema.tableName}"); @@ -342,9 +342,9 @@ namespace BCG { myConsoleRichTextBox.Clear(); // this was was very very slow //myConsoleRichTextBox.Text = BlueDataUtility.get_printed_rows(dp.newRows, (col) => 30); - _timer.stop(); - var msecs = _timer.lastIntervalTime; - bts.Log.Gen.info($"DSQA MainForm.myCallback table: {dp.schema.tableName} rows: {dp.newRows.Count} time: {msecs}"); + timer.Stop(); + Log.Gen.info( + $"DSQA MainForm.myCallback table: {dp.schema.tableName} rows: {dp.newRows.Count} time: {timer.Elapsed.TotalMilliseconds:0.00}ms"); } private Crono myCrono; @@ -646,7 +646,7 @@ namespace BCG { } private void testGridToolStripMenuItem_Click(object sender, EventArgs e) { - HighPerformanceTimer timer = new HighPerformanceTimer(); + var timer = new Stopwatch(); //while (true) //{ // Random rnd = new Random(DateTime.Now.Millisecond); @@ -668,12 +668,12 @@ namespace BCG { rows.Add(r); for (int i = 0; i < 100; i++) { - timer.start(); + timer.Restart(); rows.Sort(delegate (IBlueRow r1, IBlueRow r2) { return _sort_dicts_index(r1, r2, 5, true); }); - timer.stop(); - Log.Ui.diag($"nrows = {rows.Count} sorting time = {timer.lastIntervalTime}"); + timer.Stop(); + Log.Ui.diag($"nrows = {rows.Count} sorting time = {timer.Elapsed.TotalMilliseconds:0.00}ms"); } } diff --git a/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs b/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs index d3f3e1962..eefd545a8 100644 --- a/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs +++ b/src/DataStreamSuite/PubSub/Misc/MasterWorker.cs @@ -5,6 +5,8 @@ using BCG.Utilities.General; using System.Threading; using mbts; using mbts.spark; +using System.Diagnostics; +using bts; //using BCG.Utilities; namespace BCG @@ -19,7 +21,7 @@ namespace BCG protected Dictionary> myWorkers = new Dictionary>(), myJustAdded = new Dictionary>(), myJustRemoved = new Dictionary>(); - private HighPerformanceTimer myTimer = new HighPerformanceTimer(); + readonly Stopwatch timer = new(); private static MasterWorker myThreadDriven = null, myDoWorkDriven = null; internal MasterWorker() @@ -82,7 +84,7 @@ namespace BCG myJustRemoved.Clear(); if (DateTime.UtcNow - myLastDoWorkTime >= myPeriod) { - myTimer.start(); + timer.Restart(); //myLogEntries(); myLastDoWorkTime = DateTime.UtcNow; foreach (KeyValuePair> p in myWorkers) @@ -93,9 +95,9 @@ namespace BCG p.Value.second = DateTime.UtcNow; } } - myTimer.stop(); - if (myTimer.lastIntervalTime > 100){ - //BlueEventLog.logEvent(String.Format("{0} : took {1} msecs", GetType().ToString(), myTimer.lastIntervalTime), PS3LogLevels.performance); + timer.Stop(); + if (timer.ElapsedMilliseconds >= 100){ + Log.DataFlow.diag($"{GetType()}: took {timer.ElapsedMilliseconds}ms"); } } diff --git a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs index b41b08199..7666c52e0 100644 --- a/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs +++ b/src/DataStreamSuite/PubSub/Server/PubSubMessageReactor.cs @@ -1,12 +1,7 @@ using System; -using System.IO; -using System.Text; using System.Threading; -using System.Collections.Generic; -using mbts; -using mbts.spark; + using bts; -using System.Data; namespace BCG { // This is the main PubSub Server class where it all starts (called by Program.cs). Its constructed and start is called and thats it. @@ -27,7 +22,6 @@ namespace BCG { private delegate void SingleTagTimeSliceDelegate(string tag, double processingTimeSeconds); private SingleTagTimeSliceDelegate onSingleTagPub; - private HighPerformanceTimer mySingleTagTimeSliceTimer = new HighPerformanceTimer(); private static SyncQueue _internal_pub_queue = new SyncQueue(); //this thing keeps track of whether a given tag is compressed or not. It is used for logging messages. diff --git a/src/DataStreamSuite/PubSub/SubscriberCore.cs b/src/DataStreamSuite/PubSub/SubscriberCore.cs index 9fbd99d28..a317b24ca 100644 --- a/src/DataStreamSuite/PubSub/SubscriberCore.cs +++ b/src/DataStreamSuite/PubSub/SubscriberCore.cs @@ -1,17 +1,5 @@ using System; -//using System.Messaging; -using System.Collections; using System.Collections.Generic; -using System.IO; -using System.Threading; -using System.Text.RegularExpressions; - -using BCG; -using BCG.Utilities; -using BCG.Utilities.General; -using mbts; -using mbts.spark; -//using BCG.Utilities.BlueConfiguration; namespace BCG { @@ -27,7 +15,6 @@ namespace BCG { public class SubscriberCore : ISubscriberCore { private Dictionary _server_url_to_subcore_map = new Dictionary(); private IPubSubRouter _tag_router; - private HighPerformanceTimer myTimer = new HighPerformanceTimer(); private PubSubServerConnectionFactory _connection_factory = new PubSubServerConnectionFactory(true); public event PubSubConnectionChange onPubSubConnectionChange; diff --git a/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs b/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs index 02824ad5c..43caebf85 100644 --- a/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs +++ b/src/DataStreamSuite/SharedStreams/Business/MarketData/BookStream.cs @@ -1,35 +1,19 @@ namespace BCG { using BCG.Utilities.General; - using bts.api; - using bts.api.domain; - using bts.api.synthetic; using bts.md; - using bts.spark; - using bts.utils; - using mbts; - using mbts.co2; - using mbts.config; - using mbts.spark; - using mbts.ui; - - using Spark.Core.Data; - using Spark.Core.Disposables; using Spark.Core.Reactive; using System; - using System.Collections.Concurrent; using System.Collections.Generic; - using System.Collections.ObjectModel; using System.Linq; using System.Reactive.Linq; using bts.api.core; using Spark.Api.MarketData; using SecurityKey = System.UInt32; - using System.Data; - using System.Reflection.Metadata; + using System.Diagnostics; [DataStreamUserRespresentation("books")] public class BookStream : ViewStream, IWorker { @@ -39,7 +23,7 @@ namespace BCG { private int _id = 0; IReadOnlyDictionary _cached_state = null; MarketDataClient _md_client; - HighPerformanceTimer _timer = new HighPerformanceTimer(); + readonly Stopwatch _timer = new(); private readonly IBlueTableSchema _bbo_schema = BlueDataUtility.form_schema("books", Tuple.Create("id", TypeCode.Int32), Tuple.Create("sec_key", TypeCode.UInt32), @@ -106,7 +90,7 @@ namespace BCG { private void _scan_bbo() { // this shouldn be made incremental using the TableUpdate bts.Log.Gen.info("BookStream._scan_bbo"); - _timer.start(); + _timer.Restart(); if (_cached_state == null) return; BlueTable new_state = new BlueTable(_bbo_schema); @@ -131,8 +115,8 @@ namespace BCG { } if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0) return; - _timer.stop(); - bts.Log.Gen.info($"BookStream._scan_bbo time: {_timer.lastIntervalTime} msecs"); + _timer.Stop(); + bts.Log.Gen.info($"BookStream._scan_bbo time: {_timer.Elapsed} msecs"); myFireEvent(dp); } @@ -143,4 +127,4 @@ namespace BCG { } } } -} \ No newline at end of file +}