From 5f12e2f1b27ed3d8772ce66a23ca144adcb996bd Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Wed, 5 Dec 2018 22:03:39 +0000 Subject: [PATCH] Retrofied new perf log to stats web page. --- src/lib/CMakeDeps.cmake | 2 +- src/lib/tracing/PerfLog.cpp | 433 ++++++++++++++++++++++++++++++++---- src/lib/tracing/PerfLog.h | 44 ++-- 3 files changed, 420 insertions(+), 59 deletions(-) diff --git a/src/lib/CMakeDeps.cmake b/src/lib/CMakeDeps.cmake index f4f65e19d..4c370f0e6 100644 --- a/src/lib/CMakeDeps.cmake +++ b/src/lib/CMakeDeps.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamicOrderRegistry.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/PerfTraceLog.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/CoverOrder.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/OffsetOrder.cpp;orderman/orders/book_algos/TickOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;orderman2/OrderManager2.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/persistence_utils.cpp;persistence/tests/PersistenceServiceTests.cpp;plugin/SparkServices.cpp;plugin/plugin.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;tracing/PerfLog.cpp;tracing/bts_trace.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamicOrderRegistry.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/PerfTraceLog.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/CoverOrder.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/OffsetOrder.cpp;orderman/orders/book_algos/TickOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;orderman2/orders/NativeOrder.cpp;orderman2/orders/Order.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/persistence_utils.cpp;persistence/tests/PersistenceServiceTests.cpp;plugin/SparkServices.cpp;plugin/plugin.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;tracing/PerfLog.cpp;tracing/bts_trace.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/thread_manager.cpp) \ No newline at end of file diff --git a/src/lib/tracing/PerfLog.cpp b/src/lib/tracing/PerfLog.cpp index 849020d85..34b21909a 100644 --- a/src/lib/tracing/PerfLog.cpp +++ b/src/lib/tracing/PerfLog.cpp @@ -1,27 +1,28 @@ - - #include "lib/tracing/PerfLog.h" -#include "lib/persistence/persistence_utils.h" -#include "lib/utils/datetime.h" #include "lib/config/config.h" #include "lib/config/protos/system.pb.h" - +#include "lib/persistence/persistence_utils.h" +#include "lib/utils/datetime.h" +#include "lib/utils/stats.h" + #include #include #include +#include +#include #include #include - + namespace bts { namespace trace { - + auto create_session = "create table session (" "version TEXT, " "host TEXT, " "build_type TEXT, " "session_start TIMESTAMP PRIMARY KEY" ")"; - + auto create_trace = "create table trace (" "session_start TIMESTAMP," "ts INTEGER," @@ -35,7 +36,7 @@ auto create_trace = "create table trace (" "uint64_1 INTEGER," "thread TEXT" ")"; - + auto insert_trace = "insert into trace (" "session_start," "ts," @@ -61,7 +62,7 @@ auto insert_trace = "insert into trace (" ":uint64_1," ":thread" ")"; - + auto insert_session = "insert into session (" "version," "host," @@ -73,27 +74,122 @@ auto insert_session = "insert into session (" ":build_type," ":session_start" ")"; - + +auto hedge_events_view = + "drop view if exists hedge_events; create view hedge_events as " + "select " + " trace.session_start, " + " channel, " + " sn, " + " min(hts) hts, " + " (max(ts)-min(ts))/1000 t " + "from " + " trace, " + " session " + "where " + " trace.session_start = session.session_start and " + " (entry_kind='fix_socket_r' or entry_kind='before_fix_socket_w') " + "group by " + " trace.session_start, " + " channel, " + " sn " + "having " + " count(*) = 2 and " + " min(entry_kind)<>max(entry_kind)"; + +auto hedge_trace_view = + "drop view if exists hedge_trace; create view hedge_trace as " + "select " + " session.version, " + " session.host, " + " session.build_type, " + " session.session_start, " + " trace.hts trace_time, " + " trace.uint32_1 order_key, " + " hedge_events.t total " + "from " + " trace, " + " hedge_events, " + " session " + "where " + " entry_kind='on_activate_start' and " + " trace.channel = hedge_events.channel and " + " trace.session_start = hedge_events.session_start and " + " trace.sn = hedge_events.sn and " + " trace.session_start = session.session_start "; + +auto requote_events_view = + "drop view if exists requote_events; create view requote_events as " + "select " + " trace.session_start, " + " channel, " + " sn, " + " min(hts) hts, " + " (max(ts)-min(ts))/1000 t " + "from " + " trace, " + " session " + "where " + " trace.session_start = session.session_start and " + " (entry_kind='md' or entry_kind='before_fix_socket_w') " + "group by " + " trace.session_start, " + " channel, " + " sn " + "having " + " count(*) = 2 and " + " min(entry_kind)<>max(entry_kind) "; + +auto requote_trace_view = + "drop view if exists requote_trace; create view requote_trace as " + "select " + " session.version, " + " session.host, " + " session.build_type, " + " session.session_start, " + " trace.hts trace_time, " + " trace.uint32_1 order_key, " + " requote_events.t total " + "from " + " trace, " + " requote_events, " + " session " + "where " + " entry_kind='order_update' and " + " trace.channel = requote_events.channel and " + " trace.session_start = requote_events.session_start and " + " trace.sn = requote_events.sn and " + " trace.session_start = session.session_start "; + +auto requote_query = + "select * from requote_trace order by trace_time desc limit ?"; + +auto hedge_query = "select * from hedge_trace order by trace_time desc limit ?"; + struct PerfLog::PImpl { PImpl(std::string const &loc) : db(loc, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE), - insert_trace_statement(), insert_session_statement(), session_start() {} - + insert_trace_statement(), insert_session_statement(), session_start(), + disabled(false), lock() {} + SQLite::Database db; std::unique_ptr insert_trace_statement; std::unique_ptr insert_session_statement; + std::unique_ptr requote_query_statement; + std::unique_ptr hedge_query_statement; std::string session_start; bool disabled; + std::mutex lock; }; - + PerfLog::PerfLog() : _pending(), _disabled(false), me() { auto loc = init_storage_location(); - + if (loc.empty()) { _disabled = true; return; } - + try { me = std::make_unique(loc); if (!me->db.tableExists("session")) { @@ -102,17 +198,26 @@ PerfLog::PerfLog() : _pending(), _disabled(false), me() { me->db.exec(create_session); me->db.exec(create_trace); } - + + me->db.exec(hedge_events_view); + me->db.exec(hedge_trace_view); + me->db.exec(requote_events_view); + me->db.exec(requote_trace_view); + me->insert_trace_statement = std::make_unique(me->db, insert_trace); me->insert_session_statement = std::make_unique(me->db, insert_session); + me->requote_query_statement = + std::make_unique(me->db, requote_query); + me->hedge_query_statement = + std::make_unique(me->db, hedge_query); me->session_start = datetime::FIX_timestamp(datetime::local_now()); - - me->insert_session_statement->bind(":version", - config::get().version()); - me->insert_session_statement->bind(":host", - config::get().host_name()); + + me->insert_session_statement->bind( + ":version", config::get().version()); + me->insert_session_statement->bind( + ":host", config::get().host_name()); #ifdef DEBUG me->insert_session_statement->bind(":build_type", "debug"); #else @@ -127,30 +232,29 @@ PerfLog::PerfLog() : _pending(), _disabled(false), me() { _disabled = true; } } - + PerfLog::~PerfLog() {} - + std::string PerfLog::init_storage_location() { - auto loc = persistence::storage_location("perf_trace2"); + auto loc = persistence::storage_location("perf_log"); auto parent = boost::filesystem::path(loc).parent_path(); - + if (!boost::filesystem::is_directory(parent)) { if (boost::filesystem::create_directories(parent)) return loc; else return ""; - } - else + } else return loc; } - + size_t PerfLog::store_log_entries() { if (_disabled) return 0; - + PerfLogEntry ev; size_t count = 0; - while (_pending.try_pop(ev)) + while (_pending.try_pop(ev)) try { me->insert_trace_statement->bind(":session_start", me->session_start); me->insert_trace_statement->bind(":ts", (long long)ev.ts.to_uint64()); @@ -164,21 +268,274 @@ size_t PerfLog::store_log_entries() { me->insert_trace_statement->bind(":uint32_1", ev.uint32_1); me->insert_trace_statement->bind(":uint32_2", ev.uint32_2); me->insert_trace_statement->bind(":uint64_1", (long long)ev.uint64_1); - + char tn[17]; - pthread_getname_np(pthread_self(), tn,16); + pthread_getname_np(ev.thread_id, tn, 16); me->insert_trace_statement->bind(":thread", tn); me->insert_trace_statement->exec(); me->insert_trace_statement->reset(); - count ++; + count++; } catch (SQLite::Exception const &e) { LOG(persist, Error, "Unable to insert perf trace in {}. Error: {}", std::string(me->db.getFilename()), std::string(e.getErrorStr())); - - _disabled = true; + + _disabled = true; } return count; } - -} // namespace trace + +std::string encode_string(SQLite::Column const &c) { + return fmt::format("\"{}\"", std::string(c)); +} + +std::string encode_number(SQLite::Column const &c) { + return fmt::format("{}", (long long)c); +} + +std::vector> +PerfLog::get_latest_requote_traces(size_t n) { + std::vector> result; + + std::lock_guard l(me->lock); + try { + me->requote_query_statement->bind(1, (long long)n); + + while (me->requote_query_statement->executeStep()) { + auto &row = result.emplace_back(); + row["version"] = + encode_string(me->requote_query_statement->getColumn("version")); + row["host"] = + encode_string(me->requote_query_statement->getColumn("host")); + row["build_type"] = encode_string( + me->requote_query_statement->getColumn("build_type")); + row["session_start"] = encode_string( + me->requote_query_statement->getColumn("session_start")); + row["trace_time"] = encode_string( + me->requote_query_statement->getColumn("trace_time")); + + row["order_key"] = + encode_number(me->requote_query_statement->getColumn("order_key")); + row["total"] = + encode_number(me->requote_query_statement->getColumn("total")); + } + me->requote_query_statement->reset(); + } catch (SQLite::Exception const &e) { + LOG(persist, Error, "Unable to retrieve quote traces {}. Error: {}", + std::string(me->db.getFilename()), std::string(e.getErrorStr())); + } + return result; +} + +std::vector> +PerfLog::get_latest_hedge_traces(size_t n) { + std::vector> result; + + std::lock_guard l(me->lock); + try { + me->hedge_query_statement->bind(1, (long long)n); + + while (me->hedge_query_statement->executeStep()) { + auto &row = result.emplace_back(); + row["version"] = + encode_string(me->hedge_query_statement->getColumn("version")); + row["host"] = + encode_string(me->hedge_query_statement->getColumn("host")); + row["build_type"] = + encode_string(me->hedge_query_statement->getColumn("build_type")); + row["session_start"] = encode_string( + me->hedge_query_statement->getColumn("session_start")); + row["trace_time"] = + encode_string(me->hedge_query_statement->getColumn("trace_time")); + + row["order_key"] = + encode_number(me->hedge_query_statement->getColumn("order_key")); + row["total"] = + encode_number(me->hedge_query_statement->getColumn("total")); + } + me->hedge_query_statement->reset(); + } catch (SQLite::Exception const &e) { + LOG(persist,Error,"Unable to retrieve hedge traces. Error: {}", + std::string(e.getErrorStr())); + } + return result; +} + +namespace perf_stat { + +using Row = std::map; + +struct Summary { + std::string name; + double min; + double max; + double median; + double ninenty_nine; + double stddev; + std::vector values; +}; + +struct Session { + std::string start; + std::string host; + std::string version; + std::string build_type; + uint32_t n_rows; + std::map summaries; + std::vector rows; +}; + +} // namespace perf_stat + +std::map structure_quote_traces( + std::vector> "es) { + + std::map result; + + for (auto &q : quotes) { + auto &session = result[q["session_start"]]; + if (session.start.empty()) { + session.start = q["session_start"]; + session.host = q["host"]; + session.version = q["version"]; + session.build_type = q["build_type"]; + } + for (auto st : {"total"}) { + auto &summary = session.summaries[st]; + if (summary.name.empty()) + summary.name = st; + summary.values.push_back(std::stod(q[st])); + } + session.rows.push_back(q); + session.n_rows++; + } + + for (auto &session_kvp : result) + for (auto &stat_kvp : session_kvp.second.summaries) { + auto &summary = stat_kvp.second; + summary.min = stats::min(summary.values); + summary.max = stats::max(summary.values); + summary.median = stats::percentile(summary.values, 0.5); + summary.ninenty_nine = stats::percentile(summary.values, 0.99); + summary.stddev = stats::stdev(summary.values); + } + + return result; +} + +std::map structure_hedge_traces( + std::vector> "es) { + + std::map result; + + for (auto &q : quotes) { + auto &session = result[q["session_start"]]; + if (session.start.empty()) { + session.start = q["session_start"]; + session.host = q["host"]; + session.version = q["version"]; + session.build_type = q["build_type"]; + } + for (auto st : {"total"}) { + auto &summary = session.summaries[st]; + if (summary.name.empty()) + summary.name = st; + summary.values.push_back(std::stod(q[st])); + } + session.rows.push_back(q); + session.n_rows++; + } + + for (auto &session_kvp : result) + for (auto &stat_kvp : session_kvp.second.summaries) { + auto &summary = stat_kvp.second; + summary.min = stats::min(summary.values); + summary.max = stats::max(summary.values); + summary.median = stats::percentile(summary.values, 0.5); + summary.ninenty_nine = stats::percentile(summary.values, 0.99); + summary.stddev = stats::stdev(summary.values); + } + + return result; +} + +void print_summary(std::ostream &ss, perf_stat::Summary const &s, bool last) { + ss << " { " + << "\"value\":" + << " \"" << s.name << "\"" + << ", \"min\":" << s.min << ", \"max\":" << s.max + << ", \"median\":" << s.median << ", \"ninenty_nine\":" << s.ninenty_nine + << ", \"stdev\":" << s.stddev; + if (!last) + ss << " },\n"; + else + ss << " }\n"; +} + +void print_row(std::ostream &ss, perf_stat::Row const &r, bool last) { + ss << " { "; + size_t i = 0; + for (auto const &kvp : r) + if (kvp.first != "version" && kvp.first != "host" && + kvp.first != "build_type" && kvp.first != "session_start") { + ss << "\"" << kvp.first << "\":" << kvp.second; + if (++i != r.size() - 4) + ss << ","; + } + if (!last) + ss << " },\n"; + else + ss << " }\n"; +} + +void print_session(std::ostream &ss, perf_stat::Session const &s, bool last) { + ss << "{\n" + << " \"session_name\": " << s.start << ",\n" + << " \"host\": " << s.host << ",\n" + << " \"version\": " << s.version << ",\n" + << " \"build_type\": " << s.build_type << ",\n" + << " \"n_rows\": " << s.n_rows << ",\n"; + + ss << " \"summaries\":\n [\n"; + size_t i = 0; + for (auto &summaries_kvp : s.summaries) + print_summary(ss, summaries_kvp.second, ++i == s.summaries.size()); + ss << " ],\n"; + + ss << " \"rows\":\n [\n"; + i = 0; + for (auto &r : s.rows) + print_row(ss, r, ++i == s.rows.size()); + ss << " ]\n"; + if (!last) + ss << "},\n"; + else + ss << "}\n"; +} + +void PerfLog::print_latest_traces(std::ostream &ss, size_t n) { + { + auto rows = get_latest_requote_traces(n); + auto structured = structure_quote_traces(rows); + + ss << "{\n\"requotes\": [\n"; + size_t i = 0; + for (auto &session_kvp : boost::adaptors::reverse(structured)) + print_session(ss, session_kvp.second, ++i == structured.size()); + ss << "],\n"; + } + + { + auto rows = get_latest_hedge_traces(n); + auto structured = structure_hedge_traces(rows); + + ss << "\"hedges\": [\n"; + + size_t i = 0; + for (auto &session_kvp : boost::adaptors::reverse(structured)) + print_session(ss, session_kvp.second, ++i == structured.size()); + ss << "]\n}\n"; + } +} + } // namespace trace +} // namespace bts diff --git a/src/lib/tracing/PerfLog.h b/src/lib/tracing/PerfLog.h index cc0f31ad8..c1bd7a012 100644 --- a/src/lib/tracing/PerfLog.h +++ b/src/lib/tracing/PerfLog.h @@ -1,18 +1,18 @@ - #pragma once - + #include "lib/utils/TimeStamp.h" -#include #include - +#include +#include + namespace bts { namespace trace { - + struct CorID { const char *channel = ""; uint32_t sn = 0; }; - + struct PerfLogEntry { utils::TimeStamp ts; const char *channel; @@ -22,39 +22,43 @@ struct PerfLogEntry { uint32_t uint32_1; uint32_t uint32_2; uint64_t uint64_1; - std::thread::id thread_id; + pthread_t thread_id; }; - + class PerfLog { tbb::concurrent_queue _pending; bool _disabled; struct PImpl; std::unique_ptr me; - + public: - PerfLog() ; + PerfLog(); ~PerfLog(); size_t store_log_entries(); - + void log(CorID cid, const char *entry_kind, uint32_t uint32_1 = 0, uint32_t uint32_2 = 0, const char *txt_1 = "", uint64_t uint64_1 = 0) { _pending.push({utils::TimeStamp::now(), cid.channel, entry_kind, txt_1, - cid.sn, uint32_1, uint32_2, uint64_1, - std::this_thread::get_id()}); + cid.sn, uint32_1, uint32_2, uint64_1, pthread_self()}); } - + void log(CorID cid, utils::TimeStamp ts, const char *entry_kind, uint32_t uint32_1 = 0, uint32_t uint32_2 = 0, - const char *txt_1 = nullptr, uint64_t uint64_1 = 0) { - _pending.push({ts,cid.channel, entry_kind, txt_1, - cid.sn, uint32_1, uint32_2, uint64_1, - std::this_thread::get_id()}); + const char *txt_1 = "", uint64_t uint64_1 = 0) { + _pending.push({ts, cid.channel, entry_kind, txt_1, cid.sn, uint32_1, + uint32_2, uint64_1, pthread_self()}); } - + + // return the N last traces + std::vector> get_latest_requote_traces(size_t n=0); + std::vector> get_latest_hedge_traces(size_t n=0); + + void print_latest_traces(std::ostream &,size_t n = 0); + private: std::string init_storage_location(); }; - + } // namespace trace } // namespace bts