Improved speed and robustness of perf log.
This commit is contained in:
1 parent
8cfca8e40e
commit
5e1c4143db
4 files changed
+142
-100
No files matched your search
@@ -1 +1 @@
|
||||
set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamicOrderRegistry.cpp;orderman/orLine truncated
|
||||
set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynaLine truncated
|
||||
@@ -0,0 +1,8 @@
|
||||
syntax='proto2';
|
||||
package bts.config;
|
||||
|
||||
message PerfLogCfg {
|
||||
optional bool active = 1 [default=true];
|
||||
optional uint32 max_rows_for_query = 2 [default=140000];
|
||||
optional uint32 max_rows = 3 [default=10000000];
|
||||
}
|
||||
+128
-97
@@ -1,28 +1,30 @@
|
||||
#include "lib/tracing/PerfLog.h"
|
||||
#include "lib/config/config.h"
|
||||
#include "lib/config/protos/perflog_cfg.pb.h"
|
||||
#include "lib/config/protos/system.pb.h"
|
||||
#include "lib/persistence/persistence_utils.h"
|
||||
#include "lib/utils/datetime.h"
|
||||
#include "lib/utils/stats.h"
|
||||
|
||||
|
||||
#include <SQLiteCpp/SQLiteCpp.h>
|
||||
#include <boost/date_time/posix_time/ptime.hpp>
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <boost/range/adaptor/reversed.hpp>
|
||||
#include <fmt/format.h>
|
||||
#include <mutex>
|
||||
#include <pthread.h>
|
||||
#include <thread>
|
||||
|
||||
|
||||
namespace bts {
|
||||
namespace trace {
|
||||
|
||||
|
||||
auto create_session = "create table session ("
|
||||
"version TEXT, "
|
||||
"host TEXT, "
|
||||
"build_type TEXT, "
|
||||
"session_start TIMESTAMP PRIMARY KEY"
|
||||
")";
|
||||
|
||||
|
||||
auto create_trace = "create table trace ("
|
||||
"session_start TIMESTAMP,"
|
||||
"ts INTEGER,"
|
||||
@@ -36,7 +38,7 @@ auto create_trace = "create table trace ("
|
||||
"uint64_1 INTEGER,"
|
||||
"thread TEXT"
|
||||
")";
|
||||
|
||||
|
||||
auto insert_trace = "insert into trace ("
|
||||
"session_start,"
|
||||
"ts,"
|
||||
@@ -62,7 +64,7 @@ auto insert_trace = "insert into trace ("
|
||||
":uint64_1,"
|
||||
":thread"
|
||||
")";
|
||||
|
||||
|
||||
auto insert_session = "insert into session ("
|
||||
"version,"
|
||||
"host,"
|
||||
@@ -74,29 +76,37 @@ auto insert_session = "insert into session ("
|
||||
":build_type,"
|
||||
":session_start"
|
||||
")";
|
||||
|
||||
|
||||
auto create_recent_trace_view =
|
||||
"drop view if exists recent_trace; create view recent_trace as "
|
||||
"select * from trace order by rowid desc limit {}";
|
||||
|
||||
auto delete_old_records =
|
||||
"delete from trace where "
|
||||
"rowid in (select rowid from trace order by rowid desc limit -1 offset {})";
|
||||
|
||||
auto hedge_events_view =
|
||||
"drop view if exists hedge_events; create view hedge_events as "
|
||||
"select "
|
||||
" trace.session_start, "
|
||||
" recent_trace.session_start, "
|
||||
" channel, "
|
||||
" sn, "
|
||||
" min(hts) hts, "
|
||||
" (max(ts)-min(ts))/1000 t "
|
||||
"from "
|
||||
" trace, "
|
||||
" recent_trace, "
|
||||
" session "
|
||||
"where "
|
||||
" trace.session_start = session.session_start and "
|
||||
" recent_trace.session_start = session.session_start and "
|
||||
" (entry_kind='fix_socket_r' or entry_kind='before_fix_socket_w') "
|
||||
"group by "
|
||||
" trace.session_start, "
|
||||
" recent_trace.session_start, "
|
||||
" channel, "
|
||||
" sn "
|
||||
"having "
|
||||
" count(*) = 2 and "
|
||||
" min(entry_kind)<>max(entry_kind)";
|
||||
|
||||
|
||||
auto hedge_trace_view =
|
||||
"drop view if exists hedge_trace; create view hedge_trace as "
|
||||
"select "
|
||||
@@ -104,42 +114,42 @@ auto hedge_trace_view =
|
||||
" session.host, "
|
||||
" session.build_type, "
|
||||
" session.session_start, "
|
||||
" trace.hts trace_time, "
|
||||
" trace.uint32_1 order_key, "
|
||||
" recent_trace.hts trace_time, "
|
||||
" recent_trace.uint32_1 order_key, "
|
||||
" hedge_events.t total "
|
||||
"from "
|
||||
" trace, "
|
||||
" recent_trace, "
|
||||
" hedge_events, "
|
||||
" session "
|
||||
"where "
|
||||
" entry_kind='on_activate_start' and "
|
||||
" trace.channel = hedge_events.channel and "
|
||||
" trace.session_start = hedge_events.session_start and "
|
||||
" trace.sn = hedge_events.sn and "
|
||||
" trace.session_start = session.session_start ";
|
||||
|
||||
" recent_trace.channel = hedge_events.channel and "
|
||||
" recent_trace.session_start = hedge_events.session_start and "
|
||||
" recent_trace.sn = hedge_events.sn and "
|
||||
" recent_trace.session_start = session.session_start ";
|
||||
|
||||
auto requote_events_view =
|
||||
"drop view if exists requote_events; create view requote_events as "
|
||||
"select "
|
||||
" trace.session_start, "
|
||||
" recent_trace.session_start, "
|
||||
" channel, "
|
||||
" sn, "
|
||||
" min(hts) hts, "
|
||||
" (max(ts)-min(ts))/1000 t "
|
||||
"from "
|
||||
" trace, "
|
||||
" recent_trace, "
|
||||
" session "
|
||||
"where "
|
||||
" trace.session_start = session.session_start and "
|
||||
" recent_trace.session_start = session.session_start and "
|
||||
" (entry_kind='md' or entry_kind='before_fix_socket_w') "
|
||||
"group by "
|
||||
" trace.session_start, "
|
||||
" recent_trace.session_start, "
|
||||
" channel, "
|
||||
" sn "
|
||||
"having "
|
||||
" count(*) = 2 and "
|
||||
" min(entry_kind)<>max(entry_kind) ";
|
||||
|
||||
|
||||
auto requote_trace_view =
|
||||
"drop view if exists requote_trace; create view requote_trace as "
|
||||
"select "
|
||||
@@ -147,49 +157,53 @@ auto requote_trace_view =
|
||||
" session.host, "
|
||||
" session.build_type, "
|
||||
" session.session_start, "
|
||||
" trace.hts trace_time, "
|
||||
" trace.uint32_1 order_key, "
|
||||
" recent_trace.hts trace_time, "
|
||||
" recent_trace.uint32_1 order_key, "
|
||||
" requote_events.t total "
|
||||
"from "
|
||||
" trace, "
|
||||
" recent_trace, "
|
||||
" requote_events, "
|
||||
" session "
|
||||
"where "
|
||||
" entry_kind='order_update' and "
|
||||
" trace.channel = requote_events.channel and "
|
||||
" trace.session_start = requote_events.session_start and "
|
||||
" trace.sn = requote_events.sn and "
|
||||
" trace.session_start = session.session_start ";
|
||||
|
||||
" recent_trace.channel = requote_events.channel and "
|
||||
" recent_trace.session_start = requote_events.session_start and "
|
||||
" recent_trace.sn = requote_events.sn and "
|
||||
" recent_trace.session_start = session.session_start ";
|
||||
|
||||
auto requote_query =
|
||||
"select * from requote_trace order by trace_time desc limit ?";
|
||||
|
||||
|
||||
auto hedge_query = "select * from hedge_trace order by trace_time desc limit ?";
|
||||
|
||||
|
||||
struct PerfLog::PImpl {
|
||||
PImpl(std::string const &loc)
|
||||
: db(loc, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE),
|
||||
insert_trace_statement(), insert_session_statement(), session_start(),
|
||||
disabled(false), lock() {}
|
||||
|
||||
lock() {}
|
||||
|
||||
SQLite::Database db;
|
||||
std::unique_ptr<SQLite::Statement> insert_trace_statement;
|
||||
std::unique_ptr<SQLite::Statement> insert_session_statement;
|
||||
std::unique_ptr<SQLite::Statement> requote_query_statement;
|
||||
std::unique_ptr<SQLite::Statement> hedge_query_statement;
|
||||
std::string session_start;
|
||||
bool disabled;
|
||||
std::mutex lock;
|
||||
};
|
||||
|
||||
|
||||
PerfLog::PerfLog() : _pending(), _disabled(false), me() {
|
||||
_disabled = !config::get<config::PerfLogCfg>().active();
|
||||
|
||||
if (_disabled)
|
||||
return;
|
||||
|
||||
auto loc = init_storage_location();
|
||||
|
||||
|
||||
if (loc.empty()) {
|
||||
_disabled = true;
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
try {
|
||||
me = std::make_unique<PImpl>(loc);
|
||||
if (!me->db.tableExists("session")) {
|
||||
@@ -198,12 +212,17 @@ PerfLog::PerfLog() : _pending(), _disabled(false), me() {
|
||||
me->db.exec(create_session);
|
||||
me->db.exec(create_trace);
|
||||
}
|
||||
|
||||
|
||||
me->db.exec(fmt::format(delete_old_records,
|
||||
config::get<config::PerfLogCfg>().max_rows()));
|
||||
me->db.exec(
|
||||
fmt::format(create_recent_trace_view,
|
||||
config::get<config::PerfLogCfg>().max_rows_for_query()));
|
||||
me->db.exec(hedge_events_view);
|
||||
me->db.exec(hedge_trace_view);
|
||||
me->db.exec(requote_events_view);
|
||||
me->db.exec(requote_trace_view);
|
||||
|
||||
|
||||
me->insert_trace_statement =
|
||||
std::make_unique<SQLite::Statement>(me->db, insert_trace);
|
||||
me->insert_session_statement =
|
||||
@@ -213,7 +232,7 @@ PerfLog::PerfLog() : _pending(), _disabled(false), me() {
|
||||
me->hedge_query_statement =
|
||||
std::make_unique<SQLite::Statement>(me->db, hedge_query);
|
||||
me->session_start = datetime::FIX_timestamp(datetime::local_now());
|
||||
|
||||
|
||||
me->insert_session_statement->bind(
|
||||
":version", config::get<config::System>().version());
|
||||
me->insert_session_statement->bind(
|
||||
@@ -232,13 +251,13 @@ PerfLog::PerfLog() : _pending(), _disabled(false), me() {
|
||||
_disabled = true;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
PerfLog::~PerfLog() {}
|
||||
|
||||
|
||||
std::string PerfLog::init_storage_location() {
|
||||
auto loc = persistence::storage_location("perf_log");
|
||||
auto parent = boost::filesystem::path(loc).parent_path();
|
||||
|
||||
|
||||
if (!boost::filesystem::is_directory(parent)) {
|
||||
if (boost::filesystem::create_directories(parent))
|
||||
return loc;
|
||||
@@ -247,15 +266,19 @@ std::string PerfLog::init_storage_location() {
|
||||
} else
|
||||
return loc;
|
||||
}
|
||||
|
||||
|
||||
size_t PerfLog::store_log_entries() {
|
||||
if (_disabled)
|
||||
if (_disabled) {
|
||||
_pending.clear();
|
||||
return 0;
|
||||
|
||||
PerfLogEntry ev;
|
||||
}
|
||||
|
||||
size_t count = 0;
|
||||
while (_pending.try_pop(ev))
|
||||
try {
|
||||
try {
|
||||
PerfLogEntry ev;
|
||||
while (_pending.try_pop(ev)) {
|
||||
if (count == 0)
|
||||
me->db.exec("begin");
|
||||
me->insert_trace_statement->bind(":session_start", me->session_start);
|
||||
me->insert_trace_statement->bind(":ts", (long long)ev.ts.to_uint64());
|
||||
me->insert_trace_statement->bind(
|
||||
@@ -268,38 +291,43 @@ size_t PerfLog::store_log_entries() {
|
||||
me->insert_trace_statement->bind(":uint32_1", ev.uint32_1);
|
||||
me->insert_trace_statement->bind(":uint32_2", ev.uint32_2);
|
||||
me->insert_trace_statement->bind(":uint64_1", (long long)ev.uint64_1);
|
||||
|
||||
|
||||
char tn[17];
|
||||
pthread_getname_np(ev.thread_id, tn, 16);
|
||||
me->insert_trace_statement->bind(":thread", tn);
|
||||
me->insert_trace_statement->exec();
|
||||
me->insert_trace_statement->reset();
|
||||
count++;
|
||||
} catch (SQLite::Exception const &e) {
|
||||
LOG(persist, Error, "Unable to insert perf trace in {}. Error: {}",
|
||||
std::string(me->db.getFilename()), std::string(e.getErrorStr()));
|
||||
|
||||
_disabled = true;
|
||||
me->insert_trace_statement->reset();
|
||||
}
|
||||
if (count)
|
||||
me->db.exec("commit");
|
||||
} catch (SQLite::Exception const &e) {
|
||||
if (count)
|
||||
try {me->db.exec("rollback");}
|
||||
catch (SQLite::Exception const &e2) {}
|
||||
LOG(persist, Error, "Unable to insert perf trace in. Error: {}",
|
||||
std::string(e.getErrorStr()));
|
||||
_pending.clear();
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
|
||||
std::string encode_string(SQLite::Column const &c) {
|
||||
return fmt::format("\"{}\"", std::string(c));
|
||||
}
|
||||
|
||||
|
||||
std::string encode_number(SQLite::Column const &c) {
|
||||
return fmt::format("{}", (long long)c);
|
||||
}
|
||||
|
||||
|
||||
std::vector<std::map<std::string, std::string>>
|
||||
PerfLog::get_latest_requote_traces(size_t n) {
|
||||
std::vector<std::map<std::string, std::string>> result;
|
||||
|
||||
|
||||
std::lock_guard<std::mutex> l(me->lock);
|
||||
try {
|
||||
me->requote_query_statement->bind(1, (long long)n);
|
||||
|
||||
|
||||
while (me->requote_query_statement->executeStep()) {
|
||||
auto &row = result.emplace_back();
|
||||
row["version"] =
|
||||
@@ -312,7 +340,7 @@ PerfLog::get_latest_requote_traces(size_t n) {
|
||||
me->requote_query_statement->getColumn("session_start"));
|
||||
row["trace_time"] = encode_string(
|
||||
me->requote_query_statement->getColumn("trace_time"));
|
||||
|
||||
|
||||
row["order_key"] =
|
||||
encode_number(me->requote_query_statement->getColumn("order_key"));
|
||||
row["total"] =
|
||||
@@ -325,15 +353,15 @@ PerfLog::get_latest_requote_traces(size_t n) {
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
std::vector<std::map<std::string, std::string>>
|
||||
PerfLog::get_latest_hedge_traces(size_t n) {
|
||||
std::vector<std::map<std::string, std::string>> result;
|
||||
|
||||
|
||||
std::lock_guard<std::mutex> l(me->lock);
|
||||
try {
|
||||
me->hedge_query_statement->bind(1, (long long)n);
|
||||
|
||||
|
||||
while (me->hedge_query_statement->executeStep()) {
|
||||
auto &row = result.emplace_back();
|
||||
row["version"] =
|
||||
@@ -346,7 +374,7 @@ PerfLog::get_latest_hedge_traces(size_t n) {
|
||||
me->hedge_query_statement->getColumn("session_start"));
|
||||
row["trace_time"] =
|
||||
encode_string(me->hedge_query_statement->getColumn("trace_time"));
|
||||
|
||||
|
||||
row["order_key"] =
|
||||
encode_number(me->hedge_query_statement->getColumn("order_key"));
|
||||
row["total"] =
|
||||
@@ -354,16 +382,16 @@ PerfLog::get_latest_hedge_traces(size_t n) {
|
||||
}
|
||||
me->hedge_query_statement->reset();
|
||||
} catch (SQLite::Exception const &e) {
|
||||
LOG(persist,Error,"Unable to retrieve hedge traces. Error: {}",
|
||||
LOG(persist, Error, "Unable to retrieve hedge traces. Error: {}",
|
||||
std::string(e.getErrorStr()));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
namespace perf_stat {
|
||||
|
||||
|
||||
using Row = std::map<std::string, std::string>;
|
||||
|
||||
|
||||
struct Summary {
|
||||
std::string name;
|
||||
double min;
|
||||
@@ -373,7 +401,7 @@ struct Summary {
|
||||
double stddev;
|
||||
std::vector<double> values;
|
||||
};
|
||||
|
||||
|
||||
struct Session {
|
||||
std::string start;
|
||||
std::string host;
|
||||
@@ -383,14 +411,14 @@ struct Session {
|
||||
std::map<std::string, Summary> summaries;
|
||||
std::vector<Row> rows;
|
||||
};
|
||||
|
||||
|
||||
} // namespace perf_stat
|
||||
|
||||
|
||||
std::map<std::string, perf_stat::Session> structure_quote_traces(
|
||||
std::vector<std::map<std::string, std::string>> "es) {
|
||||
|
||||
|
||||
std::map<std::string, perf_stat::Session> result;
|
||||
|
||||
|
||||
for (auto &q : quotes) {
|
||||
auto &session = result[q["session_start"]];
|
||||
if (session.start.empty()) {
|
||||
@@ -408,7 +436,7 @@ std::map<std::string, perf_stat::Session> structure_quote_traces(
|
||||
session.rows.push_back(q);
|
||||
session.n_rows++;
|
||||
}
|
||||
|
||||
|
||||
for (auto &session_kvp : result)
|
||||
for (auto &stat_kvp : session_kvp.second.summaries) {
|
||||
auto &summary = stat_kvp.second;
|
||||
@@ -418,15 +446,15 @@ std::map<std::string, perf_stat::Session> structure_quote_traces(
|
||||
summary.ninenty_nine = stats::percentile(summary.values, 0.99);
|
||||
summary.stddev = stats::stdev(summary.values);
|
||||
}
|
||||
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
std::map<std::string, perf_stat::Session> structure_hedge_traces(
|
||||
std::vector<std::map<std::string, std::string>> "es) {
|
||||
|
||||
|
||||
std::map<std::string, perf_stat::Session> result;
|
||||
|
||||
|
||||
for (auto &q : quotes) {
|
||||
auto &session = result[q["session_start"]];
|
||||
if (session.start.empty()) {
|
||||
@@ -444,7 +472,7 @@ std::map<std::string, perf_stat::Session> structure_hedge_traces(
|
||||
session.rows.push_back(q);
|
||||
session.n_rows++;
|
||||
}
|
||||
|
||||
|
||||
for (auto &session_kvp : result)
|
||||
for (auto &stat_kvp : session_kvp.second.summaries) {
|
||||
auto &summary = stat_kvp.second;
|
||||
@@ -454,10 +482,10 @@ std::map<std::string, perf_stat::Session> structure_hedge_traces(
|
||||
summary.ninenty_nine = stats::percentile(summary.values, 0.99);
|
||||
summary.stddev = stats::stdev(summary.values);
|
||||
}
|
||||
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
void print_summary(std::ostream &ss, perf_stat::Summary const &s, bool last) {
|
||||
ss << " { "
|
||||
<< "\"value\":"
|
||||
@@ -470,7 +498,7 @@ void print_summary(std::ostream &ss, perf_stat::Summary const &s, bool last) {
|
||||
else
|
||||
ss << " }\n";
|
||||
}
|
||||
|
||||
|
||||
void print_row(std::ostream &ss, perf_stat::Row const &r, bool last) {
|
||||
ss << " { ";
|
||||
size_t i = 0;
|
||||
@@ -486,7 +514,7 @@ void print_row(std::ostream &ss, perf_stat::Row const &r, bool last) {
|
||||
else
|
||||
ss << " }\n";
|
||||
}
|
||||
|
||||
|
||||
void print_session(std::ostream &ss, perf_stat::Session const &s, bool last) {
|
||||
ss << "{\n"
|
||||
<< " \"session_name\": " << s.start << ",\n"
|
||||
@@ -494,13 +522,13 @@ void print_session(std::ostream &ss, perf_stat::Session const &s, bool last) {
|
||||
<< " \"version\": " << s.version << ",\n"
|
||||
<< " \"build_type\": " << s.build_type << ",\n"
|
||||
<< " \"n_rows\": " << s.n_rows << ",\n";
|
||||
|
||||
|
||||
ss << " \"summaries\":\n [\n";
|
||||
size_t i = 0;
|
||||
for (auto &summaries_kvp : s.summaries)
|
||||
print_summary(ss, summaries_kvp.second, ++i == s.summaries.size());
|
||||
ss << " ],\n";
|
||||
|
||||
|
||||
ss << " \"rows\":\n [\n";
|
||||
i = 0;
|
||||
for (auto &r : s.rows)
|
||||
@@ -511,31 +539,34 @@ void print_session(std::ostream &ss, perf_stat::Session const &s, bool last) {
|
||||
else
|
||||
ss << "}\n";
|
||||
}
|
||||
|
||||
|
||||
void PerfLog::print_latest_traces(std::ostream &ss, size_t n) {
|
||||
if (_disabled)
|
||||
return;
|
||||
|
||||
{
|
||||
auto rows = get_latest_requote_traces(n);
|
||||
auto structured = structure_quote_traces(rows);
|
||||
|
||||
|
||||
ss << "{\n\"requotes\": [\n";
|
||||
size_t i = 0;
|
||||
for (auto &session_kvp : boost::adaptors::reverse(structured))
|
||||
print_session(ss, session_kvp.second, ++i == structured.size());
|
||||
ss << "],\n";
|
||||
}
|
||||
|
||||
|
||||
{
|
||||
auto rows = get_latest_hedge_traces(n);
|
||||
auto structured = structure_hedge_traces(rows);
|
||||
|
||||
|
||||
ss << "\"hedges\": [\n";
|
||||
|
||||
|
||||
size_t i = 0;
|
||||
for (auto &session_kvp : boost::adaptors::reverse(structured))
|
||||
print_session(ss, session_kvp.second, ++i == structured.size());
|
||||
ss << "]\n}\n";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} // namespace trace
|
||||
} // namespace bts
|
||||
@@ -826,8 +826,11 @@ BENCHMARK(fix_time_stamp);
|
||||
|
||||
static void perf_log(benchmark::State &state) {
|
||||
trace::PerfLog l;
|
||||
for (auto _ : state)
|
||||
l.log(trace::CorID{"sim", 0}, "this", 1, 2, "txt", 3);
|
||||
for (auto _ : state) {
|
||||
for (int i = 0; i < 10000; i++)
|
||||
l.log(trace::CorID{"sim", 0}, "this", i, 2, "txt", 3);
|
||||
l.store_log_entries();
|
||||
}
|
||||
}
|
||||
BENCHMARK(perf_log);
|
||||
|
||||
|
||||
Reference in new issue
Block a user