From d0647fb8d92bd16ceb756dcc20004c121b14c541 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Wed, 12 Dec 2018 23:18:45 +0000 Subject: [PATCH 1/2] Perf tools. --- src/data_files/files/perf_trace/perf_trace.sql | 7 ++++++- src/lib/orderman/actors/MDSource.h | 2 +- src/lib/orderman/perf_analysis/perf_log.sh | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/data_files/files/perf_trace/perf_trace.sql b/src/data_files/files/perf_trace/perf_trace.sql index e0cfc7d48..74f275801 100644 --- a/src/data_files/files/perf_trace/perf_trace.sql +++ b/src/data_files/files/perf_trace/perf_trace.sql @@ -123,9 +123,14 @@ select event_type, event_id, ts, - hts + hts, + session_start from event_log; +drop view if exists last_pretty_log; +create view last_pretty_log as +select * from pretty_log where session_start=(select max(session_start) from session); + /* drop view if exists summary; create view summary as select session_start,event_type,count(*) count, avg(t) avg, max(t) max, min(t) min, median(t) median, stdev(t) stdev from events group by session_start,event_type; diff --git a/src/lib/orderman/actors/MDSource.h b/src/lib/orderman/actors/MDSource.h index 03a527495..aab32b503 100644 --- a/src/lib/orderman/actors/MDSource.h +++ b/src/lib/orderman/actors/MDSource.h @@ -80,7 +80,7 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor { auto &channel = _symbol_to_channel[handle.get_usersymbol()]; if (channel.empty()) - channel = "sim"; + channel = "sim_md"; _handles.push_back( {utils::TimeStamp(), handle, channel.c_str()}); diff --git a/src/lib/orderman/perf_analysis/perf_log.sh b/src/lib/orderman/perf_analysis/perf_log.sh index 057e302d8..f6fc9aad1 100755 --- a/src/lib/orderman/perf_analysis/perf_log.sh +++ b/src/lib/orderman/perf_analysis/perf_log.sh @@ -17,7 +17,7 @@ $SQLLITE file:$HOME/sparkdata/data/databases/perf_log/perf_log.sqlite?mode=ro << .mode column .width 60 .headers on -select * from pretty_log; +select * from last_pretty_log EOF fi @@ -25,6 +25,7 @@ if [ $1 == "summary" ]; then $SQLLITE file:$HOME/sparkdata/data/databases/perf_log/perf_log.sqlite?mode=ro < Date: Thu, 13 Dec 2018 07:10:16 -0500 Subject: [PATCH 2/2] Perf tools --- docker/linux | 2 +- .../files/perf_trace/perf_trace.sql | 287 +++++++++--------- src/web/CMakeDeps.cmake | 2 +- 3 files changed, 148 insertions(+), 143 deletions(-) diff --git a/docker/linux b/docker/linux index 8ee969259..f6d8d5986 160000 --- a/docker/linux +++ b/docker/linux @@ -1 +1 @@ -Subproject commit 8ee9692597d9e59adf0a3ab77610b711892f79a6 +Subproject commit f6d8d59860bc40672a4ac9ab4e69a54541c4d236 diff --git a/src/data_files/files/perf_trace/perf_trace.sql b/src/data_files/files/perf_trace/perf_trace.sql index 74f275801..53d7f38c2 100644 --- a/src/data_files/files/perf_trace/perf_trace.sql +++ b/src/data_files/files/perf_trace/perf_trace.sql @@ -27,158 +27,163 @@ drop view if exists recent_trace; create view recent_trace as select * from trace order by rowid desc limit 140123; -drop view if exists ordered_trace; -create view ordered_trace as select recent_trace.*,row_number() over (order by session_start,ts) as entry_idx from recent_trace; -drop view if exists delta_trace; -create view delta_trace as -select current.*, (current.ts - previous.ts) / 1000.0 delta from -ordered_trace current, ordered_trace previous where current.entry_idx = previous.entry_idx +1; -drop view if exists log; -create view log as -select - delta_trace.*, - case cor_event>0 when 1 then (ts-cor_event)/1000.0 else 0 end pair_delta, - case cor_event>0 when 1 then "end of " || entry_kind else entry_kind end paired_entry_kind -from - delta_trace; -drop view if exists hedge_events; -CREATE VIEW hedge_events as -select - recent_trace.session_start, - channel, - sn, - min(ts) ts, - min(hts) hts, - (max(ts)-min(ts))/1000.0 t -from - recent_trace, - session -where - recent_trace.session_start = session.session_start and - (entry_kind='fix_socket_r' or entry_kind='before_fix_socket_w') -group by - recent_trace.session_start, channel, sn -having - count(*) = 2 and min(entry_kind)<>max(entry_kind); -drop view if exists requote_events; -CREATE VIEW requote_events as -select - recent_trace.session_start, - channel, - sn, - min(ts) ts, - min(hts) hts, - (max(ts)-min(ts))/1000.0 t -from - recent_trace, - session -where - recent_trace.session_start = session.session_start and - (entry_kind='md' or entry_kind='before_fix_socket_w') -group by - recent_trace.session_start, channel, sn having count(*) = 2 and - min(entry_kind)<>max(entry_kind); + +-- drop view if exists ordered_trace; +-- create view ordered_trace as select recent_trace.*,row_number() over (order by session_start,ts) as entry_idx from recent_trace; + +-- drop view if exists delta_trace; +-- create view delta_trace as +-- select current.*, (current.ts - previous.ts) / 1000.0 delta from +-- ordered_trace current, ordered_trace previous where current.entry_idx = previous.entry_idx +1; + +-- drop view if exists log; +-- create view log as +-- select +-- delta_trace.*, +-- case cor_event>0 when 1 then (ts-cor_event)/1000.0 else 0 end pair_delta, +-- case cor_event>0 when 1 then "end of " || entry_kind else entry_kind end paired_entry_kind +-- from +-- delta_trace; + +-- drop view if exists hedge_events; +-- CREATE VIEW hedge_events as +-- select +-- recent_trace.session_start, +-- channel, +-- sn, +-- min(ts) ts, +-- min(hts) hts, +-- (max(ts)-min(ts))/1000.0 t +-- from +-- recent_trace, +-- session +-- where +-- recent_trace.session_start = session.session_start and +-- (entry_kind='fix_socket_r' or entry_kind='before_fix_socket_w') +-- group by +-- recent_trace.session_start, channel, sn +-- having +-- count(*) = 2 and min(entry_kind)<>max(entry_kind); + +-- drop view if exists requote_events; +-- CREATE VIEW requote_events as +-- select +-- recent_trace.session_start, +-- channel, +-- sn, +-- min(ts) ts, +-- min(hts) hts, +-- (max(ts)-min(ts))/1000.0 t +-- from +-- recent_trace, +-- session +-- where +-- recent_trace.session_start = session.session_start and +-- (entry_kind='md' or entry_kind='before_fix_socket_w') +-- group by +-- recent_trace.session_start, channel, sn having count(*) = 2 and +-- min(entry_kind)<>max(entry_kind); -drop view if exists events; -CREATE VIEW events as -select *,row_number () over (order by session_start,channel, sn) as event_id from ( -select - "requote" event_type, - requote_events.* -from - requote_events -union -select - "hedge" event_type, - hedge_events.* -from - hedge_events); +-- drop view if exists events; +-- CREATE VIEW events as +-- select *,row_number () over (order by session_start,channel, sn) as event_id from ( +-- select +-- "requote" event_type, +-- requote_events.* +-- from +-- requote_events +-- union +-- select +-- "hedge" event_type, +-- hedge_events.* +-- from +-- hedge_events); -drop view if exists event_log; -create view event_log as -select - log.*, - event_type, - events.t event_duration, - event_id -from - log, - events -where - log.session_start = events.session_start and - log.channel = events.channel and - log.sn = events.sn; +-- drop view if exists event_log; +-- create view event_log as +-- select +-- log.*, +-- event_type, +-- events.t event_duration, +-- event_id +-- from +-- log, +-- events +-- where +-- log.session_start = events.session_start and +-- log.channel = events.channel and +-- log.sn = events.sn; -drop view if exists pretty_log; -create view pretty_log as -select - paired_entry_kind entry_kind, - delta, - pair_delta, - event_duration, - event_type, - event_id, - ts, - hts, - session_start -from event_log; +-- drop view if exists pretty_log; +-- create view pretty_log as +-- select +-- paired_entry_kind entry_kind, +-- delta, +-- pair_delta, +-- event_duration, +-- event_type, +-- event_id, +-- ts, +-- hts, +-- session_start +-- from event_log; -drop view if exists last_pretty_log; -create view last_pretty_log as -select * from pretty_log where session_start=(select max(session_start) from session); +-- drop view if exists last_pretty_log; +-- create view last_pretty_log as +-- select * from pretty_log where session_start=(select max(session_start) from session); -/* -drop view if exists summary; -create view summary as select session_start,event_type,count(*) count, avg(t) avg, max(t) max, min(t) min, median(t) median, stdev(t) stdev from events group by session_start,event_type; -*/ +-- /* +-- drop view if exists summary; +-- create view summary as select session_start,event_type,count(*) count, avg(t) avg, max(t) max, min(t) min, median(t) median, stdev(t) stdev from events group by session_start,event_type; +-- */ -drop view if exists hedge_trace; -create view hedge_trace as -select - session.version, - session.host, - session.build_type, - session.session_start, - recent_trace.hts trace_time, - recent_trace.uint64_1 order_key, - hedge_events.t total -from - recent_trace, - hedge_events, - session -where - entry_kind='NativeOrder::on_activate' and - recent_trace.channel = hedge_events.channel and - recent_trace.session_start = hedge_events.session_start and - recent_trace.sn = hedge_events.sn and - recent_trace.session_start = session.session_start; +-- drop view if exists hedge_trace; +-- create view hedge_trace as +-- select +-- session.version, +-- session.host, +-- session.build_type, +-- session.session_start, +-- recent_trace.hts trace_time, +-- recent_trace.uint64_1 order_key, +-- hedge_events.t total +-- from +-- recent_trace, +-- hedge_events, +-- session +-- where +-- entry_kind='NativeOrder::on_activate' and +-- recent_trace.channel = hedge_events.channel and +-- recent_trace.session_start = hedge_events.session_start and +-- recent_trace.sn = hedge_events.sn and +-- recent_trace.session_start = session.session_start; -drop view if exists requote_trace; -create view requote_trace as -select - session.version, - session.host, - session.build_type, - session.session_start, - recent_trace.hts trace_time, - recent_trace.uint64_1 order_key, - requote_events.t total -from - recent_trace, - requote_events, - session -where - entry_kind='NativeOrder::on_update' and - recent_trace.channel = requote_events.channel and - recent_trace.session_start = requote_events.session_start and - recent_trace.sn = requote_events.sn and - recent_trace.session_start = session.session_start ; +-- drop view if exists requote_trace; +-- create view requote_trace as +-- select +-- session.version, +-- session.host, +-- session.build_type, +-- session.session_start, +-- recent_trace.hts trace_time, +-- recent_trace.uint64_1 order_key, +-- requote_events.t total +-- from +-- recent_trace, +-- requote_events, +-- session +-- where +-- entry_kind='NativeOrder::on_update' and +-- recent_trace.channel = requote_events.channel and +-- recent_trace.session_start = requote_events.session_start and +-- recent_trace.sn = requote_events.sn and +-- recent_trace.session_start = session.session_start ; -delete from trace where -rowid in (select rowid from trace order by rowid desc limit -1 offset 10000123); +-- delete from trace where +-- rowid in (select rowid from trace order by rowid desc limit -1 offset 10000123); diff --git a/src/web/CMakeDeps.cmake b/src/web/CMakeDeps.cmake index 050dc8062..63b61fd31 100644 --- a/src/web/CMakeDeps.cmake +++ b/src/web/CMakeDeps.cmake @@ -1 +1 @@ -set(sources ../../externals/linux/include/google/protobuf/descriptor.proto;../../externals/linux/include/google/protobuf/timestamp.proto;../lib/api/apicore.proto;../lib/api/apigateway.proto;../lib/api/apioptions.proto;../lib/api/services/AdminService.proto;../lib/api/services/ApiUnitTestService.proto;../lib/api/services/ErrorReportService.proto;../lib/api/services/HelloService.proto;../lib/api/services/MarketDataAuxiliaryService.proto;../lib/api/services/MarketDataRelay.proto;../lib/api/services/OrderService.proto;../lib/api/services/OutOfProcessURLListener.proto;../lib/api/services/PersistenceService.proto;../lib/api/services/PositionService.proto;../lib/api/services/ShellDiagnosticService.proto;../lib/api/services/ShellLatencyService.proto;../lib/api/services/SyntheticSecurityService.proto;../lib/api/services/UserService.proto;../lib/domain_model/Security.proto;../lib/md/protos/co2.proto;../lib/sparkproto/SparkProtos.proto;api/build.sh;api/node_modules/cssstyle/scripts/run_tests.sh;api/node_modules/exit/test/fixtures/create-files.sh;api/node_modules/protobufjs/cli/node_modules/acorn-jsx/node_modules/acorn/bin/update_authors.sh;api/node_modules/watch/release.sh;api/quiet_build.sh;api/src/DynamicService.ts;api/src/ServiceContext.ts;api/src/bts_assert.ts;ui/src/Access.fs;ui/src/Accounts/ClipLimits.fs;ui/src/Accounts/ProductLimits.fs;ui/src/Accounts/SecurityLimits.fs;ui/src/Accounts/State.fs;ui/src/Accounts/Types.fs;ui/src/Accounts/View.fs;ui/src/App.fs;ui/src/Cmd.fs;ui/src/Controls/Button.fs;ui/src/Controls/DataGrid.fs;ui/src/Controls/Fields.fs;ui/src/Controls/Modal.fs;ui/src/Controls/ReactGrid.fs;ui/src/Controls/Table.fs;ui/src/Controls/nbsp.fs;ui/src/Global.fs;ui/src/Imports.fs;ui/src/Lens.fs;ui/src/Login/Login.fs;ui/src/Nav.fs;ui/src/State.fs;ui/src/Types.fs;ui/src/Users/State.fs;ui/src/Users/Types.fs;ui/src/Users/View.fs;ui/src/Util/EnumUtils.fs;ui/src/Util/JSMap.fs) \ No newline at end of file +set(sources ../../externals/linux/include/google/protobuf/descriptor.proto;../../externals/linux/include/google/protobuf/timestamp.proto;../lib/api/apicore.proto;../lib/api/apigateway.proto;../lib/api/apioptions.proto;../lib/api/services/AdminService.proto;../lib/api/services/ApiUnitTestService.proto;../lib/api/services/ErrorReportService.proto;../lib/api/services/HelloService.proto;../lib/api/services/MarketDataAuxiliaryService.proto;../lib/api/services/MarketDataRelay.proto;../lib/api/services/OrderService.proto;../lib/api/services/OutOfProcessURLListener.proto;../lib/api/services/PersistenceService.proto;../lib/api/services/PositionService.proto;../lib/api/services/ShellDiagnosticService.proto;../lib/api/services/ShellLatencyService.proto;../lib/api/services/SyntheticSecurityService.proto;../lib/api/services/UserService.proto;../lib/domain_model/Security.proto;../lib/md/protos/co2.proto;../lib/sparkproto/SparkProtos.proto;api/build.sh;api/node_modules/exit/test/fixtures/create-files.sh;api/node_modules/protobufjs/cli/node_modules/acorn-jsx/node_modules/acorn/bin/update_authors.sh;api/node_modules/watch/release.sh;api/quiet_build.sh;api/src/DynamicService.ts;api/src/ServiceContext.ts;api/src/bts_assert.ts;ui/src/Access.fs;ui/src/Accounts/ClipLimits.fs;ui/src/Accounts/ProductLimits.fs;ui/src/Accounts/SecurityLimits.fs;ui/src/Accounts/State.fs;ui/src/Accounts/Types.fs;ui/src/Accounts/View.fs;ui/src/App.fs;ui/src/Cmd.fs;ui/src/Controls/Button.fs;ui/src/Controls/DataGrid.fs;ui/src/Controls/Fields.fs;ui/src/Controls/Modal.fs;ui/src/Controls/ReactGrid.fs;ui/src/Controls/Table.fs;ui/src/Controls/nbsp.fs;ui/src/Global.fs;ui/src/Imports.fs;ui/src/Lens.fs;ui/src/Login/Login.fs;ui/src/Nav.fs;ui/src/State.fs;ui/src/Types.fs;ui/src/Users/State.fs;ui/src/Users/Types.fs;ui/src/Users/View.fs;ui/src/Util/EnumUtils.fs;ui/src/Util/JSMap.fs;ui/src/obj/Debug/netstandard2.0/ui.AssemblyInfo.fs) \ No newline at end of file