Compare commits

..
16 Commits
Author SHA1 Message Date
Eric Harding f3dedfc4e3 Merged in SPARK-1966-add-cme-ticker-trade-details (pull request #143)
RTM: SPARK-1966 add cme ticker trade details
2019-01-14 21:35:51 +00:00
Jake Norbut 326bcb4544 Merged in SPARK-1960 (pull request #140)
RTM: locking _resources ResourceDictionary prior to indexer access

Approved-by: Eric Harding <eric.harding@bluetradesys.com>
2019-01-14 21:35:17 +00:00
Pedro Pinto 884cdef97b Merged in improve_perf_log_latency (pull request #139)
RTM. Improve perf log performance.
2019-01-14 21:34:48 +00:00
Eric Harding 5e3467e3b6 Add release note 2019-01-14 14:41:47 -05:00
Eric Harding 979c6e078a Remove misleading comments 2019-01-14 14:39:05 -05:00
Eric Harding 69f38daad7 SPARK-1966 Add CME ticker trade details 2019-01-14 14:37:51 -05:00
Eric Harding 14012a16da Add "all" option to sparkd md tail for emumba testing 2019-01-11 17:29:51 -05:00
Pedro Pinto 3bf086c73a Merged in SPARK-1970-manual-order-indicator (pull request #142)
RTM. SPARK-1970 manual order indicator
2019-01-10 19:39:31 +00:00
Pedro Pinto 9838d123c7 Updated release notes. 2019-01-10 14:15:04 -05:00
Pedro Pinto 9d5b80c990 Forced manual order indicator to false on algos. 2019-01-10 14:12:27 -05:00
Jake Norbut ae200b374b PR suggestion 2019-01-04 12:38:56 -06:00
Jake Norbut 64b94c58e7 locking _resources ResourceDictionary prior to indexer access 2019-01-04 11:20:21 -06:00
Pedro Pinto 8e907d7395 Fixed perf_log.sql 2019-01-04 12:04:10 -05:00
Pedro Pinto c567283d3f More performance improvements on perf_log 2019-01-04 11:31:36 -05:00
Pedro Pinto 6b84dc433a Making log analysis faster. 2019-01-04 09:23:59 -05:00
Pedro Pinto a833944ec5 Improved perf log latency. 2019-01-03 19:07:18 -05:00
17 changed files with 400 additions and 172 deletions

No files matched your search

+8
View File
@@ -1,4 +1,12 @@
# Release Notes
## 1.4.8
### Marketdata
- Additional trade details for CME ticker
## 1.4.7
### Bug Fixes
- Corrected FIX tag 1028, ManualOrderIndicator, when orders are triggered by algos
## 1.4.6
### Ladder Usability Improvements
- The ladder can now be zoomed in/out according to a user's preference
+7 -2
View File
@@ -31,7 +31,9 @@ namespace mbts.ui {
}
private static ImageSource load_resource_and_freeze(string resource_name) {
return ((DrawingImage)_resources[resource_name]).frozen();
lock (_resources) {
return ((DrawingImage)_resources[resource_name]).frozen();
}
}
#pragma warning disable CS0618 // Type or member is obsolete
@@ -152,7 +154,10 @@ namespace mbts.ui {
}
static IEnumerable<string> _get_geometric_path(string geometric_path_key) {
object obj = _resources[geometric_path_key];
object obj;
lock (_resources) {
obj = _resources[geometric_path_key];
}
if (obj is string path) {
if (string.IsNullOrEmpty(path)) {
Log.Api.error($"Resource lookup of '{geometric_path_key}' did not return a valid string.", nameof(geometric_path_key));
+242 -102
View File
@@ -1,3 +1,6 @@
/* -------------------------------------------------------------------------------- */
/* Main tables */
/* -------------------------------------------------------------------------------- */
create table if not exists
session (
version TEXT,
@@ -5,6 +8,7 @@ session (
build_type TEXT,
session_start TIMESTAMP PRIMARY KEY
);
create index if not exists session_idx on session(session_start);
create table if not exists
trace (
@@ -20,55 +24,46 @@ trace (
sn INTEGER,
uint32_1 INTEGER
);
drop view if exists recent_trace;
create view recent_trace as
select * from trace order by rowid desc limit 140123;
create index if not exists trace_idx on trace(session_start,channel,sn,ts);
create index if not exists trace_t_idx on trace(session_start,ts);
drop view if exists ordered_trace;
create view ordered_trace as
select
recent_trace.*,
row_number() over (order by session_start,ts) entry_rank
from
recent_trace;
/* -------------------------------------------------------------------------------- */
/* Main Views */
/* -------------------------------------------------------------------------------- */
drop view if exists delta_trace;
create view delta_trace as
/* An ordered view of the traces, with extra columns for the delta
betwen consecutive traces as well as the delta between related
(scoped) traces */
drop view if exists log_view;
create view log_view as
select
current.*,
(current.ts - previous.ts) / 1000.0 delta
from
ordered_trace current,
ordered_trace previous
where
current.entry_rank = previous.entry_rank+1;
drop view if exists log;
create view log as
select
delta_trace.*,
trace.*,
trace.rowid entry_id,
(ts-lag(ts) over by_session_ts)/1000.0 delta,
case cor_event>0 when 1 then (ts-cor_event)/1000.0 else 0 end scope_delta,
case cor_event>0 when 1 then "end of " || entry_kind else entry_kind end scoped_entry
from
delta_trace;
drop view if exists events;
create view events as
trace
window
by_session_ts as (order by session_start,ts);
/* A view identifying each high level event (requote or hedge) and
annotating it with its latency */
drop view if exists event_view;
create view event_view as
select
fix_reads.session_start,
"hedge" event_type,
fix_reads.hts hts_start,
fix_reads.entry_id event_id,
fix_reads.ts start,
fix_writes.ts stop,
(fix_writes.ts - fix_reads.ts) / 1000.0 duration,
row_number() over (order by fix_reads.session_start,fix_reads.ts,fix_writes.ts) event_rank ,
fix_reads.channel,
fix_reads.sn
from
(select session_start, channel, sn, ts from recent_trace where entry_kind='fix_msg_r') fix_reads,
(select session_start, channel, sn, ts from recent_trace where entry_kind='before_fix_socket_w') fix_writes
(select session_start, channel, sn, hts,ts,rowid entry_id from trace where entry_kind='fix_msg_r') fix_reads,
(select session_start, channel, sn, hts,ts,rowid entry_id from trace where entry_kind='before_fix_socket_w') fix_writes
where
fix_reads.session_start = fix_writes.session_start and
fix_reads.channel = fix_writes.channel and
@@ -77,35 +72,168 @@ union
select
md_reads.session_start,
"requote" event_type,
md_reads.hts hts_start,
md_reads.entry_id event_id,
md_reads.ts start,
fix_writes.ts stop,
(fix_writes.ts - md_reads.ts) / 1000.0 duration,
row_number() over (order by md_reads.session_start,md_reads.ts,fix_writes.ts) event_rank ,
md_reads.channel,
md_reads.sn
from
(select session_start, channel, sn, ts from recent_trace where entry_kind='md') md_reads,
(select session_start, channel, sn, ts from recent_trace where entry_kind='before_fix_socket_w') fix_writes
(select session_start, channel, sn, hts,ts,rowid entry_id from trace where entry_kind='md') md_reads,
(select session_start, channel, sn, hts,ts,rowid entry_id from trace where entry_kind='before_fix_socket_w') fix_writes
where
md_reads.session_start = fix_writes.session_start and
md_reads.channel = fix_writes.channel and
md_reads.sn = fix_writes.sn;
drop view if exists event_log;
create view event_log as
/* A view on hedge events, identifying each order sent as part of the event*/
drop view if exists hedge_trace_view;
create view hedge_trace_view as
select
log.*,
event_type,
event_rank,
events.duration event_duration,
events.start event_start,
events.stop event_stop
session.version,
session.host,
session.build_type,
session.session_start,
trace.hts trace_time,
uint64_1 order_key,
duration total,
event_id
from
log,
events
event_view,
trace,
session
where
log.session_start = events.session_start and
log.ts >= events.start and log.ts <= events.stop;
session.session_start = trace.session_start and
session.session_start = event_view.session_start and
event_view.channel=trace.channel and event_view.sn=trace.sn and
event_type='hedge' and
entry_kind='NativeOrder::on_activate' and
cor_event=0;
/* A view on requote events, identifying each order sent as part of the event*/
drop view if exists requote_trace_view;
create view requote_trace_view as
select
session.version,
session.host,
session.build_type,
session.session_start,
trace.hts trace_time,
uint64_1 order_key,
duration total,
event_id
from
event_view,
trace,
session
where
session.session_start = trace.session_start and
session.session_start = event_view.session_start and
event_view.channel=trace.channel and event_view.sn=trace.sn and
event_type='requote' and
entry_kind='NativeOrder::on_update' and
cor_event=0;
/* -------------------------------------------------------------------------------- */
/* Archive tables/view */
/* -------------------------------------------------------------------------------- */
/* Auxiliary tables and views, used to archive the results of the main
views to speed up analysis as data accumulates */
create table if not exists
event_archive (
session_start TIMESTAMP,
event_type TEXT,
hts_start TIMESTAMP,
event_id INTEGER,
start INTEGER,
stop INTEGER,
duration float,
channel TEXT,
sn INTEGER
);
create index if not exists event_archive_idx on event_archive(session_start,start,stop);
create table if not exists
log_archive(
session_start TIMESTAMP,
ts INTEGER,
hts timestamp,
channel TEXT,
entry_kind TEXT,
txt_1 TEXT,
uint64_1 INTEGER,
thread TEXT,
cor_event INTEGER,
sn INTEGER,
uint32_1 INTEGER,
entry_id INTEGER,
delta float,
scope_delta float,
scoped_entry TEXT
);
create index if not exists log_archive_idx on log_archive(session_start,ts);
create table if not exists
hedge_trace_archive (
version TEXT,
host TEXT,
build_type TEXT,
session_start TIMESTAMP,
trace_time TIMESTAMP,
order_key INTEGER,
total float,
event_id integer
);
drop view if exists hedge_trace;
create view hedge_trace as
select * from hedge_trace_view union select * from hedge_trace_archive;
create table if not exists
requote_trace_archive (
version TEXT,
host TEXT,
build_type TEXT,
session_start TIMESTAMP,
trace_time TIMESTAMP,
order_key INTEGER,
total float,
event_id integer
);
drop view if exists requote_trace;
create view requote_trace as
select * from requote_trace_view union select * from requote_trace_archive;
/* a synthetis of the log and events, focusing only on traces that are
part of hedge/requote events */
create table if not exists
event_log_archive (
session_start TIMESTAMP,
ts INTEGER,
hts timestamp,
channel TEXT,
entry_kind TEXT,
txt_1 TEXT,
uint64_1 INTEGER,
thread TEXT,
cor_event INTEGER,
sn INTEGER,
uint32_1 INTEGER,
entry_id INTEGER,
delta float,
scope_delta float,
scoped_entry TEXT,
event_type TEXT,
event_id INTEGER,
event_duration float,
hts_start TIMESTAMP,
event_start INTEGER,
event_stop INTEGER
);
create index if not exists event_log_archive_idx on event_log_archive(session_start);
drop view if exists pretty_log;
create view pretty_log as
@@ -115,69 +243,81 @@ select
scope_delta ,
event_duration,
event_type,
event_rank,
event_id,
hts,
ts,
session_start,
entry_rank
entry_id
from
event_log;
event_log_archive;
drop view if exists last_pretty_log;
create view last_pretty_log as
select * from pretty_log where session_start=(select max(session_start) from session);
select
*
from
pretty_log
where
session_start=(select max(session_start) from session);
-- drop view if exists summary;
-- create view summary as
-- select session_start,event_type,count(*) count, avg(duration) avg, max(duration) max, min(duration) min, median(duration) median, stdev(duration) stdev from events group by session_start,event_type;
/*
drop view if exists event_stats;
create view event_stats as
select
session_start,
event_type,
count(*) count,
avg(duration) avg,
max(duration) max,
min(duration) min,
median(duration) median,
stdev(duration) stdev
from
event_archive
group by
session_start,
event_type;
*/
drop view if exists event_log_aux;
create view event_log_aux as
select
log_archive.*,
event_type,
event_id,
event_archive.duration event_duration,
event_archive.hts_start hts_start,
event_archive.start event_start,
event_archive.stop event_stop
from
log_archive,
event_archive
where
log_archive.session_start = event_archive.session_start and
log_archive.ts >= event_archive.start and log_archive.ts <= event_archive.stop;
begin transaction;
insert into hedge_trace_archive select * from hedge_trace_view;
insert into requote_trace_archive select * from requote_trace_view;
insert into event_archive select * from event_view;
insert into log_archive select * from log_view;
insert into event_log_archive select * from event_log_aux;
drop view if exists hedge_trace;
create view hedge_trace as
select
session.version,
session.host,
session.build_type,
session.session_start,
recent_trace.hts trace_time,
recent_trace.uint64_1 order_key,
hedge_events.duration total,
hedge_events.event_rank rank
from
recent_trace,
(select * from events where event_type='hedge') hedge_events,
session
where
entry_kind='NativeOrder::on_activate' and
recent_trace.cor_event = 0 and
recent_trace.channel = hedge_events.channel and
recent_trace.session_start = hedge_events.session_start and
recent_trace.sn = hedge_events.sn and
recent_trace.session_start = session.session_start;
delete from trace;
drop view if exists requote_trace;
create view requote_trace as
select
session.version,
session.host,
session.build_type,
session.session_start,
recent_trace.hts trace_time,
recent_trace.uint64_1 order_key,
requote_events.duration total,
requote_events.event_rank rank
from
recent_trace,
(select * from events where event_type='requote') requote_events,
session
where
entry_kind='NativeOrder::on_update' and
recent_trace.cor_event = 0 and
recent_trace.channel = requote_events.channel and
recent_trace.session_start = requote_events.session_start and
recent_trace.sn = requote_events.sn and
recent_trace.session_start = session.session_start ;
delete from event_log_archive where
rowid in (select rowid from event_log_archive order by rowid desc limit -1 offset 10000123);
delete from trace where
rowid in (select rowid from trace order by rowid desc limit -1 offset 10000123);
delete from hedge_trace_archive where
rowid in (select rowid from hedge_trace_archive order by rowid desc limit -1 offset 10000123);
delete from requote_trace_archive where
rowid in (select rowid from requote_trace_archive order by rowid desc limit -1 offset 10000123);
delete from log_archive where
rowid in (select rowid from log_archive order by rowid desc limit -1 offset 10000123);
delete from event_log_archive where
rowid in (select rowid from event_log_archive order by rowid desc limit -1 offset 10000123);
commit;
+1 -1
View File
@@ -1 +1 @@
set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynaLine truncated
set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp)
+1
View File
@@ -110,6 +110,7 @@ message CMEOptions
// Prevents parsing trades/rfqs/doing anything not required for quoting
optional bool quoter_mode = 6;
optional bool verbose_ticker = 7 [default=false];
}
message RecordOptions {
-1
View File
@@ -3,7 +3,6 @@ package bts.config;
message PerfLogCfg {
optional bool active = 1 [default=true];
optional uint32 recent_traces_size = 2 [default=140123];
optional uint32 max_rows = 3 [default=10000123];
optional uint32 entry_batch_size = 4 [default=200];
}
@@ -109,7 +109,8 @@ int SocketConnection::write(WriteVec const &vec, size_t vec_size,
_perf_log.log(trace,"before_fix_socket_w");
auto result = ::writev(_fd, iv, (int)vec_size);
_perf_log.log(trace,"after_fix_socket_w");
_perf_log.flush();
if (trace.sn)
_perf_log.flush();
if (_log && result > 0) {
for (size_t i = 0; i < vec_size; i++)
+4 -2
View File
@@ -150,7 +150,7 @@ void write_header(PrintFormat format, long seq_mod, const std::vector<Entry>& en
}
}
void run_periodic_dumper(const std::set<UserSymbol>& symbols, uint32_t seq_mod, PrintFormat format) {
void run_periodic_dumper(const std::set<UserSymbol>& symbols, uint32_t seq_mod, PrintFormat format, bool monitor_all) {
MDConsumer consumer(symbols, nullptr);
@@ -160,7 +160,9 @@ void run_periodic_dumper(const std::set<UserSymbol>& symbols, uint32_t seq_mod,
entries.push_back({ 0, handle, std::make_unique<api::Security>(sec) });
});
entries = filter_entries(entries);
if (!monitor_all) {
entries = filter_entries(entries);
}
write_header(format, seq_mod, entries);
+2 -1
View File
@@ -10,7 +10,8 @@ enum class PrintFormat {
CSV
};
void run_periodic_dumper(const std::set<UserSymbol>&, uint32_t seq_mod, PrintFormat format);
void run_periodic_dumper(const std::set<UserSymbol>&, uint32_t seq_mod, PrintFormat format, bool monitor_all);
}}//ns
@@ -72,8 +72,22 @@ Order &OrderFactory::create(OrderKey parent,
auto k = gen_order_id();
auto pk = parent;
auto &order = allocate(k, pk, def);
order.on_init_new(def,
PreActivationOrderServices{me->sec_db, *this, me->router});
// For non native orders we force the ManalOrderIndicator to be 'N'
// since all non native order are algos and we would rather not
// rely on the order creator (be it shell or another algo) to set
// this correctly for every single algo.
if (dynamic_cast<NativeOrder *>(&order) == nullptr &&
def.manual_order_indicator() ==
api::orderman::MANUAL_ORDER_INDICATOR_YES) {
auto corrected_def = def;
corrected_def.set_manual_order_indicator(
api::orderman::MANUAL_ORDER_INDICATOR_NO);
order.on_init_new(corrected_def, PreActivationOrderServices{
me->sec_db, *this, me->router});
} else
order.on_init_new(
def, PreActivationOrderServices{me->sec_db, *this, me->router});
return order;
}
+29 -8
View File
@@ -19,7 +19,7 @@ fi
SQLLITE=../../../../externals/linux/bin/sqlite3
SQLITE_EXT=../../../../externals/linux/bin/libsqlitefunctions
DB=file:$HOME/sparkdata/data/databases/perf_log/perf_log.sqlite?mode=ro
DB=file:$HOME/sparkdata/data/databases/perf_log/perf_log.sqlite
if [ $# -eq 1 ]
then
@@ -28,6 +28,19 @@ else
SESSION=\'$2\'
fi
echo Generating stats...
$SQLLITE $DB<<EOF
begin transaction;
insert into hedge_trace_archive select * from hedge_trace_view;
insert into requote_trace_archive select * from requote_trace_view;
insert into event_archive select * from event_view;
insert into log_archive select * from log_view;
insert into event_log_archive select * from event_log_aux;
delete from trace;
commit;
EOF
echo Done
if [ $1 == "log" ]; then
$SQLLITE $DB<<EOF
.mode column
@@ -52,15 +65,23 @@ $SQLLITE $DB<<EOF
.mode column
.headers on
.load $SQLITE_EXT
select
session_start,event_type,count(*) count, avg(duration) avg,
max(duration) max, min(duration) min, median(duration) median,
stdev(duration) stdev
from events where session_start=$SESSION group by session_start,event_type;
select
session_start,
event_type,
count(*) count,
avg(duration) avg,
max(duration) max,
min(duration) min,
median(duration) median,
stdev(duration) stdev
from
event_archive
where
session_start=$SESSION
group by
event_type
EOF
exit 0
fi
help()
+55 -33
View File
@@ -3,9 +3,10 @@
#include "lib/config/protos/perflog_cfg.pb.h"
#include "lib/config/protos/system.pb.h"
#include "lib/persistence/persistence_utils.h"
#include "lib/utils/bts_array_size.h"
#include "lib/utils/datetime.h"
#include "lib/utils/stats.h"
#include "lib/utils/bts_array_size.h"
#include "lib/utils/thread_manager.h"
#include <SQLiteCpp/SQLiteCpp.h>
#include <boost/algorithm/string/replace.hpp>
@@ -81,6 +82,8 @@ struct PerfLog::PImpl {
std::string session_start;
boost::circular_buffer<PerfLogEntry> entry_batch;
std::mutex lock;
std::thread init_thread;
std::atomic<bool> init_completed;
};
PerfLog::PerfLog() : _pending(), _disabled(false), me() {
@@ -98,38 +101,56 @@ PerfLog::PerfLog() : _pending(), _disabled(false), me() {
try {
me = std::make_unique<PImpl>(loc);
init_db();
me->insert_trace_statement =
std::make_unique<SQLite::Statement>(me->db, insert_trace);
me->insert_session_statement =
std::make_unique<SQLite::Statement>(me->db, insert_session);
me->requote_query_statement =
std::make_unique<SQLite::Statement>(me->db, requote_query);
me->hedge_query_statement =
std::make_unique<SQLite::Statement>(me->db, hedge_query);
me->session_start = datetime::FIX_timestamp(datetime::local_now());
me->insert_session_statement->bind(
":version", config::get<config::System>().version());
me->insert_session_statement->bind(
":host", config::get<config::System>().host_name());
#ifdef DEBUG
me->insert_session_statement->bind(":build_type", "debug");
#else
me->insert_session_statement->bind(":build_type", "release");
#endif
me->insert_session_statement->bind(":session_start", me->session_start);
me->insert_session_statement->exec();
me->insert_session_statement->reset();
} catch (SQLite::Exception const &e) {
LOG(persist, Error, "Error initializing perf_trace DB {}. Error: {}", loc,
std::string(e.getErrorStr()));
_disabled = true;
return;
}
me->init_completed = false;
me->init_thread = threading::make_thread("perf_log_ini", [loc,this]() {
try {
LOG(persist,Info, "Started initialization of perf_log db");
init_db();
me->insert_trace_statement =
std::make_unique<SQLite::Statement>(me->db, insert_trace);
me->insert_session_statement =
std::make_unique<SQLite::Statement>(me->db, insert_session);
me->requote_query_statement =
std::make_unique<SQLite::Statement>(me->db, requote_query);
me->hedge_query_statement =
std::make_unique<SQLite::Statement>(me->db, hedge_query);
me->session_start = datetime::FIX_timestamp(datetime::local_now());
me->insert_session_statement->bind(
":version", config::get<config::System>().version());
me->insert_session_statement->bind(
":host", config::get<config::System>().host_name());
#ifdef DEBUG
me->insert_session_statement->bind(":build_type", "debug");
#else
me->insert_session_statement->bind(":build_type", "release");
#endif
me->insert_session_statement->bind(":session_start",
me->session_start);
me->insert_session_statement->exec();
me->insert_session_statement->reset();
} catch (SQLite::Exception const &e) {
LOG(persist, Error,
"Error initializing perf_trace DB (2) {}. Error: {}", loc,
std::string(e.getErrorStr()));
_disabled = true;
}
LOG(persist,Info, "Finished initialization of perf_log db");
me->init_completed = true;
});
}
PerfLog::~PerfLog() {}
PerfLog::~PerfLog() {
me->init_thread.join();
}
std::string PerfLog::init_storage_location() {
auto loc = persistence::storage_location("perf_log");
@@ -145,7 +166,7 @@ std::string PerfLog::init_storage_location() {
}
size_t PerfLog::store_log_entries() {
if (_disabled) {
if (!me->init_completed || _disabled) {
_pending.clear();
return 0;
}
@@ -167,9 +188,6 @@ void PerfLog::init_db() {
std::string str((std::istreambuf_iterator<char>(f)),
(std::istreambuf_iterator<char>()));
config::PerfLogCfg defaults;
boost::replace_all(
str, std::to_string(defaults.recent_traces_size()),
std::to_string(config::get<config::PerfLogCfg>().recent_traces_size()));
boost::replace_all(
str, std::to_string(defaults.max_rows()),
std::to_string(config::get<config::PerfLogCfg>().max_rows()));
@@ -187,7 +205,9 @@ std::string encode_number(SQLite::Column const &c) {
std::vector<std::map<std::string, std::string>>
PerfLog::get_latest_requote_traces(size_t n) {
std::vector<std::map<std::string, std::string>> result;
if (!me->init_completed)
return result;
std::lock_guard<std::mutex> l(me->lock);
try {
me->requote_query_statement->bind(1, (long long)n);
@@ -221,7 +241,9 @@ PerfLog::get_latest_requote_traces(size_t n) {
std::vector<std::map<std::string, std::string>>
PerfLog::get_latest_hedge_traces(size_t n) {
std::vector<std::map<std::string, std::string>> result;
if (!me->init_completed)
return result;
std::lock_guard<std::mutex> l(me->lock);
try {
me->hedge_query_statement->bind(1, (long long)n);
@@ -276,7 +298,7 @@ size_t PerfLog::flush_current_batch_to_disk() {
me->insert_trace_statement->bind(":cor_event",
(long long)ev.cor_event.to_uint64());
char tn[17];
pthread_getname_np(ev.thread_id, tn, BTS_ARRAYSIZE(tn)-1);
pthread_getname_np(ev.thread_id, tn, BTS_ARRAYSIZE(tn) - 1);
me->insert_trace_statement->bind(":thread", tn);
me->insert_trace_statement->exec();
count++;
@@ -19,6 +19,7 @@ MDP3Listener::MDP3Listener(
const std::vector<MDProducerChannel> channels) :
_index(index),
_producer(producer),
_verbose_ticker(config::get<CMEOptions>().verbose_ticker()),
_packet_arrived(),
_latest_sequence(0),
// _settlement_recorder(settles),
@@ -182,6 +182,7 @@ private:
CMESecurityIndex& _index;
MDProducer& _producer;
bool _verbose_ticker;
utils::TimeStamp _packet_arrived;
uint32_t _latest_sequence;
// co2::SettlementRecorder& _settlement_recorder;
@@ -416,7 +417,17 @@ void MDP3Listener::onTradeSummary(const onixs::Security& sec, const TradeSummary
onixs::Decimal price = arg.entry().entryPx();
onixs::AggressorSide::Enum agside = onixs::AggressorSide::NoAggressor;
arg.entry().aggressorSide(agside);
on_trade(data, price, size, agside, arg.message().transactTime());
if (_verbose_ticker) {
const auto& details = arg.orderIds();
auto sz = details.size();
for (size_t x = 0; x < sz; ++x) {
const auto& d = details[x];
on_trade(data, price, d.lastQty(), agside, arg.message().transactTime());
}
}
else {
on_trade(data, price, size, agside, arg.message().transactTime());
}
}
}
@@ -442,7 +453,6 @@ void MDP3Listener::onSettlementPrice(const onixs::Security& sec, const Statistic
onixs::Timestamp time;
args.entry().entryPx(price);
args.entry().tradingReferenceDate(time);
// args.entry().
auto display_factor = data->handle.security().display_factor;
data->handle.update_settlement(
utils::TimeStamp::now(),
@@ -454,19 +464,16 @@ void MDP3Listener::onSettlementPrice(const onixs::Security& sec, const Statistic
template <typename SessionArgs>
void MDP3Listener::onSessionOpeningPrice(const onixs::Security& sec, const SessionArgs& args) {
tracepoint (bts_trace,trace_order,(char*) "onOpeningPrice.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
// auto type = args.entry().settlPriceType();
// if (type.final() && type.actual()) {
auto* data = _index.get_security(sec.id());
if (data != nullptr) {
onixs::Decimal price = args.entry().entryPx();
onixs::Timestamp time = args.message().transactTime();
auto display_factor = data->handle.security().display_factor;
data->handle.update_opening(
utils::TimeStamp::now(),
double_of_price(price, display_factor),
date_of_timestamp(time));
}
//
auto* data = _index.get_security(sec.id());
if (data != nullptr) {
onixs::Decimal price = args.entry().entryPx();
onixs::Timestamp time = args.message().transactTime();
auto display_factor = data->handle.security().display_factor;
data->handle.update_opening(
utils::TimeStamp::now(),
double_of_price(price, display_factor),
date_of_timestamp(time));
}
}
template <typename RecoveryArgs>
void MDP3Listener::onRecoveryOpeningPrice(const onixs::Security& sec, const RecoveryArgs& args) {
@@ -269,7 +269,10 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(const string& channel_id) {
// Misc settings
settings.bookManagement().updateNotification(onixs::BookUpdateNotification::OnEndOfEvent);
settings.tradeProcessing(onixs::TradeProcessing::SummaryLevel);
settings.tradeProcessing(
config::get<CMEOptions>().verbose_ticker() ?
onixs::TradeProcessing::OrderDetails :
onixs::TradeProcessing::SummaryLevel);
//
// Logging
+5 -2
View File
@@ -252,14 +252,17 @@ void dump_db(const vector<string>& args) {
void run_tail(const vector<string>& args) {
std::set<UserSymbol> products;
auto conf = parse_md_args(args);
bool monitor_all = false;
for (const auto& b : args) { if (b=="all") monitor_all = true; }
PlatformConfig config;
config.set_type(api::SecurityType_Name(api::SECURITYTYPE_FUTURE));
if (!monitor_all) config.set_type(api::SecurityType_Name(api::SECURITYTYPE_FUTURE));
config.set_platform(co2::Platform_Name(conf.platform));
get_system_md_symbols(config, products);
auto mod = conf.number > 0 ? conf.number : 1000;
PrintFormat fmt = PrintFormat::CSV;
for (const auto& a : args) { if (a == "longform") fmt = PrintFormat::Human; }
md::run_periodic_dumper(products, mod, fmt);
md::run_periodic_dumper(products, mod, fmt, monitor_all);
}
void run_channels() {
+1 -1
View File
@@ -1 +1 @@
set(sources ../../externals/linux/include/google/protobuf/descriptor.proto;../../externals/linux/include/google/protobuf/timestamp.proto;../lib/api/apicore.proto;../lib/api/apigateway.proto;../lib/api/apioptions.proto;../lib/api/services/AdminService.proto;../lib/api/services/ApiUnitTestService.proto;../lib/api/services/AuthService.proto;../lib/api/services/ErrorReportService.proto;../lib/api/services/HelloService.proto;../lib/api/services/MarketDataAuxiliaryService.proto;../lib/api/services/MarketDataRelay.proto;../lib/api/services/OrderService.proto;../lib/api/services/OutOfProcessURLListener.proto;../lib/api/services/PersistenceService.proto;../lib/api/services/PositionService.proto;../lib/api/services/ShellDiagnosticService.proto;../lib/api/services/ShellLatencyService.proto;../lib/api/services/SyntheticSecurityService.proto;../lib/domain_model/Security.proto;../lib/md/protos/co2.proto;../lib/sparkproto/SparkProtos.proto;api/build.sh;api/node_modules/cssstyle/scripts/run_tests.sh;api/node_modules/exit/test/fixtures/create-files.sh;api/node_modules/protobufjs/cli/node_modules/acorn-jsx/node_modules/acorn/bin/update_authors.sh;api/node_modules/watch/release.sh;api/quiet_build.sh;api/src/DynamicService.ts;api/src/ServiceContext.ts;api/src/bts_assert.ts;ui/src/Access.fs;ui/src/Accounts/ClipLimits.fs;ui/src/Accounts/ProductLimits.fs;ui/src/Accounts/SecurityLimits.fs;ui/src/Accounts/State.fs;ui/src/Accounts/Types.fs;ui/src/Accounts/View.fs;ui/src/App.fs;ui/src/Cmd.fs;ui/src/Controls/Button.fs;ui/src/Controls/DataGrid.fs;ui/src/Controls/Fields.fs;ui/src/Controls/Modal.fs;ui/src/Controls/ReactGrid.fs;ui/src/Controls/Table.fs;ui/src/Controls/nbsp.fs;ui/src/Global.fs;ui/src/Imports.fs;ui/src/Lens.fs;ui/src/Login/Login.fs;ui/src/Nav.fs;ui/src/State.fs;ui/src/Types.fs;ui/src/Users/State.fs;ui/src/Users/Types.fs;ui/src/Users/View.fs;ui/src/Util/EnumUtils.fs;ui/src/Util/JSMap.fs;ui/src/obj/Debug/netstandard2.0/ui.AssemblyInfo.fs)
set(sources ../../externals/linux/include/google/protobuf/descriptor.proto;../../externals/linux/include/google/protobuf/timestamp.proto;../lib/api/apicore.proto;../lib/api/apigateway.proto;../lib/api/apioptions.proto;../lib/api/services/AdminService.proto;../lib/api/services/ApiUnitTestService.proto;../lib/api/services/AuthService.proto;../lib/api/services/ErrorReportService.proto;../lib/api/services/HelloService.proto;../lib/api/services/MarketDataAuxiliaryService.proto;../lib/api/services/MarketDataRelay.proto;../lib/api/services/OrderService.proto;../lib/api/services/OutOfProcessURLListener.proto;../lib/api/services/PersistenceService.proto;../lib/api/services/PositionService.proto;../lib/api/services/ShellDiagnosticService.proto;../lib/api/services/ShellLatencyService.proto;../lib/api/services/SyntheticSecurityService.proto;../lib/domain_model/Security.proto;../lib/md/protos/co2.proto;../lib/sparkproto/SparkProtos.proto;api/build.sh;api/node_modules/exit/test/fixtures/create-files.sh;api/node_modules/protobufjs/cli/node_modules/acorn-jsx/node_modules/acorn/bin/update_authors.sh;api/node_modules/watch/release.sh;api/quiet_build.sh;api/src/DynamicService.ts;api/src/ServiceContext.ts;api/src/bts_assert.ts;ui/src/Access.fs;ui/src/Accounts/ClipLimits.fs;ui/src/Accounts/ProductLimits.fs;ui/src/Accounts/SecurityLimits.fs;ui/src/Accounts/State.fs;ui/src/Accounts/Types.fs;ui/src/Accounts/View.fs;ui/src/App.fs;ui/src/Cmd.fs;ui/src/Controls/Button.fs;ui/src/Controls/DataGrid.fs;ui/src/Controls/Fields.fs;ui/src/Controls/Modal.fs;ui/src/Controls/ReactGrid.fs;ui/src/Controls/Table.fs;ui/src/Controls/nbsp.fs;ui/src/Global.fs;ui/src/Imports.fs;ui/src/Lens.fs;ui/src/Login/Login.fs;ui/src/Nav.fs;ui/src/State.fs;ui/src/Types.fs;ui/src/Users/State.fs;ui/src/Users/Types.fs;ui/src/Users/View.fs;ui/src/Util/EnumUtils.fs;ui/src/Util/JSMap.fs;ui/src/obj/Debug/netstandard2.0/ui.AssemblyInfo.fs)