Metric for latency between spark shell and server

This commit is contained in:
Eric Harding committed 2018-06-28 11:07:45 -04:00
1 parent 03eff1f388
commit 3da1bfaeb1
8 files changed
+185 -3

No files matched your search

@@ -0,0 +1,44 @@
using bts.api.core;
using bts.metrics;
using mbts.config;
using mbts.ui;
using System;
using System.Collections.Generic;
using System.ComponentModel.Composition;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace bts.perf {
[Export, Export(typeof(IShellDaemon))]
class ShellLatencyDaemon : IShellDaemon, IDisposable {
readonly ShellLatencyService _service;
readonly string _hostname;
readonly string _version;
IDisposable _subs;
[ImportingConstructor]
public ShellLatencyDaemon(ShellLatencyService service) {
_service = service;
_version = Configuration.get<bts.config.System>().version;
_hostname = Environment.MachineName;
_subs = _service.subscribei_clock().Subscribe(on_tick);
}
void on_tick(Update<TickImmutable, int> update) {
if (update.added.Count > 0) {
var tick = update.added.Last();
_service.report_async(new Report() {
HostName = _hostname,
Version = _version,
Timestamp = tick.Timestamp
});
}
}
public void Dispose() {
throw new NotImplementedException();
}
}
}
+6 -1
View File
@@ -12,6 +12,7 @@ using bts.api.orderman;
using bts.api.synthetic;
using bts.api.diagnostic;
using bts.api.admin;
using bts.metrics;
namespace Spark
{
@@ -41,5 +42,9 @@ namespace Spark
[Export]
public SyntheticSecurityService synthetic_security_service => context.locate<SyntheticSecurityService>();
}
[Export]
public ShellLatencyService shell_latency_service => context.locate<ShellLatencyService>();
}
}
+1 -1
View File
@@ -1 +1 @@
set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/ShellDiagnosticService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/quickbus_cfg.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/windows_test_driver.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp)
set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/quickbus_cfg.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/windows_test_driver.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp)
@@ -0,0 +1,21 @@
syntax="proto3";
package bts.metrics;
option csharp_namespace="bts.metrics";
import "lib/api/apioptions.proto";
import "lib/api/apicore.proto";
message Tick {
int32 key = 1 [(bts.api.key)=true];
uint64 timestamp = 2;
}
message Report {
string host_name = 1;
string version = 2;
uint64 timestamp = 10;
}
service ShellLatencyService {
rpc clock(bts.api.VoidMessage) returns (stream Tick);
rpc report(Report) returns (bts.api.VoidMessage);
}
+2
View File
@@ -15,6 +15,7 @@
#include "server/sparkd/orderman/server/OrderServiceServer.h"
#include "server/sparkd/persistence/PersistenceServiceServer.h"
#include "server/sparkd/support/ErrorReportService.h"
#include "server/sparkd/support/ShellLatencyServer.h"
#include "server/sparkd/http/WebServer.h"
#include "server/sparkd/metrics/metrics.h"
#include "server/sparkd/metrics/system_metrics.h"
@@ -53,6 +54,7 @@ int Run::execute(std::vector<std::string> const &) {
orderman::OrderServiceServer order_service_server(sc, synthetic_service);
support::ErrorReportServer error_report_service(sc);
admin::AdminServiceServer admin_service(sc);
metrics::ShellLatencyServer shell_latency_service(sc, stats, utils::Dispatcher::background());
std::thread([]() {
os::set_thread_name("ExchangeSim");
+20 -1
View File
@@ -42,15 +42,17 @@ struct CounterMetric : public Metric {
void decrement(T val) { _val -= val; }
void set(T val) { _val = val; }
};
typedef CounterMetric<int> IntCounterMetric;
template <class T>
class AtomicCounterMetric : public Metric {
protected:
std::atomic<T> _val;
public:
AtomicCounterMetric(const std::string& name) : Metric(name, {}), _val() {}
AtomicCounterMetric(const std::string& name, const std::vector<Label>& labels) : Metric(name, labels), _val() {}
AtomicCounterMetric(const AtomicCounterMetric& o) : Metric(o.name, o.labels), _val(o._val.load()) { }
~AtomicCounterMetric() {};
~AtomicCounterMetric() {}
void print(std::ostream& out) const override {
out << name << labels << " " << _val << std::endl;
}
@@ -61,6 +63,23 @@ class AtomicCounterMetric : public Metric {
void set(T val) { _val = val; }
};
class AutoResetMetric : public AtomicCounterMetric<int64_t> {
public:
AutoResetMetric(const std::string& name) : AtomicCounterMetric(name, {}) {}
AutoResetMetric(const std::string& name, const std::vector<Label>& labels) : AtomicCounterMetric(name, labels) {}
~AutoResetMetric(){}
void print(std::ostream& out) const override {
out << name << labels << " " << _val << std::endl;
const_cast<AutoResetMetric*>(this)->set(0);
}
void max(int64_t val) {
_val = std::max(val, _val.load());
}
void min(int64_t val) {
_val = std::min(val, _val.load());
}
};
template <class T>
class LazyCounterMetric : public Metric {
const std::function<T()> _get;
@@ -0,0 +1,57 @@
#include "ShellLatencyServer.h"
#include "lib/utils/TimeStamp.h"
namespace bts {
namespace metrics {
ShellLatencyServer::ShellLatencyServer(bts::api::ServiceContext& ctx, metrics::MetricsContainer& mc, utils::Dispatcher& d) :
ShellLatencyServiceBase(ctx),
_counter(0),
_pub(make_clock_stateful_publisher()),
_timer(utils::DispatcherTimer(d, 2500, [=]() { tick(); })),
_mc(mc),
_metrics()
{
_timer.start();
}
ShellLatencyServer::~ShellLatencyServer(){}
// This happens on the Dispatcher thread
void ShellLatencyServer::tick() {
_counter++;
Tick t;
t.set_key(_counter);
t.set_timestamp(utils::TimeStamp::now().to_uint64());
_pub.add_or_update(t);
_pub.erase(_counter - 1);
_pub.flush();
}
// This happens on the quickbus thread
void ShellLatencyServer::report(
const ::bts::metrics::Report& request,
const api::Response<::bts::api::VoidMessage>& reply) {
reply(bts::api::VoidMessage());
// look up or create the metric
auto key = request.host_name() + "-" + request.version();
auto iter = _metrics.find(key);
std::shared_ptr<AutoResetMetric> metric;
auto delta = (utils::TimeStamp::now() - utils::TimeStamp(request.timestamp())).milliseconds();
if (iter == _metrics.end()) {
// create metric
auto metric = std::make_shared<AutoResetMetric>("spark_shell_latency", std::vector<Label>({{"shell",key}}));
_metrics.insert(std::make_pair(key, metric));
metric->max(delta);
_mc.publish(metric);
}
else {
iter->second->max(delta);
}
}
}}
@@ -0,0 +1,34 @@
#pragma once
#include "server/sparkd/metrics/metrics.h"
#include "lib/utils/Dispatcher.h"
#include "lib/api/services/ShellLatencyService.pb.h"
#include "lib/api/services/ShellLatencyService.server.pb.h"
#include <map>
#include <string>
#include <memory>
namespace bts {
namespace metrics {
class ShellLatencyServer : public ShellLatencyServiceBase {
int _counter;
ClockStatefulPublisher _pub;
utils::DispatcherTimer _timer;
MetricsContainer& _mc;
std::map<std::string, std::shared_ptr<AutoResetMetric>> _metrics;
public:
ShellLatencyServer(bts::api::ServiceContext&, metrics::MetricsContainer&, utils::Dispatcher&);
~ShellLatencyServer();
void tick();
void report(
const ::bts::metrics::Report& request,
const api::Response<::bts::api::VoidMessage>& reply) override;
};
}}