Metric for latency between spark shell and server
This commit is contained in:
1 parent
03eff1f388
commit
3da1bfaeb1
8 files changed
+185
-3
No files matched your search
@@ -0,0 +1,44 @@
|
||||
using bts.api.core;
|
||||
using bts.metrics;
|
||||
using mbts.config;
|
||||
using mbts.ui;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.ComponentModel.Composition;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace bts.perf {
|
||||
|
||||
[Export, Export(typeof(IShellDaemon))]
|
||||
class ShellLatencyDaemon : IShellDaemon, IDisposable {
|
||||
readonly ShellLatencyService _service;
|
||||
readonly string _hostname;
|
||||
readonly string _version;
|
||||
IDisposable _subs;
|
||||
|
||||
[ImportingConstructor]
|
||||
public ShellLatencyDaemon(ShellLatencyService service) {
|
||||
_service = service;
|
||||
_version = Configuration.get<bts.config.System>().version;
|
||||
_hostname = Environment.MachineName;
|
||||
_subs = _service.subscribei_clock().Subscribe(on_tick);
|
||||
}
|
||||
|
||||
void on_tick(Update<TickImmutable, int> update) {
|
||||
if (update.added.Count > 0) {
|
||||
var tick = update.added.Last();
|
||||
_service.report_async(new Report() {
|
||||
HostName = _hostname,
|
||||
Version = _version,
|
||||
Timestamp = tick.Timestamp
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose() {
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ using bts.api.orderman;
|
||||
using bts.api.synthetic;
|
||||
using bts.api.diagnostic;
|
||||
using bts.api.admin;
|
||||
using bts.metrics;
|
||||
|
||||
namespace Spark
|
||||
{
|
||||
@@ -41,5 +42,9 @@ namespace Spark
|
||||
[Export]
|
||||
public SyntheticSecurityService synthetic_security_service => context.locate<SyntheticSecurityService>();
|
||||
|
||||
}
|
||||
[Export]
|
||||
public ShellLatencyService shell_latency_service => context.locate<ShellLatencyService>();
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1 +1 @@
|
||||
set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/ShellDiagnosticService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/quickbus_cfg.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/windows_test_driver.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp)
|
||||
set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/quickbus_cfg.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/windows_test_driver.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp)
|
||||
@@ -0,0 +1,21 @@
|
||||
syntax="proto3";
|
||||
package bts.metrics;
|
||||
option csharp_namespace="bts.metrics";
|
||||
import "lib/api/apioptions.proto";
|
||||
import "lib/api/apicore.proto";
|
||||
|
||||
message Tick {
|
||||
int32 key = 1 [(bts.api.key)=true];
|
||||
uint64 timestamp = 2;
|
||||
}
|
||||
|
||||
message Report {
|
||||
string host_name = 1;
|
||||
string version = 2;
|
||||
uint64 timestamp = 10;
|
||||
}
|
||||
|
||||
service ShellLatencyService {
|
||||
rpc clock(bts.api.VoidMessage) returns (stream Tick);
|
||||
rpc report(Report) returns (bts.api.VoidMessage);
|
||||
}
|
||||
@@ -15,6 +15,7 @@
|
||||
#include "server/sparkd/orderman/server/OrderServiceServer.h"
|
||||
#include "server/sparkd/persistence/PersistenceServiceServer.h"
|
||||
#include "server/sparkd/support/ErrorReportService.h"
|
||||
#include "server/sparkd/support/ShellLatencyServer.h"
|
||||
#include "server/sparkd/http/WebServer.h"
|
||||
#include "server/sparkd/metrics/metrics.h"
|
||||
#include "server/sparkd/metrics/system_metrics.h"
|
||||
@@ -53,6 +54,7 @@ int Run::execute(std::vector<std::string> const &) {
|
||||
orderman::OrderServiceServer order_service_server(sc, synthetic_service);
|
||||
support::ErrorReportServer error_report_service(sc);
|
||||
admin::AdminServiceServer admin_service(sc);
|
||||
metrics::ShellLatencyServer shell_latency_service(sc, stats, utils::Dispatcher::background());
|
||||
|
||||
std::thread([]() {
|
||||
os::set_thread_name("ExchangeSim");
|
||||
|
||||
@@ -42,15 +42,17 @@ struct CounterMetric : public Metric {
|
||||
void decrement(T val) { _val -= val; }
|
||||
void set(T val) { _val = val; }
|
||||
};
|
||||
typedef CounterMetric<int> IntCounterMetric;
|
||||
|
||||
template <class T>
|
||||
class AtomicCounterMetric : public Metric {
|
||||
protected:
|
||||
std::atomic<T> _val;
|
||||
public:
|
||||
AtomicCounterMetric(const std::string& name) : Metric(name, {}), _val() {}
|
||||
AtomicCounterMetric(const std::string& name, const std::vector<Label>& labels) : Metric(name, labels), _val() {}
|
||||
AtomicCounterMetric(const AtomicCounterMetric& o) : Metric(o.name, o.labels), _val(o._val.load()) { }
|
||||
~AtomicCounterMetric() {};
|
||||
~AtomicCounterMetric() {}
|
||||
void print(std::ostream& out) const override {
|
||||
out << name << labels << " " << _val << std::endl;
|
||||
}
|
||||
@@ -61,6 +63,23 @@ class AtomicCounterMetric : public Metric {
|
||||
void set(T val) { _val = val; }
|
||||
};
|
||||
|
||||
class AutoResetMetric : public AtomicCounterMetric<int64_t> {
|
||||
public:
|
||||
AutoResetMetric(const std::string& name) : AtomicCounterMetric(name, {}) {}
|
||||
AutoResetMetric(const std::string& name, const std::vector<Label>& labels) : AtomicCounterMetric(name, labels) {}
|
||||
~AutoResetMetric(){}
|
||||
void print(std::ostream& out) const override {
|
||||
out << name << labels << " " << _val << std::endl;
|
||||
const_cast<AutoResetMetric*>(this)->set(0);
|
||||
}
|
||||
void max(int64_t val) {
|
||||
_val = std::max(val, _val.load());
|
||||
}
|
||||
void min(int64_t val) {
|
||||
_val = std::min(val, _val.load());
|
||||
}
|
||||
};
|
||||
|
||||
template <class T>
|
||||
class LazyCounterMetric : public Metric {
|
||||
const std::function<T()> _get;
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
#include "ShellLatencyServer.h"
|
||||
#include "lib/utils/TimeStamp.h"
|
||||
|
||||
namespace bts {
|
||||
namespace metrics {
|
||||
|
||||
ShellLatencyServer::ShellLatencyServer(bts::api::ServiceContext& ctx, metrics::MetricsContainer& mc, utils::Dispatcher& d) :
|
||||
ShellLatencyServiceBase(ctx),
|
||||
_counter(0),
|
||||
_pub(make_clock_stateful_publisher()),
|
||||
_timer(utils::DispatcherTimer(d, 2500, [=]() { tick(); })),
|
||||
_mc(mc),
|
||||
_metrics()
|
||||
{
|
||||
_timer.start();
|
||||
}
|
||||
|
||||
ShellLatencyServer::~ShellLatencyServer(){}
|
||||
|
||||
// This happens on the Dispatcher thread
|
||||
void ShellLatencyServer::tick() {
|
||||
_counter++;
|
||||
Tick t;
|
||||
t.set_key(_counter);
|
||||
t.set_timestamp(utils::TimeStamp::now().to_uint64());
|
||||
_pub.add_or_update(t);
|
||||
_pub.erase(_counter - 1);
|
||||
_pub.flush();
|
||||
}
|
||||
|
||||
// This happens on the quickbus thread
|
||||
void ShellLatencyServer::report(
|
||||
const ::bts::metrics::Report& request,
|
||||
const api::Response<::bts::api::VoidMessage>& reply) {
|
||||
|
||||
reply(bts::api::VoidMessage());
|
||||
|
||||
// look up or create the metric
|
||||
auto key = request.host_name() + "-" + request.version();
|
||||
auto iter = _metrics.find(key);
|
||||
std::shared_ptr<AutoResetMetric> metric;
|
||||
auto delta = (utils::TimeStamp::now() - utils::TimeStamp(request.timestamp())).milliseconds();
|
||||
if (iter == _metrics.end()) {
|
||||
// create metric
|
||||
auto metric = std::make_shared<AutoResetMetric>("spark_shell_latency", std::vector<Label>({{"shell",key}}));
|
||||
_metrics.insert(std::make_pair(key, metric));
|
||||
metric->max(delta);
|
||||
_mc.publish(metric);
|
||||
}
|
||||
else {
|
||||
iter->second->max(delta);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
}}
|
||||
@@ -0,0 +1,34 @@
|
||||
#pragma once
|
||||
|
||||
#include "server/sparkd/metrics/metrics.h"
|
||||
#include "lib/utils/Dispatcher.h"
|
||||
#include "lib/api/services/ShellLatencyService.pb.h"
|
||||
#include "lib/api/services/ShellLatencyService.server.pb.h"
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
|
||||
namespace bts {
|
||||
namespace metrics {
|
||||
|
||||
class ShellLatencyServer : public ShellLatencyServiceBase {
|
||||
int _counter;
|
||||
ClockStatefulPublisher _pub;
|
||||
utils::DispatcherTimer _timer;
|
||||
MetricsContainer& _mc;
|
||||
std::map<std::string, std::shared_ptr<AutoResetMetric>> _metrics;
|
||||
|
||||
public:
|
||||
ShellLatencyServer(bts::api::ServiceContext&, metrics::MetricsContainer&, utils::Dispatcher&);
|
||||
~ShellLatencyServer();
|
||||
|
||||
void tick();
|
||||
|
||||
void report(
|
||||
const ::bts::metrics::Report& request,
|
||||
const api::Response<::bts::api::VoidMessage>& reply) override;
|
||||
};
|
||||
|
||||
|
||||
}}
|
||||
Reference in new issue
Block a user