Added central control of threading.

This commit is contained in:
Pedro Pinto committed 2018-10-02 16:24:00 +00:00
1 parent ef3b39361a
commit 2769de6f7e
25 files changed
+259 -110

No files matched your search

+1 -1
View File
@@ -1 +1 @@
set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cpp_config.conf;config/defaults/dynamic_orders.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cpp_config.proto;config/protos/dynamic_orders.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/OrderFactory.cpp;orderman/ordeLine truncated
set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cpp_config.conf;config/defaults/dynamic_orders.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cpp_config.proto;config/protos/dynamic_orders.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.Line truncated
+9 -12
View File
@@ -1,22 +1,19 @@
OrdermanCfg {
value {
thread {
name: "orderman"
#busy_poll: true
core_affinity: 0
executor {
thread_name: "oman_orders"
poll: false
hosted_actor: "MDSource"
hosted_actor: "OrderManager"
}
thread {
name: "gateway"
#busy_poll: true
core_affinity: 1
executor {
thread_name: "oman_gw"
poll: false
hosted_actor: "ExecutionGateway"
}
thread {
name: "slow"
busy_poll: false
core_affinity: 2
executor {
thread_name: "oman_slow"
poll: false
hosted_actor: "*"
}
}
+7
View File
@@ -0,0 +1,7 @@
Threads {
value {
thread_group {name_rx: "oman_orders" core: 0}
thread_group {name_rx: "oman_gw" core: 1}
thread_group {name_rx: ".*" exclude:true core:0 core:1}
}
}
+6 -7
View File
@@ -2,11 +2,10 @@ syntax='proto2';
package bts.config;
message OrdermanCfg {
message OrdermanThread {
required string name = 1;
optional bool busy_poll = 2 [default = false];
optional int32 core_affinity = 3 [default = -1];
repeated string hosted_actor = 4;
message Executor {
required string thread_name = 1;
optional bool poll = 2 [ default = false];
repeated string hosted_actor = 3;
}
repeated OrdermanThread thread = 1;
}
repeated Executor executor = 1;
}
+13
View File
@@ -0,0 +1,13 @@
syntax='proto2';
package bts.config;
message ThreadGroupCfg {
required string name_rx = 1 [default = "*"];
optional bool exclude = 2 [default = false];
repeated uint32 core = 3;
}
message Threads {
repeated ThreadGroupCfg thread_group = 1;
}
+3 -2
View File
@@ -8,6 +8,7 @@
#include "lib/os/os.h"
#include "lib/tracing/bts_trace.h"
#include "lib/orderman/orderman_utils/TickToTradeCorID.h"
#include "lib/utils/thread_manager.h"
#include <fstream>
@@ -112,8 +113,8 @@ AsyncReader::AsyncReader (IConnection &connection,
void AsyncReader::start ()
{
me->reader_thread = std::thread ([this] () {
os::set_thread_name(me->connection_name+"_asyncr");
auto thread_name = fmt::format ("jf_ar_{}", me->connection_name);
me->reader_thread = threading::make_thread (thread_name,[this] () {
dispatch();
});
}
+7 -2
View File
@@ -13,6 +13,7 @@
#include "lib/orderman/orderman_utils/orderman_common.h"
#include "lib/tracing/bts_trace.h"
#include <boost/lockfree/queue.hpp>
#include "lib/utils/thread_manager.h"
#include <atomic>
#include <boost/date_time/posix_time/posix_time.hpp>
@@ -121,12 +122,17 @@ struct CoreGateway::PImpl {
uint32_t connection_attempts;
};
std::thread CoreGateway::make_timer_thread() {
auto thread_name = fmt::format ("jf_tmr_{}", me->cfg.name());
return threading::make_thread(thread_name, std::bind(&CoreGateway::call_time, this));
}
CoreGateway::CoreGateway(config::JetFIX::Connection const &cfg,
ICoreGatewayObserver &observer,
ConnectionFactory const &conn_factory,
StoreFactory const &store_factory)
: me(new PImpl(cfg, observer, conn_factory, store_factory)) {
me->timer_thread = std::thread(std::bind(&CoreGateway::call_time, this));
me->timer_thread = make_timer_thread();
}
CoreGateway::~CoreGateway() {
@@ -202,7 +208,6 @@ void CoreGateway::dispatch_events() {
}
void CoreGateway::call_time() {
os::set_thread_name(fmt::format("{0}_timer", me->cfg.name()).c_str());
while (!me->destroy) {
std::this_thread::sleep_for(10ms);
auto ts = std::make_shared<std::string>(
+2 -1
View File
@@ -67,7 +67,8 @@ private:
void transition (State new_state);
void check_state_invariants ();
const char* state_name (State st);
std::thread make_timer_thread();
std::unique_ptr<PImpl> me;
};
+4 -2
View File
@@ -9,6 +9,7 @@
#include "lib/utils/bts_chrono.h"
#include "lib/utils/datetime.h"
#include "lib/utils/file_utils.h"
#include "lib/utils/thread_manager.h"
#include <boost/date_time.hpp>
#include <boost/date_time/gregorian/gregorian.hpp>
@@ -204,7 +205,9 @@ FIXFileStore::FIXFileStore(config::JetFIX::Connection const &cfg,
}
}
me->fix_store.open(fix_log_p, std::ofstream::app);
me->writer_thread = std::thread(std::bind(&FIXFileStore::write, this));
auto thread_name = fmt::format("jf_store_{}", me->cfg.name().c_str());
me->writer_thread = threading::make_thread(
thread_name, std::bind(&FIXFileStore::write, this));
}
}
@@ -270,7 +273,6 @@ void FIXFileStore::set_last_seq_num(uint32_t last_seq_num) {
void FIXFileStore::write() {
Event ev;
os::set_thread_name(fmt::format("{0}_store", me->cfg.name()).c_str());
while (true)
if (me->event_queue.try_pop(ev)) {
+7 -4
View File
@@ -4,6 +4,7 @@
#include "lib/os/os.h"
#include "lib/utils/bts_assert.h"
#include "lib/utils/bts_chrono.h"
#include "lib/utils/thread_manager.h"
#include <algorithm>
#include <atomic>
@@ -25,7 +26,7 @@ struct LoggerHub::PImpl {
PImpl(std::string const &logger_name)
: lock(), sinks(), loggers(), level_names(), subsystem_names(), worker(),
done(false), logger_initial_queue_size(), logger_max_queue_growth(),
logger_name(logger_name) {}
logger_name(logger_name), thread_cfg() {}
std::mutex lock;
std::vector<SinkRec> sinks;
@@ -37,6 +38,7 @@ struct LoggerHub::PImpl {
size_t logger_initial_queue_size;
size_t logger_max_queue_growth;
std::string logger_name;
config::ThreadGroupCfg thread_cfg;
};
LoggerHub::LoggerHub(std::string const &logger_name)
@@ -49,7 +51,6 @@ LoggerHub::LoggerHub(std::string const &logger_name)
void LoggerHub::config(const LoggingConfig &cfg) {
std::lock_guard<std::mutex> l(me->lock);
me->logger_initial_queue_size = cfg.logger_initial_queue_size;
me->logger_max_queue_growth = cfg.logger_max_queue_growth;
@@ -74,11 +75,14 @@ void LoggerHub::config(const LoggingConfig &cfg) {
}
rebuild_logger_filters();
update_logger_queue_limits();
me->thread_cfg = cfg.thread_cfg;
}
void LoggerHub::start() {
bts_assert(!me->worker);
me->worker = std::make_unique<std::thread>(&LoggerHub::do_work, this);
me->worker = std::make_unique<std::thread>(
threading::make_thread(me->logger_name.c_str(),
std::bind(&LoggerHub::do_work, this),me->thread_cfg));
}
LoggerHub::~LoggerHub() {
@@ -96,7 +100,6 @@ Logger &LoggerHub::make_logger() {
}
void LoggerHub::do_work() {
os::set_thread_name(me->logger_name.c_str());
while (!me->done) {
size_t consumed = 0;
{
-1
View File
@@ -33,5 +33,4 @@ private:
Filter get_base_filter() const;
};
void start_logging_hub();
}}
+12 -7
View File
@@ -4,18 +4,22 @@
#include "Common.h"
#include <vector>
#include <map>
#include "lib/config/protos/threads.pb.h"
namespace bts {
namespace log {
struct LoggingConfig {
LoggingConfig() : logger_initial_queue_size(120000),
logger_max_queue_growth(4),
sinks(),
level_names(),
subsystem_names(),
deployment_type("dev")
{}
LoggingConfig() :
logger_initial_queue_size(120000),
logger_max_queue_growth(4),
sinks(),
level_names(),
subsystem_names(),
deployment_type("dev"),
thread_cfg()
{
}
size_t logger_initial_queue_size = 0;
size_t logger_max_queue_growth = 0;
@@ -23,6 +27,7 @@ struct LoggingConfig {
std::map<Level,std::string> level_names;
std::map<Subsystem,std::string> subsystem_names;
std::string deployment_type;
config::ThreadGroupCfg thread_cfg;
};
}}
+10 -7
View File
@@ -9,6 +9,7 @@
#include "lib/utils/file_utils.h"
#include "lib/utils/Dispatcher.h"
#include "lib/config/protos/system.pb.h"
#include "lib/utils/thread_manager.h"
#include <google/protobuf/text_format.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>
@@ -86,8 +87,8 @@ Filter get_filter (LoggingConfig &cfg,
}
void configure_logging(LoggingConfig const&cfg) {_default_logger_hub.config(cfg);}
void configure_logging(bts::config::Logging const&cfg)
void configure_logging(bts::config::Logging const&cfg) {configure_logging(cfg,bts::config::ThreadGroupCfg ());}
void configure_logging(bts::config::Logging const&cfg, bts::config::ThreadGroupCfg const&thread_cfg)
{
LoggingConfig real_cfg;
real_cfg.deployment_type = config::get<config::DeploymentType>().type();
@@ -124,6 +125,7 @@ void configure_logging(bts::config::Logging const&cfg)
std::make_pair (syslog_sink(real_cfg),get_filter(real_cfg,cfg.syslog()));
real_cfg.sinks["file"] =
std::make_pair (file_sink(real_cfg),get_filter(real_cfg,cfg.file()));
real_cfg.thread_cfg = thread_cfg;
configure_logging(real_cfg);
}
@@ -136,19 +138,20 @@ boost::filesystem::path log_cache_file_path ()
return result;
}
LoggingConfigPoller::LoggingConfigPoller (bts::config::Logging const&cfg)
LoggingConfigPoller::LoggingConfigPoller (bts::config::Logging const&cfg, bts::config::ThreadGroupCfg const&thread_cfg)
: _poller(utils::Dispatcher::background(), 1000, [=]() { poll_once(); }),
_last_ts(std::time(nullptr)),
_log_cache_file(log_cache_file_path()),
_done(false)
{
init(cfg);
init(cfg,thread_cfg);
}
LoggingConfigPoller::LoggingConfigPoller () : LoggingConfigPoller(config::get<config::Logging>())
LoggingConfigPoller::LoggingConfigPoller () : LoggingConfigPoller(config::get<config::Logging>(),
threading::get_thread_cfg("logger"))
{ }
void LoggingConfigPoller::init (bts::config::Logging const&cfg)
void LoggingConfigPoller::init (bts::config::Logging const&cfg,bts::config::ThreadGroupCfg const&thread_cfg)
{
auto parent = _log_cache_file.parent_path();
if (!boost::filesystem::is_directory(parent))
@@ -160,7 +163,7 @@ void LoggingConfigPoller::init (bts::config::Logging const&cfg)
// initialize logging
configure_logging(cfg);
configure_logging(cfg,thread_cfg);
_poller.start();
}
+4 -3
View File
@@ -10,7 +10,7 @@
namespace bts {
namespace config {class Logging;}
namespace config {class Logging; class ThreadGroupCfg;}
// Please keep this list in sync with Log.cs
constexpr log::Level Diag {0};
@@ -55,17 +55,18 @@ struct LoggerHandle {
extern thread_local LoggerHandle _thread_logger;
void configure_logging(LoggingConfig const&cfg);
void configure_logging(bts::config::Logging const&cfg,bts::config::ThreadGroupCfg const&);
void configure_logging(bts::config::Logging const&cfg);
void start_default_logging_thread();
class LoggingConfigPoller {
public:
LoggingConfigPoller ();
LoggingConfigPoller (bts::config::Logging const&cfg);
LoggingConfigPoller (bts::config::Logging const&cfg, bts::config::ThreadGroupCfg const&thread_cfg);
~LoggingConfigPoller ();
private:
void init(bts::config::Logging const&cfg);
void init(bts::config::Logging const&cfg, bts::config::ThreadGroupCfg const&thread_cfg);
void poll_once();
utils::DispatcherTimer _poller;
std::time_t _last_ts;
+2 -2
View File
@@ -30,10 +30,10 @@ class Ping : public Actor, public mercury::IPollableActor {
private:
void on_poll() override {
_i++;
// if (_i % 100000 == 0) {
if (_i % 100000 == 0) {
tracepoint(bts_trace, trace_order, "ping_send", 0, 0, _i);
publish(_stream, _i);
// }
}
}
mercury::Stream<PingEvent> _stream;
+13 -29
View File
@@ -18,6 +18,7 @@
#include "lib/orderman/actors/order_manager/OrderManager.h"
#include "lib/utils/bts_chrono.h"
#include "lib/utils/type_name.h"
#include "lib/utils/thread_manager.h"
#include <boost/algorithm/string/predicate.hpp>
@@ -25,7 +26,7 @@ namespace bts {
namespace orderman {
struct ActorThread {
ActorThread(const config::OrdermanCfg::OrdermanThread &cfg,metrics::MetricsContainer &stats)
ActorThread(const config::OrdermanCfg::Executor &cfg,metrics::MetricsContainer &stats)
: cfg(cfg), thread(), executor(), loop_metric(), done(false) {
std::array<double, 30> buckets;
auto i = 0;
@@ -33,14 +34,12 @@ struct ActorThread {
b = (i += 1000);
loop_metric = std::make_shared<metrics::HistogramMetric<30>>(
fmt::format("spark_metrics_{}_thread_loop_ns", cfg.name()), buckets);
fmt::format("spark_metrics_{}_thread_loop_ns", cfg.thread_name()), buckets);
stats.publish(loop_metric);
}
void start() {
thread = std::thread([this]() {
os::set_thread_name(cfg.name());
thread = threading::make_thread(cfg.thread_name(),[this]() {
const auto inner_iter = 1000;
while (!done) {
auto start = Timer::now();
@@ -48,25 +47,10 @@ struct ActorThread {
executor.do_work();
auto stop = Timer::now();
loop_metric->observe(ns_diff(stop, start) / inner_iter);
if (!cfg.busy_poll())
if (!cfg.poll())
std::this_thread::sleep_for(10ms);
}
});
if (cfg.core_affinity() >= 0) {
LOG(OrderMan, Info, "Setting {} core affinity to {}", cfg.name(),
cfg.core_affinity());
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(cfg.core_affinity(), &cpuset);
int rc = pthread_setaffinity_np(thread.native_handle(),
sizeof(cpu_set_t), &cpuset);
if (rc != 0)
LOG(OrderMan, Error,
"Could not set core affinity to {} for {}. Error "
"code={}",
cfg.core_affinity(), cfg.name(), rc);
}
}
~ActorThread() {
@@ -74,7 +58,7 @@ struct ActorThread {
thread.join();
}
config::OrdermanCfg::OrdermanThread cfg;
config::OrdermanCfg::Executor cfg;
std::thread thread;
mercury::Executor executor;
std::shared_ptr<metrics::HistogramMetric<30>> loop_metric;
@@ -83,12 +67,12 @@ struct ActorThread {
ActorThread *find_thread_for_actor(
std::string const &actor_name,
std::vector<std::unique_ptr<ActorThread>> const &all_threads) {
for (auto const &at : all_threads)
for (auto const &hosted : at->cfg.hosted_actor())
std::vector<std::unique_ptr<ActorThread>> const &all_executors) {
for (auto const &ex : all_executors)
for (auto const &hosted : ex->cfg.hosted_actor())
if (hosted == "*" || boost::algorithm::to_lower_copy(hosted) ==
boost::algorithm::to_lower_copy(actor_name))
return at.get();
boost::algorithm::to_lower_copy(actor_name))
return ex.get();
return nullptr;
}
@@ -126,8 +110,8 @@ OrderServiceServer::OrderServiceServer(
me->publish_fills = make_fills_publisher();
std::vector<mercury::Executor*> all_executors;
for (auto const &thread_cfg : config::get<config::OrdermanCfg>().thread()) {
auto at = std::make_unique<ActorThread>(thread_cfg,stats);
for (auto const &ex_cfg : config::get<config::OrdermanCfg>().executor()) {
auto at = std::make_unique<ActorThread>(ex_cfg,stats);
all_executors.push_back(&at->executor);
me->actor_threads.push_back(std::move(at));
}
+3 -2
View File
@@ -5,6 +5,7 @@
#include "lib/config/protos/quickbus_cfg.pb.h"
#include "lib/os/os.h"
#include "lib/utils/bts_assert.h"
#include "lib/utils/thread_manager.h"
#include <boost/asio.hpp>
#include <thread>
@@ -71,8 +72,8 @@ Router& Quickbus::router() const
void Quickbus::init()
{
if (me->thread_options == ThreadOptions::own_thread) {
me->worker = std::make_unique<std::thread> (std::thread ([this] () {
os::set_thread_name("quickbus");
me->worker = std::make_unique<std::thread> (
threading::make_thread("quickbus", [this] () {
me->io_service.run();
}));
me->do_work_tid = me->worker->get_id();
+5 -9
View File
@@ -7,6 +7,7 @@
#include "defer.h"
#include "lib/utils/bts_chrono.h"
#include "lib/logging/logging.h"
#include "lib/utils/thread_manager.h"
#include <memory>
@@ -83,10 +84,7 @@ void Dispatcher::run(size_t threads)
void Dispatcher::add_thread() {
_me->is_pool = true;
boost::thread t([=]() {
os::set_thread_name("DispatcherPool");
_me->service.run();
});
auto t = threading::make_thread("DispatcherPool",[=] () {_me->service.run();});
t.detach();
}
@@ -113,9 +111,8 @@ bool Dispatcher::check_access()
boost::thread_specific_ptr<Dispatcher> _tl_dispatchers;
Dispatcher& Dispatcher::current(bool set_name) {
Dispatcher& Dispatcher::current() {
if (_tl_dispatchers.get() == nullptr) {
if(set_name) os::set_thread_name("Dispatcher");
_tl_dispatchers.reset(new Dispatcher());
}
return *(_tl_dispatchers.get());
@@ -153,9 +150,8 @@ Dispatcher& Dispatcher::create_new(const char* name) {
condition_variable c;
// Take the lock before starting the thread to ensure c.notify_all() cannot happen before we wait on the condition
unique_lock<mutex> lock(m);
boost::thread t([&]() {
Dispatcher& temp = current(false);
os::set_thread_name(name);
auto t = threading::make_thread(name,[&]() {
Dispatcher& temp = current();
try {
temp.post([&]() {
unique_lock<mutex> lock(m);
+1 -1
View File
@@ -77,7 +77,7 @@ public:
bool is_threadpool();
// Get/create the dispatcher for the current thread
static Dispatcher& current(bool set_name=true);
static Dispatcher& current();
// Get a default background thread to dispatch work to
static Dispatcher& background();
// Create a new background dispatcher thread and return the dispatcher for it
+2
View File
@@ -1,6 +1,7 @@
#include "init_spark_lib.h"
#include "lib/logging/logging.h"
#include "lib/utils/bts_assert.h"
#include "lib/utils/thread_manager.h"
namespace bts {
namespace utils {
@@ -9,6 +10,7 @@ static log::LoggingConfigPoller *poller=nullptr;
void init_spark_lib()
{
threading::configure_current_thread("main");
log::start_default_logging_thread();
poller = new log::LoggingConfigPoller ();
}
+101
View File
@@ -0,0 +1,101 @@
#include "thread_manager.h"
#include "lib/config/config.h"
#include "lib/config/protos/threads.pb.h"
#include "lib/logging/logging.h"
#include "lib/utils/bts_assert.h"
#include <boost/regex.hpp>
#include <fmt/format.h>
#ifndef WIN32
#include <pthread.h>
#endif
namespace bts {
namespace threading {
config::ThreadGroupCfg get_thread_cfg(std::string const &name) {
for (auto const &cfg : config::get<config::Threads>().thread_group()) {
boost::regex rx(cfg.name_rx());
if (boost::regex_match(name, rx))
return cfg;
}
bts_assert_msg(false,
fmt::format("No configuration found for thread {}", name));
}
#ifndef WIN32
void configure_thread(std::string const &name,
config::ThreadGroupCfg const &cfg, pthread_t t) {
if (cfg.core().size()) {
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
if (cfg.exclude()) {
auto total_cpus = sysconf(_SC_NPROCESSORS_ONLN);
for (int i = 0; i < total_cpus; i++)
CPU_SET(i, &cpuset);
for (auto const &core : cfg.core())
CPU_CLR(core, &cpuset);
} else {
for (auto const &core : cfg.core())
CPU_SET(core, &cpuset);
}
int rc = pthread_setaffinity_np(t, sizeof(cpu_set_t), &cpuset);
if (rc != 0)
LOG(Gen, Error,
"Could not set core affinity for {} on thread {}. Error "
"code={}",
name, cfg.name_rx(), rc);
}
}
#endif
void configure_current_thread(std::string const &name) {
#ifndef WIN32
configure_thread(name, get_thread_cfg(name), pthread_self());
#endif
os::set_thread_name(name);
}
void configure_current_thread(std::string const &name,
config::ThreadGroupCfg const &cfg) {
#ifndef WIN32
configure_thread(name, cfg, pthread_self());
#endif
os::set_thread_name(name);
}
std::thread make_thread_from_function(std::string const &name,
std::function<void()> const &c) {
auto result = std::thread(
[c,name] () {
os::set_thread_name(name);
c();
});
#ifndef WIN32
configure_thread(name, get_thread_cfg(name), result.native_handle());
#endif
return result;
}
std::thread make_thread_from_function(std::string const &name,
std::function<void()> const &c,
config::ThreadGroupCfg const &cfg) {
auto result = std::thread(
[c,name] () {
os::set_thread_name(name);
c();
});
#ifndef WIN32
configure_thread(name, cfg, result.native_handle());
#endif
return result;
}
} // namespace threading
} // namespace bts
+30
View File
@@ -0,0 +1,30 @@
#pragma once
#include <string>
#include <thread>
#include <functional>
#include "lib/config/protos/threads.pb.h"
namespace bts {
namespace config {class ThreadGroupCfg;}
namespace threading {
std::thread make_thread_from_function(std::string const&name, std::function<void()> const&c, config::ThreadGroupCfg const&cfg);
std::thread make_thread_from_function(std::string const&name, std::function<void()> const&c);
template <class C>
std::thread make_thread(std::string const &name, C callable) {
return make_thread_from_function (name, std::function<void()> (callable));
}
template <class C>
std::thread make_thread(std::string const &name, C callable,config::ThreadGroupCfg const&cfg) {
return make_thread_from_function (name, std::function<void()> (callable),cfg);
}
void configure_current_thread(std::string const&name);
void configure_current_thread(std::string const&name, config::ThreadGroupCfg const&);
config::ThreadGroupCfg get_thread_cfg(std::string const &);
}}
+11 -11
View File
@@ -20,20 +20,20 @@ void test_ping_pong_mercury2()
ex2.init_neighbours(std::vector<Executor*> {&ex1});
ex1.spawn<Ping>();
ex1.spawn<Pong>();
//ex2.spawn<Pong>();
//ex1.spawn<Pong>();
ex2.spawn<Pong>();
// auto t = std::thread([&ex2]() {
// cpu_set_t cpuset;
// CPU_ZERO(&cpuset);
// CPU_SET(1, &cpuset);
// pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
auto t = std::thread([&ex2]() {
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(1, &cpuset);
pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
// while (true)
// ex2.do_work();
// });
while (true)
ex2.do_work();
});
// bts::sleep_for(2s);
bts::sleep_for(2s);
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
+3 -5
View File
@@ -23,6 +23,7 @@
#include "server/sparkd/user/AdminServiceServer.h"
#include "server/sparkd/user/UserServiceServer.h"
#include "server/sparkd/position/PositionServiceServer.h"
#include "lib/utils/thread_manager.h"
#include "lib/md/relay/QueryServer.h"
@@ -63,11 +64,8 @@ int Run::execute(std::vector<std::string> const &) {
utils::DispatcherTimer timer(utils::Dispatcher::background(), 5000, [=]() { bg_health->increment(); });
timer.start();
stats.publish(bg_health);
std::thread([]() {
os::set_thread_name("ExchangeSim");
sim::start_exchange_sim();
}).detach();
threading::make_thread("ExchangeSim", []() {sim::start_exchange_sim();}).detach();
std::this_thread::sleep_until(
std::chrono::time_point<std::chrono::system_clock>::max());
+3 -2
View File
@@ -1,4 +1,4 @@
#include "lib/os/os.h"
#include "lib/utils/thread_manager.h"
#include "lib/utils/init_spark_lib.h"
#include "server/sparkd/cmds/CmdRegistry.h"
@@ -20,8 +20,9 @@
int main(int argc, char **argv) {
using namespace bts;
os::set_thread_name("main");
utils::init_spark_lib();
BEGIN_CMD;
REGISTER(Test);
REGISTER(Help);