Made thread allocation configurable for order management.

This commit is contained in:
Pedro Pinto committed 2018-10-02 16:22:04 +00:00
1 parent 6e3c880ea2
commit 75568e6c75
5 files changed
+179 -73

No files matched your search

+146 -69
View File
@@ -18,29 +18,96 @@
#include "lib/orderman/actors/ServiceBridge.h"
#include "lib/orderman/actors/order_manager/OrderManager.h"
#include "lib/utils/bts_chrono.h"
#include "lib/utils/type_name.h"
#include <boost/algorithm/string/predicate.hpp>
namespace bts {
namespace orderman {
struct ActorThread {
ActorThread(const config::OrdermanCfg::OrdermanThread &cfg,metrics::MetricsContainer &stats)
: cfg(cfg), thread(), event_broker(new mercury::AsyncEventBroker()),
executor(*event_broker), loop_metric(), done(false) {
std::array<double, 30> buckets;
auto i = 0;
for (auto &b : buckets)
b = (i += 1000);
loop_metric = std::make_shared<metrics::HistogramMetric<30>>(
fmt::format("spark_metrics_{}_thread_loop_ns", cfg.name()), buckets);
stats.publish(loop_metric);
}
void start() {
thread = std::thread([this]() {
os::set_thread_name(cfg.name());
const auto inner_iter = 1000;
while (!done) {
auto start = Timer::now();
for (auto i = 0; i < inner_iter; i++)
executor.do_work();
auto stop = Timer::now();
loop_metric->observe(ns_diff(stop, start) / inner_iter);
if (!cfg.busy_poll())
std::this_thread::sleep_for(10ms);
}
});
if (cfg.core_affinity() >= 0) {
LOG(OrderMan, Info, "Setting {} core affinity to {}", cfg.name(),
cfg.core_affinity());
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(cfg.core_affinity(), &cpuset);
int rc = pthread_setaffinity_np(thread.native_handle(),
sizeof(cpu_set_t), &cpuset);
if (rc != 0)
LOG(OrderMan, Error,
"Could not set core affinity to {} for {}. Error "
"code={}",
cfg.core_affinity(), cfg.name(), rc);
}
}
~ActorThread() {
done= true;
thread.join();
}
config::OrdermanCfg::OrdermanThread cfg;
std::thread thread;
std::shared_ptr<mercury::AsyncEventBroker> event_broker;
mercury::Executor executor;
std::shared_ptr<metrics::HistogramMetric<30>> loop_metric;
std::atomic<bool> done;
};
ActorThread *find_thread_for_actor(
std::string const &actor_name,
std::vector<std::unique_ptr<ActorThread>> const &all_threads) {
for (auto const &at : all_threads)
for (auto const &hosted : at->cfg.hosted_actor())
if (hosted == "*" || boost::algorithm::to_lower_copy(hosted) ==
boost::algorithm::to_lower_copy(actor_name))
return at.get();
return nullptr;
}
struct OrderServiceServer::PImpl {
PImpl(bts::api::ServiceContext &ctx,
std::shared_ptr<api::ISyntheticSecurityService> &ss,
metrics::MetricsContainer &stats)
: fast_broker(std::make_shared<mercury::AsyncEventBroker>()),
slow_broker(std::make_shared<mercury::AsyncEventBroker>()),
service_bridge(), fast_exe(*fast_broker), slow_exe(*slow_broker),
fast_thread(), slow_thread(), service_ready(false), known_orders(),
known_fills(), publish_orders(), publish_fills(),
: service_bridge(), service_ready(false), known_orders(), known_fills(),
publish_orders(), publish_fills(),
admin_service(ctx.locate<api::admin::AdminService>()), md_consumer(ss),
stats(stats) {}
std::shared_ptr<mercury::AsyncEventBroker> fast_broker;
std::shared_ptr<mercury::AsyncEventBroker> slow_broker;
std::shared_ptr<ServiceBridge> service_bridge;
mercury::Executor fast_exe;
mercury::Executor slow_exe;
std::thread fast_thread;
std::thread slow_thread;
std::vector<std::unique_ptr<ActorThread>> actor_threads;
std::atomic<bool> service_ready;
std::unordered_set<OrderKey> known_orders;
std::unordered_set<std::string> known_fills;
@@ -72,66 +139,76 @@ OrderServiceServer::OrderServiceServer(
ctx.post([this, fill] { this->on_fills(fill); });
});
me->fast_broker->init_neighbours({me->slow_broker});
me->slow_broker->init_neighbours({me->fast_broker});
// fast actors
me->fast_exe.spawn<MDSource>(me->md_consumer, me->stats);
me->fast_exe.spawn<OrderManager>();
auto &all_connections = config::get<config::JetFIX>().connection();
for (auto const &c : all_connections)
me->fast_exe.spawn<ExecutionGateway>(c, stats);
me->fast_thread = std::thread([this]() {
os::set_thread_name("fast_exe");
std::array<double, 30> buckets;
auto i = 1;
for (auto &b : buckets)
b = (i *= 2);
auto loop_timer = std::make_shared<metrics::HistogramMetric<30>>(
"spark_metrics_fast_thread_loop_ns", buckets);
me->stats.publish(loop_timer);
for (;;) {
auto start = Timer::now();
me->fast_exe.do_work();
auto stop = Timer::now();
loop_timer->observe(ns_diff(stop, start));
}
});
auto fast_thread_core =
config::get<config::OrdermanCfg>().fast_thread_affinity();
if (fast_thread_core >= 0) {
LOG(OrderMan, Info, "Setting fast thread core affinity to {}",
fast_thread_core);
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(fast_thread_core, &cpuset);
int rc = pthread_setaffinity_np(me->fast_thread.native_handle(),
sizeof(cpu_set_t), &cpuset);
if (rc != 0)
LOG(OrderMan, Error,
"Could not set core affinity to {} for fast thread. Error code={}",
fast_thread_core, rc);
std::vector<mercury::IEventBrokerPtr> all_brokers;
for (auto const &thread_cfg : config::get<config::OrdermanCfg>().thread()) {
auto at = std::make_unique<ActorThread>(thread_cfg,stats);
all_brokers.push_back(at->event_broker);
me->actor_threads.push_back(std::move(at));
}
// slow actors
me->slow_exe.activate(me->service_bridge);
me->slow_exe.spawn<mercury::Timer>();
me->slow_exe.spawn<AsyncFactory>();
me->slow_exe.spawn<OrderStore>();
me->slow_exe.spawn<FillStore>();
me->slow_exe.spawn<SecuritySrc>(me->md_consumer);
me->slow_thread = std::thread([this]() {
os::set_thread_name("slow_exe");
for (;;)
if (!me->slow_exe.do_work())
std::this_thread::sleep_for(10ms);
});
for (auto &t : me->actor_threads)
t->event_broker->init_neighbours(all_brokers);
// MDSource
auto thread = find_thread_for_actor("MDSource", me->actor_threads);
bts_assert_msg(thread, "Could not find a configured thread for MDSource. See the "
"OrdermanCfg key");
thread->executor.spawn<MDSource>(me->md_consumer, me->stats);
// OrderManager
thread = find_thread_for_actor("OrderManager", me->actor_threads);
bts_assert_msg(thread, "Could not find a configured thread for OrderManager. See "
"the OrdermanCfg key");
thread->executor.spawn<OrderManager>();
// ExecutionGateway's
thread = find_thread_for_actor("ExecutionGateway", me->actor_threads);
bts_assert_msg(thread, "Could not find a configured thread for ExecutionGateway "
"instances. See the OrdermanCfg key");
auto &all_connections = config::get<config::JetFIX>().connection();
for (auto const &c : all_connections)
thread->executor.spawn<ExecutionGateway>(c, stats);
// Timer
thread = find_thread_for_actor("Timer", me->actor_threads);
bts_assert_msg(
thread,
"Could not find a configured thread for Timer. See the OrdermanCfg key");
thread->executor.spawn<mercury::Timer>();
// AsyncFactory
thread = find_thread_for_actor("AsyncFactory", me->actor_threads);
bts_assert_msg(thread, "Could not find a configured thread for AsyncFactory. See "
"the OrdermanCfg key");
thread->executor.spawn<AsyncFactory>();
// OrderStore
thread = find_thread_for_actor("OrderStore", me->actor_threads);
bts_assert_msg(thread, "Could not find a configured thread for OrderStore. See "
"the OrdermanCfg key");
thread->executor.spawn<OrderStore>();
// FillStore
thread = find_thread_for_actor("FillStore", me->actor_threads);
bts_assert_msg(thread, "Could not find a configured thread for FilllStore. See "
"the OrdermanCfg key");
thread->executor.spawn<FillStore>();
// SecuritySrc
thread = find_thread_for_actor("SecuritySrc", me->actor_threads);
bts_assert_msg(thread, "Could not find a configured thread for SecuritySrc. See "
"the OrdermanCfg key");
thread->executor.spawn<SecuritySrc>(me->md_consumer);
// ServiceBridge
thread = find_thread_for_actor("ServiceBridge", me->actor_threads);
bts_assert_msg(thread, "Could not find a configured thread for ServiceBridge. See "
"the OrdermanCfg key");
thread->executor.activate(me->service_bridge);
// spin up threads
for (auto &t : me->actor_threads)
t->start();
// wait for the service bridge to be ready
me->service_bridge->block_until_ready();