Made thread allocation configurable for order management.
This commit is contained in:
1 parent
6e3c880ea2
commit
75568e6c75
5 files changed
+179
-73
No files matched your search
@@ -18,29 +18,96 @@
|
||||
#include "lib/orderman/actors/ServiceBridge.h"
|
||||
#include "lib/orderman/actors/order_manager/OrderManager.h"
|
||||
#include "lib/utils/bts_chrono.h"
|
||||
#include "lib/utils/type_name.h"
|
||||
|
||||
#include <boost/algorithm/string/predicate.hpp>
|
||||
|
||||
namespace bts {
|
||||
namespace orderman {
|
||||
|
||||
struct ActorThread {
|
||||
ActorThread(const config::OrdermanCfg::OrdermanThread &cfg,metrics::MetricsContainer &stats)
|
||||
: cfg(cfg), thread(), event_broker(new mercury::AsyncEventBroker()),
|
||||
executor(*event_broker), loop_metric(), done(false) {
|
||||
std::array<double, 30> buckets;
|
||||
auto i = 0;
|
||||
for (auto &b : buckets)
|
||||
b = (i += 1000);
|
||||
|
||||
loop_metric = std::make_shared<metrics::HistogramMetric<30>>(
|
||||
fmt::format("spark_metrics_{}_thread_loop_ns", cfg.name()), buckets);
|
||||
stats.publish(loop_metric);
|
||||
}
|
||||
|
||||
void start() {
|
||||
thread = std::thread([this]() {
|
||||
os::set_thread_name(cfg.name());
|
||||
|
||||
const auto inner_iter = 1000;
|
||||
while (!done) {
|
||||
auto start = Timer::now();
|
||||
for (auto i = 0; i < inner_iter; i++)
|
||||
executor.do_work();
|
||||
auto stop = Timer::now();
|
||||
loop_metric->observe(ns_diff(stop, start) / inner_iter);
|
||||
if (!cfg.busy_poll())
|
||||
std::this_thread::sleep_for(10ms);
|
||||
}
|
||||
});
|
||||
|
||||
if (cfg.core_affinity() >= 0) {
|
||||
LOG(OrderMan, Info, "Setting {} core affinity to {}", cfg.name(),
|
||||
cfg.core_affinity());
|
||||
cpu_set_t cpuset;
|
||||
CPU_ZERO(&cpuset);
|
||||
CPU_SET(cfg.core_affinity(), &cpuset);
|
||||
int rc = pthread_setaffinity_np(thread.native_handle(),
|
||||
sizeof(cpu_set_t), &cpuset);
|
||||
if (rc != 0)
|
||||
LOG(OrderMan, Error,
|
||||
"Could not set core affinity to {} for {}. Error "
|
||||
"code={}",
|
||||
cfg.core_affinity(), cfg.name(), rc);
|
||||
}
|
||||
}
|
||||
|
||||
~ActorThread() {
|
||||
done= true;
|
||||
thread.join();
|
||||
}
|
||||
|
||||
config::OrdermanCfg::OrdermanThread cfg;
|
||||
std::thread thread;
|
||||
std::shared_ptr<mercury::AsyncEventBroker> event_broker;
|
||||
mercury::Executor executor;
|
||||
std::shared_ptr<metrics::HistogramMetric<30>> loop_metric;
|
||||
std::atomic<bool> done;
|
||||
};
|
||||
|
||||
ActorThread *find_thread_for_actor(
|
||||
std::string const &actor_name,
|
||||
std::vector<std::unique_ptr<ActorThread>> const &all_threads) {
|
||||
for (auto const &at : all_threads)
|
||||
for (auto const &hosted : at->cfg.hosted_actor())
|
||||
if (hosted == "*" || boost::algorithm::to_lower_copy(hosted) ==
|
||||
boost::algorithm::to_lower_copy(actor_name))
|
||||
return at.get();
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
struct OrderServiceServer::PImpl {
|
||||
PImpl(bts::api::ServiceContext &ctx,
|
||||
std::shared_ptr<api::ISyntheticSecurityService> &ss,
|
||||
metrics::MetricsContainer &stats)
|
||||
: fast_broker(std::make_shared<mercury::AsyncEventBroker>()),
|
||||
slow_broker(std::make_shared<mercury::AsyncEventBroker>()),
|
||||
service_bridge(), fast_exe(*fast_broker), slow_exe(*slow_broker),
|
||||
fast_thread(), slow_thread(), service_ready(false), known_orders(),
|
||||
known_fills(), publish_orders(), publish_fills(),
|
||||
: service_bridge(), service_ready(false), known_orders(), known_fills(),
|
||||
publish_orders(), publish_fills(),
|
||||
admin_service(ctx.locate<api::admin::AdminService>()), md_consumer(ss),
|
||||
stats(stats) {}
|
||||
|
||||
std::shared_ptr<mercury::AsyncEventBroker> fast_broker;
|
||||
std::shared_ptr<mercury::AsyncEventBroker> slow_broker;
|
||||
std::shared_ptr<ServiceBridge> service_bridge;
|
||||
mercury::Executor fast_exe;
|
||||
mercury::Executor slow_exe;
|
||||
std::thread fast_thread;
|
||||
std::thread slow_thread;
|
||||
|
||||
std::vector<std::unique_ptr<ActorThread>> actor_threads;
|
||||
|
||||
std::atomic<bool> service_ready;
|
||||
std::unordered_set<OrderKey> known_orders;
|
||||
std::unordered_set<std::string> known_fills;
|
||||
@@ -72,66 +139,76 @@ OrderServiceServer::OrderServiceServer(
|
||||
ctx.post([this, fill] { this->on_fills(fill); });
|
||||
});
|
||||
|
||||
me->fast_broker->init_neighbours({me->slow_broker});
|
||||
me->slow_broker->init_neighbours({me->fast_broker});
|
||||
|
||||
// fast actors
|
||||
me->fast_exe.spawn<MDSource>(me->md_consumer, me->stats);
|
||||
me->fast_exe.spawn<OrderManager>();
|
||||
|
||||
auto &all_connections = config::get<config::JetFIX>().connection();
|
||||
for (auto const &c : all_connections)
|
||||
me->fast_exe.spawn<ExecutionGateway>(c, stats);
|
||||
|
||||
me->fast_thread = std::thread([this]() {
|
||||
os::set_thread_name("fast_exe");
|
||||
|
||||
std::array<double, 30> buckets;
|
||||
auto i = 1;
|
||||
for (auto &b : buckets)
|
||||
b = (i *= 2);
|
||||
|
||||
auto loop_timer = std::make_shared<metrics::HistogramMetric<30>>(
|
||||
"spark_metrics_fast_thread_loop_ns", buckets);
|
||||
|
||||
me->stats.publish(loop_timer);
|
||||
for (;;) {
|
||||
auto start = Timer::now();
|
||||
me->fast_exe.do_work();
|
||||
auto stop = Timer::now();
|
||||
loop_timer->observe(ns_diff(stop, start));
|
||||
}
|
||||
});
|
||||
|
||||
auto fast_thread_core =
|
||||
config::get<config::OrdermanCfg>().fast_thread_affinity();
|
||||
if (fast_thread_core >= 0) {
|
||||
LOG(OrderMan, Info, "Setting fast thread core affinity to {}",
|
||||
fast_thread_core);
|
||||
cpu_set_t cpuset;
|
||||
CPU_ZERO(&cpuset);
|
||||
CPU_SET(fast_thread_core, &cpuset);
|
||||
int rc = pthread_setaffinity_np(me->fast_thread.native_handle(),
|
||||
sizeof(cpu_set_t), &cpuset);
|
||||
if (rc != 0)
|
||||
LOG(OrderMan, Error,
|
||||
"Could not set core affinity to {} for fast thread. Error code={}",
|
||||
fast_thread_core, rc);
|
||||
std::vector<mercury::IEventBrokerPtr> all_brokers;
|
||||
for (auto const &thread_cfg : config::get<config::OrdermanCfg>().thread()) {
|
||||
auto at = std::make_unique<ActorThread>(thread_cfg,stats);
|
||||
all_brokers.push_back(at->event_broker);
|
||||
me->actor_threads.push_back(std::move(at));
|
||||
}
|
||||
|
||||
// slow actors
|
||||
me->slow_exe.activate(me->service_bridge);
|
||||
me->slow_exe.spawn<mercury::Timer>();
|
||||
me->slow_exe.spawn<AsyncFactory>();
|
||||
me->slow_exe.spawn<OrderStore>();
|
||||
me->slow_exe.spawn<FillStore>();
|
||||
me->slow_exe.spawn<SecuritySrc>(me->md_consumer);
|
||||
me->slow_thread = std::thread([this]() {
|
||||
os::set_thread_name("slow_exe");
|
||||
for (;;)
|
||||
if (!me->slow_exe.do_work())
|
||||
std::this_thread::sleep_for(10ms);
|
||||
});
|
||||
for (auto &t : me->actor_threads)
|
||||
t->event_broker->init_neighbours(all_brokers);
|
||||
|
||||
// MDSource
|
||||
auto thread = find_thread_for_actor("MDSource", me->actor_threads);
|
||||
bts_assert_msg(thread, "Could not find a configured thread for MDSource. See the "
|
||||
"OrdermanCfg key");
|
||||
thread->executor.spawn<MDSource>(me->md_consumer, me->stats);
|
||||
|
||||
// OrderManager
|
||||
thread = find_thread_for_actor("OrderManager", me->actor_threads);
|
||||
bts_assert_msg(thread, "Could not find a configured thread for OrderManager. See "
|
||||
"the OrdermanCfg key");
|
||||
thread->executor.spawn<OrderManager>();
|
||||
|
||||
// ExecutionGateway's
|
||||
thread = find_thread_for_actor("ExecutionGateway", me->actor_threads);
|
||||
bts_assert_msg(thread, "Could not find a configured thread for ExecutionGateway "
|
||||
"instances. See the OrdermanCfg key");
|
||||
auto &all_connections = config::get<config::JetFIX>().connection();
|
||||
for (auto const &c : all_connections)
|
||||
thread->executor.spawn<ExecutionGateway>(c, stats);
|
||||
|
||||
// Timer
|
||||
thread = find_thread_for_actor("Timer", me->actor_threads);
|
||||
bts_assert_msg(
|
||||
thread,
|
||||
"Could not find a configured thread for Timer. See the OrdermanCfg key");
|
||||
thread->executor.spawn<mercury::Timer>();
|
||||
|
||||
// AsyncFactory
|
||||
thread = find_thread_for_actor("AsyncFactory", me->actor_threads);
|
||||
bts_assert_msg(thread, "Could not find a configured thread for AsyncFactory. See "
|
||||
"the OrdermanCfg key");
|
||||
thread->executor.spawn<AsyncFactory>();
|
||||
|
||||
// OrderStore
|
||||
thread = find_thread_for_actor("OrderStore", me->actor_threads);
|
||||
bts_assert_msg(thread, "Could not find a configured thread for OrderStore. See "
|
||||
"the OrdermanCfg key");
|
||||
thread->executor.spawn<OrderStore>();
|
||||
|
||||
// FillStore
|
||||
thread = find_thread_for_actor("FillStore", me->actor_threads);
|
||||
bts_assert_msg(thread, "Could not find a configured thread for FilllStore. See "
|
||||
"the OrdermanCfg key");
|
||||
thread->executor.spawn<FillStore>();
|
||||
|
||||
// SecuritySrc
|
||||
thread = find_thread_for_actor("SecuritySrc", me->actor_threads);
|
||||
bts_assert_msg(thread, "Could not find a configured thread for SecuritySrc. See "
|
||||
"the OrdermanCfg key");
|
||||
thread->executor.spawn<SecuritySrc>(me->md_consumer);
|
||||
|
||||
// ServiceBridge
|
||||
thread = find_thread_for_actor("ServiceBridge", me->actor_threads);
|
||||
bts_assert_msg(thread, "Could not find a configured thread for ServiceBridge. See "
|
||||
"the OrdermanCfg key");
|
||||
thread->executor.activate(me->service_bridge);
|
||||
|
||||
// spin up threads
|
||||
for (auto &t : me->actor_threads)
|
||||
t->start();
|
||||
|
||||
// wait for the service bridge to be ready
|
||||
me->service_bridge->block_until_ready();
|
||||
|
||||
Reference in new issue
Block a user