diff --git a/src/lib/config/defaults/Executor.conf b/src/lib/config/defaults/Executor.conf new file mode 100644 index 000000000..2fed6e7bc --- /dev/null +++ b/src/lib/config/defaults/Executor.conf @@ -0,0 +1,13 @@ +ExecutorCfg +{ + value + { + # executor for latency sensitive code + executor { affinity: 0 yield_cpu: false } + + # normal executor + executor { affinity: -1 yield_cpu: true } + } +} + + diff --git a/src/lib/quickbus/core/Quickbus.cpp b/src/lib/quickbus/core/Quickbus.cpp index cae949b2d..b620df2b4 100644 --- a/src/lib/quickbus/core/Quickbus.cpp +++ b/src/lib/quickbus/core/Quickbus.cpp @@ -4,6 +4,7 @@ #include "lib/quickbus/network/QBConnectionManager.h" #include "lib/config/protos/quickbus_cfg.pb.h" #include "lib/os/os.h" +#include "lib/utils/bts_assert.h" #include #include @@ -13,35 +14,41 @@ namespace qbus { struct Quickbus::PImpl { - PImpl(config::QuickbusNodeCfg const&cfg) : + PImpl(config::QuickbusNodeCfg const&cfg,ThreadOptions options) : io_service(), router(make_node_id()), connection_manager(io_service,router,cfg), - worker() + worker(), + thread_options(options), + worker_id() {} - boost::asio::io_service io_service; Router router; QBConnectionManager connection_manager; std::thread worker; + ThreadOptions thread_options; + std::thread::id worker_id; }; -Quickbus::Quickbus () : - me (new PImpl(config::QuickbusNodeCfg())) +Quickbus::Quickbus(ThreadOptions options) : + me (new PImpl(config::QuickbusNodeCfg(),options)) {init();} -Quickbus::Quickbus (config::QuickbusNodeCfg const&cfg) : - me (new PImpl(cfg)) +Quickbus::Quickbus(config::QuickbusNodeCfg const&cfg, ThreadOptions options) : + me (new PImpl(cfg,options)) {init();} Quickbus::~Quickbus () { me->io_service.stop(); - me->worker.join(); + if (me->thread_options == ThreadOptions::own_thread) + me->worker.join(); } bool Quickbus::on_qb_thread() const -{return std::this_thread::get_id() == me->worker.get_id();} +{ + return std::this_thread::get_id() == me->worker_id; +} void Quickbus::post (std::function const &action) {me->io_service.post(action);} @@ -51,10 +58,22 @@ Router& Quickbus::router() const void Quickbus::init() { - me->worker = std::thread ([this] () { - os::set_thread_name("quickbus"); - me->io_service.run(); - });} + if (me->thread_options == ThreadOptions::own_thread) { + me->worker = std::thread ([this] () { + os::set_thread_name("quickbus"); + me->io_service.run(); + }); + me->worker_id = me->worker.get_id(); + } + else + me->worker_id = std::this_thread::get_id() ; +} +bool Quickbus::do_work() +{ + bts_assert(me->thread_options == ThreadOptions::outside_thread); + return me->io_service.poll(); +} + }} diff --git a/src/lib/quickbus/core/Quickbus.h b/src/lib/quickbus/core/Quickbus.h index 3bca21858..494ce8156 100644 --- a/src/lib/quickbus/core/Quickbus.h +++ b/src/lib/quickbus/core/Quickbus.h @@ -31,12 +31,19 @@ namespace qbus { class BTS_EXPORT Quickbus { public: - Quickbus(); - Quickbus(config::QuickbusNodeCfg const&cfg); + enum class ThreadOptions { + own_thread, + outside_thread + }; + + Quickbus(ThreadOptions options = ThreadOptions::own_thread); + Quickbus(config::QuickbusNodeCfg const&cfg, ThreadOptions = ThreadOptions::own_thread); ~Quickbus(); + // call this only if ThreadOptions = own_thread + // returns wheter any work was done + bool do_work(); bool on_qb_thread() const; - protected: friend class LocalNode; void post (std::function const &); diff --git a/src/lib/utils/type_name.h b/src/lib/utils/type_name.h index 8117882b5..776c098b0 100644 --- a/src/lib/utils/type_name.h +++ b/src/lib/utils/type_name.h @@ -9,9 +9,8 @@ namespace utils { template const char* type_name() { - T* t(nullptr); int status; - return abi::__cxa_demangle(typeid (*t).name (), 0, 0, &status); + return abi::__cxa_demangle(typeid (T).name (), 0, 0, &status); } }} diff --git a/src/server/sparkd/CMakeLists.txt b/src/server/sparkd/CMakeLists.txt index b6ace8f11..d44c0b490 100644 --- a/src/server/sparkd/CMakeLists.txt +++ b/src/server/sparkd/CMakeLists.txt @@ -72,6 +72,10 @@ set (Sparkd_Sources # testing utilities testing/catch_test_runner.cpp + # setup + setup/BootstrapActor.cpp + setup/ServiceHostActor.cpp + # md, mdp3 onixs platform # md/platforms/mdp3_onixs/MDP3Listener.cpp # md/platforms/mdp3_onixs/MDP3Main.cpp diff --git a/src/server/sparkd/fabric/Actor.h b/src/server/sparkd/fabric/Actor.h index e20702cd2..8cee12b0b 100644 --- a/src/server/sparkd/fabric/Actor.h +++ b/src/server/sparkd/fabric/Actor.h @@ -214,6 +214,12 @@ public: Base::_executor->send_request(Base::_my_address,ref.address,mid,on_reply,[](auto){}); } + template + void spawn(ExecutorID target_exe) + { + spawn (target_exe,[](auto,auto) {}); + } + template void spawn(ExecutorID target_exe, Callable on_address, ConsArgs&&... args) { diff --git a/src/server/sparkd/fabric/Domain.cpp b/src/server/sparkd/fabric/Domain.cpp index 651f358d2..24218a6c2 100644 --- a/src/server/sparkd/fabric/Domain.cpp +++ b/src/server/sparkd/fabric/Domain.cpp @@ -3,9 +3,11 @@ #include "Executor.h" #include "Msg.h" #include "lib/config/protos/Executor.pb.h" +#include "lib/os/os.h" #include #include +#include namespace bts { @@ -23,6 +25,8 @@ struct Domain::ExecutorRecord { {} void run(config::Executor const&cfg) { + auto tn = fmt::format("executor_{}",executor.address().executor().value()); + os::set_thread_name(tn.c_str()); if (cfg.affinity() >= 0) { cpu_set_t cpuset; CPU_ZERO(&cpuset); @@ -33,8 +37,9 @@ struct Domain::ExecutorRecord { Msg m; while (true ) { - executor.on_poll(); + auto executed = executor.on_poll(); if (queue.try_dequeue(m)) { + executed++; switch (m.type) { case Msg::REQUEST: executor.on_request(m.request); @@ -55,7 +60,7 @@ struct Domain::ExecutorRecord { return; }; } - else if (yield) + if (executed == 0 && yield) std::this_thread::yield(); } } diff --git a/src/server/sparkd/fabric/Domain.h b/src/server/sparkd/fabric/Domain.h index 5f258e15d..1abc99d9d 100644 --- a/src/server/sparkd/fabric/Domain.h +++ b/src/server/sparkd/fabric/Domain.h @@ -8,8 +8,8 @@ namespace bts { namespace config { class ExecutorCfg; }} - // end fws + namespace bts { namespace fabric { diff --git a/src/server/sparkd/fabric/Executor.cpp b/src/server/sparkd/fabric/Executor.cpp index 61e35e6d7..c6c790cd5 100644 --- a/src/server/sparkd/fabric/Executor.cpp +++ b/src/server/sparkd/fabric/Executor.cpp @@ -106,10 +106,12 @@ Executor::Executor(Address my_address, Executor::~Executor() {} -void Executor::on_poll() +int Executor::on_poll() { + auto actions_executed = 0; for (auto &ipoll : me->pollable_actors) - ipoll.second->on_poll(); + actions_executed += ipoll.second->on_poll(); + return actions_executed; } void Executor::on_request(Request const&r) { diff --git a/src/server/sparkd/fabric/Executor.h b/src/server/sparkd/fabric/Executor.h index 16c71610e..ed809b00c 100644 --- a/src/server/sparkd/fabric/Executor.h +++ b/src/server/sparkd/fabric/Executor.h @@ -21,7 +21,7 @@ public: ~Executor(); void bootstrap(ActorPtr &&with_actor); - void on_poll(); + int on_poll(); void on_request(Request const&r) ; void on_reply(Reply const&r) ; diff --git a/src/server/sparkd/fabric/IPollable.h b/src/server/sparkd/fabric/IPollable.h index cf247cbe7..8092ac7ef 100644 --- a/src/server/sparkd/fabric/IPollable.h +++ b/src/server/sparkd/fabric/IPollable.h @@ -6,6 +6,6 @@ namespace fabric { class IPollable { public: virtual ~IPollable() {}; - virtual void on_poll() = 0; + virtual bool on_poll() = 0; }; }} diff --git a/src/server/sparkd/fabric/Msg.h b/src/server/sparkd/fabric/Msg.h index db92abd2d..9f2789330 100644 --- a/src/server/sparkd/fabric/Msg.h +++ b/src/server/sparkd/fabric/Msg.h @@ -151,3 +151,4 @@ GCC_DIAG_RESTORE() } } + diff --git a/src/server/sparkd/fabric/tests/fabric_test.cpp b/src/server/sparkd/fabric/tests/fabric_test.cpp index b3e76318c..04801cc4e 100644 --- a/src/server/sparkd/fabric/tests/fabric_test.cpp +++ b/src/server/sparkd/fabric/tests/fabric_test.cpp @@ -107,12 +107,15 @@ struct Pub : public Actor<>, public IPollable { ts_topic = lookup(); } - void on_poll() override { + bool on_poll() override { auto now = Timer::now(); if (ns_diff(now, last_pub) > 2000) { publish(ts_topic,now); last_pub = Timer::now(); + return true; } + else + return false; } }; diff --git a/src/server/sparkd/main.cpp b/src/server/sparkd/main.cpp index 41254012b..8eaaa86dc 100644 --- a/src/server/sparkd/main.cpp +++ b/src/server/sparkd/main.cpp @@ -3,8 +3,12 @@ #include "lib/config/config.h" #include "lib/config/protos/logging.pb.h" +#include "lib/config/protos/Executor.pb.h" +#include "setup/BootstrapActor.h" +#include "fabric/Domain.h" #include "testing/catch_test_runner.h" + int main( int argc, char*argv [] ) { using namespace bts; @@ -12,9 +16,12 @@ int main( int argc, char*argv [] ) if (argc > 1 && std::string(argv[1]) == "test") bts::run_tests(--argc, ++argv); - else { + else { log::LoggingConfigPoller poller; - LOG(Gen,Warn, "Nothing here yet"); + auto executor_cfg = config::get (); + + fabric::Domain d (executor_cfg,std::make_unique(executor_cfg)); + std::this_thread::sleep_until(std::chrono::time_point::max()); } return 0; } diff --git a/src/server/sparkd/setup/BootstrapActor.cpp b/src/server/sparkd/setup/BootstrapActor.cpp new file mode 100644 index 000000000..1e9a79fe9 --- /dev/null +++ b/src/server/sparkd/setup/BootstrapActor.cpp @@ -0,0 +1,41 @@ +#include "BootstrapActor.h" + +#include "lib/config/protos/Executor.pb.h" +#include "lib/config/config.h" +#include "lib/utils/type_name.h" +#include "server/sparkd/setup/ServiceHostActor.h" + +namespace bts { +namespace setup { + +struct BootstrapActor::PImpl +{ + PImpl(config::ExecutorCfg const&cfg) : cfg(cfg) {} + config::ExecutorCfg const cfg; + +}; + +template +fabric::ExecutorID find_executor () +{ + auto cfg = config::get(); + auto tn = utils::type_name(); + size_t exe_idx = 0; + for (auto const &e : cfg.executor()) { + for (auto const &exe_actor : e.actor()) + if (exe_actor == tn) + return fabric::ExecutorID(exe_idx); + exe_idx++; + } + return fabric::ExecutorID(cfg.executor_size()-1); +} + +BootstrapActor::BootstrapActor (config::ExecutorCfg const&cfg) : me (new PImpl(cfg)) {} +BootstrapActor::~BootstrapActor () {} + +void BootstrapActor::init() +{ + spawn (find_executor()); +} + +}} diff --git a/src/server/sparkd/setup/BootstrapActor.h b/src/server/sparkd/setup/BootstrapActor.h new file mode 100644 index 000000000..7db6610f4 --- /dev/null +++ b/src/server/sparkd/setup/BootstrapActor.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include "server/sparkd/fabric/Actor.h" + +// fwd +namespace bts { +namespace config { +class ExecutorCfg; +}} +// end fws + +namespace bts { +namespace setup { + +class BootstrapActor : public fabric::Actor<> +{ +public: + BootstrapActor(config::ExecutorCfg const&cfg); + ~BootstrapActor (); + void init() override; +private: + struct PImpl; + std::unique_ptr me; +}; + +}} diff --git a/src/server/sparkd/setup/ServiceHostActor.cpp b/src/server/sparkd/setup/ServiceHostActor.cpp new file mode 100644 index 000000000..c2aa3951c --- /dev/null +++ b/src/server/sparkd/setup/ServiceHostActor.cpp @@ -0,0 +1,29 @@ +#include "ServiceHostActor.h" + +#include "lib/config/protos/Executor.pb.h" +#include "lib/quickbus/quickbus.h" +#include "lib/api/public/ServiceContext.h" +#include "server/sparkd/persistence/PersistenceServiceServer.h" + +namespace bts { +namespace setup { + +struct ServiceHostActor::PImpl +{ + PImpl() : quickbus(qbus::Quickbus::ThreadOptions::outside_thread), + service_context(&quickbus), + settings_saver(service_context,"settings") + {} + qbus::Quickbus quickbus; + api::ServiceContext service_context; + + // hosted services + persistence::PersistenceServiceServer settings_saver; +}; + +ServiceHostActor::ServiceHostActor () : me (new PImpl()) {} +ServiceHostActor::~ServiceHostActor () {} +bool ServiceHostActor::on_poll() {return me->quickbus.do_work(); +} + +}} diff --git a/src/server/sparkd/setup/ServiceHostActor.h b/src/server/sparkd/setup/ServiceHostActor.h new file mode 100644 index 000000000..29c92227d --- /dev/null +++ b/src/server/sparkd/setup/ServiceHostActor.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include "server/sparkd/fabric/Actor.h" +#include "server/sparkd/fabric/IPollable.h" + +namespace bts { +namespace setup { + +class ServiceHostActor : public fabric::Actor<>, public fabric::IPollable +{ +public: + ServiceHostActor(); + ~ServiceHostActor (); + bool on_poll() override; +private: + struct PImpl; + std::unique_ptr me; +}; + +}}