Issue SPARK-153, Fabric and Service integration

This commit is contained in:
Pedro Pinto committed 2017-08-18 15:27:48 +00:00
1 parent dace321b21
commit d4445fe3e3
18 files changed
+212 -28

No files matched your search

+13
View File
@@ -0,0 +1,13 @@
ExecutorCfg
{
value
{
# executor for latency sensitive code
executor { affinity: 0 yield_cpu: false }
# normal executor
executor { affinity: -1 yield_cpu: true }
}
}
+32 -13
View File
@@ -4,6 +4,7 @@
#include "lib/quickbus/network/QBConnectionManager.h"
#include "lib/config/protos/quickbus_cfg.pb.h"
#include "lib/os/os.h"
#include "lib/utils/bts_assert.h"
#include <boost/asio.hpp>
#include <thread>
@@ -13,35 +14,41 @@ namespace qbus {
struct Quickbus::PImpl
{
PImpl(config::QuickbusNodeCfg const&cfg) :
PImpl(config::QuickbusNodeCfg const&cfg,ThreadOptions options) :
io_service(),
router(make_node_id()),
connection_manager(io_service,router,cfg),
worker()
worker(),
thread_options(options),
worker_id()
{}
boost::asio::io_service io_service;
Router router;
QBConnectionManager connection_manager;
std::thread worker;
ThreadOptions thread_options;
std::thread::id worker_id;
};
Quickbus::Quickbus () :
me (new PImpl(config::QuickbusNodeCfg()))
Quickbus::Quickbus(ThreadOptions options) :
me (new PImpl(config::QuickbusNodeCfg(),options))
{init();}
Quickbus::Quickbus (config::QuickbusNodeCfg const&cfg) :
me (new PImpl(cfg))
Quickbus::Quickbus(config::QuickbusNodeCfg const&cfg, ThreadOptions options) :
me (new PImpl(cfg,options))
{init();}
Quickbus::~Quickbus ()
{
me->io_service.stop();
me->worker.join();
if (me->thread_options == ThreadOptions::own_thread)
me->worker.join();
}
bool Quickbus::on_qb_thread() const
{return std::this_thread::get_id() == me->worker.get_id();}
{
return std::this_thread::get_id() == me->worker_id;
}
void Quickbus::post (std::function<void()> const &action)
{me->io_service.post(action);}
@@ -51,10 +58,22 @@ Router& Quickbus::router() const
void Quickbus::init()
{
me->worker = std::thread ([this] () {
os::set_thread_name("quickbus");
me->io_service.run();
});}
if (me->thread_options == ThreadOptions::own_thread) {
me->worker = std::thread ([this] () {
os::set_thread_name("quickbus");
me->io_service.run();
});
me->worker_id = me->worker.get_id();
}
else
me->worker_id = std::this_thread::get_id() ;
}
bool Quickbus::do_work()
{
bts_assert(me->thread_options == ThreadOptions::outside_thread);
return me->io_service.poll();
}
}}
+10 -3
View File
@@ -31,12 +31,19 @@ namespace qbus {
class BTS_EXPORT Quickbus
{
public:
Quickbus();
Quickbus(config::QuickbusNodeCfg const&cfg);
enum class ThreadOptions {
own_thread,
outside_thread
};
Quickbus(ThreadOptions options = ThreadOptions::own_thread);
Quickbus(config::QuickbusNodeCfg const&cfg, ThreadOptions = ThreadOptions::own_thread);
~Quickbus();
// call this only if ThreadOptions = own_thread
// returns wheter any work was done
bool do_work();
bool on_qb_thread() const;
protected:
friend class LocalNode;
void post (std::function<void()> const &);
+1 -2
View File
@@ -9,9 +9,8 @@ namespace utils {
template <class T>
const char* type_name()
{
T* t(nullptr);
int status;
return abi::__cxa_demangle(typeid (*t).name (), 0, 0, &status);
return abi::__cxa_demangle(typeid (T).name (), 0, 0, &status);
}
}}
+4
View File
@@ -72,6 +72,10 @@ set (Sparkd_Sources
# testing utilities
testing/catch_test_runner.cpp
# setup
setup/BootstrapActor.cpp
setup/ServiceHostActor.cpp
# md, mdp3 onixs platform
# md/platforms/mdp3_onixs/MDP3Listener.cpp
# md/platforms/mdp3_onixs/MDP3Main.cpp
+6
View File
@@ -214,6 +214,12 @@ public:
Base::_executor->send_request(Base::_my_address,ref.address,mid,on_reply,[](auto){});
}
template <class ActorT>
void spawn(ExecutorID target_exe)
{
spawn<ActorT> (target_exe,[](auto,auto) {});
}
template <class ActorT, typename Callable, typename... ConsArgs>
void spawn(ExecutorID target_exe, Callable on_address, ConsArgs&&... args)
{
+7 -2
View File
@@ -3,9 +3,11 @@
#include "Executor.h"
#include "Msg.h"
#include "lib/config/protos/Executor.pb.h"
#include "lib/os/os.h"
#include <moodycamel/concurrentqueue.h>
#include <thread>
#include <fmt/format.h>
namespace bts {
@@ -23,6 +25,8 @@ struct Domain::ExecutorRecord {
{}
void run(config::Executor const&cfg) {
auto tn = fmt::format("executor_{}",executor.address().executor().value());
os::set_thread_name(tn.c_str());
if (cfg.affinity() >= 0) {
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
@@ -33,8 +37,9 @@ struct Domain::ExecutorRecord {
Msg m;
while (true ) {
executor.on_poll();
auto executed = executor.on_poll();
if (queue.try_dequeue(m)) {
executed++;
switch (m.type) {
case Msg::REQUEST:
executor.on_request(m.request);
@@ -55,7 +60,7 @@ struct Domain::ExecutorRecord {
return;
};
}
else if (yield)
if (executed == 0 && yield)
std::this_thread::yield();
}
}
+1 -1
View File
@@ -8,8 +8,8 @@ namespace bts {
namespace config {
class ExecutorCfg;
}}
// end fws
namespace bts {
namespace fabric {
+4 -2
View File
@@ -106,10 +106,12 @@ Executor::Executor(Address my_address,
Executor::~Executor() {}
void Executor::on_poll()
int Executor::on_poll()
{
auto actions_executed = 0;
for (auto &ipoll : me->pollable_actors)
ipoll.second->on_poll();
actions_executed += ipoll.second->on_poll();
return actions_executed;
}
void Executor::on_request(Request const&r) {
+1 -1
View File
@@ -21,7 +21,7 @@ public:
~Executor();
void bootstrap(ActorPtr &&with_actor);
void on_poll();
int on_poll();
void on_request(Request const&r) ;
void on_reply(Reply const&r) ;
+1 -1
View File
@@ -6,6 +6,6 @@ namespace fabric {
class IPollable {
public:
virtual ~IPollable() {};
virtual void on_poll() = 0;
virtual bool on_poll() = 0;
};
}}
+1
View File
@@ -151,3 +151,4 @@ GCC_DIAG_RESTORE()
}
}
@@ -107,12 +107,15 @@ struct Pub : public Actor<>, public IPollable {
ts_topic = lookup<TimeStamp>();
}
void on_poll() override {
bool on_poll() override {
auto now = Timer::now();
if (ns_diff(now, last_pub) > 2000) {
publish(ts_topic,now);
last_pub = Timer::now();
return true;
}
else
return false;
}
};
+9 -2
View File
@@ -3,8 +3,12 @@
#include "lib/config/config.h"
#include "lib/config/protos/logging.pb.h"
#include "lib/config/protos/Executor.pb.h"
#include "setup/BootstrapActor.h"
#include "fabric/Domain.h"
#include "testing/catch_test_runner.h"
int main( int argc, char*argv [] )
{
using namespace bts;
@@ -12,9 +16,12 @@ int main( int argc, char*argv [] )
if (argc > 1 && std::string(argv[1]) == "test")
bts::run_tests(--argc, ++argv);
else {
else {
log::LoggingConfigPoller poller;
LOG(Gen,Warn, "Nothing here yet");
auto executor_cfg = config::get<config::ExecutorCfg> ();
fabric::Domain d (executor_cfg,std::make_unique<setup::BootstrapActor>(executor_cfg));
std::this_thread::sleep_until(std::chrono::time_point<std::chrono::system_clock>::max());
}
return 0;
}
@@ -0,0 +1,41 @@
#include "BootstrapActor.h"
#include "lib/config/protos/Executor.pb.h"
#include "lib/config/config.h"
#include "lib/utils/type_name.h"
#include "server/sparkd/setup/ServiceHostActor.h"
namespace bts {
namespace setup {
struct BootstrapActor::PImpl
{
PImpl(config::ExecutorCfg const&cfg) : cfg(cfg) {}
config::ExecutorCfg const cfg;
};
template <class A>
fabric::ExecutorID find_executor ()
{
auto cfg = config::get<config::ExecutorCfg>();
auto tn = utils::type_name<A>();
size_t exe_idx = 0;
for (auto const &e : cfg.executor()) {
for (auto const &exe_actor : e.actor())
if (exe_actor == tn)
return fabric::ExecutorID(exe_idx);
exe_idx++;
}
return fabric::ExecutorID(cfg.executor_size()-1);
}
BootstrapActor::BootstrapActor (config::ExecutorCfg const&cfg) : me (new PImpl(cfg)) {}
BootstrapActor::~BootstrapActor () {}
void BootstrapActor::init()
{
spawn<ServiceHostActor> (find_executor<ServiceHostActor>());
}
}}
+27
View File
@@ -0,0 +1,27 @@
#pragma once
#include <memory>
#include "server/sparkd/fabric/Actor.h"
// fwd
namespace bts {
namespace config {
class ExecutorCfg;
}}
// end fws
namespace bts {
namespace setup {
class BootstrapActor : public fabric::Actor<>
{
public:
BootstrapActor(config::ExecutorCfg const&cfg);
~BootstrapActor ();
void init() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me;
};
}}
@@ -0,0 +1,29 @@
#include "ServiceHostActor.h"
#include "lib/config/protos/Executor.pb.h"
#include "lib/quickbus/quickbus.h"
#include "lib/api/public/ServiceContext.h"
#include "server/sparkd/persistence/PersistenceServiceServer.h"
namespace bts {
namespace setup {
struct ServiceHostActor::PImpl
{
PImpl() : quickbus(qbus::Quickbus::ThreadOptions::outside_thread),
service_context(&quickbus),
settings_saver(service_context,"settings")
{}
qbus::Quickbus quickbus;
api::ServiceContext service_context;
// hosted services
persistence::PersistenceServiceServer settings_saver;
};
ServiceHostActor::ServiceHostActor () : me (new PImpl()) {}
ServiceHostActor::~ServiceHostActor () {}
bool ServiceHostActor::on_poll() {return me->quickbus.do_work();
}
}}
@@ -0,0 +1,21 @@
#pragma once
#include <memory>
#include "server/sparkd/fabric/Actor.h"
#include "server/sparkd/fabric/IPollable.h"
namespace bts {
namespace setup {
class ServiceHostActor : public fabric::Actor<>, public fabric::IPollable
{
public:
ServiceHostActor();
~ServiceHostActor ();
bool on_poll() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me;
};
}}