Issue SPARK-153, Fabric and Service integration
This commit is contained in:
1 parent
dace321b21
commit
d4445fe3e3
18 files changed
+212
-28
No files matched your search
@@ -0,0 +1,13 @@
|
||||
ExecutorCfg
|
||||
{
|
||||
value
|
||||
{
|
||||
# executor for latency sensitive code
|
||||
executor { affinity: 0 yield_cpu: false }
|
||||
|
||||
# normal executor
|
||||
executor { affinity: -1 yield_cpu: true }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
#include "lib/quickbus/network/QBConnectionManager.h"
|
||||
#include "lib/config/protos/quickbus_cfg.pb.h"
|
||||
#include "lib/os/os.h"
|
||||
#include "lib/utils/bts_assert.h"
|
||||
|
||||
#include <boost/asio.hpp>
|
||||
#include <thread>
|
||||
@@ -13,35 +14,41 @@ namespace qbus {
|
||||
|
||||
struct Quickbus::PImpl
|
||||
{
|
||||
PImpl(config::QuickbusNodeCfg const&cfg) :
|
||||
PImpl(config::QuickbusNodeCfg const&cfg,ThreadOptions options) :
|
||||
io_service(),
|
||||
router(make_node_id()),
|
||||
connection_manager(io_service,router,cfg),
|
||||
worker()
|
||||
worker(),
|
||||
thread_options(options),
|
||||
worker_id()
|
||||
{}
|
||||
|
||||
boost::asio::io_service io_service;
|
||||
Router router;
|
||||
QBConnectionManager connection_manager;
|
||||
std::thread worker;
|
||||
ThreadOptions thread_options;
|
||||
std::thread::id worker_id;
|
||||
};
|
||||
|
||||
Quickbus::Quickbus () :
|
||||
me (new PImpl(config::QuickbusNodeCfg()))
|
||||
Quickbus::Quickbus(ThreadOptions options) :
|
||||
me (new PImpl(config::QuickbusNodeCfg(),options))
|
||||
{init();}
|
||||
|
||||
Quickbus::Quickbus (config::QuickbusNodeCfg const&cfg) :
|
||||
me (new PImpl(cfg))
|
||||
Quickbus::Quickbus(config::QuickbusNodeCfg const&cfg, ThreadOptions options) :
|
||||
me (new PImpl(cfg,options))
|
||||
{init();}
|
||||
|
||||
Quickbus::~Quickbus ()
|
||||
{
|
||||
me->io_service.stop();
|
||||
me->worker.join();
|
||||
if (me->thread_options == ThreadOptions::own_thread)
|
||||
me->worker.join();
|
||||
}
|
||||
|
||||
bool Quickbus::on_qb_thread() const
|
||||
{return std::this_thread::get_id() == me->worker.get_id();}
|
||||
{
|
||||
return std::this_thread::get_id() == me->worker_id;
|
||||
}
|
||||
|
||||
void Quickbus::post (std::function<void()> const &action)
|
||||
{me->io_service.post(action);}
|
||||
@@ -51,10 +58,22 @@ Router& Quickbus::router() const
|
||||
|
||||
void Quickbus::init()
|
||||
{
|
||||
me->worker = std::thread ([this] () {
|
||||
os::set_thread_name("quickbus");
|
||||
me->io_service.run();
|
||||
});}
|
||||
if (me->thread_options == ThreadOptions::own_thread) {
|
||||
me->worker = std::thread ([this] () {
|
||||
os::set_thread_name("quickbus");
|
||||
me->io_service.run();
|
||||
});
|
||||
me->worker_id = me->worker.get_id();
|
||||
}
|
||||
else
|
||||
me->worker_id = std::this_thread::get_id() ;
|
||||
}
|
||||
|
||||
bool Quickbus::do_work()
|
||||
{
|
||||
bts_assert(me->thread_options == ThreadOptions::outside_thread);
|
||||
return me->io_service.poll();
|
||||
}
|
||||
|
||||
}}
|
||||
|
||||
@@ -31,12 +31,19 @@ namespace qbus {
|
||||
class BTS_EXPORT Quickbus
|
||||
{
|
||||
public:
|
||||
Quickbus();
|
||||
Quickbus(config::QuickbusNodeCfg const&cfg);
|
||||
enum class ThreadOptions {
|
||||
own_thread,
|
||||
outside_thread
|
||||
};
|
||||
|
||||
Quickbus(ThreadOptions options = ThreadOptions::own_thread);
|
||||
Quickbus(config::QuickbusNodeCfg const&cfg, ThreadOptions = ThreadOptions::own_thread);
|
||||
~Quickbus();
|
||||
|
||||
// call this only if ThreadOptions = own_thread
|
||||
// returns wheter any work was done
|
||||
bool do_work();
|
||||
bool on_qb_thread() const;
|
||||
|
||||
protected:
|
||||
friend class LocalNode;
|
||||
void post (std::function<void()> const &);
|
||||
|
||||
@@ -9,9 +9,8 @@ namespace utils {
|
||||
template <class T>
|
||||
const char* type_name()
|
||||
{
|
||||
T* t(nullptr);
|
||||
int status;
|
||||
return abi::__cxa_demangle(typeid (*t).name (), 0, 0, &status);
|
||||
return abi::__cxa_demangle(typeid (T).name (), 0, 0, &status);
|
||||
}
|
||||
|
||||
}}
|
||||
@@ -72,6 +72,10 @@ set (Sparkd_Sources
|
||||
# testing utilities
|
||||
testing/catch_test_runner.cpp
|
||||
|
||||
# setup
|
||||
setup/BootstrapActor.cpp
|
||||
setup/ServiceHostActor.cpp
|
||||
|
||||
# md, mdp3 onixs platform
|
||||
# md/platforms/mdp3_onixs/MDP3Listener.cpp
|
||||
# md/platforms/mdp3_onixs/MDP3Main.cpp
|
||||
|
||||
@@ -214,6 +214,12 @@ public:
|
||||
Base::_executor->send_request(Base::_my_address,ref.address,mid,on_reply,[](auto){});
|
||||
}
|
||||
|
||||
template <class ActorT>
|
||||
void spawn(ExecutorID target_exe)
|
||||
{
|
||||
spawn<ActorT> (target_exe,[](auto,auto) {});
|
||||
}
|
||||
|
||||
template <class ActorT, typename Callable, typename... ConsArgs>
|
||||
void spawn(ExecutorID target_exe, Callable on_address, ConsArgs&&... args)
|
||||
{
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
#include "Executor.h"
|
||||
#include "Msg.h"
|
||||
#include "lib/config/protos/Executor.pb.h"
|
||||
#include "lib/os/os.h"
|
||||
|
||||
#include <moodycamel/concurrentqueue.h>
|
||||
#include <thread>
|
||||
#include <fmt/format.h>
|
||||
|
||||
|
||||
namespace bts {
|
||||
@@ -23,6 +25,8 @@ struct Domain::ExecutorRecord {
|
||||
{}
|
||||
|
||||
void run(config::Executor const&cfg) {
|
||||
auto tn = fmt::format("executor_{}",executor.address().executor().value());
|
||||
os::set_thread_name(tn.c_str());
|
||||
if (cfg.affinity() >= 0) {
|
||||
cpu_set_t cpuset;
|
||||
CPU_ZERO(&cpuset);
|
||||
@@ -33,8 +37,9 @@ struct Domain::ExecutorRecord {
|
||||
|
||||
Msg m;
|
||||
while (true ) {
|
||||
executor.on_poll();
|
||||
auto executed = executor.on_poll();
|
||||
if (queue.try_dequeue(m)) {
|
||||
executed++;
|
||||
switch (m.type) {
|
||||
case Msg::REQUEST:
|
||||
executor.on_request(m.request);
|
||||
@@ -55,7 +60,7 @@ struct Domain::ExecutorRecord {
|
||||
return;
|
||||
};
|
||||
}
|
||||
else if (yield)
|
||||
if (executed == 0 && yield)
|
||||
std::this_thread::yield();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,8 +8,8 @@ namespace bts {
|
||||
namespace config {
|
||||
class ExecutorCfg;
|
||||
}}
|
||||
|
||||
// end fws
|
||||
|
||||
namespace bts {
|
||||
namespace fabric {
|
||||
|
||||
|
||||
@@ -106,10 +106,12 @@ Executor::Executor(Address my_address,
|
||||
|
||||
Executor::~Executor() {}
|
||||
|
||||
void Executor::on_poll()
|
||||
int Executor::on_poll()
|
||||
{
|
||||
auto actions_executed = 0;
|
||||
for (auto &ipoll : me->pollable_actors)
|
||||
ipoll.second->on_poll();
|
||||
actions_executed += ipoll.second->on_poll();
|
||||
return actions_executed;
|
||||
}
|
||||
|
||||
void Executor::on_request(Request const&r) {
|
||||
|
||||
@@ -21,7 +21,7 @@ public:
|
||||
~Executor();
|
||||
void bootstrap(ActorPtr &&with_actor);
|
||||
|
||||
void on_poll();
|
||||
int on_poll();
|
||||
void on_request(Request const&r) ;
|
||||
void on_reply(Reply const&r) ;
|
||||
|
||||
|
||||
@@ -6,6 +6,6 @@ namespace fabric {
|
||||
class IPollable {
|
||||
public:
|
||||
virtual ~IPollable() {};
|
||||
virtual void on_poll() = 0;
|
||||
virtual bool on_poll() = 0;
|
||||
};
|
||||
}}
|
||||
@@ -151,3 +151,4 @@ GCC_DIAG_RESTORE()
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,12 +107,15 @@ struct Pub : public Actor<>, public IPollable {
|
||||
ts_topic = lookup<TimeStamp>();
|
||||
}
|
||||
|
||||
void on_poll() override {
|
||||
bool on_poll() override {
|
||||
auto now = Timer::now();
|
||||
if (ns_diff(now, last_pub) > 2000) {
|
||||
publish(ts_topic,now);
|
||||
last_pub = Timer::now();
|
||||
return true;
|
||||
}
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
@@ -3,8 +3,12 @@
|
||||
|
||||
#include "lib/config/config.h"
|
||||
#include "lib/config/protos/logging.pb.h"
|
||||
#include "lib/config/protos/Executor.pb.h"
|
||||
#include "setup/BootstrapActor.h"
|
||||
#include "fabric/Domain.h"
|
||||
#include "testing/catch_test_runner.h"
|
||||
|
||||
|
||||
int main( int argc, char*argv [] )
|
||||
{
|
||||
using namespace bts;
|
||||
@@ -12,9 +16,12 @@ int main( int argc, char*argv [] )
|
||||
|
||||
if (argc > 1 && std::string(argv[1]) == "test")
|
||||
bts::run_tests(--argc, ++argv);
|
||||
else {
|
||||
else {
|
||||
log::LoggingConfigPoller poller;
|
||||
LOG(Gen,Warn, "Nothing here yet");
|
||||
auto executor_cfg = config::get<config::ExecutorCfg> ();
|
||||
|
||||
fabric::Domain d (executor_cfg,std::make_unique<setup::BootstrapActor>(executor_cfg));
|
||||
std::this_thread::sleep_until(std::chrono::time_point<std::chrono::system_clock>::max());
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
#include "BootstrapActor.h"
|
||||
|
||||
#include "lib/config/protos/Executor.pb.h"
|
||||
#include "lib/config/config.h"
|
||||
#include "lib/utils/type_name.h"
|
||||
#include "server/sparkd/setup/ServiceHostActor.h"
|
||||
|
||||
namespace bts {
|
||||
namespace setup {
|
||||
|
||||
struct BootstrapActor::PImpl
|
||||
{
|
||||
PImpl(config::ExecutorCfg const&cfg) : cfg(cfg) {}
|
||||
config::ExecutorCfg const cfg;
|
||||
|
||||
};
|
||||
|
||||
template <class A>
|
||||
fabric::ExecutorID find_executor ()
|
||||
{
|
||||
auto cfg = config::get<config::ExecutorCfg>();
|
||||
auto tn = utils::type_name<A>();
|
||||
size_t exe_idx = 0;
|
||||
for (auto const &e : cfg.executor()) {
|
||||
for (auto const &exe_actor : e.actor())
|
||||
if (exe_actor == tn)
|
||||
return fabric::ExecutorID(exe_idx);
|
||||
exe_idx++;
|
||||
}
|
||||
return fabric::ExecutorID(cfg.executor_size()-1);
|
||||
}
|
||||
|
||||
BootstrapActor::BootstrapActor (config::ExecutorCfg const&cfg) : me (new PImpl(cfg)) {}
|
||||
BootstrapActor::~BootstrapActor () {}
|
||||
|
||||
void BootstrapActor::init()
|
||||
{
|
||||
spawn<ServiceHostActor> (find_executor<ServiceHostActor>());
|
||||
}
|
||||
|
||||
}}
|
||||
@@ -0,0 +1,27 @@
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include "server/sparkd/fabric/Actor.h"
|
||||
|
||||
// fwd
|
||||
namespace bts {
|
||||
namespace config {
|
||||
class ExecutorCfg;
|
||||
}}
|
||||
// end fws
|
||||
|
||||
namespace bts {
|
||||
namespace setup {
|
||||
|
||||
class BootstrapActor : public fabric::Actor<>
|
||||
{
|
||||
public:
|
||||
BootstrapActor(config::ExecutorCfg const&cfg);
|
||||
~BootstrapActor ();
|
||||
void init() override;
|
||||
private:
|
||||
struct PImpl;
|
||||
std::unique_ptr<PImpl> me;
|
||||
};
|
||||
|
||||
}}
|
||||
@@ -0,0 +1,29 @@
|
||||
#include "ServiceHostActor.h"
|
||||
|
||||
#include "lib/config/protos/Executor.pb.h"
|
||||
#include "lib/quickbus/quickbus.h"
|
||||
#include "lib/api/public/ServiceContext.h"
|
||||
#include "server/sparkd/persistence/PersistenceServiceServer.h"
|
||||
|
||||
namespace bts {
|
||||
namespace setup {
|
||||
|
||||
struct ServiceHostActor::PImpl
|
||||
{
|
||||
PImpl() : quickbus(qbus::Quickbus::ThreadOptions::outside_thread),
|
||||
service_context(&quickbus),
|
||||
settings_saver(service_context,"settings")
|
||||
{}
|
||||
qbus::Quickbus quickbus;
|
||||
api::ServiceContext service_context;
|
||||
|
||||
// hosted services
|
||||
persistence::PersistenceServiceServer settings_saver;
|
||||
};
|
||||
|
||||
ServiceHostActor::ServiceHostActor () : me (new PImpl()) {}
|
||||
ServiceHostActor::~ServiceHostActor () {}
|
||||
bool ServiceHostActor::on_poll() {return me->quickbus.do_work();
|
||||
}
|
||||
|
||||
}}
|
||||
@@ -0,0 +1,21 @@
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include "server/sparkd/fabric/Actor.h"
|
||||
#include "server/sparkd/fabric/IPollable.h"
|
||||
|
||||
namespace bts {
|
||||
namespace setup {
|
||||
|
||||
class ServiceHostActor : public fabric::Actor<>, public fabric::IPollable
|
||||
{
|
||||
public:
|
||||
ServiceHostActor();
|
||||
~ServiceHostActor ();
|
||||
bool on_poll() override;
|
||||
private:
|
||||
struct PImpl;
|
||||
std::unique_ptr<PImpl> me;
|
||||
};
|
||||
|
||||
}}
|
||||
Reference in new issue
Block a user