diff --git a/CMakeLists.txt b/CMakeLists.txt index 937b06c00..f40f0dcf1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,7 +58,6 @@ else () COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/sdk/examples/order_plugin_example.so ${CMAKE_BINARY_DIR}/server/plugins/ COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/sdk/examples/ai_order_plugin_example.so ${CMAKE_BINARY_DIR}/server/plugins/ - COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libtensorflow_cc.so ${CMAKE_BINARY_DIR}/server COMMENT "Creating server runtime layout" DEPENDS sparkd data_files_dir) diff --git a/externals b/externals index e23695418..c0486ec0f 160000 --- a/externals +++ b/externals @@ -1 +1 @@ -Subproject commit e2369541811cb67504d4543889feccebbe58a4f3 +Subproject commit c0486ec0f5aaabcf7e01a9f5490a52005a6fdcc8 diff --git a/remote-sparkvm.code-workspace b/remote-sparkvm.code-workspace new file mode 100644 index 000000000..03c875df7 --- /dev/null +++ b/remote-sparkvm.code-workspace @@ -0,0 +1,27 @@ +{ + "folders": [ + { + "path": "." + }, + { + "uri": "sftp://bts:BT5u53r@sparkvm:22/home/bts/", + "name": "sparkvm" + } + ], + "settings": { + "files.associations": { + "utility": "cpp", + "initializer_list": "cpp", + "list": "cpp", + "type_traits": "cpp", + "vector": "cpp", + "xhash": "cpp", + "xstring": "cpp", + "system_error": "cpp", + "array": "cpp", + "string_view": "cpp", + "xutility": "cpp", + "*.ipp": "cpp" + } + } +} \ No newline at end of file diff --git a/spark.code-workspace b/spark.code-workspace new file mode 100644 index 000000000..63e8255aa --- /dev/null +++ b/spark.code-workspace @@ -0,0 +1,77 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": { + "files.associations": { + "utility": "cpp", + "initializer_list": "cpp", + "list": "cpp", + "type_traits": "cpp", + "vector": "cpp", + "xhash": "cpp", + "xstring": "cpp", + "system_error": "cpp", + "array": "cpp", + "string_view": "cpp", + "xutility": "cpp", + "*.ipp": "cpp", + "atomic": "cpp", + "hash_map": "cpp", + "hash_set": "cpp", + "*.tcc": "cpp", + "bitset": "cpp", + "cctype": "cpp", + "cfenv": "cpp", + "chrono": "cpp", + "cinttypes": "cpp", + "clocale": "cpp", + "cmath": "cpp", + "complex": "cpp", + "condition_variable": "cpp", + "csignal": "cpp", + "cstdarg": "cpp", + "cstddef": "cpp", + "cstdint": "cpp", + "cstdio": "cpp", + "cstdlib": "cpp", + "cstring": "cpp", + "ctime": "cpp", + "cwchar": "cpp", + "cwctype": "cpp", + "deque": "cpp", + "forward_list": "cpp", + "unordered_map": "cpp", + "unordered_set": "cpp", + "exception": "cpp", + "any": "cpp", + "optional": "cpp", + "fstream": "cpp", + "functional": "cpp", + "future": "cpp", + "iomanip": "cpp", + "iosfwd": "cpp", + "iostream": "cpp", + "istream": "cpp", + "limits": "cpp", + "memory": "cpp", + "mutex": "cpp", + "new": "cpp", + "ostream": "cpp", + "numeric": "cpp", + "ratio": "cpp", + "scoped_allocator": "cpp", + "sstream": "cpp", + "stdexcept": "cpp", + "streambuf": "cpp", + "thread": "cpp", + "tuple": "cpp", + "typeindex": "cpp", + "typeinfo": "cpp", + "valarray": "cpp", + "variant": "cpp" + } + } +} \ No newline at end of file diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt index 2bc9cd150..f5705d6df 100644 --- a/src/lib/CMakeLists.txt +++ b/src/lib/CMakeLists.txt @@ -130,6 +130,7 @@ else () cryptopp pugixml lttng-ust + tensorflow-core -lrt -ldl -static-libgcc diff --git a/src/sdk/examples/ai_order_plugin_example.cpp b/src/sdk/examples/ai_order_plugin_example.cpp index 2fffd05d6..763079f9d 100644 --- a/src/sdk/examples/ai_order_plugin_example.cpp +++ b/src/sdk/examples/ai_order_plugin_example.cpp @@ -15,7 +15,7 @@ using namespace bts::api::orderman; using namespace bts::orderman; - +using std::vector; namespace bts { namespace order_example { @@ -82,8 +82,16 @@ void AISampleOrder::on_init_new( { std::unordered_set tags; tensorflow::SavedModelBundle bundle; - bts::ml::load_tf_model("", tags, &bundle); - } + auto status = bts::ml::load_tf_model("", tags, &bundle); + if(status.ok()) + { + vector > inputs; + vector output_tensor_names; + vector target_node_names; + vector outputs; + bts::ml::execute(bundle, inputs, output_tensor_names, target_node_names, &outputs); + } + } me->def = def; me->state.mutable_dynamic(); @@ -220,7 +228,7 @@ void AISampleOrder::on_cancel(trace::CorID const &) { case ORDER_STATUS_CANCELED: case ORDER_STATUS_ERROR: transition(me->state.status(), "Cannot cancel order"); - break; + break; case ORDER_STATUS_INACTIVE: case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_: diff --git a/src/sdk/examples/build_examples.sh b/src/sdk/examples/build_examples.sh index 4c646a0a8..cedaa18ab 100755 --- a/src/sdk/examples/build_examples.sh +++ b/src/sdk/examples/build_examples.sh @@ -12,36 +12,58 @@ g++ -isystem ../externals/include \ -llttng-ust \ -ldl -g++ -isystem ../externals/include \ - -I ../include \ - -I ../include/spark \ - -std=c++1z \ - -fPIC \ - -g \ - -shared \ - -o order_plugin_example.so \ - order_plugin_example.cpp \ - -Wl,-rpath,.. \ - -L ../lib \ - -l spark +# g++ -isystem ../externals/include \ +# -I ../include \ +# -I ../include/spark \ +# -std=c++1z \ +# -fPIC \ +# -g \ +# -shared \ +# -o order_plugin_example.so \ +# order_plugin_example.cpp \ +# -Wl,-rpath,.. \ +# -L ../lib \ +# -l spark +# g++ -isystem ../externals/include \ +# -I ../include \ +# -I ../include/spark \ +# -I ../externals/include \ +# -std=c++1z \ +# -fPIC \ +# -g \ +# -shared \ +# -o ai_order_plugin_example.so \ +# ai_order_plugin_example.cpp \ +# ../tensorflow/tfexecute.cpp \ +# ../tensorflow/tfloader.cpp \ +# -Wl,-rpath,. \ +# -Wl,-rpath,.. \ +# -Wl,-rpath,../lib \ +# -L ../lib \ +# -l spark +# # -L ../externals/lib \ +# # -l tensorflow-core + g++ -isystem ../externals/include \ -I ../include \ -I ../include/spark \ -I ../externals/include \ -std=c++1z \ - -fPIC \ - -g \ - -shared \ - -o ai_order_plugin_example.so \ - ai_order_plugin_example.cpp \ + -o tf_test \ + tf_test.cpp \ ../tensorflow/tfexecute.cpp \ ../tensorflow/tfloader.cpp \ -Wl,-rpath,. \ -Wl,-rpath,.. \ -Wl,-rpath,../lib \ -L ../lib \ - -l tensorflow_cc \ - -l spark - + -l spark \ + -L ../externals/lib \ + -l tensorflow \ + -l tensorflow_framework \ + -l nsync \ + -lpthread \ + -llttng-ust \ + -ldl diff --git a/src/sdk/examples/tf_test.cpp b/src/sdk/examples/tf_test.cpp new file mode 100644 index 000000000..10878fa48 --- /dev/null +++ b/src/sdk/examples/tf_test.cpp @@ -0,0 +1,18 @@ +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/public/session.h" + +#include + +using namespace std; +using namespace tensorflow; + +int main() +{ + Session* session; + Status status = NewSession(SessionOptions(), &session); + if (!status.ok()) { + cout << status.ToString() << "\n"; + return 1; + } + cout << "Session successfully created.\n"; +} \ No newline at end of file diff --git a/src/sdk/tensorflow/saved_model_bundle_factory.h b/src/sdk/tensorflow/saved_model_bundle_factory.h new file mode 100644 index 000000000..14b947874 --- /dev/null +++ b/src/sdk/tensorflow/saved_model_bundle_factory.h @@ -0,0 +1,93 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SAVED_MODEL_BUNDLE_FACTORY_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SAVED_MODEL_BUNDLE_FACTORY_H_ + +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" + +namespace tensorflow { +namespace serving { + +/// A factory that creates SavedModelBundles from SavedModel or SessionBundle +/// export paths. +/// +/// The emitted sessions only support Run(), and although not enforced it is +/// expected that the client will only make non-mutating Run() calls. (If this +/// restriction, which we've added as a safety measure, is problematic for your +/// use-case please contact the TensorFlow Serving team to discuss disabling +/// it.) +/// +/// If the config calls for batching, the emitted sessions automatically batch +/// Run() calls behind the scenes, using a SharedBatchScheduler owned by the +/// factory. The 'config.num_batch_threads' threads are shared across all +/// session instances created by this factory. However, each session has its own +/// dedicated queue of size 'config.max_enqueued_batches'. +/// +/// The factory can also estimate the resource (e.g. RAM) requirements of a +/// SavedModelBundle based on the SavedModel (i.e. prior to loading the +/// session). +/// +/// This class is thread-safe. +class SavedModelBundleFactory { + public: + /// Instantiates a SavedModelBundleFactory using a config. + /// + /// @param config Config with initialization options. + /// @param factory Newly created factory if the returned Status is OK. + static Status Create(const SessionBundleConfig& config, + std::unique_ptr* factory); + + /// Instantiates a bundle from a given export or SavedModel path. + /// + /// @param path Path to the model. + /// @param bundle Newly created SavedModelBundle if the returned Status is + /// OK. + Status CreateSavedModelBundle(const string& path, + std::unique_ptr* bundle); + + /// Estimates the resources a SavedModel bundle will use once loaded, from its + /// export path. + /// + /// @param path Path to the model. + /// @param estimate Output resource usage estimates. Different kinds of + /// resources (e.g. CPU, RAM, etc.) may get populated. +// Status EstimateResourceRequirement(const string& path, +// ResourceAllocation* estimate) const; + +// const SessionBundleConfig& config() const { return config_; } + + private: +// using Batcher = SharedBatchScheduler; + +// SavedModelBundleFactory(const SessionBundleConfig& config, +// std::shared_ptr batch_scheduler); + +// const SessionBundleConfig config_; + +// // A shared batch scheduler. One queue is used for each session this factory +// // emits. If batching is not configured, this remains null. +// std::shared_ptr batch_scheduler_; + + TF_DISALLOW_COPY_AND_ASSIGN(SavedModelBundleFactory); +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SAVED_MODEL_BUNDLE_FACTORY_H_ diff --git a/src/sdk/tensorflow/tfexecute.cpp b/src/sdk/tensorflow/tfexecute.cpp index c1f1f88af..03525a047 100644 --- a/src/sdk/tensorflow/tfexecute.cpp +++ b/src/sdk/tensorflow/tfexecute.cpp @@ -4,6 +4,7 @@ #include "tensorflow/cc/saved_model/loader.h" using namespace tensorflow; +using std::vector; namespace bts { @@ -11,10 +12,10 @@ namespace ml { tensorflow::Status execute(const tensorflow::SavedModelBundle& bundle, - const std::vector >& inputs, - const std::vector& output_tensor_names, - const std::vector& target_node_names, - std::vector* outputs) + const vector >& inputs, + const vector& output_tensor_names, + const vector& target_node_names, + vector* outputs) { tensorflow::RunMetadata* run_metadata = nullptr; tensorflow::RunOptions run_options; diff --git a/src/sdk/tensorflow/tfloader.cpp b/src/sdk/tensorflow/tfloader.cpp index 69698b56d..6acd6dfac 100644 --- a/src/sdk/tensorflow/tfloader.cpp +++ b/src/sdk/tensorflow/tfloader.cpp @@ -1,29 +1,75 @@ #include "tfloader.h" - -#include "tensorflow/cc/saved_model/loader.h" #include "lib/utils/helper_macros.h" + +//no c++ yet +//https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/label_image/main.cc +#include "tensorflow/cc/saved_model/loader.h" + using namespace tensorflow; +using namespace tensorflow::c; +using std::string; +using std::vector; namespace bts { namespace ml { + +// tensorflow::Status load_tf_model(const std::string& export_dir, const std::unordered_set& tags, SavedModelBundle* const bundle) +// { +// tensorflow::SessionOptions session_options; //defaults +// tensorflow::RunOptions run_options; +// UNUSED(export_dir); +// UNUSED(tags); +// UNUSED(bundle); + +// Status status = LoadSavedModel(session_options, run_options, export_dir, tags, bundle); +// return status; + +// //return tensorflow::Status(); +// } -tensorflow::Status load_tf_model(const std::string& export_dir, const std::unordered_set& tags, SavedModelBundle* const bundle) +// tensorflow::Status load_tf_model(const string& export_dir, const std::unordered_set& tags, std::unique_ptr* bundle) +// { +// SessionBundleConfig config; +// for(auto& i : tags) +// config.add_saved_model_tags(i); +// std::unique_ptr factory; +// Status stat = SavedModelBundleFactory::Create(config, &factory); +// if(stat.ok()) +// { +// stat = factory->CreateSavedModelBundle(export_dir, bundle); +// } +// return stat; +// } + +tensorflow::c::TFStatus load_tf_model(const std::string& export_dir, const std::unordered_set& tags, + TF_Session** session) { - tensorflow::SessionOptions session_options; //defaults - tensorflow::RunOptions run_options; - UNUSED(export_dir); - UNUSED(tags); - UNUSED(bundle); - - Status status = LoadSavedModel(session_options, run_options, export_dir, tags, bundle); - return status; - - //return tensorflow::Status(); -} + vector ctags; + for(auto& t : tags) + ctags.push_back(const_cast(t.c_str())); + TF_SessionOptions* session_options = TF_NewSessionOptions(); + TF_Buffer* run_options = nullptr; + TF_Buffer* meta_graph_def = TF_NewBuffer(); //TODO get the protos here + TF_Graph* graph = TF_NewGraph(); + TF_Status* status = TF_NewStatus(); + *session = TF_LoadSessionFromSavedModel(session_options, run_options, export_dir.c_str(), &ctags[0], ctags.size(), + graph, meta_graph_def, status); + std::string msg(TF_Message(status)); + TF_Code code = TF_GetCode(status); + TFStatus stat; + stat.error = msg; + stat.code = code; + + TF_DeleteSessionOptions(session_options); + TF_DeleteBuffer(run_options); + TF_DeleteBuffer(meta_graph_def); + TF_DeleteStatus(status); + return stat; +} } } \ No newline at end of file diff --git a/src/sdk/tensorflow/tfloader.h b/src/sdk/tensorflow/tfloader.h index 3870d3d40..cf8a17284 100644 --- a/src/sdk/tensorflow/tfloader.h +++ b/src/sdk/tensorflow/tfloader.h @@ -1,12 +1,24 @@ #pragma once +#include "tensorflow/core/platform/env.h" #include #include -#include "tensorflow/core/platform/env.h" +#include + namespace tensorflow { struct SavedModelBundle; + namespace c + { + #include "tensorflow/c/c_api.h" + struct TFStatus + { + std::string error; + TF_Code code; + bool ok() { return (int)code == 0; } + }; + } } namespace bts @@ -14,8 +26,13 @@ namespace bts namespace ml { ///call Status.ok() for a boolean - tensorflow::Status load_tf_model(const std::string& export_dir, const std::unordered_set& tags, - tensorflow::SavedModelBundle* const bundle); + // tensorflow::Status load_tf_model(const std::string& export_dir, const std::unordered_set& tags, + // tensorflow::SavedModelBundle* const bundle); + // tensorflow::Status load_tf_model(const std::string& export_dir, const std::unordered_set& tags, + // std::unique_ptr* bundle); + + tensorflow::c::TFStatus load_tf_model(const std::string& export_dir, const std::unordered_set& tags, + tensorflow::c::TF_Session** session); } }