tf
This commit is contained in:
1 parent
eb1b42a5e8
commit
458eb2aef1
1283 files changed
+323373
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,61 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// This file contains APIs that assume a StreamExecutor is backed by CUDA.
|
||||
// It reaches into the CUDA implementation to activate an underlying CUDA
|
||||
// context.
|
||||
//
|
||||
// Having this file separate from cuda_gpu_executor.h means that dependent
|
||||
// code does not also have to depend on cuda.h.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_ACTIVATION_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_ACTIVATION_H_
|
||||
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class StreamExecutor;
|
||||
|
||||
namespace cuda {
|
||||
|
||||
class CUDAExecutor;
|
||||
class ScopedActivateContext;
|
||||
|
||||
// Activates a CUDA context within an enclosing scope.
|
||||
class ScopedActivateExecutorContext {
|
||||
public:
|
||||
// Form that takes a CUDA executor implementation.
|
||||
explicit ScopedActivateExecutorContext(CUDAExecutor* cuda_exec);
|
||||
|
||||
// Form that takes a pImpl executor and extracts a CUDA implementation --
|
||||
// fatal failure if it is not CUDA inside.
|
||||
explicit ScopedActivateExecutorContext(StreamExecutor* stream_exec);
|
||||
|
||||
ScopedActivateExecutorContext(ScopedActivateExecutorContext&& other);
|
||||
|
||||
~ScopedActivateExecutorContext();
|
||||
|
||||
private:
|
||||
// The cuda.h-using datatype that we wrap.
|
||||
ScopedActivateContext* driver_scoped_activate_context_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(ScopedActivateExecutorContext);
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_ACTIVATION_H_
|
||||
@@ -0,0 +1,162 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// CUDA-specific support for BLAS functionality -- this wraps the cuBLAS library
|
||||
// capabilities, and is only included into CUDA implementation code -- it will
|
||||
// not introduce cuda headers into other code.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_BLAS_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_BLAS_H_
|
||||
|
||||
#include "tensorflow/stream_executor/blas.h"
|
||||
#include "tensorflow/stream_executor/host_or_device_scalar.h"
|
||||
#include "tensorflow/stream_executor/lib/stringpiece.h"
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/platform/thread_annotations.h"
|
||||
#include "tensorflow/stream_executor/plugin_registry.h"
|
||||
|
||||
typedef struct cublasContext *cublasHandle_t;
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class Stream;
|
||||
|
||||
namespace cuda {
|
||||
|
||||
// Opaque and unique identifier for the cuBLAS plugin.
|
||||
extern const PluginId kCuBlasPlugin;
|
||||
|
||||
class CUDAExecutor;
|
||||
|
||||
// BLAS plugin for CUDA platform via cuBLAS library.
|
||||
//
|
||||
// This satisfies the platform-agnostic BlasSupport interface.
|
||||
//
|
||||
// Note that the cuBLAS handle that this encapsulates is implicitly tied to the
|
||||
// context (and, as a result, the device) that the parent CUDAExecutor is tied
|
||||
// to. This simply happens as an artifact of creating the cuBLAS handle when a
|
||||
// CUDA context is active.
|
||||
//
|
||||
// Thread-safe post-initialization.
|
||||
class CUDABlas : public blas::BlasSupport {
|
||||
public:
|
||||
explicit CUDABlas(CUDAExecutor *parent);
|
||||
|
||||
// Allocates a cuBLAS handle.
|
||||
bool Init();
|
||||
|
||||
// Releases the cuBLAS handle, if present.
|
||||
~CUDABlas() override;
|
||||
|
||||
TENSORFLOW_STREAM_EXECUTOR_GPU_BLAS_SUPPORT_OVERRIDES
|
||||
|
||||
private:
|
||||
// Tells cuBLAS to enqueue the BLAS operation onto a particular Stream.
|
||||
//
|
||||
// cuBLAS is stateful, and only be associated with one stream (in order to
|
||||
// enqueue dispatch) at a given time. As a result, this generally must be
|
||||
// invoked before calling into cuBLAS.
|
||||
bool SetStream(Stream *stream) EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
||||
|
||||
// A helper function that calls the real cuBLAS function together with error
|
||||
// handling.
|
||||
//
|
||||
// cublas_func: cuBLAS function pointer.
|
||||
// cublas_name: cuBLAS function name.
|
||||
// stream: Stream to enqueue the BLAS operation onto.
|
||||
// pointer_mode_host: Indicate if the pointer to a scalar value is from host
|
||||
// (true) or device (false).
|
||||
// err_on_failure: Whether to print an error if the cublas function fails.
|
||||
// args: Arguments of cuBLAS function.
|
||||
template <typename FuncT, typename... Args>
|
||||
bool DoBlasInternalImpl(FuncT cublas_func, Stream *stream,
|
||||
bool pointer_mode_host, bool err_on_failure,
|
||||
bool use_tensor_op_math, Args... args);
|
||||
|
||||
// Convenience functions that call DoBlasInternalImpl with different values
|
||||
// for err_on_failure.
|
||||
template <typename FuncT, typename... Args>
|
||||
bool DoBlasInternal(FuncT cublas_func, Stream *stream, bool pointer_mode_host,
|
||||
Args... args) {
|
||||
return DoBlasInternalImpl(cublas_func, stream, pointer_mode_host,
|
||||
/*err_on_failure=*/true, /*use_tensor_ops=*/false,
|
||||
args...);
|
||||
}
|
||||
template <typename FuncT, typename... Args>
|
||||
bool DoBlasInternalFailureOK(FuncT cublas_func, Stream *stream,
|
||||
bool pointer_mode_host, Args... args) {
|
||||
// Tensor ops are hard-coded off in this path, but can still be enabled with
|
||||
// a specific algorithm choice as in DoBlasGemmWithAlgorithmImpl().
|
||||
return DoBlasInternalImpl(cublas_func, stream, pointer_mode_host,
|
||||
/*err_on_failure=*/false,
|
||||
/*use_tensor_ops=*/false, args...);
|
||||
}
|
||||
|
||||
// A helper function to implement DoBlasGemmBatched interfaces for generic
|
||||
// types.
|
||||
template <typename T, typename Scalar, typename FuncT>
|
||||
port::Status DoBlasGemmBatchedInternal(
|
||||
FuncT cublas_func, Stream *stream, blas::Transpose transa,
|
||||
blas::Transpose transb, uint64 m, uint64 n, uint64 k, Scalar alpha,
|
||||
const port::ArraySlice<DeviceMemory<T> *> &a_array, int lda,
|
||||
const port::ArraySlice<DeviceMemory<T> *> &b_array, int ldb, Scalar beta,
|
||||
const port::ArraySlice<DeviceMemory<T> *> &c_array, int ldc,
|
||||
int batch_count, ScratchAllocator *scratch_allocator);
|
||||
|
||||
// Helper function for implementing DoBlasGemmWithAlgorithm.
|
||||
template <typename InT, typename OutT, typename CompT>
|
||||
bool DoBlasGemmWithAlgorithmImpl(
|
||||
Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m,
|
||||
uint64 n, uint64 k, const HostOrDeviceScalar<CompT> &alpha,
|
||||
const DeviceMemory<InT> &a, int lda, const DeviceMemory<InT> &b, int ldb,
|
||||
const HostOrDeviceScalar<CompT> &beta, DeviceMemory<OutT> *c, int ldc,
|
||||
blas::ComputationType computation_type, blas::AlgorithmType algorithm,
|
||||
blas::ProfileResult *output_profile_result);
|
||||
|
||||
// Helper function for implementing DoBlasGemmWithProfiling.
|
||||
template <typename T, typename ParamType>
|
||||
bool DoBlasGemmWithProfilingImpl(
|
||||
Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m,
|
||||
uint64 n, uint64 k, const ParamType &alpha, const DeviceMemory<T> &a,
|
||||
int lda, const DeviceMemory<T> &b, int ldb, const ParamType &beta,
|
||||
DeviceMemory<T> *c, int ldc, blas::ProfileResult *output_profile_result);
|
||||
|
||||
// Helper function for implementing DoBlasGemvWithProfiling.
|
||||
template <typename T>
|
||||
bool DoBlasGemvWithProfilingImpl(Stream *stream, blas::Transpose trans,
|
||||
uint64 m, uint64 n, const T &alpha,
|
||||
const DeviceMemory<T> &a, int lda,
|
||||
const DeviceMemory<T> &x, int incx,
|
||||
const T &beta, DeviceMemory<T> *y, int incy,
|
||||
blas::ProfileResult *output_profile_result);
|
||||
|
||||
// mutex that guards the cuBLAS handle for this device.
|
||||
mutex mu_;
|
||||
|
||||
// CUDAExecutor which instantiated this CUDABlas.
|
||||
// Immutable post-initialization.
|
||||
CUDAExecutor *parent_;
|
||||
|
||||
// cuBLAS library handle on the device.
|
||||
cublasHandle_t blas_ GUARDED_BY(mu_);
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CUDABlas);
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_BLAS_H_
|
||||
@@ -0,0 +1,97 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DIAGNOSTICS_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DIAGNOSTICS_H_
|
||||
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include <tuple>
|
||||
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
// e.g. DriverVersion{346, 3, 4}
|
||||
using DriverVersion = std::tuple<int, int, int>;
|
||||
|
||||
// Converts a parsed driver version to string form.
|
||||
string DriverVersionToString(DriverVersion version);
|
||||
|
||||
// Converts a parsed driver version or status value to natural string form.
|
||||
string DriverVersionStatusToString(port::StatusOr<DriverVersion> version);
|
||||
|
||||
// Converts a string of a form like "331.79" to a DriverVersion{331, 79}.
|
||||
port::StatusOr<DriverVersion> StringToDriverVersion(const string &value);
|
||||
|
||||
class Diagnostician {
|
||||
public:
|
||||
// Logs diagnostic information when CUDA appears to be misconfigured (e.g. is
|
||||
// not initializing).
|
||||
//
|
||||
// Note: if we're running on a machine that has no GPUs, we don't want to
|
||||
// produce very much log spew beyond saying, "looks like there's no CUDA
|
||||
// kernel
|
||||
// module running".
|
||||
//
|
||||
// Note: we use non-Google-File:: API here because we may be called before
|
||||
// InitGoogle has completed.
|
||||
static void LogDiagnosticInformation();
|
||||
|
||||
// Given the driver version file contents, finds the kernel module version and
|
||||
// returns it as a string.
|
||||
//
|
||||
// This is solely used for more informative log messages when the user is
|
||||
// running on a machine that happens to have a libcuda/kernel driver mismatch.
|
||||
static port::StatusOr<DriverVersion> FindKernelModuleVersion(
|
||||
const string &driver_version_file_contents);
|
||||
|
||||
// Extracts the kernel driver version from the current host.
|
||||
static port::StatusOr<DriverVersion> FindKernelDriverVersion();
|
||||
|
||||
// Iterates through loaded DSOs with DlIteratePhdrCallback to find the
|
||||
// driver-interfacing DSO version number. Returns it as a string.
|
||||
static port::StatusOr<DriverVersion> FindDsoVersion();
|
||||
|
||||
// Logs information about the kernel driver version and userspace driver
|
||||
// library version.
|
||||
static void LogDriverVersionInformation();
|
||||
|
||||
private:
|
||||
|
||||
// Given the DSO version number and the driver version file contents, extracts
|
||||
// the driver version and compares, warning the user in the case of
|
||||
// incompatibility.
|
||||
//
|
||||
// This is solely used for more informative log messages when the user is
|
||||
// running on a machine that happens to have a libcuda/kernel driver mismatch.
|
||||
static void WarnOnDsoKernelMismatch(
|
||||
port::StatusOr<DriverVersion> dso_version,
|
||||
port::StatusOr<DriverVersion> kernel_version);
|
||||
|
||||
// Logs information about the dev nodes present on this machine: their
|
||||
// existence, permissions, accessibility from this uid/gid.
|
||||
static void LogDevNodeDiagnosticInformation();
|
||||
|
||||
static string GetDevNodePath(int dev_node_ordinal);
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(Diagnostician);
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DIAGNOSTICS_H_
|
||||
@@ -0,0 +1,778 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// The CUDA-specific DNN library support, implementing the general DnnSupport
|
||||
// interface.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DNN_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DNN_H_
|
||||
|
||||
#include "tensorflow/stream_executor/cuda/cuda_activation.h"
|
||||
#include "tensorflow/stream_executor/dnn.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/platform/thread_annotations.h"
|
||||
#include "tensorflow/stream_executor/plugin_registry.h"
|
||||
#include "tensorflow/stream_executor/temporary_device_memory.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
class CUDAExecutor;
|
||||
class CudnnRnnDescriptor;
|
||||
class CudnnRnnSequenceTensorDescriptor;
|
||||
class CudnnRnnStateTensorDescriptor;
|
||||
|
||||
// Opaque and unique identifier for the cuDNN plugin.
|
||||
extern const PluginId kCuDnnPlugin;
|
||||
|
||||
// cudnn-library based DNN support. For details on overridden interface
|
||||
// functions, see dnn.h.
|
||||
class CudnnSupport : public dnn::DnnSupport {
|
||||
public:
|
||||
explicit CudnnSupport(CUDAExecutor* parent);
|
||||
|
||||
port::Status Init() override;
|
||||
port::StatusOr<perftools::gputools::dnn::VersionInfo> GetVersion() override;
|
||||
|
||||
port::StatusOr<std::unique_ptr<dnn::RnnDescriptor>> createRnnDescriptor(
|
||||
int num_layers, int hidden_size, int input_size, int batch_size,
|
||||
dnn::RnnInputMode input_mode, dnn::RnnDirectionMode direction_mode,
|
||||
dnn::RnnMode rnn_mode, dnn::DataType data_type,
|
||||
const dnn::AlgorithmConfig& algorithm_config, float dropout, uint64 seed,
|
||||
ScratchAllocator* state_allocator) override;
|
||||
|
||||
port::StatusOr<std::unique_ptr<dnn::RnnSequenceTensorDescriptor>>
|
||||
createRnnSequenceTensorDescriptor(int seq_length, int batch_size,
|
||||
int data_size,
|
||||
dnn::DataType data_type) override;
|
||||
|
||||
port::StatusOr<std::unique_ptr<dnn::RnnStateTensorDescriptor>>
|
||||
createRnnStateTensorDescriptor(int num_layer, int batch_size, int data_size,
|
||||
dnn::DataType data_type) override;
|
||||
|
||||
bool DoRnnForward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
|
||||
const dnn::RnnSequenceTensorDescriptor& input_desc,
|
||||
const DeviceMemory<Eigen::half>& input_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_h_desc,
|
||||
const DeviceMemory<Eigen::half>& input_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_c_desc,
|
||||
const DeviceMemory<Eigen::half>& input_c_data,
|
||||
const DeviceMemory<Eigen::half>& params,
|
||||
const dnn::RnnSequenceTensorDescriptor& output_desc,
|
||||
DeviceMemory<Eigen::half>* output_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_h_desc,
|
||||
DeviceMemory<Eigen::half>* output_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_c_desc,
|
||||
DeviceMemory<Eigen::half>* output_c_data, bool is_training,
|
||||
ScratchAllocator* reserve_space_allocator,
|
||||
ScratchAllocator* workspace_allocator,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoRnnForward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
|
||||
const dnn::RnnSequenceTensorDescriptor& input_desc,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_h_desc,
|
||||
const DeviceMemory<float>& input_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_c_desc,
|
||||
const DeviceMemory<float>& input_c_data,
|
||||
const DeviceMemory<float>& params,
|
||||
const dnn::RnnSequenceTensorDescriptor& output_desc,
|
||||
DeviceMemory<float>* output_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_h_desc,
|
||||
DeviceMemory<float>* output_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_c_desc,
|
||||
DeviceMemory<float>* output_c_data, bool is_training,
|
||||
ScratchAllocator* reserve_space_allocator,
|
||||
ScratchAllocator* workspace_allocator,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoRnnForward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
|
||||
const dnn::RnnSequenceTensorDescriptor& input_desc,
|
||||
const DeviceMemory<double>& input_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_h_desc,
|
||||
const DeviceMemory<double>& input_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_c_desc,
|
||||
const DeviceMemory<double>& input_c_data,
|
||||
const DeviceMemory<double>& params,
|
||||
const dnn::RnnSequenceTensorDescriptor& output_desc,
|
||||
DeviceMemory<double>* output_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_h_desc,
|
||||
DeviceMemory<double>* output_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_c_desc,
|
||||
DeviceMemory<double>* output_c_data, bool is_training,
|
||||
ScratchAllocator* reserve_space_allocator,
|
||||
ScratchAllocator* workspace_allocator,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoRnnBackward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
|
||||
const dnn::RnnSequenceTensorDescriptor& input_desc,
|
||||
const DeviceMemory<Eigen::half>& input_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_h_desc,
|
||||
const DeviceMemory<Eigen::half>& input_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_c_desc,
|
||||
const DeviceMemory<Eigen::half>& input_c_data,
|
||||
const DeviceMemory<Eigen::half>& params,
|
||||
const dnn::RnnSequenceTensorDescriptor& output_desc,
|
||||
const DeviceMemory<Eigen::half>& output_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_h_desc,
|
||||
const DeviceMemory<Eigen::half>& output_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_c_desc,
|
||||
const DeviceMemory<Eigen::half>& output_c_data,
|
||||
const DeviceMemory<Eigen::half>& output_backprop_data,
|
||||
const DeviceMemory<Eigen::half>& output_h_backprop_data,
|
||||
const DeviceMemory<Eigen::half>& output_c_backprop_data,
|
||||
DeviceMemory<Eigen::half>* input_backprop_data,
|
||||
DeviceMemory<Eigen::half>* input_h_backprop_data,
|
||||
DeviceMemory<Eigen::half>* input_c_backprop_data,
|
||||
DeviceMemory<Eigen::half>* params_backprop_data,
|
||||
DeviceMemory<uint8>* reserve_space_data,
|
||||
ScratchAllocator* workspace_allocator,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoRnnBackward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
|
||||
const dnn::RnnSequenceTensorDescriptor& input_desc,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_h_desc,
|
||||
const DeviceMemory<float>& input_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_c_desc,
|
||||
const DeviceMemory<float>& input_c_data,
|
||||
const DeviceMemory<float>& params,
|
||||
const dnn::RnnSequenceTensorDescriptor& output_desc,
|
||||
const DeviceMemory<float>& output_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_h_desc,
|
||||
const DeviceMemory<float>& output_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_c_desc,
|
||||
const DeviceMemory<float>& output_c_data,
|
||||
const DeviceMemory<float>& output_backprop_data,
|
||||
const DeviceMemory<float>& output_h_backprop_data,
|
||||
const DeviceMemory<float>& output_c_backprop_data,
|
||||
DeviceMemory<float>* input_backprop_data,
|
||||
DeviceMemory<float>* input_h_backprop_data,
|
||||
DeviceMemory<float>* input_c_backprop_data,
|
||||
DeviceMemory<float>* params_backprop_data,
|
||||
DeviceMemory<uint8>* reserve_space_data,
|
||||
ScratchAllocator* workspace_allocator,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoRnnBackward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
|
||||
const dnn::RnnSequenceTensorDescriptor& input_desc,
|
||||
const DeviceMemory<double>& input_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_h_desc,
|
||||
const DeviceMemory<double>& input_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_c_desc,
|
||||
const DeviceMemory<double>& input_c_data,
|
||||
const DeviceMemory<double>& params,
|
||||
const dnn::RnnSequenceTensorDescriptor& output_desc,
|
||||
const DeviceMemory<double>& output_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_h_desc,
|
||||
const DeviceMemory<double>& output_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_c_desc,
|
||||
const DeviceMemory<double>& output_c_data,
|
||||
const DeviceMemory<double>& output_backprop_data,
|
||||
const DeviceMemory<double>& output_h_backprop_data,
|
||||
const DeviceMemory<double>& output_c_backprop_data,
|
||||
DeviceMemory<double>* input_backprop_data,
|
||||
DeviceMemory<double>* input_h_backprop_data,
|
||||
DeviceMemory<double>* input_c_backprop_data,
|
||||
DeviceMemory<double>* params_backprop_data,
|
||||
DeviceMemory<uint8>* reserve_space_data,
|
||||
ScratchAllocator* workspace_allocator,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool GetConvolveAlgorithms(
|
||||
bool with_winograd_nonfused, int cc_major, int cc_minor,
|
||||
std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
|
||||
|
||||
bool GetRnnAlgorithms(
|
||||
std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
|
||||
|
||||
bool GetConvolveBackwardDataAlgorithms(
|
||||
bool with_winograd_nonfused, int cc_major, int cc_minor,
|
||||
std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
|
||||
|
||||
bool GetConvolveBackwardFilterAlgorithms(
|
||||
bool with_winograd_nonfused, int cc_major, int cc_minor,
|
||||
std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
|
||||
|
||||
bool DoBatchNormalizationForward(
|
||||
Stream* stream, const DeviceMemory<float>& x,
|
||||
const DeviceMemory<float>& scale, const DeviceMemory<float>& offset,
|
||||
const DeviceMemory<float>& estimated_mean,
|
||||
const DeviceMemory<float>& estimated_variance,
|
||||
const dnn::BatchDescriptor& x_desc,
|
||||
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
|
||||
DeviceMemory<float>* y, DeviceMemory<float>* batch_mean,
|
||||
DeviceMemory<float>* batch_var, DeviceMemory<float>* saved_mean,
|
||||
DeviceMemory<float>* saved_inv_var, bool is_training,
|
||||
std::function<const DeviceMemory<float>&()> var_to_inv_var,
|
||||
std::function<void()> inv_var_to_var) override;
|
||||
|
||||
bool DoBatchNormalizationForward(
|
||||
Stream* stream, const DeviceMemory<Eigen::half>& x,
|
||||
const DeviceMemory<float>& scale, const DeviceMemory<float>& offset,
|
||||
const DeviceMemory<float>& estimated_mean,
|
||||
const DeviceMemory<float>& estimated_variance,
|
||||
const dnn::BatchDescriptor& x_desc,
|
||||
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
|
||||
DeviceMemory<Eigen::half>* y, DeviceMemory<float>* batch_mean,
|
||||
DeviceMemory<float>* batch_var, DeviceMemory<float>* saved_mean,
|
||||
DeviceMemory<float>* saved_inv_var, bool is_training,
|
||||
std::function<const DeviceMemory<float>&()> var_to_inv_var,
|
||||
std::function<void()> inv_var_to_var) override;
|
||||
|
||||
bool DoBatchNormalizationBackward(
|
||||
Stream* stream, const DeviceMemory<float>& y_backprop,
|
||||
const DeviceMemory<float>& x, const DeviceMemory<float>& scale,
|
||||
const DeviceMemory<float>& mean, const DeviceMemory<float>& inv_var,
|
||||
const dnn::BatchDescriptor& x_desc,
|
||||
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
|
||||
DeviceMemory<float>* x_backprop, DeviceMemory<float>* scale_backprop,
|
||||
DeviceMemory<float>* offset_backprop) override;
|
||||
|
||||
bool DoBatchNormalizationBackward(
|
||||
Stream* stream, const DeviceMemory<Eigen::half>& y_backprop,
|
||||
const DeviceMemory<Eigen::half>& x, const DeviceMemory<float>& scale,
|
||||
const DeviceMemory<float>& mean, const DeviceMemory<float>& inv_var,
|
||||
const dnn::BatchDescriptor& x_desc,
|
||||
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
|
||||
DeviceMemory<Eigen::half>* x_backprop,
|
||||
DeviceMemory<float>* scale_backprop,
|
||||
DeviceMemory<float>* offset_backprop) override;
|
||||
|
||||
bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<float>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<float>* output_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
|
||||
const DeviceMemory<double>& input_data,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<double>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<double>* output_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
|
||||
const DeviceMemory<Eigen::half>& input_data,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<Eigen::half>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<Eigen::half>* output_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoFusedConvolve(
|
||||
Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
|
||||
const DeviceMemory<double>& conv_input_data, double conv_input_scale,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<double>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const DeviceMemory<double>& side_input_data, double side_input_scale,
|
||||
const dnn::BatchDescriptor& bias_descriptor,
|
||||
const DeviceMemory<double>& biases, dnn::ActivationMode activation_mode,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoFusedConvolve(
|
||||
Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
|
||||
const DeviceMemory<float>& conv_input_data, float conv_input_scale,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<float>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const DeviceMemory<float>& side_input_data, float side_input_scale,
|
||||
const dnn::BatchDescriptor& bias_descriptor,
|
||||
const DeviceMemory<float>& biases, dnn::ActivationMode activation_mode,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoFusedConvolve(Stream* stream,
|
||||
const dnn::BatchDescriptor& conv_input_descriptor,
|
||||
const DeviceMemory<Eigen::half>& conv_input_data,
|
||||
float conv_input_scale,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<Eigen::half>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const DeviceMemory<Eigen::half>& side_input_data,
|
||||
float side_input_scale,
|
||||
const dnn::BatchDescriptor& bias_descriptor,
|
||||
const DeviceMemory<Eigen::half>& biases,
|
||||
dnn::ActivationMode activation_mode,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<Eigen::half>* output_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoFusedConvolve(
|
||||
Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
|
||||
const DeviceMemory<int8>& conv_input_data, float conv_input_scale,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<int8>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const DeviceMemory<int8>& side_input_data, float side_input_scale,
|
||||
const dnn::BatchDescriptor& bias_descriptor,
|
||||
const DeviceMemory<float>& biases, dnn::ActivationMode activation_mode,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<int8>* output_data, ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolveQuantized(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<int8>& filter_coefficients,
|
||||
const DeviceMemory<float>& coefficient_scales,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<float>* output_data) override {
|
||||
LOG(ERROR) << "DoConvolveQuantized not supported by cuDNN";
|
||||
return false;
|
||||
}
|
||||
|
||||
bool DoConvolveQuantized(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<int16>& filter_coefficients,
|
||||
const DeviceMemory<float>& coefficient_scales,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<float>* output_data) override {
|
||||
LOG(ERROR) << "DoConvolveQuantized not supported by cuDNN";
|
||||
return false;
|
||||
}
|
||||
|
||||
bool DoSeparableConvolve(
|
||||
Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::FilterDescriptor& filter_descriptor, int depth_multiplier,
|
||||
const DeviceMemory<float>& first_weights,
|
||||
const DeviceMemory<float>& second_weights,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<float>* output_data) override {
|
||||
LOG(ERROR) << "separable convolution not supported by CUDNN";
|
||||
return false;
|
||||
}
|
||||
|
||||
bool DoConvolveBackwardData(
|
||||
Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<double>& filter_data,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<double> backward_output_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& input_descriptor,
|
||||
DeviceMemory<double>* backward_input_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolveBackwardData(
|
||||
Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<float>& filter_data,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<float> backward_output_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& input_descriptor,
|
||||
DeviceMemory<float>* backward_input_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolveBackwardData(
|
||||
Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<Eigen::half>& filter_data,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<Eigen::half> backward_output_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& input_descriptor,
|
||||
DeviceMemory<Eigen::half>* backward_input_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolveBackwardFilter(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<double>& input_data,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<double> backward_output_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
DeviceMemory<double>* backward_filter_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolveBackwardFilter(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<float> backward_output_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
DeviceMemory<float>* backward_filter_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolveBackwardFilter(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<Eigen::half>& input_data,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<Eigen::half> backward_output_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
DeviceMemory<Eigen::half>* backward_filter_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolveBackwardBias(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<double>& input_data,
|
||||
const dnn::BatchDescriptor& bias_descriptor,
|
||||
DeviceMemory<double>* backward_bias_data) override;
|
||||
|
||||
bool DoConvolveBackwardBias(Stream* stream,
|
||||
const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::BatchDescriptor& bias_descriptor,
|
||||
DeviceMemory<float>* backward_bias_data) override;
|
||||
|
||||
bool DoConvolveBackwardBias(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<Eigen::half>& input_data,
|
||||
const dnn::BatchDescriptor& bias_descriptor,
|
||||
DeviceMemory<Eigen::half>* backward_bias_data) override;
|
||||
|
||||
bool DoMatMul(Stream* stream, const DeviceMemory<float>& input_data,
|
||||
const DeviceMemory<float>& weights,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
DeviceMemory<float>* output_data) override;
|
||||
|
||||
bool DoMatMulQuantized(Stream* stream, const DeviceMemory<float>& input_data,
|
||||
const DeviceMemory<int8>& quantized_weights,
|
||||
const DeviceMemory<float>& weight_scales,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
DeviceMemory<float>* output_data) override {
|
||||
LOG(ERROR) << "DNN MatMulQuantized not supported by CUDNN";
|
||||
return false;
|
||||
}
|
||||
|
||||
bool DoMatMulQuantized(Stream* stream, const DeviceMemory<float>& input_data,
|
||||
const DeviceMemory<int16>& quantized_weights,
|
||||
const DeviceMemory<float>& weight_scales,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
DeviceMemory<float>* output_data) override {
|
||||
LOG(ERROR) << "DNN MatMulQuantized not supported by CUDNN";
|
||||
return false;
|
||||
}
|
||||
|
||||
bool DoBiasAdd(Stream* stream, const DeviceMemory<float>& input_data,
|
||||
const DeviceMemory<float>& biases,
|
||||
const dnn::BatchDescriptor& dimensions,
|
||||
DeviceMemory<float>* output_data) override;
|
||||
|
||||
bool DoActivate(Stream* stream, dnn::ActivationMode activation_mode,
|
||||
const dnn::BatchDescriptor& dimensions,
|
||||
const DeviceMemory<float>& input_data,
|
||||
DeviceMemory<float>* output_data, uint64 options) override;
|
||||
|
||||
bool DoPoolForward(Stream* stream,
|
||||
const dnn::PoolingDescriptor& pooling_dimensions,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const DeviceMemory<double>& input_data,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
DeviceMemory<double>* output_data,
|
||||
ScratchAllocator* workspace_allocator) override;
|
||||
|
||||
bool DoPoolForward(Stream* stream,
|
||||
const dnn::PoolingDescriptor& pooling_dimensions,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
DeviceMemory<float>* output_data,
|
||||
ScratchAllocator* workspace_allocator) override;
|
||||
|
||||
bool DoPoolForward(Stream* stream,
|
||||
const dnn::PoolingDescriptor& pooling_dimensions,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const DeviceMemory<Eigen::half>& input_data,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
DeviceMemory<Eigen::half>* output_data,
|
||||
ScratchAllocator* workspace_allocator) override;
|
||||
|
||||
bool DoPoolBackward(Stream* stream,
|
||||
const dnn::PoolingDescriptor& pooling_dimensions,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const DeviceMemory<double>& input_data,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
const DeviceMemory<double>& output_data,
|
||||
const DeviceMemory<double>& input_diff_data,
|
||||
DeviceMemory<double>* output_diff_data,
|
||||
ScratchAllocator* workspace_allocator) override;
|
||||
|
||||
bool DoPoolBackward(Stream* stream,
|
||||
const dnn::PoolingDescriptor& pooling_dimensions,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
const DeviceMemory<float>& output_data,
|
||||
const DeviceMemory<float>& input_diff_data,
|
||||
DeviceMemory<float>* output_diff_data,
|
||||
ScratchAllocator* workspace_allocator) override;
|
||||
|
||||
bool DoPoolBackward(Stream* stream,
|
||||
const dnn::PoolingDescriptor& pooling_dimensions,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const DeviceMemory<Eigen::half>& input_data,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
const DeviceMemory<Eigen::half>& output_data,
|
||||
const DeviceMemory<Eigen::half>& input_diff_data,
|
||||
DeviceMemory<Eigen::half>* output_diff_data,
|
||||
ScratchAllocator* workspace_allocator) override;
|
||||
|
||||
bool DoNormalize(Stream* stream,
|
||||
const dnn::NormalizeDescriptor& normalize_descriptor,
|
||||
const DeviceMemory<float>& input_data,
|
||||
DeviceMemory<float>* output_data) override;
|
||||
|
||||
bool DoNormalizeWithDimensions(
|
||||
Stream* stream, const dnn::NormalizeDescriptor& normalize_descriptor,
|
||||
const dnn::BatchDescriptor& dimensions,
|
||||
const DeviceMemory<float>& input_data,
|
||||
DeviceMemory<float>* output_data) override;
|
||||
|
||||
bool DoNormalizeBackwardWithDimensions(
|
||||
Stream* stream, const dnn::NormalizeDescriptor& normalize_descriptor,
|
||||
const dnn::BatchDescriptor& dimensions,
|
||||
const DeviceMemory<float>& raw_data,
|
||||
const DeviceMemory<float>& normalized_data,
|
||||
const DeviceMemory<float>& normalized_variable_gradient,
|
||||
DeviceMemory<float>* raw_variable_gradient,
|
||||
ScratchAllocator* workspace_allocator) override;
|
||||
|
||||
bool DoDepthConcatenate(
|
||||
Stream* stream, port::ArraySlice<dnn::BatchDescriptor> input_dimensions,
|
||||
port::ArraySlice<const DeviceMemory<float>*> input_data,
|
||||
DeviceMemory<float>* output_data) override;
|
||||
|
||||
bool DoElementwiseOperate(
|
||||
Stream* stream, dnn::ElementwiseOperation operation,
|
||||
port::ArraySlice<dnn::BatchDescriptor> input_dimensions,
|
||||
port::ArraySlice<const DeviceMemory<float>*> input_data,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
DeviceMemory<float>* output_data) override;
|
||||
|
||||
bool DoXYPad(Stream* stream, const dnn::BatchDescriptor &dimensions,
|
||||
const DeviceMemory<float> &input_data,
|
||||
int64 left_pad, int64 right_pad, int64 top_pad,
|
||||
int64 bottom_pad, DeviceMemory<float> *output_data) override;
|
||||
|
||||
bool DoXYSlice(Stream* stream, const dnn::BatchDescriptor &dimensions,
|
||||
const DeviceMemory<float> &input_data,
|
||||
int64 left_trim, int64 right_trim, int64 top_trim,
|
||||
int64 bottom_trim, DeviceMemory<float> *output_data) override;
|
||||
|
||||
bool DoMemcpyD2HQuantized(Stream* stream,
|
||||
const DeviceMemory<float>& device_unquantized_src,
|
||||
dnn::QuantizedActivationMode mode, void* host_dst,
|
||||
int64 size) override;
|
||||
|
||||
bool DoMemcpyH2DQuantized(
|
||||
Stream* stream, const void* host_src, int64 size,
|
||||
dnn::QuantizedActivationMode mode,
|
||||
DeviceMemory<float>* device_unquantized_dst) override;
|
||||
|
||||
// Derives an output batch descriptor from an input batch and convolution
|
||||
// descriptors.
|
||||
bool DeriveOutputBatchDescriptor(
|
||||
const dnn::BatchDescriptor& batch_descriptor,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
dnn::BatchDescriptor* output_batch_descriptor);
|
||||
|
||||
bool DoTransformTensor(Stream* stream, const dnn::BatchDescriptor& input_desc,
|
||||
dnn::DataType input_type,
|
||||
const DeviceMemoryBase& input_data,
|
||||
const dnn::BatchDescriptor& output_desc,
|
||||
dnn::DataType output_type, float scale,
|
||||
DeviceMemoryBase* output_data) override;
|
||||
|
||||
private:
|
||||
CUDAExecutor* parent_; // Parent executor object. Not owned.
|
||||
|
||||
// Provides access to the cuDNN handle.
|
||||
std::unique_ptr<class CudnnAccess> cudnn_;
|
||||
|
||||
template <class T, class U>
|
||||
port::Status DoBatchNormalizationForwardImpl(
|
||||
Stream* stream, dnn::DataType input_data_type,
|
||||
dnn::DataType scale_data_type, const DeviceMemory<T>& x,
|
||||
const DeviceMemory<U>& scale, const DeviceMemory<U>& offset,
|
||||
const DeviceMemory<U>& estimated_mean,
|
||||
const DeviceMemory<U>& estimated_variance,
|
||||
const dnn::BatchDescriptor& x_desc,
|
||||
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
|
||||
DeviceMemory<T>* y, DeviceMemory<U>* batch_mean,
|
||||
DeviceMemory<U>* batch_var, DeviceMemory<U>* saved_mean,
|
||||
DeviceMemory<U>* saved_inv_var, bool is_training,
|
||||
std::function<const DeviceMemory<U>&()> var_to_inv_var,
|
||||
std::function<void()> inv_var_to_var);
|
||||
|
||||
template <class T, class U>
|
||||
port::Status DoBatchNormalizationBackwardImpl(
|
||||
Stream* stream, int cudnn_input_type, int cudnn_scale_type,
|
||||
const DeviceMemory<T>& y_backprop, const DeviceMemory<T>& x,
|
||||
const DeviceMemory<U>& scale, const DeviceMemory<U>& mean,
|
||||
const DeviceMemory<U>& inv_var, const dnn::BatchDescriptor& x_desc,
|
||||
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
|
||||
DeviceMemory<T>* x_backprop, DeviceMemory<U>* scale_backprop,
|
||||
DeviceMemory<U>* offset_backprop);
|
||||
|
||||
template <class T>
|
||||
port::Status DoConvolveImpl(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<T>& input_data,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<T>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<T>* output_data, ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result);
|
||||
|
||||
template <typename AccumulatorType, typename ElementType, typename BiasType,
|
||||
typename ScaleType>
|
||||
port::Status DoFusedConvolveImpl(
|
||||
Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
|
||||
const DeviceMemory<ElementType>& conv_input_data,
|
||||
ScaleType conv_input_scale,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<ElementType>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const DeviceMemory<ElementType>& side_input_data,
|
||||
ScaleType side_input_scale, const dnn::BatchDescriptor& bias_descriptor,
|
||||
const DeviceMemory<BiasType>& biases, dnn::ActivationMode activation_mode,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<ElementType>* output_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result);
|
||||
|
||||
template <class T>
|
||||
port::Status DoConvolveBackwardDataImpl(
|
||||
Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<T>& filter_data,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<T> backward_output_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& input_descriptor,
|
||||
DeviceMemory<T>* backward_input_data, ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result);
|
||||
|
||||
template <class T>
|
||||
port::Status DoConvolveBackwardFilterImpl(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<T>& input_data,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<T> backward_output_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
DeviceMemory<T>* backward_filter_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result);
|
||||
|
||||
template <class T>
|
||||
port::Status DoConvolveBackwardBiasImpl(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<T>& input_data,
|
||||
const dnn::BatchDescriptor& bias_descriptor,
|
||||
DeviceMemory<T>* backward_bias_data);
|
||||
|
||||
template <class T>
|
||||
port::Status DoRnnForwardImpl(
|
||||
Stream* stream, const CudnnRnnDescriptor& rnn_desc,
|
||||
const CudnnRnnSequenceTensorDescriptor& input_desc,
|
||||
const DeviceMemory<T>& input_data,
|
||||
const CudnnRnnStateTensorDescriptor& input_h_desc,
|
||||
const DeviceMemory<T>& input_h_data,
|
||||
const CudnnRnnStateTensorDescriptor& input_c_desc,
|
||||
const DeviceMemory<T>& input_c_data, const DeviceMemory<T>& params,
|
||||
const CudnnRnnSequenceTensorDescriptor& output_desc,
|
||||
DeviceMemory<T>* output_data,
|
||||
const CudnnRnnStateTensorDescriptor& output_h_desc,
|
||||
DeviceMemory<T>* output_h_data,
|
||||
const CudnnRnnStateTensorDescriptor& output_c_desc,
|
||||
DeviceMemory<T>* output_c_data, bool is_training,
|
||||
ScratchAllocator* reserve_space_allocator,
|
||||
ScratchAllocator* workspace_allocator,
|
||||
dnn::ProfileResult* output_profile_result);
|
||||
|
||||
template <class T>
|
||||
port::Status DoRnnBackwardImpl(
|
||||
Stream* stream, const CudnnRnnDescriptor& rnn_desc,
|
||||
const CudnnRnnSequenceTensorDescriptor& input_desc,
|
||||
const DeviceMemory<T>& input_data,
|
||||
const CudnnRnnStateTensorDescriptor& input_h_desc,
|
||||
const DeviceMemory<T>& input_h_data,
|
||||
const CudnnRnnStateTensorDescriptor& input_c_desc,
|
||||
const DeviceMemory<T>& input_c_data, const DeviceMemory<T>& params,
|
||||
const CudnnRnnSequenceTensorDescriptor& output_desc,
|
||||
const DeviceMemory<T>& output_data,
|
||||
const CudnnRnnStateTensorDescriptor& output_h_desc,
|
||||
const DeviceMemory<T>& output_h_data,
|
||||
const CudnnRnnStateTensorDescriptor& output_c_desc,
|
||||
const DeviceMemory<T>& output_c_data,
|
||||
const DeviceMemory<T>& output_backprop_data,
|
||||
const DeviceMemory<T>& output_h_backprop_data,
|
||||
const DeviceMemory<T>& output_c_backprop_data,
|
||||
DeviceMemory<T>* input_backprop_data,
|
||||
DeviceMemory<T>* input_h_backprop_data,
|
||||
DeviceMemory<T>* input_c_backprop_data,
|
||||
DeviceMemory<T>* params_backprop_data,
|
||||
DeviceMemory<uint8>* reserve_space_data,
|
||||
ScratchAllocator* workspace_allocator,
|
||||
dnn::ProfileResult* output_profile_result);
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CudnnSupport);
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DNN_H_
|
||||
@@ -0,0 +1,520 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// CUDA userspace driver library wrapper functionality.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_H_
|
||||
|
||||
#include <stddef.h>
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
#include "tensorflow/stream_executor/device_options.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "cuda/include/cuda.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
// Identifies the memory space where an allocation resides. See
|
||||
// CUDADriver::GetPointerMemorySpace().
|
||||
enum class MemorySpace { kHost, kDevice };
|
||||
|
||||
// Returns a casual string, such as "host" for the provided memory space.
|
||||
string MemorySpaceString(MemorySpace memory_space);
|
||||
|
||||
class CudaContext;
|
||||
|
||||
// CUDADriver contains wrappers for calls to the userspace library driver. It's
|
||||
// useful to isolate these calls and put basic wrappers around them to separate
|
||||
// userspace library driver behaviors from the rest of the program.
|
||||
//
|
||||
// At the moment it's simply used as a namespace.
|
||||
//
|
||||
// The calls log any specific errors internally and return whether the operation
|
||||
// was successful to the caller.
|
||||
//
|
||||
// The order of parameters is generally kept symmetric with the underlying CUDA
|
||||
// driver API.
|
||||
//
|
||||
// Links on functions are to specific documentation under
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/
|
||||
//
|
||||
// Thread safety: these functions should not be used from signal handlers.
|
||||
class CUDADriver {
|
||||
public:
|
||||
// Wraps a call to cuInit with logging to help indicate what has gone wrong in
|
||||
// the case of failure. Safe to call multiple times; will be fast on all calls
|
||||
// after the first.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__INITIALIZE.html#group__CUDA__INITIALIZE_1g0a2f1517e1bd8502c7194c3a8c134bc3
|
||||
static port::Status Init();
|
||||
|
||||
// Returns the device associated with the given context.
|
||||
// device is an outparam owned by the caller, must not be null.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g4e84b109eba36cdaaade167f34ae881e
|
||||
static port::StatusOr<CUdevice> DeviceFromContext(CudaContext* context);
|
||||
|
||||
// Creates a new CUDA stream associated with the given context via
|
||||
// cuStreamCreate.
|
||||
// stream is an outparam owned by the caller, must not be null.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1ga581f0c5833e21ded8b5a56594e243f4
|
||||
static bool CreateStream(CudaContext* context, CUstream *stream);
|
||||
|
||||
// Destroys a CUDA stream associated with the given context.
|
||||
// stream is owned by the caller, must not be null, and *stream is set to null
|
||||
// if the stream is successfully destroyed.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g244c8833de4596bcd31a06cdf21ee758
|
||||
static void DestroyStream(CudaContext* context, CUstream *stream);
|
||||
|
||||
// CUDA events can explicitly disable event TSC retrieval for some presumed
|
||||
// performance improvement if timing is unnecessary.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g450687e75f3ff992fe01662a43d9d3db
|
||||
enum class EventFlags { kDefault, kDisableTiming };
|
||||
|
||||
// Creates a new event associated with the given context.
|
||||
// result is an outparam owned by the caller and must not be null.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g450687e75f3ff992fe01662a43d9d3db
|
||||
static port::Status CreateEvent(CudaContext* context, CUevent *result,
|
||||
EventFlags flags);
|
||||
|
||||
// Destroys *event and turns it into a nullptr. event may not be null, but
|
||||
// *event may be, via cuEventDestroy
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g593ec73a8ec5a5fc031311d3e4dca1ef
|
||||
static port::Status DestroyEvent(CudaContext* context, CUevent *event);
|
||||
|
||||
// Allocates a GPU memory space of size bytes associated with the given
|
||||
// context via cuMemAlloc.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gb82d2a09844a58dd9e744dc31e8aa467
|
||||
static void *DeviceAllocate(CudaContext* context, uint64 bytes);
|
||||
|
||||
// Deallocates a GPU memory space of size bytes associated with the given
|
||||
// context via cuMemFree.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g89b3f154e17cc89b6eea277dbdf5c93a
|
||||
static void DeviceDeallocate(CudaContext* context, void *location);
|
||||
|
||||
// Allocates a unified memory space of size bytes associated with the given
|
||||
// context via cuMemAllocManaged.
|
||||
// https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gb347ded34dc326af404aa02af5388a32
|
||||
static void* UnifiedMemoryAllocate(CudaContext* context, uint64 bytes);
|
||||
|
||||
// Deallocates a unified memory space of size bytes associated with the given
|
||||
// context via cuMemFree.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g89b3f154e17cc89b6eea277dbdf5c93a
|
||||
static void UnifiedMemoryDeallocate(CudaContext* context, void* location);
|
||||
|
||||
// Allocates page-locked and CUDA-registered memory on the host via
|
||||
// cuMemAllocHost.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gdd8311286d2c2691605362c689bc64e0
|
||||
static void *HostAllocate(CudaContext* context, uint64 bytes);
|
||||
|
||||
// Deallocates a location created by HostAllocate, via cuMemFreeHost.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g62e0fdbe181dab6b1c90fa1a51c7b92c
|
||||
static void HostDeallocate(CudaContext* context, void *location);
|
||||
|
||||
// Registers a memory region at location of size bytes via cuMemHostRegister.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gf0a9fe11544326dabd743b7aa6b54223
|
||||
static bool HostRegister(CudaContext* context, void *location, uint64 bytes);
|
||||
|
||||
// Unregisters a memory region that was previously registered at location via
|
||||
// cuMemHostUnregister.
|
||||
//
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g63f450c8125359be87b7623b1c0b2a14
|
||||
//
|
||||
// TODO(leary) verify an error will be returned if the location wasn't
|
||||
// previously registered.
|
||||
static bool HostUnregister(CudaContext* context, void *location);
|
||||
|
||||
// Given a device ordinal, returns a device handle into the device outparam,
|
||||
// which must not be null.
|
||||
//
|
||||
// N.B. these device handles do not have a corresponding destroy function in
|
||||
// the CUDA driver API.
|
||||
static port::Status GetDevice(int device_ordinal, CUdevice *device);
|
||||
|
||||
// Given a device handle, returns the name reported by the driver for the
|
||||
// device.
|
||||
static bool GetDeviceName(CUdevice device, string *name_out);
|
||||
|
||||
// Given a device to create a context for, returns a context handle into the
|
||||
// context outparam, which must not be null.
|
||||
//
|
||||
// N.B. CUDA contexts are weird. They are implicitly associated with the
|
||||
// calling thread. Current documentation on contexts and their influence on
|
||||
// userspace processes is given here:
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g65dc0012348bc84810e2103a40d8e2cf
|
||||
static port::Status CreateContext(CUdevice device,
|
||||
const DeviceOptions& device_options,
|
||||
CudaContext** context);
|
||||
|
||||
// Destroys the provided context via cuCtxDestroy.
|
||||
// Don't do this while clients could still be using the context, per the docs
|
||||
// bad things will happen.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g27a365aebb0eb548166309f58a1e8b8e
|
||||
static void DestroyContext(CudaContext* context);
|
||||
|
||||
// Queries the runtime for the specified attribute of the specified function.
|
||||
// cuFuncGetAttribute (the underlying CUDA driver API routine) only operates
|
||||
// in terms of integer-sized values, so there's no potential for overrun (as
|
||||
// of CUDA 5.5).
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1g5e92a1b0d8d1b82cb00dcfb2de15961b
|
||||
static bool FuncGetAttribute(CUfunction_attribute attribute,
|
||||
CUfunction function, int *attribute_value);
|
||||
|
||||
// Sets the preferred cache configuration for the specified function.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1g40f8c11e81def95dc0072a375f965681
|
||||
static bool FuncSetCacheConfig(CUfunction function,
|
||||
CUfunc_cache cache_config);
|
||||
|
||||
// Gets the preferred shared memory bank configuration for the specified
|
||||
// CONTEXT (not function!), either default or four- or eight-byte bank size.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g17153a1b8b8c756f7ab8505686a4ad74
|
||||
static port::StatusOr<CUsharedconfig> ContextGetSharedMemConfig(
|
||||
CudaContext* context);
|
||||
|
||||
// Sets the preferred shared memory bank configuration for the specified
|
||||
// CONTEXT (not function!), either default or four- or eight-byte bank size.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g2574235fa643f8f251bf7bc28fac3692
|
||||
static port::Status ContextSetSharedMemConfig(
|
||||
CudaContext* context, CUsharedconfig shared_mem_config);
|
||||
|
||||
// Launches a CUDA kernel via cuLaunchKernel.
|
||||
// TODO(leary) describe the structure of kernel_params and extra in a readable
|
||||
// way.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1gb8f3dc3031b40da29d5f9a7139e52e15
|
||||
static bool LaunchKernel(CudaContext* context, CUfunction function,
|
||||
unsigned int grid_dim_x, unsigned int grid_dim_y,
|
||||
unsigned int grid_dim_z, unsigned int block_dim_x,
|
||||
unsigned int block_dim_y, unsigned int block_dim_z,
|
||||
unsigned int shared_mem_bytes, CUstream stream,
|
||||
void **kernel_params, void **extra);
|
||||
|
||||
// Loads ptx_contents with the CUDA driver's PTX JIT and stores the resulting
|
||||
// handle in "module". Any error logs that are produced are logged internally.
|
||||
static bool LoadPtx(CudaContext* context, const char *ptx_contents,
|
||||
CUmodule *module);
|
||||
|
||||
// Loads cubin_bytes with the CUDA driver's blob loading interface and stores
|
||||
// the resulting handle in "module".
|
||||
static port::Status LoadCubin(CudaContext* context, const char *cubin_bytes,
|
||||
CUmodule *module);
|
||||
|
||||
// Retrieves a named kernel from a loaded module, and places the resulting
|
||||
// handle into function (outparam) on success. Neither kernel_name nor
|
||||
// function may be null. No ownership is taken of kernel_name.
|
||||
static bool GetModuleFunction(CudaContext* context, CUmodule module,
|
||||
const char *kernel_name, CUfunction *function);
|
||||
|
||||
// Retrieves a named global/constant symbol from a loaded module, and returns
|
||||
// a device pointer and size of the symbol on success. symbol_name may not be
|
||||
// null. At least one of dptr or bytes should not be null. No ownership is
|
||||
// taken of symbol_name.
|
||||
static bool GetModuleSymbol(CudaContext* context, CUmodule module,
|
||||
const char *symbol_name, CUdeviceptr *dptr,
|
||||
size_t *bytes);
|
||||
|
||||
// Unloads module from the current context via cuModuleUnload.
|
||||
// TODO(leary) the documentation doesn't say what kind of disasters happen
|
||||
// if you try to unload a module while its CUfunctions are in use.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MODULE.html#group__CUDA__MODULE_1g8ea3d716524369de3763104ced4ea57b
|
||||
static void UnloadModule(CudaContext* context, CUmodule module);
|
||||
|
||||
// Performs a synchronous memset of the device memory segment via cuMemsetD8.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g6e582bf866e9e2fb014297bfaf354d7b
|
||||
static bool SynchronousMemsetUint8(CudaContext* context, CUdeviceptr location,
|
||||
uint8 value, size_t size);
|
||||
|
||||
// Performs a synchronous memset of the device memory segment via cuMemsetD32.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g983e8d8759acd1b64326317481fbf132
|
||||
static bool SynchronousMemsetUint32(CudaContext* context,
|
||||
CUdeviceptr location, uint32 value,
|
||||
size_t uint32_count);
|
||||
|
||||
// Performs an asynchronous memset of the device memory segment via
|
||||
// cuMemsetD8Async.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gaef08a7ccd61112f94e82f2b30d43627
|
||||
static bool AsynchronousMemsetUint8(CudaContext* context, CUdeviceptr location,
|
||||
uint8 value, size_t uint32_count,
|
||||
CUstream stream);
|
||||
|
||||
// Performs an asynchronous memset of the device memory segment via
|
||||
// cuMemsetD32Async.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g58229da5d30f1c0cdf667b320ec2c0f5
|
||||
static bool AsynchronousMemsetUint32(CudaContext* context,
|
||||
CUdeviceptr location, uint32 value,
|
||||
size_t uint32_count, CUstream stream);
|
||||
|
||||
// -- Synchronous memcopies.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g4d32266788c440b0220b1a9ba5795169
|
||||
|
||||
static port::Status SynchronousMemcpyD2H(CudaContext* context, void* host_dst,
|
||||
CUdeviceptr gpu_src, uint64 size);
|
||||
static port::Status SynchronousMemcpyH2D(CudaContext* context,
|
||||
CUdeviceptr gpu_dst,
|
||||
const void* host_src, uint64 size);
|
||||
static port::Status SynchronousMemcpyD2D(CudaContext* context,
|
||||
CUdeviceptr gpu_dst,
|
||||
CUdeviceptr gpu_src, uint64 size);
|
||||
|
||||
// -- Asynchronous memcopies.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g56f30236c7c5247f8e061b59d3268362
|
||||
|
||||
static bool AsynchronousMemcpyD2H(CudaContext* context, void *host_dst,
|
||||
CUdeviceptr gpu_src, uint64 size,
|
||||
CUstream stream);
|
||||
static bool AsynchronousMemcpyH2D(CudaContext* context, CUdeviceptr gpu_dst,
|
||||
const void *host_src, uint64 size,
|
||||
CUstream stream);
|
||||
static bool AsynchronousMemcpyD2D(CudaContext* context, CUdeviceptr gpu_dst,
|
||||
CUdeviceptr gpu_src, uint64 size,
|
||||
CUstream stream);
|
||||
|
||||
// The CUDA stream callback type signature.
|
||||
// The data passed to AddStreamCallback is subsequently passed to this
|
||||
// callback when it fires.
|
||||
//
|
||||
// Some notable things:
|
||||
// * Callbacks must not make any CUDA API calls.
|
||||
// * Callbacks from independent streams execute in an undefined order and may
|
||||
// be serialized.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g613d97a277d7640f4cb1c03bd51c2483
|
||||
typedef void (*StreamCallback)(CUstream stream, CUresult status, void *data);
|
||||
|
||||
// Enqueues a callback operation into stream.
|
||||
// See StreamCallback above and the NVIDIA documentation for additional
|
||||
// details.
|
||||
static bool AddStreamCallback(CudaContext* context, CUstream stream,
|
||||
StreamCallback callback, void *data);
|
||||
|
||||
// Causes stream to wait for event to trigger before proceeding via
|
||||
// cuStreamWaitEvent.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#axzz334nAXAhM
|
||||
static bool WaitStreamOnEvent(CudaContext* context, CUstream stream,
|
||||
CUevent event);
|
||||
|
||||
// Blocks the calling thread until the operations enqueued onto stream have
|
||||
// been completed, via cuStreamSynchronize.
|
||||
//
|
||||
// TODO(leary) if a pathological thread enqueues operations onto the stream
|
||||
// while another thread blocks like this, can you wind up waiting an unbounded
|
||||
// amount of time?
|
||||
//
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g15e49dd91ec15991eb7c0a741beb7dad
|
||||
static port::Status SynchronizeStream(CudaContext* context, CUstream stream);
|
||||
|
||||
// Blocks the calling thread until the operations associated with the context
|
||||
// have been completed, via cuCtxSynchronize.
|
||||
//
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g7a54725f28d34b8c6299f0c6ca579616
|
||||
static bool SynchronizeContext(CudaContext* context);
|
||||
|
||||
// Returns true if all stream tasks have completed at time of the call. Note
|
||||
// the potential for races around this call (if another thread adds work to
|
||||
// the stream immediately after this returns).
|
||||
static bool IsStreamIdle(CudaContext* context, CUstream stream);
|
||||
|
||||
// Returns whether code in the from context can access memory in the to
|
||||
// context via cuDeviceCanAccessPeer.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__PEER__ACCESS.html#group__CUDA__PEER__ACCESS_1g496bdaae1f632ebfb695b99d2c40f19e
|
||||
static bool CanEnablePeerAccess(CudaContext* from, CudaContext* to);
|
||||
|
||||
// Enables peer access per CanEnablePeerAccess, via cuCtxEnablePeerAccess.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__PEER__ACCESS.html#group__CUDA__PEER__ACCESS_1g0889ec6728e61c05ed359551d67b3f5a
|
||||
static port::Status EnablePeerAccess(CudaContext* from, CudaContext* to);
|
||||
|
||||
// Returns the elapsed milliseconds between start and stop via
|
||||
// cuEventElapsedTime.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1gdfb1178807353bbcaa9e245da497cf97
|
||||
static bool GetEventElapsedTime(CudaContext* context,
|
||||
float *elapsed_milliseconds, CUevent start,
|
||||
CUevent stop);
|
||||
|
||||
// Records that an event occurred when execution reaches the current point in
|
||||
// thestream via cuEventRecord.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g95424d3be52c4eb95d83861b70fb89d1
|
||||
static port::Status RecordEvent(CudaContext* context, CUevent event,
|
||||
CUstream stream);
|
||||
|
||||
// Polls (without blocking) to determine the status of an event - pending or
|
||||
// complete (or an error status).
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g6f0704d755066b0ee705749ae911deef
|
||||
static port::StatusOr<CUresult> QueryEvent(CudaContext* context,
|
||||
CUevent event);
|
||||
|
||||
// -- Pointer-specific calls.
|
||||
|
||||
// Returns the context in which pointer was allocated or registered.
|
||||
static port::StatusOr<CudaContext*> GetPointerContext(CUdeviceptr pointer);
|
||||
|
||||
// Returns the device associated with the context from GetPointerContext().
|
||||
static port::StatusOr<CUdevice> GetPointerDevice(CUdeviceptr pointer);
|
||||
|
||||
// Returns the memory space addressed by pointer.
|
||||
static port::StatusOr<MemorySpace> GetPointerMemorySpace(CUdeviceptr pointer);
|
||||
|
||||
// Returns the base address and size of the device pointer dptr.
|
||||
static port::Status GetPointerAddressRange(CUdeviceptr dptr,
|
||||
CUdeviceptr *base, size_t *size);
|
||||
|
||||
// -- Device-specific calls.
|
||||
|
||||
// Returns the compute capability for the device; i.e (3, 5).
|
||||
// This is currently done via the deprecated device API.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE__DEPRECATED.html#group__CUDA__DEVICE__DEPRECATED_1ge2091bbac7e1fb18c2821612115607ea
|
||||
static port::Status GetComputeCapability(int *cc_major, int *cc_minor,
|
||||
CUdevice device);
|
||||
|
||||
// Returns the number of multiprocessors on the device (note that the device
|
||||
// may be multi-GPU-per-board).
|
||||
static port::StatusOr<int> GetMultiprocessorCount(CUdevice device);
|
||||
|
||||
// Returns the limit on number of threads that can be resident in a single
|
||||
// multiprocessor.
|
||||
static port::StatusOr<int64> GetMaxThreadsPerMultiprocessor(CUdevice device);
|
||||
|
||||
// Returns the limit on number of threads which may be resident for a single
|
||||
// block (cooperative thread array).
|
||||
static port::StatusOr<int64> GetMaxThreadsPerBlock(CUdevice device);
|
||||
|
||||
// Returns the amount of shared memory available on a single GPU core (i.e.
|
||||
// SM on NVIDIA devices).
|
||||
static port::StatusOr<int64> GetMaxSharedMemoryPerCore(CUdevice device);
|
||||
|
||||
// Returns the amount of shared memory available for a single block
|
||||
// (cooperative thread array).
|
||||
static port::StatusOr<int64> GetMaxSharedMemoryPerBlock(CUdevice device);
|
||||
|
||||
// Returns the maximum supported number of registers per block.
|
||||
static port::StatusOr<int64> GetMaxRegistersPerBlock(CUdevice device);
|
||||
|
||||
// Returns the number of threads per warp.
|
||||
static port::StatusOr<int64> GetThreadsPerWarp(CUdevice device);
|
||||
|
||||
// Queries the grid limits for device with cuDeviceGetAttribute calls.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
|
||||
static bool GetGridLimits(int *x, int *y, int *z, CUdevice device);
|
||||
|
||||
// Returns a grab-bag of device properties in a caller-owned device_properties
|
||||
// structure for device_ordinal via cuDeviceGetProperties.
|
||||
//
|
||||
// This call is deprecated in the NVIDIA driver API; its replacement is
|
||||
// GetDeviceAttribute
|
||||
//
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE__DEPRECATED.html#group__CUDA__DEVICE__DEPRECATED_1g65a5b4e25186bd257df80b98c98cffe6
|
||||
static bool GetDeviceProperties(CUdevprop *device_properties,
|
||||
int device_ordinal);
|
||||
|
||||
// Gets a specific integer-valued property about the given device.
|
||||
//
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
|
||||
static port::StatusOr<int> GetDeviceAttribute(CUdevice_attribute attribute,
|
||||
CUdevice device);
|
||||
|
||||
// Returns whether ECC is enabled for the given CUdevice via
|
||||
// cuDeviceGetattribute with CU_DEVICE_ATTRIBUTE_ECC_ENABLED.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
|
||||
static bool IsEccEnabled(CUdevice device, bool *result);
|
||||
|
||||
// Returns the total amount of memory available for allocation by the CUDA
|
||||
// context, in bytes, via cuDeviceTotalMem.
|
||||
static bool GetDeviceTotalMemory(CUdevice device, uint64 *result);
|
||||
|
||||
// Returns the free amount of memory and total amount of memory, as reported
|
||||
// by cuMemGetInfo.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g808f555540d0143a331cc42aa98835c0
|
||||
static bool GetDeviceMemoryInfo(CudaContext* context, int64* free,
|
||||
int64* total);
|
||||
|
||||
// Returns a PCI bus id string for the device.
|
||||
// [domain]:[bus]:[device].[function]
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g85295e7d9745ab8f0aa80dd1e172acfc
|
||||
static string GetPCIBusID(CUdevice device);
|
||||
|
||||
// -- Context- and device-independent calls.
|
||||
|
||||
// Returns the number of visible CUDA device via cuDeviceGetCount.
|
||||
// This should correspond to the set of device ordinals available.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g52b5ce05cb8c5fb6831b2c0ff2887c74
|
||||
static int GetDeviceCount();
|
||||
|
||||
// Returns the driver version number via cuDriverGetVersion.
|
||||
// This is, surprisingly, NOT the actual driver version (e.g. 331.79) but,
|
||||
// instead, the CUDA toolkit release number that this driver is compatible
|
||||
// with; e.g. 6000 (for a CUDA 6.0 compatible driver) or 6050 (for a CUDA 6.5
|
||||
// compatible driver).
|
||||
//
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__VERSION.html#group__CUDA__VERSION_1g8b7a10395392e049006e61bcdc8ebe71
|
||||
static bool GetDriverVersion(int *driver_version);
|
||||
|
||||
// -- Other calls
|
||||
|
||||
// Returns the maximum number of blocks (per multiprocessor) occupied by the
|
||||
// specified kernel/CUfunction when launched with the specified parameters.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__OCCUPANCY.html#group__CUDA__OCCUPANCY_1gcc6e1094d05cba2cee17fe33ddd04a98
|
||||
static port::StatusOr<int> GetMaxOccupiedBlocksPerCore(
|
||||
CudaContext* context, CUfunction kernel, int threads_per_block,
|
||||
size_t dynamic_shared_memory_bytes);
|
||||
|
||||
// Returns the current context set in CUDA. This is done by calling the cuda
|
||||
// driver (e.g., this value is not our cached view of the current context).
|
||||
static CUcontext CurrentContextOrDie();
|
||||
|
||||
// Seam for injecting an error at CUDA initialization time for testing
|
||||
// purposes.
|
||||
static bool driver_inject_init_error_;
|
||||
};
|
||||
|
||||
// Ensures a context is activated within a scope.
|
||||
class ScopedActivateContext {
|
||||
public:
|
||||
// Activates the context via cuCtxSetCurrent, if it is not the currently
|
||||
// active context (a la cuCtxGetCurrent). Note the alternative push/pop
|
||||
// mechanism is said by NVIDIA to be relatively slow and deprecated.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1gbe562ee6258b4fcc272ca6478ca2a2f7
|
||||
explicit ScopedActivateContext(CudaContext* context);
|
||||
|
||||
// Checks that the context has remained activated for the duration of the
|
||||
// scope.
|
||||
~ScopedActivateContext();
|
||||
|
||||
private:
|
||||
CudaContext* to_restore_ = nullptr;
|
||||
};
|
||||
|
||||
// CudaContext wraps a cuda CUcontext handle, and includes a unique id. The
|
||||
// unique id is positive, and ids are not repeated within the process.
|
||||
class CudaContext {
|
||||
public:
|
||||
CudaContext(CUcontext context, int64 id) : context_(context), id_(id) { }
|
||||
|
||||
CUcontext context() const { return context_; }
|
||||
int64 id() const { return id_; }
|
||||
|
||||
// Disallow copying and moving.
|
||||
CudaContext(CudaContext&&) = delete;
|
||||
CudaContext(const CudaContext&) = delete;
|
||||
CudaContext& operator=(CudaContext&&) = delete;
|
||||
CudaContext& operator=(const CudaContext&) = delete;
|
||||
|
||||
private:
|
||||
CUcontext const context_;
|
||||
const int64 id_;
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_H_
|
||||
@@ -0,0 +1,62 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_EVENT_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_EVENT_H_
|
||||
|
||||
#include "tensorflow/stream_executor/cuda/cuda_driver.h"
|
||||
#include "tensorflow/stream_executor/cuda/cuda_stream.h"
|
||||
#include "tensorflow/stream_executor/event.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
// CUDAEvent wraps a CUevent in the platform-independent EventInterface
|
||||
// interface.
|
||||
class CUDAEvent : public internal::EventInterface {
|
||||
public:
|
||||
explicit CUDAEvent(CUDAExecutor* parent);
|
||||
|
||||
~CUDAEvent() override;
|
||||
|
||||
// Populates the CUDA-platform-specific elements of this object.
|
||||
port::Status Init();
|
||||
|
||||
// Deallocates any platform-specific elements of this object. This is broken
|
||||
// out (not part of the destructor) to allow for error reporting.
|
||||
port::Status Destroy();
|
||||
|
||||
// Inserts the event at the current position into the specified stream.
|
||||
port::Status Record(CUDAStream* stream);
|
||||
|
||||
// Polls the CUDA platform for the event's current status.
|
||||
Event::Status PollForStatus();
|
||||
|
||||
// The underlying CUDA event element.
|
||||
const CUevent& cuda_event();
|
||||
|
||||
private:
|
||||
// The Executor used to which this object and CUevent are bound.
|
||||
CUDAExecutor* parent_;
|
||||
|
||||
// The underlying CUDA event element.
|
||||
CUevent cuda_event_;
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_EVENT_H_
|
||||
@@ -0,0 +1,137 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// CUDA-specific support for FFT functionality -- this wraps the cuFFT library
|
||||
// capabilities, and is only included into CUDA implementation code -- it will
|
||||
// not introduce cuda headers into other code.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_FFT_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_FFT_H_
|
||||
|
||||
#include "cuda/include/cufft.h"
|
||||
#include "tensorflow/stream_executor/fft.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/plugin_registry.h"
|
||||
#include "tensorflow/stream_executor/scratch_allocator.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class Stream;
|
||||
|
||||
namespace cuda {
|
||||
|
||||
class CUDAExecutor;
|
||||
|
||||
// Opaque and unique indentifier for the cuFFT plugin.
|
||||
extern const PluginId kCuFftPlugin;
|
||||
|
||||
// CUDAFftPlan uses deferred initialization. Only a single call of
|
||||
// Initialize() is allowed to properly create cufft plan and set member
|
||||
// variable is_initialized_ to true. Newly added interface that uses member
|
||||
// variables should first check is_initialized_ to make sure that the values of
|
||||
// member variables are valid.
|
||||
class CUDAFftPlan : public fft::Plan {
|
||||
public:
|
||||
CUDAFftPlan()
|
||||
: parent_(nullptr),
|
||||
plan_(-1),
|
||||
fft_type_(fft::Type::kInvalid),
|
||||
scratch_(nullptr),
|
||||
scratch_size_bytes_(0),
|
||||
is_initialized_(false) {}
|
||||
~CUDAFftPlan() override;
|
||||
|
||||
// Get FFT direction in cuFFT based on FFT type.
|
||||
int GetFftDirection() const;
|
||||
cufftHandle GetPlan() const {
|
||||
if (IsInitialized()) {
|
||||
return plan_;
|
||||
} else {
|
||||
LOG(FATAL) << "Try to get cufftHandle value before initialization.";
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize function for batched plan
|
||||
port::Status Initialize(CUDAExecutor *parent, Stream *stream, int rank,
|
||||
uint64 *elem_count, uint64 *input_embed,
|
||||
uint64 input_stride, uint64 input_distance,
|
||||
uint64 *output_embed, uint64 output_stride,
|
||||
uint64 output_distance, fft::Type type,
|
||||
int batch_count, ScratchAllocator *scratch_allocator);
|
||||
|
||||
// Initialize function for 1d,2d, and 3d plan
|
||||
port::Status Initialize(CUDAExecutor *parent, Stream *stream, int rank,
|
||||
uint64 *elem_count, fft::Type type,
|
||||
ScratchAllocator *scratch_allocator);
|
||||
|
||||
port::Status UpdateScratchAllocator(Stream *stream,
|
||||
ScratchAllocator *scratch_allocator);
|
||||
|
||||
protected:
|
||||
bool IsInitialized() const { return is_initialized_; }
|
||||
|
||||
private:
|
||||
CUDAExecutor *parent_;
|
||||
cufftHandle plan_;
|
||||
fft::Type fft_type_;
|
||||
DeviceMemory<uint8> scratch_;
|
||||
size_t scratch_size_bytes_;
|
||||
bool is_initialized_;
|
||||
};
|
||||
|
||||
// FFT support for CUDA platform via cuFFT library.
|
||||
//
|
||||
// This satisfies the platform-agnostic FftSupport interface.
|
||||
//
|
||||
// Note that the cuFFT handle that this encapsulates is implicitly tied to the
|
||||
// context (and, as a result, the device) that the parent CUDAExecutor is tied
|
||||
// to. This simply happens as an artifact of creating the cuFFT handle when a
|
||||
// CUDA context is active.
|
||||
//
|
||||
// Thread-safe. The CUDA context associated with all operations is the CUDA
|
||||
// context of parent_, so all context is explicit.
|
||||
class CUDAFft : public fft::FftSupport {
|
||||
public:
|
||||
explicit CUDAFft(CUDAExecutor *parent) : parent_(parent) {}
|
||||
~CUDAFft() override {}
|
||||
|
||||
TENSORFLOW_STREAM_EXECUTOR_GPU_FFT_SUPPORT_OVERRIDES
|
||||
|
||||
private:
|
||||
CUDAExecutor *parent_;
|
||||
|
||||
// Two helper functions that execute dynload::cufftExec?2?.
|
||||
|
||||
// This is for complex to complex FFT, when the direction is required.
|
||||
template <typename FuncT, typename InputT, typename OutputT>
|
||||
bool DoFftWithDirectionInternal(Stream *stream, fft::Plan *plan,
|
||||
FuncT cufft_exec,
|
||||
const DeviceMemory<InputT> &input,
|
||||
DeviceMemory<OutputT> *output);
|
||||
|
||||
// This is for complex to real or real to complex FFT, when the direction
|
||||
// is implied.
|
||||
template <typename FuncT, typename InputT, typename OutputT>
|
||||
bool DoFftInternal(Stream *stream, fft::Plan *plan, FuncT cufft_exec,
|
||||
const DeviceMemory<InputT> &input,
|
||||
DeviceMemory<OutputT> *output);
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CUDAFft);
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_FFT_H_
|
||||
@@ -0,0 +1,299 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// The CUDA implementation of the StreamExecutorInterface functionality.
|
||||
// CUDA inclusions are ideally confined to this implementation file.
|
||||
//
|
||||
// The notions from the StreamExecutor basically correspond to the CUDA streams
|
||||
// programming model provided by the libcuda.so driver APIs, so we don't have
|
||||
// to do much more than wrap the calls to the libraries appropriately.
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_GPU_EXECUTOR_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_GPU_EXECUTOR_H_
|
||||
|
||||
#include <set>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "tensorflow/stream_executor/cuda/cuda_kernel.h"
|
||||
#include "tensorflow/stream_executor/event.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/platform.h"
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/platform/thread_annotations.h"
|
||||
#include "tensorflow/stream_executor/stream_executor_internal.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
// CUDA-platform implementation of the platform-agnostic
|
||||
// StreamExecutorInferface.
|
||||
class CUDAExecutor : public internal::StreamExecutorInterface {
|
||||
public:
|
||||
// sub_platform indicates the subplatform used in this executor; it must
|
||||
// be a CUDA type.
|
||||
explicit CUDAExecutor(const PluginConfig &plugin_config)
|
||||
: device_(0),
|
||||
context_(nullptr),
|
||||
device_ordinal_(0),
|
||||
cc_major_(0),
|
||||
cc_minor_(0),
|
||||
plugin_config_(plugin_config) {}
|
||||
|
||||
// See the corresponding StreamExecutor methods for method comments on the
|
||||
// following overrides.
|
||||
|
||||
~CUDAExecutor() override;
|
||||
|
||||
port::Status Init(int device_ordinal, DeviceOptions device_options) override;
|
||||
|
||||
bool GetKernel(const MultiKernelLoaderSpec &spec,
|
||||
KernelBase *kernel) override;
|
||||
void UnloadKernel(const KernelBase *kernel) override;
|
||||
bool LoadModule(const MultiModuleLoaderSpec &spec,
|
||||
ModuleHandle *module_handle) override;
|
||||
bool UnloadModule(ModuleHandle module_handle) override;
|
||||
|
||||
bool Launch(Stream *stream, const ThreadDim &thread_dims,
|
||||
const BlockDim &block_dims, const KernelBase &k,
|
||||
const KernelArgsArrayBase &args) override;
|
||||
|
||||
void *Allocate(uint64 size) override;
|
||||
|
||||
void *AllocateSubBuffer(DeviceMemoryBase *mem, uint64 offset_bytes,
|
||||
uint64 size_bytes) override;
|
||||
|
||||
void Deallocate(DeviceMemoryBase *mem) override;
|
||||
|
||||
void *UnifiedMemoryAllocate(uint64 size) override {
|
||||
return CUDADriver::UnifiedMemoryAllocate(context_, size);
|
||||
}
|
||||
|
||||
void UnifiedMemoryDeallocate(void *location) override {
|
||||
return CUDADriver::UnifiedMemoryDeallocate(context_, location);
|
||||
}
|
||||
|
||||
// CUDA allocation/registration functions are necessary because the driver
|
||||
// internally sets up buffers for DMA operations (and page locks them).
|
||||
// There's no external interface for us to otherwise control these DMA
|
||||
// settings.
|
||||
void *HostMemoryAllocate(uint64 size) override {
|
||||
return CUDADriver::HostAllocate(context_, size);
|
||||
}
|
||||
|
||||
void HostMemoryDeallocate(void *location) override {
|
||||
return CUDADriver::HostDeallocate(context_, location);
|
||||
}
|
||||
|
||||
bool HostMemoryRegister(void *location, uint64 size) override;
|
||||
|
||||
bool HostMemoryUnregister(void *location) override;
|
||||
|
||||
bool SynchronizeAllActivity() override;
|
||||
|
||||
bool SynchronousMemZero(DeviceMemoryBase *location, uint64 size) override;
|
||||
|
||||
bool SynchronousMemSet(DeviceMemoryBase *location, int value,
|
||||
uint64 size) override;
|
||||
|
||||
port::Status SynchronousMemcpy(DeviceMemoryBase *gpu_dst,
|
||||
const void *host_src, uint64 size) override;
|
||||
|
||||
port::Status SynchronousMemcpy(void *host_dst,
|
||||
const DeviceMemoryBase &gpu_src,
|
||||
uint64 size) override;
|
||||
|
||||
port::Status SynchronousMemcpyDeviceToDevice(DeviceMemoryBase *gpu_dst,
|
||||
const DeviceMemoryBase &gpu_src,
|
||||
uint64 size) override;
|
||||
|
||||
bool MemZero(Stream *stream, DeviceMemoryBase *location,
|
||||
uint64 size) override;
|
||||
bool Memset(Stream *stream, DeviceMemoryBase *location, uint8 pattern,
|
||||
uint64 size) override;
|
||||
bool Memset32(Stream *stream, DeviceMemoryBase *location, uint32 pattern,
|
||||
uint64 size) override;
|
||||
|
||||
bool Memcpy(Stream *stream, void *host_dst, const DeviceMemoryBase &gpu_src,
|
||||
uint64 size) override;
|
||||
|
||||
bool Memcpy(Stream *stream, DeviceMemoryBase *gpu_dst, const void *host_src,
|
||||
uint64 size) override;
|
||||
|
||||
bool MemcpyDeviceToDevice(Stream *stream, DeviceMemoryBase *gpu_dst,
|
||||
const DeviceMemoryBase &gpu_src,
|
||||
uint64 size) override;
|
||||
|
||||
bool HostCallback(Stream *stream, std::function<void()> callback) override;
|
||||
|
||||
bool AllocateStream(Stream *stream) override;
|
||||
|
||||
void DeallocateStream(Stream *stream) override;
|
||||
|
||||
bool CreateStreamDependency(Stream *dependent, Stream *other) override;
|
||||
|
||||
bool AllocateTimer(Timer *timer) override;
|
||||
|
||||
void DeallocateTimer(Timer *timer) override;
|
||||
|
||||
bool StartTimer(Stream *stream, Timer *timer) override;
|
||||
|
||||
bool StopTimer(Stream *stream, Timer *timer) override;
|
||||
|
||||
port::Status AllocateEvent(Event *event) override;
|
||||
|
||||
port::Status DeallocateEvent(Event *event) override;
|
||||
|
||||
port::Status RecordEvent(Stream *stream, Event *event) override;
|
||||
|
||||
port::Status WaitForEvent(Stream *stream, Event *event) override;
|
||||
|
||||
Event::Status PollForEventStatus(Event *event) override;
|
||||
|
||||
port::Status BlockHostUntilDone(Stream *stream) override;
|
||||
|
||||
int PlatformDeviceCount() override { return CUDADriver::GetDeviceCount(); }
|
||||
|
||||
port::Status EnablePeerAccessTo(StreamExecutorInterface *other) override;
|
||||
|
||||
bool CanEnablePeerAccessTo(StreamExecutorInterface *other) override;
|
||||
|
||||
SharedMemoryConfig GetDeviceSharedMemoryConfig() override;
|
||||
|
||||
port::Status SetDeviceSharedMemoryConfig(SharedMemoryConfig config) override;
|
||||
|
||||
bool DeviceMemoryUsage(int64 *free, int64 *total) const override;
|
||||
|
||||
// Search for the symbol and returns a device pointer and size.
|
||||
// Returns false if symbol does not exist.
|
||||
bool GetSymbol(const string &symbol_name, ModuleHandle module_handle,
|
||||
void **mem, size_t *bytes) override;
|
||||
|
||||
DeviceDescription *PopulateDeviceDescription() const override;
|
||||
|
||||
// Populates the block_dim_limit by querying the device driver API. If an
|
||||
// error occurs at any point while asking the driver for block dim limits, it
|
||||
// will be only partially populated as a result, and an error will be logged.
|
||||
bool FillBlockDimLimit(BlockDim *block_dim_limit) const;
|
||||
|
||||
bool SupportsBlas() const override;
|
||||
|
||||
blas::BlasSupport *CreateBlas() override;
|
||||
|
||||
bool SupportsFft() const override;
|
||||
|
||||
fft::FftSupport *CreateFft() override;
|
||||
|
||||
bool SupportsRng() const override;
|
||||
|
||||
rng::RngSupport *CreateRng() override;
|
||||
|
||||
bool SupportsDnn() const override;
|
||||
|
||||
dnn::DnnSupport *CreateDnn() override;
|
||||
|
||||
std::unique_ptr<internal::EventInterface> CreateEventImplementation()
|
||||
override;
|
||||
|
||||
std::unique_ptr<internal::KernelInterface> CreateKernelImplementation()
|
||||
override;
|
||||
|
||||
std::unique_ptr<internal::StreamInterface> GetStreamImplementation() override;
|
||||
|
||||
std::unique_ptr<internal::TimerInterface> GetTimerImplementation() override;
|
||||
|
||||
void *GpuContextHack() override;
|
||||
|
||||
CudaContext* cuda_context();
|
||||
|
||||
private:
|
||||
// Attempts to find a more specific version of the file indicated by
|
||||
// filename by looking for compute-capability-specific suffixed versions; i.e.
|
||||
// looking for "foo.ptx" will check to see if "foo.ptx.cc30.ptx" is present if
|
||||
// we're on a compute capability 3.0 machine.
|
||||
bool FindOnDiskForComputeCapability(port::StringPiece filename,
|
||||
port::StringPiece canonical_suffix,
|
||||
string *found_filename) const;
|
||||
|
||||
// Host callback landing routine invoked by CUDA.
|
||||
// data: User-provided callback provided to HostCallback() above, captured
|
||||
// as a std::function<void()>. Allocated/initialized inside
|
||||
// HostCallback() and owned and deleted by this call.
|
||||
static void InternalHostCallback(CUstream stream, CUresult status,
|
||||
void *data);
|
||||
|
||||
// Collects metadata for the specified kernel.
|
||||
bool GetKernelMetadata(CUDAKernel *cuda_kernel,
|
||||
KernelMetadata *kernel_metadata);
|
||||
|
||||
// Prints to VLOG(2) information about the kernel's occupancy and how it might
|
||||
// be improved.
|
||||
void VlogOccupancyInfo(const KernelBase &kernel, const ThreadDim &thread_dims,
|
||||
const BlockDim &block_dims);
|
||||
|
||||
bool LoadModuleFromCuBin(const char *cubin, CUmodule *module)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
|
||||
|
||||
// Loads the PTX text `ptx` as a CUDA module. `ptx` must be null terminated.
|
||||
bool LoadModuleFromPtx(const char *ptx, CUmodule *module)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
|
||||
|
||||
bool UnloadGpuBinary(const void *gpu_binary)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
|
||||
|
||||
// Guards the in-memory-module mapping.
|
||||
mutex in_memory_modules_mu_;
|
||||
|
||||
// Kernel -> loaded GPU binary. Many kernels may load the same binary.
|
||||
std::unordered_map<const KernelBase *, const void *> kernel_to_gpu_binary_
|
||||
GUARDED_BY(in_memory_modules_mu_);
|
||||
// GPU binary (PTX or CUBIN) -> {CUDA module, reference count}.
|
||||
std::unordered_map<const void *, std::pair<CUmodule, uint64>>
|
||||
gpu_binary_to_module_ GUARDED_BY(in_memory_modules_mu_);
|
||||
|
||||
// Guards the launched kernel set.
|
||||
mutex launched_kernels_mu_;
|
||||
|
||||
// Keeps track of the set of launched kernels. Currently used to suppress the
|
||||
// occupancy check on subsequent launches.
|
||||
std::set<CUfunction> launched_kernels_ GUARDED_BY(launched_kernels_mu_);
|
||||
|
||||
// Handle for the CUDA device being operated on. Immutable
|
||||
// post-initialization.
|
||||
CUdevice device_;
|
||||
|
||||
// Handle for session with the library/driver. Immutable post-initialization.
|
||||
CudaContext* context_;
|
||||
|
||||
// The device ordinal value that this executor was initialized with; recorded
|
||||
// for use in getting device metadata. Immutable post-initialization.
|
||||
int device_ordinal_;
|
||||
|
||||
// The major verion of the compute capability for device_.
|
||||
int cc_major_;
|
||||
|
||||
// The minor verion of the compute capability for device_.
|
||||
int cc_minor_;
|
||||
|
||||
// The plugin configuration associated with this instance.
|
||||
PluginConfig plugin_config_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CUDAExecutor);
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_GPU_EXECUTOR_H_
|
||||
@@ -0,0 +1,105 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Common helper functions used for dealing with CUDA API datatypes.
|
||||
//
|
||||
// These are typically placed here for use by multiple source components (for
|
||||
// example, BLAS and executor components).
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_HELPERS_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_HELPERS_H_
|
||||
|
||||
#include <stddef.h>
|
||||
#include <complex>
|
||||
|
||||
#include "cuda/include/cuComplex.h"
|
||||
#include "cuda/include/cuda.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
template <typename ElemT>
|
||||
class DeviceMemory;
|
||||
|
||||
namespace cuda {
|
||||
|
||||
// Converts a const DeviceMemory reference to its underlying typed pointer in
|
||||
// CUDA
|
||||
// device memory.
|
||||
template <typename T>
|
||||
const T *CUDAMemory(const DeviceMemory<T> &mem) {
|
||||
return static_cast<const T *>(mem.opaque());
|
||||
}
|
||||
|
||||
// Converts a (non-const) DeviceMemory pointer reference to its underlying typed
|
||||
// pointer in CUDA device memory.
|
||||
template <typename T>
|
||||
T *CUDAMemoryMutable(DeviceMemory<T> *mem) {
|
||||
return static_cast<T *>(mem->opaque());
|
||||
}
|
||||
|
||||
static_assert(sizeof(std::complex<float>) == sizeof(cuComplex),
|
||||
"std::complex<float> and cuComplex should have the same size");
|
||||
static_assert(offsetof(cuComplex, x) == 0,
|
||||
"The real part of cuComplex should appear first.");
|
||||
static_assert(sizeof(std::complex<double>) == sizeof(cuDoubleComplex),
|
||||
"std::complex<double> and cuDoubleComplex should have the same "
|
||||
"size");
|
||||
static_assert(offsetof(cuDoubleComplex, x) == 0,
|
||||
"The real part of cuDoubleComplex should appear first.");
|
||||
|
||||
// Type traits to get CUDA complex types from std::complex<>.
|
||||
|
||||
template <typename T>
|
||||
struct CUDAComplexT {
|
||||
typedef T type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct CUDAComplexT<std::complex<float>> {
|
||||
typedef cuComplex type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct CUDAComplexT<std::complex<double>> {
|
||||
typedef cuDoubleComplex type;
|
||||
};
|
||||
|
||||
// Converts pointers of std::complex<> to pointers of
|
||||
// cuComplex/cuDoubleComplex. No type conversion for non-complex types.
|
||||
|
||||
template <typename T>
|
||||
inline const typename CUDAComplexT<T>::type *CUDAComplex(const T *p) {
|
||||
return reinterpret_cast<const typename CUDAComplexT<T>::type *>(p);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline typename CUDAComplexT<T>::type *CUDAComplex(T *p) {
|
||||
return reinterpret_cast<typename CUDAComplexT<T>::type *>(p);
|
||||
}
|
||||
|
||||
// Converts values of std::complex<float/double> to values of
|
||||
// cuComplex/cuDoubleComplex.
|
||||
inline cuComplex CUDAComplexValue(std::complex<float> val) {
|
||||
return {val.real(), val.imag()};
|
||||
}
|
||||
|
||||
inline cuDoubleComplex CUDAComplexValue(std::complex<double> val) {
|
||||
return {val.real(), val.imag()};
|
||||
}
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_HELPERS_H_
|
||||
@@ -0,0 +1,128 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// The CUDA implementation of the StreamExecutorInterface functionality.
|
||||
// CUDA inclusions are ideally confined to this implementation file.
|
||||
//
|
||||
// The notions from the StreamExecutor basically correspond to the CUDA streams
|
||||
// programming model provided by the libcuda.so driver APIs, so we don't have
|
||||
// to do much more than wrap the calls to the libraries appropriately.
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_KERNEL_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_KERNEL_H_
|
||||
|
||||
#include "tensorflow/stream_executor/kernel_cache_config.h"
|
||||
#include "tensorflow/stream_executor/stream_executor_internal.h"
|
||||
#include "tensorflow/stream_executor/cuda/cuda_driver.h"
|
||||
#include "tensorflow/stream_executor/lib/casts.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/platform/logging.h"
|
||||
#include "cuda/include/cuda.h"
|
||||
|
||||
#ifdef PLATFORMS_GPUS_CUDA_DYNAMIC_LIBCUDA_DYNAMIC_LIBCUDA_H_
|
||||
#error \
|
||||
"No driver calls in this file, wrap driver functionality in cuda_driver.cc."
|
||||
#endif
|
||||
|
||||
#ifdef __CUDA_RUNTIME_H__
|
||||
#error \
|
||||
"CUDA runtime being included into CUDA GPU executor; should be driver only."
|
||||
#endif
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
// Wraps a CUfunction to implement the platform-independent KernelInterface.
|
||||
class CUDAKernel : public internal::KernelInterface {
|
||||
public:
|
||||
CUDAKernel() : cuda_function_(nullptr), arity_(0),
|
||||
preferred_cache_config_(KernelCacheConfig::kNoPreference) {}
|
||||
|
||||
// Note that the function is unloaded when the module is unloaded, and the
|
||||
// module that the function is contained in is owned by the CUDAExecutor.
|
||||
~CUDAKernel() override {}
|
||||
|
||||
// As arity cannot be reflected upon using the CUDA API, the arity is
|
||||
// explicitly set during the CUDAExecutor::GetKernel initialization process.
|
||||
void set_arity(unsigned arity) { arity_ = arity; }
|
||||
unsigned Arity() const override { return arity_; }
|
||||
|
||||
// Returns the CUfunction value for passing to the CUDA API.
|
||||
CUfunction AsCUDAFunctionValue() const {
|
||||
DCHECK(cuda_function_ != nullptr);
|
||||
return const_cast<CUfunction>(cuda_function_);
|
||||
}
|
||||
|
||||
// Returns the slot that the CUfunction is stored within for this object,
|
||||
// for the CUDA API which wants to load into a CUfunction*.
|
||||
CUfunction *cuda_function_ptr() { return &cuda_function_; }
|
||||
|
||||
// CUDA supports setting the preferred cache configuration of a CUfunction
|
||||
// (more-or-less equivalent to a CUDAKernel). We support this via the below
|
||||
// functions; users can set a preference, and that is applied when the kernel
|
||||
// is [lazy-]loaded (in CUDAExecutor::Launch). The alternative would be to
|
||||
// load the kernel & set the preference when the user calls the setter below;
|
||||
// either approach is valid.
|
||||
// Sets the current kernel cache configuration preference.
|
||||
void SetPreferredCacheConfig(KernelCacheConfig config) override {
|
||||
preferred_cache_config_ = config;
|
||||
}
|
||||
|
||||
// Returns the current kernel cache configuration preference.
|
||||
KernelCacheConfig GetPreferredCacheConfig() const override {
|
||||
return preferred_cache_config_;
|
||||
}
|
||||
|
||||
// Returns the current kernel cache configuration preference as a
|
||||
// CUfunc_cache.
|
||||
CUfunc_cache GetCUDACacheConfig() const {
|
||||
switch (preferred_cache_config_) {
|
||||
case KernelCacheConfig::kNoPreference:
|
||||
return CU_FUNC_CACHE_PREFER_NONE;
|
||||
case KernelCacheConfig::kPreferShared:
|
||||
return CU_FUNC_CACHE_PREFER_SHARED;
|
||||
case KernelCacheConfig::kPreferL1:
|
||||
return CU_FUNC_CACHE_PREFER_L1;
|
||||
case KernelCacheConfig::kPreferEqual:
|
||||
return CU_FUNC_CACHE_PREFER_EQUAL;
|
||||
default:
|
||||
LOG(FATAL) << "Unknown KernelCacheConfig"
|
||||
<< static_cast<int32>(preferred_cache_config_);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
CUfunction cuda_function_; // Wrapped CUDA kernel handle.
|
||||
unsigned arity_; // Number of formal parameters the kernel takes.
|
||||
|
||||
// Preferred (but not required) cache configuration for this kernel.
|
||||
KernelCacheConfig preferred_cache_config_;
|
||||
};
|
||||
|
||||
// Given a platform-independent kernel datatype, returns the (const) internal
|
||||
// CUDA platform implementation pointer.
|
||||
inline const CUDAKernel *AsCUDAKernel(const KernelBase *kernel) {
|
||||
return static_cast<const CUDAKernel *>(kernel->implementation());
|
||||
}
|
||||
|
||||
// Given a platform-independent kernel datatype, returns the (non-const)
|
||||
// internal CUDA platform implementation pointer.
|
||||
inline CUDAKernel *AsCUDAKernel(KernelBase *kernel) {
|
||||
return static_cast<CUDAKernel *>(kernel->implementation());
|
||||
}
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_KERNEL_H_
|
||||
@@ -0,0 +1,108 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_H_
|
||||
|
||||
#include <memory>
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/stream_executor/executor_cache.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/multi_platform_manager.h"
|
||||
#include "tensorflow/stream_executor/platform.h"
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/platform/thread_annotations.h"
|
||||
#include "tensorflow/stream_executor/stream_executor_internal.h"
|
||||
#include "tensorflow/stream_executor/stream_executor_pimpl.h"
|
||||
#include "tensorflow/stream_executor/trace_listener.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
// Opaque and unique identifier for the CUDA platform plugin.
|
||||
// This is needed so that plugins can refer to/identify this platform without
|
||||
// instantiating a CudaPlatform object.
|
||||
extern const Platform::Id kCudaPlatformId;
|
||||
|
||||
// Cuda-specific platform plugin, registered as a singleton value via module
|
||||
// initializer.
|
||||
class CudaPlatform : public Platform {
|
||||
public:
|
||||
CudaPlatform();
|
||||
~CudaPlatform() override;
|
||||
|
||||
// CudaPlatform-specific functionality
|
||||
// Returns the number of distinct buses / NUMA nodes on the machine.
|
||||
int BusCount();
|
||||
|
||||
// Returns the bus/NUMA node for the specified device ordinal.
|
||||
int DeviceToBus(int device_ordinal);
|
||||
|
||||
// Returns the lowest-ordinal-number StreamExecutor on the specified bus.
|
||||
port::StatusOr<StreamExecutor*> FirstExecutorForBus(int bus_ordinal);
|
||||
|
||||
// Platform interface implementation:
|
||||
// Returns the same value as kCudaPlatform above.
|
||||
Platform::Id id() const override;
|
||||
|
||||
// Returns -1 as a sentinel on internal failure (and logs the error).
|
||||
int VisibleDeviceCount() const override;
|
||||
|
||||
const string& Name() const override;
|
||||
|
||||
port::StatusOr<StreamExecutor*> ExecutorForDevice(int ordinal) override;
|
||||
|
||||
port::StatusOr<StreamExecutor*> ExecutorForDeviceWithPluginConfig(
|
||||
int ordinal, const PluginConfig& config) override;
|
||||
|
||||
port::StatusOr<StreamExecutor*> GetExecutor(
|
||||
const StreamExecutorConfig& config) override;
|
||||
|
||||
port::StatusOr<std::unique_ptr<StreamExecutor>> GetUncachedExecutor(
|
||||
const StreamExecutorConfig& config) override;
|
||||
|
||||
void RegisterTraceListener(std::unique_ptr<TraceListener> listener) override;
|
||||
|
||||
void UnregisterTraceListener(TraceListener* listener) override;
|
||||
|
||||
private:
|
||||
// Determines the number of NUMA nodes and the assignment of executor to each.
|
||||
void InspectNumaNodes();
|
||||
|
||||
// This platform's name.
|
||||
string name_;
|
||||
|
||||
// Cache of created executors.
|
||||
ExecutorCache executor_cache_;
|
||||
|
||||
// The smallest NUMA node value for any device managed by this machine
|
||||
// manager. Used, along with limit_numa_node_, to convert NUMA nodes into bus
|
||||
// ordinals. The NUMA node space occupied by GPUs is assumed to be dense./
|
||||
int min_numa_node_;
|
||||
|
||||
// Larger than the NUMA node value for any device managed by this machine
|
||||
// manager.
|
||||
int limit_numa_node_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CudaPlatform);
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_H_
|
||||
@@ -0,0 +1,34 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_ID_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_ID_H_
|
||||
|
||||
#include "tensorflow/stream_executor/platform.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
// Opaque and unique identifier for the cuda platform.
|
||||
// This is needed so that plugins can refer to/identify this platform without
|
||||
// instantiating a CudaPlatform object.
|
||||
// This is broken out here to avoid a circular dependency between CudaPlatform
|
||||
// and CudaExecutor.
|
||||
extern const Platform::Id kCudaPlatformId;
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_ID_H_
|
||||
@@ -0,0 +1,102 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_RNG_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_RNG_H_
|
||||
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/platform/thread_annotations.h"
|
||||
#include "tensorflow/stream_executor/plugin_registry.h"
|
||||
#include "tensorflow/stream_executor/rng.h"
|
||||
|
||||
typedef struct curandGenerator_st *curandGenerator_t;
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class Stream;
|
||||
template <typename ElemT>
|
||||
class DeviceMemory;
|
||||
|
||||
namespace cuda {
|
||||
|
||||
// Opaque and unique identifier for the cuRAND plugin.
|
||||
extern const PluginId kCuRandPlugin;
|
||||
|
||||
class CUDAExecutor;
|
||||
|
||||
// CUDA-platform implementation of the random number generation support
|
||||
// interface.
|
||||
//
|
||||
// Thread-safe post-initialization.
|
||||
class CUDARng : public rng::RngSupport {
|
||||
public:
|
||||
explicit CUDARng(CUDAExecutor *parent);
|
||||
|
||||
// Retrieves a curand library generator handle. This is necessary for
|
||||
// enqueuing random number generation work onto the device.
|
||||
// TODO(leary) provide a way for users to select the RNG algorithm.
|
||||
bool Init();
|
||||
|
||||
// Releases a curand library generator handle, if one was acquired.
|
||||
~CUDARng() override;
|
||||
|
||||
// See rng::RngSupport for details on the following overrides.
|
||||
bool DoPopulateRandUniform(Stream *stream, DeviceMemory<float> *v) override;
|
||||
bool DoPopulateRandUniform(Stream *stream, DeviceMemory<double> *v) override;
|
||||
bool DoPopulateRandUniform(Stream *stream,
|
||||
DeviceMemory<std::complex<float>> *v) override;
|
||||
bool DoPopulateRandUniform(Stream *stream,
|
||||
DeviceMemory<std::complex<double>> *v) override;
|
||||
bool DoPopulateRandGaussian(Stream *stream, float mean, float stddev,
|
||||
DeviceMemory<float> *v) override;
|
||||
bool DoPopulateRandGaussian(Stream *stream, double mean, double stddev,
|
||||
DeviceMemory<double> *v) override;
|
||||
|
||||
bool SetSeed(Stream *stream, const uint8 *seed, uint64 seed_bytes) override;
|
||||
|
||||
private:
|
||||
// Actually performs the work of generating random numbers - the public
|
||||
// methods are thin wrappers to this interface.
|
||||
template <typename T>
|
||||
bool DoPopulateRandUniformInternal(Stream *stream, DeviceMemory<T> *v);
|
||||
template <typename ElemT, typename FuncT>
|
||||
bool DoPopulateRandGaussianInternal(Stream *stream, ElemT mean, ElemT stddev,
|
||||
DeviceMemory<ElemT> *v, FuncT func);
|
||||
|
||||
// Sets the stream for the internal curand generator.
|
||||
//
|
||||
// This is a stateful operation, as the handle can only have one stream set at
|
||||
// a given time, so it is usually performed right before enqueuing work to do
|
||||
// with random number generation.
|
||||
bool SetStream(Stream *stream) EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
||||
|
||||
// mutex that guards the cuRAND handle for this device.
|
||||
mutex mu_;
|
||||
|
||||
// CUDAExecutor which instantiated this CUDARng.
|
||||
// Immutable post-initialization.
|
||||
CUDAExecutor *parent_;
|
||||
|
||||
// cuRANDalibrary handle on the device.
|
||||
curandGenerator_t rng_ GUARDED_BY(mu_);
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CUDARng);
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_RNG_H_
|
||||
@@ -0,0 +1,93 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Defines the CUDAStream type - the CUDA-specific implementation of the generic
|
||||
// StreamExecutor Stream interface.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_STREAM_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_STREAM_H_
|
||||
|
||||
#include "tensorflow/stream_executor/cuda/cuda_driver.h"
|
||||
#include "tensorflow/stream_executor/platform/thread_annotations.h"
|
||||
#include "tensorflow/stream_executor/stream_executor_internal.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
class CUDAExecutor;
|
||||
|
||||
// Wraps a CUstream in order to satisfy the platform-independent
|
||||
// StreamInterface.
|
||||
//
|
||||
// Thread-safe post-initialization.
|
||||
class CUDAStream : public internal::StreamInterface {
|
||||
public:
|
||||
explicit CUDAStream(CUDAExecutor *parent)
|
||||
: parent_(parent), cuda_stream_(nullptr), completed_event_(nullptr) {}
|
||||
|
||||
// Note: teardown is handled by a parent's call to DeallocateStream.
|
||||
~CUDAStream() override {}
|
||||
|
||||
void *GpuStreamHack() override { return cuda_stream_; }
|
||||
void **GpuStreamMemberHack() override {
|
||||
return reinterpret_cast<void **>(&cuda_stream_);
|
||||
}
|
||||
|
||||
// Explicitly initialize the CUDA resources associated with this stream, used
|
||||
// by StreamExecutor::AllocateStream().
|
||||
bool Init();
|
||||
|
||||
// Explicitly destroy the CUDA resources associated with this stream, used by
|
||||
// StreamExecutor::DeallocateStream().
|
||||
void Destroy();
|
||||
|
||||
// Returns true if no work is pending or executing on the stream.
|
||||
bool IsIdle() const;
|
||||
|
||||
// Retrieves an event which indicates that all work enqueued into the stream
|
||||
// has completed. Ownership of the event is not transferred to the caller, the
|
||||
// event is owned by this stream.
|
||||
CUevent* completed_event() { return &completed_event_; }
|
||||
|
||||
// Returns the CUstream value for passing to the CUDA API.
|
||||
//
|
||||
// Precond: this CUDAStream has been allocated (otherwise passing a nullptr
|
||||
// into the NVIDIA library causes difficult-to-understand faults).
|
||||
CUstream cuda_stream() const {
|
||||
DCHECK(cuda_stream_ != nullptr);
|
||||
return const_cast<CUstream>(cuda_stream_);
|
||||
}
|
||||
|
||||
CUDAExecutor *parent() const { return parent_; }
|
||||
|
||||
private:
|
||||
CUDAExecutor *parent_; // Executor that spawned this stream.
|
||||
CUstream cuda_stream_; // Wrapped CUDA stream handle.
|
||||
|
||||
// Event that indicates this stream has completed.
|
||||
CUevent completed_event_ = nullptr;
|
||||
};
|
||||
|
||||
// Helper functions to simplify extremely common flows.
|
||||
// Converts a Stream to the underlying CUDAStream implementation.
|
||||
CUDAStream *AsCUDAStream(Stream *stream);
|
||||
|
||||
// Extracts a CUstream from a CUDAStream-backed Stream object.
|
||||
CUstream AsCUDAStreamValue(Stream *stream);
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_STREAM_H_
|
||||
@@ -0,0 +1,90 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Defines the CUDATimer type - the CUDA-specific implementation of the generic
|
||||
// StreamExecutor Timer interface.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_TIMER_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_TIMER_H_
|
||||
|
||||
#include "tensorflow/stream_executor/stream_executor_internal.h"
|
||||
#include "tensorflow/stream_executor/cuda/cuda_driver.h"
|
||||
#include "tensorflow/stream_executor/cuda/cuda_gpu_executor.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
class CUDAExecutor;
|
||||
class CUDAStream;
|
||||
|
||||
// Wraps a pair of CUevents in order to satisfy the platform-independent
|
||||
// TimerInferface -- both a start and a stop event are present which may be
|
||||
// recorded in a stream.
|
||||
class CUDATimer : public internal::TimerInterface {
|
||||
public:
|
||||
explicit CUDATimer(CUDAExecutor *parent)
|
||||
: parent_(parent), start_event_(nullptr), stop_event_(nullptr) {}
|
||||
|
||||
// Note: teardown needs to be explicitly handled in this API by a call to
|
||||
// StreamExecutor::DeallocateTimer(), which invokes Destroy().
|
||||
// TODO(csigg): Change to RAII.
|
||||
~CUDATimer() override {}
|
||||
|
||||
// Allocates the platform-specific pieces of the timer, called as part of
|
||||
// StreamExecutor::AllocateTimer().
|
||||
bool Init();
|
||||
|
||||
// Deallocates the platform-specific pieces of the timer, called as part of
|
||||
// StreamExecutor::DeallocateTimer().
|
||||
void Destroy();
|
||||
|
||||
// Records the "timer start" event at the current point in the stream.
|
||||
bool Start(CUDAStream *stream);
|
||||
|
||||
// Records the "timer stop" event at the current point in the stream.
|
||||
bool Stop(CUDAStream *stream);
|
||||
|
||||
// Returns the elapsed time, in milliseconds, between the start and stop
|
||||
// events.
|
||||
float GetElapsedMilliseconds() const;
|
||||
|
||||
// See Timer::Microseconds().
|
||||
// TODO(leary) make this into an error code interface...
|
||||
uint64 Microseconds() const override {
|
||||
return GetElapsedMilliseconds() * 1e3;
|
||||
}
|
||||
|
||||
// See Timer::Nanoseconds().
|
||||
uint64 Nanoseconds() const override { return GetElapsedMilliseconds() * 1e6; }
|
||||
|
||||
private:
|
||||
CUDAExecutor *parent_;
|
||||
CUevent start_event_; // Event recorded to indicate the "start" timestamp
|
||||
// executing in a stream.
|
||||
CUevent stop_event_; // Event recorded to indicate the "stop" timestamp
|
||||
// executing in a stream.
|
||||
};
|
||||
|
||||
struct TimerDeleter {
|
||||
void operator()(CUDATimer *t) {
|
||||
t->Destroy();
|
||||
delete t;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_TIMER_H_
|
||||
@@ -0,0 +1,50 @@
|
||||
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDNN_VERSION_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDNN_VERSION_H_
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "tensorflow/core/lib/strings/strcat.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
struct CudnnVersion {
|
||||
CudnnVersion() = default;
|
||||
|
||||
CudnnVersion(int major, int minor, int patch)
|
||||
: major_version(major), minor_version(minor), patch_level(patch) {}
|
||||
|
||||
tensorflow::string ToString() const {
|
||||
return tensorflow::strings::StrCat(major_version, ".", minor_version, ".",
|
||||
patch_level);
|
||||
}
|
||||
|
||||
int major_version;
|
||||
int minor_version;
|
||||
int patch_level;
|
||||
};
|
||||
|
||||
// Returns true if the given source CuDNN version is compatible with the given
|
||||
// loaded version.
|
||||
bool IsSourceCompatibleWithCudnnLibrary(CudnnVersion source_version,
|
||||
CudnnVersion loaded_version);
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDNN_VERSION_H_
|
||||
@@ -0,0 +1,390 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Describes the underlying platform for a StreamExecutor; e.g. OpenCL or CUDA
|
||||
// device and platform properties. Also contains convenience functions for
|
||||
// checking/calculating launch dimensionality based on device properties.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_DEVICE_DESCRIPTION_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_DEVICE_DESCRIPTION_H_
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include "absl/base/macros.h"
|
||||
#include "tensorflow/stream_executor/launch_dim.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace internal {
|
||||
class DeviceDescriptionBuilder;
|
||||
} // namespace internal
|
||||
|
||||
// Data that describes the execution target of the StreamExecutor, in terms of
|
||||
// important logical parameters. These include dimensionality limits and
|
||||
// physical parameters of interest, such as number of cores present on the
|
||||
// device.
|
||||
//
|
||||
// Thread-safe: immutable post-initialization.
|
||||
class DeviceDescription {
|
||||
public:
|
||||
// Returns the platform being run on; this value is primarily intended for
|
||||
// printing, and comes out something like "OpenCL 1.2" or "Compute Capability
|
||||
// 3.5".
|
||||
const string &platform_version() const { return platform_version_; }
|
||||
|
||||
// Returns the driver version interfacing with the underlying platform. Vendor
|
||||
// dependent format.
|
||||
const string &driver_version() const { return driver_version_; }
|
||||
|
||||
// Return the runtime version, if one is provided by the underlying platform.
|
||||
// Vendor dependent format / usefulness.
|
||||
const string &runtime_version() const { return runtime_version_; }
|
||||
|
||||
// Returns the name that the device reports. Vendor dependent.
|
||||
const string &name() const { return name_; }
|
||||
|
||||
// Returns the PCI bus identifier for this device, of the form
|
||||
// [domain]:[bus]:[device].[function]
|
||||
const string &pci_bus_id() const { return pci_bus_id_; }
|
||||
|
||||
// Returns the NUMA node associated with this device, for use in
|
||||
// determining socket locality. If the NUMA node could not be determined, -1
|
||||
// is returned.
|
||||
int numa_node() const { return numa_node_; }
|
||||
|
||||
// Number of cores (traditional notion of core; i.e. an SM on an NVIDIA device
|
||||
// or an AMD Compute Unit.
|
||||
int core_count() const { return core_count_; }
|
||||
|
||||
// Returns the limit on the thread dimensionality values in each of the
|
||||
// respective dimensions. These limits affect what constitutes a legitimate
|
||||
// kernel launch request.
|
||||
const ThreadDim &thread_dim_limit() const { return thread_dim_limit_; }
|
||||
|
||||
// Returns the limit on the block dimensionality values in each of the
|
||||
// respective dimensions. These limits may affect what constitutes a
|
||||
// legitimate kernel launch request.
|
||||
const BlockDim &block_dim_limit() const { return block_dim_limit_; }
|
||||
|
||||
// Returns the limit on the number of simultaneously resident blocks
|
||||
// on a multiprocessor.
|
||||
uint64 blocks_per_core_limit() const { return blocks_per_core_limit_; }
|
||||
|
||||
// Returns the limit on the total number of threads that can be launched in a
|
||||
// single block; i.e. the limit on x * y * z dimensions of a ThreadDim.
|
||||
// This limit affects what constitutes a legitimate kernel launch request.
|
||||
const uint64 &threads_per_block_limit() const {
|
||||
return threads_per_block_limit_;
|
||||
}
|
||||
|
||||
// Returns the limit on the total number of threads that can be simultaneously
|
||||
// launched on a given multiprocessor.
|
||||
const uint64 &threads_per_core_limit() const {
|
||||
return threads_per_core_limit_;
|
||||
}
|
||||
|
||||
// Returns the number of threads per warp/wavefront.
|
||||
const uint64 &threads_per_warp() const { return threads_per_warp_; }
|
||||
|
||||
// Returns the limit on the total number of registers per core.
|
||||
const uint64 ®isters_per_core_limit() const {
|
||||
return registers_per_core_limit_;
|
||||
}
|
||||
|
||||
// Returns the limit on the total number of registers that can be
|
||||
// simultaneously used by a block.
|
||||
const uint64 ®isters_per_block_limit() const {
|
||||
return registers_per_block_limit_;
|
||||
}
|
||||
|
||||
// Returns the limit on the total number of registers that can be
|
||||
// allocated to a thread.
|
||||
const uint64 ®isters_per_thread_limit() const {
|
||||
return registers_per_thread_limit_;
|
||||
}
|
||||
|
||||
// Returns the granularity at which warps are allocated resources.
|
||||
const uint64 &warp_alloc_granularity() const {
|
||||
return warp_alloc_granularity_;
|
||||
}
|
||||
|
||||
// Returns the granularity at which registers are allocated to warps.
|
||||
const uint64 ®ister_alloc_granularity() const {
|
||||
return register_alloc_granularity_;
|
||||
}
|
||||
|
||||
// Returns the granularity at which shared memory is allocated to warps.
|
||||
const uint64 &shared_memory_alloc_granularity() const {
|
||||
return shared_memory_alloc_granularity_;
|
||||
}
|
||||
|
||||
// Returns the number of address bits available to kernel code running on the
|
||||
// platform. This affects things like the maximum allocation size and perhaps
|
||||
// types used in kernel code such as size_t.
|
||||
const uint64 &device_address_bits() const { return device_address_bits_; }
|
||||
|
||||
// Returns the device memory size in bytes.
|
||||
uint64 device_memory_size() const { return device_memory_size_; }
|
||||
|
||||
// Returns the device's memory bandwidth in bytes/sec. (This is for
|
||||
// reads/writes to/from the device's own memory, not for transfers between the
|
||||
// host and device.)
|
||||
uint64 memory_bandwidth() const { return memory_bandwidth_; }
|
||||
|
||||
// Returns the device's core clock rate in GHz.
|
||||
float clock_rate_ghz() const { return clock_rate_ghz_; }
|
||||
|
||||
// Returns whether ECC is enabled.
|
||||
bool ecc_enabled() const { return ecc_enabled_; }
|
||||
|
||||
// Returns the device vendor string, e.g., "NVIDIA Corporation", "Advanced
|
||||
// Micro Devices, Inc.", or "GenuineIntel".
|
||||
const string &device_vendor() const { return device_vendor_; }
|
||||
|
||||
// Returns the CUDA compute capability if we're running on the CUDA platform.
|
||||
// If a CUDA compute capability is not available, the major version will be
|
||||
// zero, and the return value will be false.
|
||||
bool cuda_compute_capability(int *major, int *minor) const;
|
||||
|
||||
// Returns the maximum amount of shared memory present on a single core
|
||||
// (i.e. Streaming Multiprocessor on NVIDIA GPUs; Compute Unit for OpenCL
|
||||
// devices). Note that some devices, such as NVIDIA's have a configurable
|
||||
// partitioning between shared memory and L1 cache.
|
||||
uint64 shared_memory_per_core() const { return shared_memory_per_core_; }
|
||||
|
||||
// Returns the maximum amount of shared memory available for a single block.
|
||||
uint64 shared_memory_per_block() const { return shared_memory_per_block_; }
|
||||
|
||||
// TODO(leary): resident blocks per core will be useful.
|
||||
|
||||
// Convenience typedef for the string-based DeviceDescription mapping.
|
||||
typedef std::map<string, string> Map;
|
||||
|
||||
// Returns a mapping from readable names to readable values that describe the
|
||||
// device. This is useful for things like printing.
|
||||
std::unique_ptr<Map> ToMap() const;
|
||||
|
||||
// For string values that are not available via the underlying platform, this
|
||||
// value will be provided.
|
||||
static const char *kUndefinedString;
|
||||
|
||||
private:
|
||||
friend class internal::DeviceDescriptionBuilder;
|
||||
|
||||
DeviceDescription();
|
||||
|
||||
// For description of the following members, see the corresponding accessor
|
||||
// above.
|
||||
//
|
||||
// N.B. If another field is added, update ToMap() above.
|
||||
string device_vendor_;
|
||||
string platform_version_;
|
||||
string driver_version_;
|
||||
string runtime_version_;
|
||||
string pci_bus_id_;
|
||||
string name_;
|
||||
|
||||
ThreadDim thread_dim_limit_;
|
||||
BlockDim block_dim_limit_;
|
||||
|
||||
uint64 blocks_per_core_limit_;
|
||||
|
||||
uint64 threads_per_core_limit_;
|
||||
uint64 threads_per_block_limit_;
|
||||
uint64 threads_per_warp_;
|
||||
|
||||
uint64 registers_per_core_limit_;
|
||||
uint64 registers_per_block_limit_;
|
||||
uint64 registers_per_thread_limit_;
|
||||
|
||||
uint64 warp_alloc_granularity_;
|
||||
uint64 register_alloc_granularity_;
|
||||
uint64 shared_memory_alloc_granularity_;
|
||||
|
||||
uint64 device_address_bits_;
|
||||
uint64 device_memory_size_;
|
||||
uint64 memory_bandwidth_;
|
||||
|
||||
// Shared memory limits on a given device.
|
||||
uint64 shared_memory_per_core_;
|
||||
uint64 shared_memory_per_block_;
|
||||
|
||||
float clock_rate_ghz_;
|
||||
|
||||
// CUDA "CC" major value, -1 if not available.
|
||||
int cuda_compute_capability_major_;
|
||||
int cuda_compute_capability_minor_;
|
||||
|
||||
int numa_node_;
|
||||
int core_count_;
|
||||
bool ecc_enabled_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(DeviceDescription);
|
||||
};
|
||||
|
||||
namespace internal {
|
||||
|
||||
// Helper class the builds a device description, given that it has a large
|
||||
// number of fields that would be easily confused in constructor form.
|
||||
class DeviceDescriptionBuilder {
|
||||
public:
|
||||
DeviceDescriptionBuilder();
|
||||
|
||||
// For descriptions of the following fields, see comments on the corresponding
|
||||
// DeviceDescription::* accessors above.
|
||||
|
||||
void set_device_vendor(const string &value) {
|
||||
device_description_->device_vendor_ = value;
|
||||
}
|
||||
void set_platform_version(const string &value) {
|
||||
device_description_->platform_version_ = value;
|
||||
}
|
||||
void set_driver_version(const string &value) {
|
||||
device_description_->driver_version_ = value;
|
||||
}
|
||||
void set_runtime_version(const string &value) {
|
||||
device_description_->runtime_version_ = value;
|
||||
}
|
||||
void set_pci_bus_id(const string &value) {
|
||||
device_description_->pci_bus_id_ = value;
|
||||
}
|
||||
void set_name(const string &value) { device_description_->name_ = value; }
|
||||
|
||||
void set_thread_dim_limit(const ThreadDim &value) {
|
||||
device_description_->thread_dim_limit_ = value;
|
||||
}
|
||||
void set_block_dim_limit(const BlockDim &value) {
|
||||
device_description_->block_dim_limit_ = value;
|
||||
}
|
||||
|
||||
void set_blocks_per_core_limit(uint64 value) {
|
||||
device_description_->blocks_per_core_limit_ = value;
|
||||
}
|
||||
|
||||
void set_threads_per_core_limit(uint64 value) {
|
||||
device_description_->threads_per_core_limit_ = value;
|
||||
}
|
||||
void set_threads_per_block_limit(uint64 value) {
|
||||
device_description_->threads_per_block_limit_ = value;
|
||||
}
|
||||
void set_threads_per_warp(uint64 value) {
|
||||
device_description_->threads_per_warp_ = value;
|
||||
}
|
||||
|
||||
void set_registers_per_core_limit(uint64 value) {
|
||||
device_description_->registers_per_core_limit_ = value;
|
||||
}
|
||||
void set_registers_per_block_limit(uint64 value) {
|
||||
device_description_->registers_per_block_limit_ = value;
|
||||
}
|
||||
void set_registers_per_thread_limit(uint64 value) {
|
||||
device_description_->registers_per_thread_limit_ = value;
|
||||
}
|
||||
|
||||
void set_warp_alloc_granularity(uint64 value) {
|
||||
device_description_->warp_alloc_granularity_ = value;
|
||||
}
|
||||
void set_register_alloc_granularity(uint64 value) {
|
||||
device_description_->register_alloc_granularity_ = value;
|
||||
}
|
||||
void set_shared_memory_alloc_granularity(uint64 value) {
|
||||
device_description_->shared_memory_alloc_granularity_ = value;
|
||||
}
|
||||
|
||||
void set_device_address_bits(uint64 value) {
|
||||
device_description_->device_address_bits_ = value;
|
||||
}
|
||||
void set_device_memory_size(uint64 value) {
|
||||
device_description_->device_memory_size_ = value;
|
||||
}
|
||||
void set_memory_bandwidth(uint64 value) {
|
||||
device_description_->memory_bandwidth_ = value;
|
||||
}
|
||||
|
||||
void set_shared_memory_per_core(int64 value) {
|
||||
device_description_->shared_memory_per_core_ = value;
|
||||
}
|
||||
void set_shared_memory_per_block(int64 value) {
|
||||
device_description_->shared_memory_per_block_ = value;
|
||||
}
|
||||
|
||||
void set_clock_rate_ghz(float value) {
|
||||
device_description_->clock_rate_ghz_ = value;
|
||||
}
|
||||
|
||||
void set_cuda_compute_capability(int major, int minor) {
|
||||
device_description_->cuda_compute_capability_major_ = major;
|
||||
device_description_->cuda_compute_capability_minor_ = minor;
|
||||
}
|
||||
|
||||
void set_numa_node(int value) { device_description_->numa_node_ = value; }
|
||||
void set_core_count(int value) { device_description_->core_count_ = value; }
|
||||
void set_ecc_enabled(bool value) {
|
||||
device_description_->ecc_enabled_ = value;
|
||||
}
|
||||
|
||||
// Returns a built DeviceDescription with ownership transferred to the
|
||||
// caller. There are currently no restrictions on which fields must be set in
|
||||
// order to build the descriptor.
|
||||
//
|
||||
// Once the description is built, this builder object should be discarded.
|
||||
std::unique_ptr<DeviceDescription> Build() {
|
||||
return std::move(device_description_);
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<DeviceDescription> device_description_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(DeviceDescriptionBuilder);
|
||||
};
|
||||
|
||||
} // namespace internal
|
||||
|
||||
// Returns whether the given thread_dim is acceptable given the limits described
|
||||
// in device_description. For detailed reasons for failing the predicate, enable
|
||||
// VLOG(2) for this module.
|
||||
bool ThreadDimOk(const DeviceDescription &device_description,
|
||||
const ThreadDim &thread_dim);
|
||||
|
||||
// Equivalent to ceil(double(element_count) / threads_per_block).
|
||||
ABSL_DEPRECATED("Use MathUtil::CeilOfRatio directly instead.")
|
||||
uint64 DivideCeil(uint64 x, uint64 y);
|
||||
|
||||
// Calculate the number of threads/blocks required to process element_count
|
||||
// elements. Note that you can still end up with more threads than
|
||||
// element_count due to rounding, so kernels often start with an "is this
|
||||
// thread id in the element_count range?" test.
|
||||
void CalculateDimensionality(const DeviceDescription &device_description,
|
||||
uint64 element_count, uint64 *threads_per_block,
|
||||
uint64 *block_count);
|
||||
|
||||
// Compute and return maximum blocks per core (occupancy) based on the
|
||||
// device description, some kernel characteristics and the number of threads per
|
||||
// block. If unable to compute occupancy, zero is returned.
|
||||
uint64 CalculateOccupancy(const DeviceDescription &device_description,
|
||||
uint64 registers_per_thread,
|
||||
uint64 shared_memory_per_block,
|
||||
const ThreadDim &thread_dims);
|
||||
|
||||
// Compute and return the maximum number of registers per thread which
|
||||
// achieves the target occupancy. If the target is not possible then
|
||||
// zero is returned.
|
||||
uint64 CalculateRegisterLimitForTargetOccupancy(
|
||||
const DeviceDescription &device_description, uint64 shared_memory_per_block,
|
||||
const ThreadDim &thread_dims, uint64 target_blocks_per_core);
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_DEVICE_DESCRIPTION_H_
|
||||
@@ -0,0 +1,295 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Suite of types that represent device memory allocations. These are
|
||||
// allocated by the StreamExecutor interface, which produces values appropriate
|
||||
// for the underlying platform (whether it be CUDA or OpenCL).
|
||||
//
|
||||
// The untyped base class (like a device void*) is DeviceMemoryBase, which can
|
||||
// be specialized for a given allocation type (like a device T*) using
|
||||
// DeviceMemory<T>.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_DEVICE_MEMORY_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_DEVICE_MEMORY_H_
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
#include "tensorflow/stream_executor/lib/casts.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace perftools {
|
||||
namespace gputools {
|
||||
|
||||
// Temporarily pull stream_executor into perftools::gputools while we migrate
|
||||
// code to the new namespace. TODO(b/77980417): Remove this once we've
|
||||
// completed the migration.
|
||||
using namespace stream_executor; // NOLINT[build/namespaces]
|
||||
|
||||
} // namespace gputools
|
||||
} // namespace perftools
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class StreamExecutor;
|
||||
|
||||
// void*-analogous device memory allocation. For the typed variation, see
|
||||
// DeviceMemory<T>.
|
||||
//
|
||||
// This is effectively a two-tuple of a pointer and size; however, note that the
|
||||
// pointer may not be to the virtual address itself -- in OpenCL the pointer is
|
||||
// to a cl_mem handle that describes the device allocation. Therefore,
|
||||
// DeviceMemoryBase::opaque does not necessarily produce a pointer that can be
|
||||
// referenced directly, so use it with caution.
|
||||
//
|
||||
// Thread-compatible.
|
||||
class DeviceMemoryBase {
|
||||
public:
|
||||
// Default constructor instantiates a null-pointed, zero-sized device memory
|
||||
// region. An opaque pointer may be provided -- see header for details on the
|
||||
// opacity of that pointer.
|
||||
explicit DeviceMemoryBase(void *opaque = nullptr, uint64 size = 0,
|
||||
bool is_sub_buffer = false)
|
||||
: opaque_(opaque), size_(size), is_sub_buffer_(is_sub_buffer) {}
|
||||
|
||||
// Returns whether the backing memory is the null pointer.
|
||||
// A `== nullptr` convenience method is also provided.
|
||||
bool is_null() const { return opaque_ == nullptr; }
|
||||
bool operator==(std::nullptr_t other) const { return is_null(); }
|
||||
bool operator!=(std::nullptr_t other) const { return !is_null(); }
|
||||
|
||||
// Provides a partial order between device memory values.
|
||||
//
|
||||
// This operator is provided so that this object can be used as a key in an
|
||||
// ordered map.
|
||||
bool operator<(const DeviceMemoryBase &other) const {
|
||||
return opaque() < other.opaque();
|
||||
}
|
||||
|
||||
// Returns the size, in bytes, for the backing memory.
|
||||
uint64 size() const { return size_; }
|
||||
|
||||
// Warning: note that the pointer returned is not necessarily directly to
|
||||
// device virtual address space, but is platform-dependent.
|
||||
void *opaque() { return opaque_; }
|
||||
const void *opaque() const { return opaque_; }
|
||||
|
||||
// Returns true if this is an offset into another primary allocation.
|
||||
bool is_sub_buffer() const { return is_sub_buffer_; }
|
||||
|
||||
// Returns whether the two DeviceMemoryBase segments are identical (both in
|
||||
// their opaque pointer and size).
|
||||
bool IsSameAs(const DeviceMemoryBase &other) const {
|
||||
return opaque() == other.opaque() && size() == other.size();
|
||||
}
|
||||
|
||||
protected:
|
||||
friend class StreamExecutor;
|
||||
|
||||
// Resets the internal values of the opaque pointer and number of bytes in the
|
||||
// memory region, just as in the constructor.
|
||||
void Reset(void *opaque, uint64 bytes) {
|
||||
opaque_ = opaque;
|
||||
size_ = bytes;
|
||||
}
|
||||
|
||||
private:
|
||||
void *opaque_; // Platform-dependent value representing allocated memory.
|
||||
uint64 size_; // Size in bytes of this allocation.
|
||||
bool is_sub_buffer_; // Is this a primary allocation or a sub-buffer?
|
||||
};
|
||||
|
||||
// Typed wrapper around "void *"-like DeviceMemoryBase.
|
||||
//
|
||||
// For example, DeviceMemory<int> is a simple wrapper around DeviceMemoryBase
|
||||
// that represents one or more integers in Device memory.
|
||||
//
|
||||
// Thread-compatible.
|
||||
template <typename ElemT>
|
||||
class DeviceMemory final : public DeviceMemoryBase {
|
||||
public:
|
||||
// Default constructor instantiates a null-pointed, zero-sized memory region.
|
||||
DeviceMemory() : DeviceMemoryBase(nullptr, 0) {}
|
||||
DeviceMemory(std::nullptr_t) : DeviceMemory() {}
|
||||
|
||||
// Typed device memory regions may be constructed from untyped device memory
|
||||
// regions, this effectively amounts to a cast from a void*.
|
||||
explicit DeviceMemory(const DeviceMemoryBase &other)
|
||||
: DeviceMemoryBase(const_cast<DeviceMemoryBase &>(other).opaque(),
|
||||
other.size(), other.is_sub_buffer()) {}
|
||||
|
||||
// Returns the number of elements of type ElemT that constitute this
|
||||
// allocation.
|
||||
uint64 ElementCount() const { return size() / sizeof(ElemT); }
|
||||
|
||||
// Returns whether this is a single-element allocation.
|
||||
bool IsScalar() const { return ElementCount() == 1; }
|
||||
|
||||
// Create a typed area of DeviceMemory with a given opaque pointer and the
|
||||
// quantity of bytes in the allocation. This function is broken out to
|
||||
// distinguish bytes from an element count.
|
||||
static DeviceMemory<ElemT> MakeFromByteSize(void *opaque, uint64 bytes) {
|
||||
return DeviceMemory<ElemT>(opaque, bytes);
|
||||
}
|
||||
|
||||
// Resets the DeviceMemory data, in MakeFromByteSize fashion.
|
||||
// This simply clobbers the prior values.
|
||||
void ResetFromByteSize(void *opaque, uint64 bytes) {
|
||||
// TODO(leary) when NVCC is eliminated we can add this check (and the
|
||||
// logging include it requires).
|
||||
// CHECK_EQ(0, bytes % sizeof(ElemT));
|
||||
DeviceMemoryBase::Reset(opaque, bytes);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
|
||||
protected:
|
||||
// This constructor is solely used from derived classes; it is made protected
|
||||
// because it accepts a byte-size instead of an element count, which could
|
||||
// potentially be misused given the ElementCount() nature of this interface.
|
||||
//
|
||||
// In order to specify the desire to use byte size instead of element count
|
||||
// explicitly, use MakeFromByteSize.
|
||||
DeviceMemory(void *opaque, uint64 size) : DeviceMemoryBase(opaque, size) {}
|
||||
};
|
||||
|
||||
// A class to encapsulate the type and size of a dynamic shared memory
|
||||
// buffer. Because the buffer exists solely on the device and is not copyable
|
||||
// to the host, memory objects of this type do not maintain buffer pointers
|
||||
// on the host.
|
||||
template <typename ElemT>
|
||||
class SharedDeviceMemory final : public DeviceMemoryBase {
|
||||
public:
|
||||
explicit SharedDeviceMemory(uint64 elem_count)
|
||||
: DeviceMemoryBase(nullptr, elem_count * kElemSize) {}
|
||||
|
||||
static constexpr size_t kElemSize = sizeof(ElemT);
|
||||
|
||||
// Returns the number of elements of type ElemT that constitute this
|
||||
// allocation.
|
||||
uint64 ElementCount() const { return size() / kElemSize; }
|
||||
|
||||
// Returns whether this is a single-element allocation.
|
||||
bool IsScalar() const { return ElementCount() == 1; }
|
||||
};
|
||||
|
||||
// Similar to the typed DeviceMemory, but is the unique owner of its
|
||||
// memory, if any. ScopedDeviceMemory is thread-compatible. It is also
|
||||
// movable and uncopyable to represent unique ownership.
|
||||
template <typename ElemT>
|
||||
class ScopedDeviceMemory {
|
||||
public:
|
||||
// Default construction initializes the internal state to nullptr. This
|
||||
// mirrors the std::unique_ptr<> functionality, where default construction
|
||||
// produces a nullptr unique_ptr, which can be assigned later.
|
||||
ScopedDeviceMemory();
|
||||
|
||||
// Parameters:
|
||||
// parent: Executor used to deallocate memory when this instance goes
|
||||
// out of scope.
|
||||
// value: Already-allocated device memory value for this scoped mechanism to
|
||||
// deallocate. This memory must have been allocated by parent.
|
||||
ScopedDeviceMemory(StreamExecutor *parent, DeviceMemoryBase value);
|
||||
|
||||
// Constructor overload that places a literal array into device memory
|
||||
ScopedDeviceMemory(StreamExecutor *parent,
|
||||
std::initializer_list<ElemT> values);
|
||||
|
||||
// Moves ownership of the memory from other to the constructed
|
||||
// object.
|
||||
//
|
||||
// Postcondition: other == nullptr.
|
||||
ScopedDeviceMemory(ScopedDeviceMemory &&other) noexcept:
|
||||
ScopedDeviceMemory(other.parent_, other.Release()) {}
|
||||
|
||||
// Releases the memory that was provided in the constructor, through the
|
||||
// "parent" StreamExecutor.
|
||||
~ScopedDeviceMemory();
|
||||
|
||||
// Moves ownership of the memory from other to this object.
|
||||
//
|
||||
// Postcondition: other == nullptr.
|
||||
ScopedDeviceMemory& operator=(ScopedDeviceMemory &&other) {
|
||||
Reset(other.Release());
|
||||
parent_ = other.parent_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
// Returns the memory that backs this scoped allocation converted to
|
||||
// DeviceMemory<T> apparent type. This is useful for cases where the
|
||||
// DeviceMemory must be passed by const-ref, as the ScopedDeviceMemory doesn't
|
||||
// allow copying, for scoped-object-lifetime reasons.
|
||||
const DeviceMemory<ElemT> &cref() const { return wrapped_; }
|
||||
|
||||
// Returns a pointer to the DeviceMemory<T> apparent type for use in mutable
|
||||
// operations. The value returned should not be used outside the scope of this
|
||||
// ScopedDeviceMemory object's lifetime.
|
||||
DeviceMemory<ElemT> *ptr() { return &wrapped_; }
|
||||
const DeviceMemory<ElemT> *ptr() const { return &wrapped_; }
|
||||
|
||||
// Smart-pointer-like operators for the wrapped DeviceMemory.
|
||||
// This reference must not be used outside the lifetime of this
|
||||
// ScopedDeviceMemory.
|
||||
const DeviceMemory<ElemT> &operator*() const { return cref(); }
|
||||
DeviceMemory<ElemT> *operator->() { return ptr(); }
|
||||
const DeviceMemory<ElemT> *operator->() const { return ptr(); }
|
||||
bool operator==(std::nullptr_t other) const { return wrapped_.is_null(); }
|
||||
bool operator!=(std::nullptr_t other) const { return !wrapped_.is_null(); }
|
||||
|
||||
// Analogous to std::unique_ptr::reset, frees the existing memory held in
|
||||
// this scoped memory container and replaces it with updated. Ownership
|
||||
// of updated is transferred to this object.
|
||||
void Reset(DeviceMemory<ElemT> updated);
|
||||
void Reset(std::nullptr_t);
|
||||
|
||||
// Analogous to std::unique_ptr::release, releases ownership of the held
|
||||
// memory and transfers it to the caller.
|
||||
//
|
||||
// Postcondition: *this == nullptr
|
||||
DeviceMemory<ElemT> Release() {
|
||||
auto tmp = wrapped_;
|
||||
wrapped_.ResetFromByteSize(nullptr, 0);
|
||||
return tmp;
|
||||
}
|
||||
|
||||
private:
|
||||
DeviceMemory<ElemT> wrapped_; // Value we wrap with scoped-release.
|
||||
StreamExecutor *parent_; // See constructor.
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(ScopedDeviceMemory);
|
||||
};
|
||||
|
||||
// Host-side representation of packed-and-aligned vector datatypes on the device
|
||||
// side. Since these can appear in device kernel signatures, we support
|
||||
// launching them with these datatypes in launch signatures.
|
||||
|
||||
struct Float2 {
|
||||
float x, y;
|
||||
};
|
||||
|
||||
struct Float4 {
|
||||
Float2 xz, yw;
|
||||
};
|
||||
|
||||
struct Double2 {
|
||||
double x, y;
|
||||
};
|
||||
|
||||
static_assert(sizeof(Float2) == 2 * sizeof(float), "Float2 must be packed");
|
||||
static_assert(sizeof(Float4) == 4 * sizeof(float), "Float4 must be packed");
|
||||
static_assert(sizeof(Double2) == 2 * sizeof(double), "Double2 must be packed");
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_DEVICE_MEMORY_H_
|
||||
@@ -0,0 +1,88 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Contains device-level options that can be specified at a platform level.
|
||||
// Example usage:
|
||||
// auto device_options = DeviceOptions::Default();
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_DEVICE_OPTIONS_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_DEVICE_OPTIONS_H_
|
||||
|
||||
#include <map>
|
||||
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/platform/logging.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
// Indicates a set of options for a device's usage, which generally must be
|
||||
// provided at StreamExecutor device-initialization time.
|
||||
//
|
||||
// These are intended to be useful-but-not-mandatorily-supported options for
|
||||
// using devices on the underlying platform. Presently, if the option requested
|
||||
// is not available on the target platform, a warning will be emitted.
|
||||
struct DeviceOptions {
|
||||
public:
|
||||
// When it is observed that more memory has to be allocated for thread stacks,
|
||||
// this flag prevents it from ever being deallocated. Potentially saves
|
||||
// thrashing the thread stack memory allocation, but at the potential cost of
|
||||
// some memory space.
|
||||
static const unsigned kDoNotReclaimStackAllocation = 0x1;
|
||||
|
||||
// The following options refer to synchronization options when
|
||||
// using SynchronizeStream or SynchronizeContext.
|
||||
|
||||
// Synchronize with spinlocks.
|
||||
static const unsigned kScheduleSpin = 0x02;
|
||||
// Synchronize with spinlocks that also call CPU yield instructions.
|
||||
static const unsigned kScheduleYield = 0x04;
|
||||
// Synchronize with a "synchronization primitive" (e.g. mutex).
|
||||
static const unsigned kScheduleBlockingSync = 0x08;
|
||||
|
||||
static const unsigned kMask = 0xf; // Mask of all available flags.
|
||||
|
||||
// Constructs an or-d together set of device options.
|
||||
explicit DeviceOptions(unsigned flags) : flags_(flags) {
|
||||
CHECK((flags & kMask) == flags);
|
||||
}
|
||||
|
||||
// Factory for the default set of device options.
|
||||
static DeviceOptions Default() { return DeviceOptions(0); }
|
||||
|
||||
unsigned flags() const { return flags_; }
|
||||
|
||||
bool operator==(const DeviceOptions& other) const {
|
||||
return flags_ == other.flags_;
|
||||
}
|
||||
|
||||
bool operator!=(const DeviceOptions& other) const {
|
||||
return !(*this == other);
|
||||
}
|
||||
|
||||
string ToString() {
|
||||
return flags_ == 0 ? "none" : "kDoNotReclaimStackAllocation";
|
||||
}
|
||||
|
||||
// Platform-specific device options. Expressed as key-value pairs to avoid
|
||||
// DeviceOptions subclass proliferation.
|
||||
std::map<string, string> non_portable_tags;
|
||||
|
||||
private:
|
||||
unsigned flags_;
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_DEVICE_OPTIONS_H_
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,118 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Common DSO loading functionality: exposes callables that dlopen DSOs
|
||||
// in either the runfiles directories
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_DSO_LOADER_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_DSO_LOADER_H_
|
||||
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/lib/stringpiece.h"
|
||||
#include "tensorflow/stream_executor/platform.h"
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace internal {
|
||||
|
||||
// Permits StreamExecutor code to dynamically load a pre-determined set of
|
||||
// relevant DSOs via dlopen.
|
||||
//
|
||||
// Thread-safe.
|
||||
class DsoLoader {
|
||||
public:
|
||||
// The following methods either load the DSO of interest and return a dlopen
|
||||
// handle or error status in the canonical namespace.
|
||||
|
||||
static port::Status GetCublasDsoHandle(void** dso_handle);
|
||||
static port::Status GetCudnnDsoHandle(void** dso_handle);
|
||||
static port::Status GetCufftDsoHandle(void** dso_handle);
|
||||
static port::Status GetCurandDsoHandle(void** dso_handle);
|
||||
static port::Status GetLibcudaDsoHandle(void** dso_handle);
|
||||
static port::Status GetLibcuptiDsoHandle(void** dso_handle);
|
||||
|
||||
// Registers a new binary-relative path to use as a dlopen search path.
|
||||
static void RegisterRpath(port::StringPiece path);
|
||||
|
||||
private:
|
||||
// Registered rpaths (singleton vector) and a mutex that guards it.
|
||||
static std::vector<string>* GetRpaths();
|
||||
|
||||
// Descriptive boolean wrapper to indicate whether symbols are made available
|
||||
// to resolve in later-loaded libraries.
|
||||
enum class LoadKind { kLocal, kGlobal };
|
||||
|
||||
// Loads a DSO from the given "path" (which can technically be any dlopen-able
|
||||
// name). If the load kind is global, the symbols in the loaded DSO are
|
||||
// visible to subsequent DSO loading operations.
|
||||
static port::Status GetDsoHandle(port::StringPiece path, void** dso_handle,
|
||||
LoadKind load_kind = LoadKind::kLocal);
|
||||
|
||||
|
||||
// Returns the binary directory (or binary path) associated with the currently
|
||||
// executing program. If strip_executable_name is true, the executable file is
|
||||
// stripped off of the path.
|
||||
static string GetBinaryDirectory(bool strip_executable_name);
|
||||
|
||||
// Invokes realpath on the original path; updates candidate and returns true
|
||||
// if it succeeds (i.e. a file exists at the path); otherwise, returns false.
|
||||
static bool TrySymbolicDereference(string* candidate);
|
||||
|
||||
// Attempts to find a path to the DSO of interest, otherwise returns the
|
||||
// bare library name:
|
||||
// Arguments:
|
||||
// library_name: the filename in tree; e.g. libOpenCL.so.1.0.0
|
||||
// runfiles_relpath: where to look for the library relative to the runfiles
|
||||
// root; e.g. third_party/gpus/cuda/lib64
|
||||
static string FindDsoPath(port::StringPiece library_name,
|
||||
port::StringPiece runfiles_relpath);
|
||||
|
||||
// Return platform dependent paths for DSOs
|
||||
static string GetCudaLibraryDirPath();
|
||||
static string GetCudaDriverLibraryPath();
|
||||
static string GetCudaCuptiLibraryPath();
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(DsoLoader);
|
||||
};
|
||||
|
||||
// Wrapper around the DsoLoader that prevents us from dlopen'ing any of the DSOs
|
||||
// more than once.
|
||||
class CachedDsoLoader {
|
||||
public:
|
||||
// Cached versions of the corresponding DsoLoader methods above.
|
||||
static port::StatusOr<void*> GetCublasDsoHandle();
|
||||
static port::StatusOr<void*> GetCudnnDsoHandle();
|
||||
static port::StatusOr<void*> GetCufftDsoHandle();
|
||||
static port::StatusOr<void*> GetCurandDsoHandle();
|
||||
static port::StatusOr<void*> GetLibcudaDsoHandle();
|
||||
static port::StatusOr<void*> GetLibcuptiDsoHandle();
|
||||
|
||||
private:
|
||||
// Fetches a DSO handle via "load_dso" and returns the StatusOr form of the
|
||||
// result.
|
||||
static port::StatusOr<void*> FetchHandleResult(
|
||||
std::function<port::Status(void**)> load_dso);
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CachedDsoLoader);
|
||||
};
|
||||
|
||||
} // namespace internal
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_DSO_LOADER_H_
|
||||
@@ -0,0 +1,83 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_EVENT_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_EVENT_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
namespace internal {
|
||||
class EventInterface;
|
||||
}
|
||||
|
||||
class Stream;
|
||||
class StreamExecutor;
|
||||
|
||||
// The Event class, when supported by a platform, enables low-overhead status
|
||||
// reporting for a Stream. An Event is inserted at a location in a stream via
|
||||
// the Stream::ThenRecordEvent() API. From then on, the Event's status can be
|
||||
// monitored via the nonblocking Event::PollForStatus() call.
|
||||
class Event {
|
||||
public:
|
||||
// Potential states for an Event. If PollForStatus() returns anything aside
|
||||
// from kPending or kComplete, an error has occurred; kUnknown is a bad state.
|
||||
// Not all implementations are able to return all enumeration values. Refer to
|
||||
// the platform-specific implementation for details.
|
||||
enum class Status {
|
||||
kUnknown,
|
||||
kError,
|
||||
kPending,
|
||||
kComplete,
|
||||
};
|
||||
|
||||
explicit Event(StreamExecutor* stream_exec); // NOLINT
|
||||
|
||||
// Releases any resources held by the Event object.
|
||||
~Event();
|
||||
|
||||
// Performs any platform-specific or potentially error-generating
|
||||
// initialization.
|
||||
bool Init();
|
||||
|
||||
// Returns the current Status for the event.
|
||||
Status PollForStatus();
|
||||
|
||||
// Returns a pointer to the underlying platform-specific implementation.
|
||||
internal::EventInterface* implementation() { return implementation_.get(); }
|
||||
|
||||
Event(Event&&) = default;
|
||||
Event& operator=(Event&&) = default;
|
||||
|
||||
private:
|
||||
friend class Stream;
|
||||
|
||||
// Pointer to the StreamExecutor interface used to create this object.
|
||||
// Not owned.
|
||||
StreamExecutor* stream_exec_;
|
||||
|
||||
// Pointer to the platform-specific EventInterface implementation underlying
|
||||
// the object. Owned.
|
||||
std::unique_ptr<internal::EventInterface> implementation_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(Event);
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_EVENT_H_
|
||||
@@ -0,0 +1,80 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_EXECUTOR_CACHE_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_EXECUTOR_CACHE_H_
|
||||
|
||||
#include <functional>
|
||||
#include <map>
|
||||
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/stream_executor_pimpl.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
// Utility class to allow Platform objects to manage cached StreamExecutors.
|
||||
// Thread-safe.
|
||||
class ExecutorCache {
|
||||
public:
|
||||
ExecutorCache() {}
|
||||
|
||||
// Looks up 'config' in the cache. Returns a pointer to the existing executor,
|
||||
// if already present, or creates it using 'factory', if it does not.
|
||||
// Factories may be executed concurrently for different device ordinals.
|
||||
typedef port::StatusOr<std::unique_ptr<StreamExecutor>> ExecutorFactory();
|
||||
port::StatusOr<StreamExecutor*> GetOrCreate(
|
||||
const StreamExecutorConfig& config,
|
||||
const std::function<ExecutorFactory>& factory);
|
||||
|
||||
// Returns a pointer to the described executor (if one with a matching config
|
||||
// has been created), or a NOT_FOUND status.
|
||||
port::StatusOr<StreamExecutor*> Get(const StreamExecutorConfig& config);
|
||||
|
||||
// Destroys all Executors and clears the cache.
|
||||
// Performs no synchronization with the executors - undefined behavior may
|
||||
// occur if any executors are active!
|
||||
void DestroyAllExecutors();
|
||||
|
||||
private:
|
||||
// Each Entry contains zero or more cached executors for a device ordinal.
|
||||
struct Entry {
|
||||
~Entry();
|
||||
|
||||
// Mutex that locks the contents of each entry. The 'mutex_' of the
|
||||
// ExecutorCache class protects both the 'cache_' and the existence of each
|
||||
// Entry, but not the Entry's contents. 'configurations_mutex' protects the
|
||||
// contents of the entry after 'mutex_' has been dropped.
|
||||
mutex configurations_mutex;
|
||||
|
||||
// Vector of cached {config, executor} pairs.
|
||||
std::vector<
|
||||
std::pair<StreamExecutorConfig, std::unique_ptr<StreamExecutor>>>
|
||||
configurations GUARDED_BY(configurations_mutex);
|
||||
};
|
||||
|
||||
// Maps ordinal number to a list of cached executors for that ordinal.
|
||||
// We key off of ordinal (instead of just looking up all fields in the
|
||||
// StreamExecutorConfig) for a slight improvement in lookup time.
|
||||
mutex mutex_;
|
||||
std::map<int, Entry> cache_ GUARDED_BY(mutex_);
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(ExecutorCache);
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_EXECUTOR_CACHE_H_
|
||||
@@ -0,0 +1,269 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Exposes the family of FFT routines as pre-canned high performance calls for
|
||||
// use in conjunction with the StreamExecutor abstraction.
|
||||
//
|
||||
// Note that this interface is optionally supported by platforms; see
|
||||
// StreamExecutor::SupportsFft() for details.
|
||||
//
|
||||
// This abstraction makes it simple to entrain FFT operations on GPU data into
|
||||
// a Stream -- users typically will not use this API directly, but will use the
|
||||
// Stream builder methods to entrain these operations "under the hood". For
|
||||
// example:
|
||||
//
|
||||
// DeviceMemory<std::complex<float>> x =
|
||||
// stream_exec->AllocateArray<std::complex<float>>(1024);
|
||||
// DeviceMemory<std::complex<float>> y =
|
||||
// stream_exec->AllocateArray<std::complex<float>>(1024);
|
||||
// // ... populate x and y ...
|
||||
// Stream stream{stream_exec};
|
||||
// std::unique_ptr<Plan> plan =
|
||||
// stream_exec.AsFft()->Create1dPlan(&stream, 1024, Type::kC2CForward);
|
||||
// stream
|
||||
// .Init()
|
||||
// .ThenFft(plan.get(), x, &y);
|
||||
// SE_CHECK_OK(stream.BlockHostUntilDone());
|
||||
//
|
||||
// By using stream operations in this manner the user can easily intermix custom
|
||||
// kernel launches (via StreamExecutor::ThenLaunch()) with these pre-canned FFT
|
||||
// routines.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_FFT_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_FFT_H_
|
||||
|
||||
#include <complex>
|
||||
#include <memory>
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class Stream;
|
||||
template <typename ElemT>
|
||||
class DeviceMemory;
|
||||
class ScratchAllocator;
|
||||
|
||||
namespace fft {
|
||||
|
||||
// Specifies FFT input and output types, and the direction.
|
||||
// R, D, C, and Z stand for SP real, DP real, SP complex, and DP complex.
|
||||
enum class Type {
|
||||
kInvalid,
|
||||
kC2CForward,
|
||||
kC2CInverse,
|
||||
kC2R,
|
||||
kR2C,
|
||||
kZ2ZForward,
|
||||
kZ2ZInverse,
|
||||
kZ2D,
|
||||
kD2Z
|
||||
};
|
||||
|
||||
// FFT plan class. Each FFT implementation should define a plan class that is
|
||||
// derived from this class. It does not provide any interface but serves
|
||||
// as a common type that is used to execute the plan.
|
||||
class Plan {
|
||||
public:
|
||||
virtual ~Plan() {}
|
||||
};
|
||||
|
||||
// FFT support interface -- this can be derived from a GPU executor when the
|
||||
// underlying platform has an FFT library implementation available. See
|
||||
// StreamExecutor::AsFft().
|
||||
//
|
||||
// This support interface is not generally thread-safe; it is only thread-safe
|
||||
// for the CUDA platform (cuFFT) usage; host side FFT support is known
|
||||
// thread-compatible, but not thread-safe.
|
||||
class FftSupport {
|
||||
public:
|
||||
virtual ~FftSupport() {}
|
||||
|
||||
// Creates a 1d FFT plan.
|
||||
virtual std::unique_ptr<Plan> Create1dPlan(Stream *stream, uint64 num_x,
|
||||
Type type, bool in_place_fft) = 0;
|
||||
|
||||
// Creates a 2d FFT plan.
|
||||
virtual std::unique_ptr<Plan> Create2dPlan(Stream *stream, uint64 num_x,
|
||||
uint64 num_y, Type type,
|
||||
bool in_place_fft) = 0;
|
||||
|
||||
// Creates a 3d FFT plan.
|
||||
virtual std::unique_ptr<Plan> Create3dPlan(Stream *stream, uint64 num_x,
|
||||
uint64 num_y, uint64 num_z,
|
||||
Type type, bool in_place_fft) = 0;
|
||||
|
||||
// Creates a 1d FFT plan with scratch allocator.
|
||||
virtual std::unique_ptr<Plan> Create1dPlanWithScratchAllocator(
|
||||
Stream *stream, uint64 num_x, Type type, bool in_place_fft,
|
||||
ScratchAllocator *scratch_allocator) = 0;
|
||||
|
||||
// Creates a 2d FFT plan with scratch allocator.
|
||||
virtual std::unique_ptr<Plan> Create2dPlanWithScratchAllocator(
|
||||
Stream *stream, uint64 num_x, uint64 num_y, Type type, bool in_place_fft,
|
||||
ScratchAllocator *scratch_allocator) = 0;
|
||||
|
||||
// Creates a 3d FFT plan with scratch allocator.
|
||||
virtual std::unique_ptr<Plan> Create3dPlanWithScratchAllocator(
|
||||
Stream *stream, uint64 num_x, uint64 num_y, uint64 num_z, Type type,
|
||||
bool in_place_fft, ScratchAllocator *scratch_allocator) = 0;
|
||||
|
||||
// Creates a batched FFT plan.
|
||||
//
|
||||
// stream: The GPU stream in which the FFT runs.
|
||||
// rank: Dimensionality of the transform (1, 2, or 3).
|
||||
// elem_count: Array of size rank, describing the size of each dimension.
|
||||
// input_embed, output_embed:
|
||||
// Pointer of size rank that indicates the storage dimensions
|
||||
// of the input/output data in memory. If set to null_ptr all
|
||||
// other advanced data layout parameters are ignored.
|
||||
// input_stride: Indicates the distance (number of elements; same below)
|
||||
// between two successive input elements.
|
||||
// input_distance: Indicates the distance between the first element of two
|
||||
// consecutive signals in a batch of the input data.
|
||||
// output_stride: Indicates the distance between two successive output
|
||||
// elements.
|
||||
// output_distance: Indicates the distance between the first element of two
|
||||
// consecutive signals in a batch of the output data.
|
||||
virtual std::unique_ptr<Plan> CreateBatchedPlan(
|
||||
Stream *stream, int rank, uint64 *elem_count, uint64 *input_embed,
|
||||
uint64 input_stride, uint64 input_distance, uint64 *output_embed,
|
||||
uint64 output_stride, uint64 output_distance, Type type,
|
||||
bool in_place_fft, int batch_count) = 0;
|
||||
|
||||
// Creates a batched FFT plan with scratch allocator.
|
||||
//
|
||||
// stream: The GPU stream in which the FFT runs.
|
||||
// rank: Dimensionality of the transform (1, 2, or 3).
|
||||
// elem_count: Array of size rank, describing the size of each dimension.
|
||||
// input_embed, output_embed:
|
||||
// Pointer of size rank that indicates the storage dimensions
|
||||
// of the input/output data in memory. If set to null_ptr all
|
||||
// other advanced data layout parameters are ignored.
|
||||
// input_stride: Indicates the distance (number of elements; same below)
|
||||
// between two successive input elements.
|
||||
// input_distance: Indicates the distance between the first element of two
|
||||
// consecutive signals in a batch of the input data.
|
||||
// output_stride: Indicates the distance between two successive output
|
||||
// elements.
|
||||
// output_distance: Indicates the distance between the first element of two
|
||||
// consecutive signals in a batch of the output data.
|
||||
virtual std::unique_ptr<Plan> CreateBatchedPlanWithScratchAllocator(
|
||||
Stream *stream, int rank, uint64 *elem_count, uint64 *input_embed,
|
||||
uint64 input_stride, uint64 input_distance, uint64 *output_embed,
|
||||
uint64 output_stride, uint64 output_distance, Type type,
|
||||
bool in_place_fft, int batch_count,
|
||||
ScratchAllocator *scratch_allocator) = 0;
|
||||
|
||||
// Updates the plan's work area with space allocated by a new scratch
|
||||
// allocator. This facilitates plan reuse with scratch allocators.
|
||||
//
|
||||
// This requires that the plan was originally created using a scratch
|
||||
// allocator, as otherwise scratch space will have been allocated internally
|
||||
// by cuFFT.
|
||||
virtual void UpdatePlanWithScratchAllocator(
|
||||
Stream *stream, Plan *plan, ScratchAllocator *scratch_allocator) = 0;
|
||||
|
||||
// Computes complex-to-complex FFT in the transform direction as specified
|
||||
// by direction parameter.
|
||||
virtual bool DoFft(Stream *stream, Plan *plan,
|
||||
const DeviceMemory<std::complex<float>> &input,
|
||||
DeviceMemory<std::complex<float>> *output) = 0;
|
||||
virtual bool DoFft(Stream *stream, Plan *plan,
|
||||
const DeviceMemory<std::complex<double>> &input,
|
||||
DeviceMemory<std::complex<double>> *output) = 0;
|
||||
|
||||
// Computes real-to-complex FFT in forward direction.
|
||||
virtual bool DoFft(Stream *stream, Plan *plan,
|
||||
const DeviceMemory<float> &input,
|
||||
DeviceMemory<std::complex<float>> *output) = 0;
|
||||
virtual bool DoFft(Stream *stream, Plan *plan,
|
||||
const DeviceMemory<double> &input,
|
||||
DeviceMemory<std::complex<double>> *output) = 0;
|
||||
|
||||
// Computes complex-to-real FFT in inverse direction.
|
||||
virtual bool DoFft(Stream *stream, Plan *plan,
|
||||
const DeviceMemory<std::complex<float>> &input,
|
||||
DeviceMemory<float> *output) = 0;
|
||||
virtual bool DoFft(Stream *stream, Plan *plan,
|
||||
const DeviceMemory<std::complex<double>> &input,
|
||||
DeviceMemory<double> *output) = 0;
|
||||
|
||||
protected:
|
||||
FftSupport() {}
|
||||
|
||||
private:
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(FftSupport);
|
||||
};
|
||||
|
||||
// Macro used to quickly declare overrides for abstract virtuals in the
|
||||
// fft::FftSupport base class. Assumes that it's emitted somewhere inside the
|
||||
// ::stream_executor namespace.
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_GPU_FFT_SUPPORT_OVERRIDES \
|
||||
std::unique_ptr<fft::Plan> Create1dPlan(Stream *stream, uint64 num_x, \
|
||||
fft::Type type, bool in_place_fft) \
|
||||
override; \
|
||||
std::unique_ptr<fft::Plan> Create2dPlan(Stream *stream, uint64 num_x, \
|
||||
uint64 num_y, fft::Type type, \
|
||||
bool in_place_fft) override; \
|
||||
std::unique_ptr<fft::Plan> Create3dPlan( \
|
||||
Stream *stream, uint64 num_x, uint64 num_y, uint64 num_z, \
|
||||
fft::Type type, bool in_place_fft) override; \
|
||||
std::unique_ptr<fft::Plan> Create1dPlanWithScratchAllocator( \
|
||||
Stream *stream, uint64 num_x, fft::Type type, bool in_place_fft, \
|
||||
ScratchAllocator *scratch_allocator) override; \
|
||||
std::unique_ptr<fft::Plan> Create2dPlanWithScratchAllocator( \
|
||||
Stream *stream, uint64 num_x, uint64 num_y, fft::Type type, \
|
||||
bool in_place_fft, ScratchAllocator *scratch_allocator) override; \
|
||||
std::unique_ptr<fft::Plan> Create3dPlanWithScratchAllocator( \
|
||||
Stream *stream, uint64 num_x, uint64 num_y, uint64 num_z, \
|
||||
fft::Type type, bool in_place_fft, ScratchAllocator *scratch_allocator) \
|
||||
override; \
|
||||
std::unique_ptr<fft::Plan> CreateBatchedPlan( \
|
||||
Stream *stream, int rank, uint64 *elem_count, uint64 *input_embed, \
|
||||
uint64 input_stride, uint64 input_distance, uint64 *output_embed, \
|
||||
uint64 output_stride, uint64 output_distance, fft::Type type, \
|
||||
bool in_place_fft, int batch_count) override; \
|
||||
std::unique_ptr<fft::Plan> CreateBatchedPlanWithScratchAllocator( \
|
||||
Stream *stream, int rank, uint64 *elem_count, uint64 *input_embed, \
|
||||
uint64 input_stride, uint64 input_distance, uint64 *output_embed, \
|
||||
uint64 output_stride, uint64 output_distance, fft::Type type, \
|
||||
bool in_place_fft, int batch_count, ScratchAllocator *scratch_allocator) \
|
||||
override; \
|
||||
void UpdatePlanWithScratchAllocator(Stream *stream, fft::Plan *plan, \
|
||||
ScratchAllocator *scratch_allocator) \
|
||||
override; \
|
||||
bool DoFft(Stream *stream, fft::Plan *plan, \
|
||||
const DeviceMemory<std::complex<float>> &input, \
|
||||
DeviceMemory<std::complex<float>> *output) override; \
|
||||
bool DoFft(Stream *stream, fft::Plan *plan, \
|
||||
const DeviceMemory<std::complex<double>> &input, \
|
||||
DeviceMemory<std::complex<double>> *output) override; \
|
||||
bool DoFft(Stream *stream, fft::Plan *plan, \
|
||||
const DeviceMemory<float> &input, \
|
||||
DeviceMemory<std::complex<float>> *output) override; \
|
||||
bool DoFft(Stream *stream, fft::Plan *plan, \
|
||||
const DeviceMemory<double> &input, \
|
||||
DeviceMemory<std::complex<double>> *output) override; \
|
||||
bool DoFft(Stream *stream, fft::Plan *plan, \
|
||||
const DeviceMemory<std::complex<float>> &input, \
|
||||
DeviceMemory<float> *output) override; \
|
||||
bool DoFft(Stream *stream, fft::Plan *plan, \
|
||||
const DeviceMemory<std::complex<double>> &input, \
|
||||
DeviceMemory<double> *output) override;
|
||||
|
||||
} // namespace fft
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_FFT_H_
|
||||
@@ -0,0 +1,23 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_GPU_LAUNCH_DIM_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_GPU_LAUNCH_DIM_H_
|
||||
|
||||
// TODO(rspringer): Temporary redirection until all users - including gcudacc -
|
||||
// are using the new file.
|
||||
#include "tensorflow/stream_executor/launch_dim.h"
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_GPU_LAUNCH_DIM_H_
|
||||
@@ -0,0 +1,214 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Declares the HostExecutor class, which is a CPU-only implementation of
|
||||
// the StreamExecutor interface. For now, this is used for testing and to
|
||||
// examine the performance of host-based StreamExecutor code.
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_GPU_EXECUTOR_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_GPU_EXECUTOR_H_
|
||||
|
||||
#include "tensorflow/stream_executor/blas.h"
|
||||
#include "tensorflow/stream_executor/host/host_stream.h"
|
||||
#include "tensorflow/stream_executor/host/host_timer.h"
|
||||
#include "tensorflow/stream_executor/lib/error.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/rng.h"
|
||||
#include "tensorflow/stream_executor/stream_executor.h"
|
||||
#include "tensorflow/stream_executor/stream_executor_internal.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace host {
|
||||
|
||||
// An implementation of StreamExecutor that does no communication or interaction
|
||||
// with a device, but DOES perform memory operations backed by the host.
|
||||
// Plugin routines (RNG, BLAS) are also supported and functional.
|
||||
// Kernel invocations will fail, but host callbacks may be enqueued on this
|
||||
// executor and its associated stream, and should follow standard ordering
|
||||
// semantics.
|
||||
//
|
||||
// This is useful for evaluating the performance of host-based or fallback
|
||||
// routines executed under the context of a GPU executor.
|
||||
// See stream_executor.h for description of the below operations.
|
||||
class HostExecutor : public internal::StreamExecutorInterface {
|
||||
public:
|
||||
explicit HostExecutor(const PluginConfig &plugin_config);
|
||||
~HostExecutor() override;
|
||||
|
||||
port::Status Init(int device_ordinal, DeviceOptions device_options) override {
|
||||
return port::Status::OK();
|
||||
}
|
||||
|
||||
bool GetKernel(const MultiKernelLoaderSpec &spec,
|
||||
KernelBase *kernel) override {
|
||||
return false;
|
||||
}
|
||||
bool Launch(Stream *stream, const ThreadDim &thread_dims,
|
||||
const BlockDim &block_dims, const KernelBase &kernel,
|
||||
const KernelArgsArrayBase &args) override {
|
||||
return false;
|
||||
}
|
||||
|
||||
void *Allocate(uint64 size) override;
|
||||
void *AllocateSubBuffer(DeviceMemoryBase *mem, uint64 offset_bytes,
|
||||
uint64 size_bytes) override;
|
||||
void Deallocate(DeviceMemoryBase *mem) override;
|
||||
|
||||
void *HostMemoryAllocate(uint64 size) override { return new char[size]; }
|
||||
void HostMemoryDeallocate(void *mem) override {
|
||||
delete[] static_cast<char *>(mem);
|
||||
}
|
||||
bool HostMemoryRegister(void *mem, uint64 size) override { return true; }
|
||||
bool HostMemoryUnregister(void *mem) override { return true; }
|
||||
|
||||
bool Memcpy(Stream *stream, void *host_dst, const DeviceMemoryBase &gpu_src,
|
||||
uint64 size) override;
|
||||
bool Memcpy(Stream *stream, DeviceMemoryBase *gpu_dst, const void *host_src,
|
||||
uint64 size) override;
|
||||
bool MemcpyDeviceToDevice(Stream *stream, DeviceMemoryBase *gpu_dst,
|
||||
const DeviceMemoryBase &host_src,
|
||||
uint64 size) override;
|
||||
|
||||
bool MemZero(Stream *stream, DeviceMemoryBase *location,
|
||||
uint64 size) override;
|
||||
bool Memset(Stream *stream, DeviceMemoryBase *location, uint8 pattern,
|
||||
uint64 size) override;
|
||||
bool Memset32(Stream *stream, DeviceMemoryBase *location, uint32 pattern,
|
||||
uint64 size) override;
|
||||
|
||||
// No "synchronize all activity" implemented for this platform at the moment.
|
||||
bool SynchronizeAllActivity() override { return true; }
|
||||
bool SynchronousMemZero(DeviceMemoryBase *location, uint64 size) override;
|
||||
|
||||
bool SynchronousMemSet(DeviceMemoryBase *location, int value,
|
||||
uint64 size) override;
|
||||
|
||||
port::Status SynchronousMemcpy(DeviceMemoryBase *gpu_dst,
|
||||
const void *host_src, uint64 size) override;
|
||||
port::Status SynchronousMemcpy(void *host_dst,
|
||||
const DeviceMemoryBase &gpu_src,
|
||||
uint64 size) override;
|
||||
port::Status SynchronousMemcpyDeviceToDevice(DeviceMemoryBase *gpu_dst,
|
||||
const DeviceMemoryBase &gpu_src,
|
||||
uint64 size) override;
|
||||
|
||||
bool HostCallback(Stream *stream, std::function<void()> callback) override;
|
||||
|
||||
port::Status AllocateEvent(Event *event) override {
|
||||
return port::Status(port::error::UNIMPLEMENTED, "");
|
||||
}
|
||||
|
||||
port::Status DeallocateEvent(Event *event) override {
|
||||
return port::Status(port::error::UNIMPLEMENTED, "");
|
||||
}
|
||||
|
||||
port::Status RecordEvent(Stream *stream, Event *event) override {
|
||||
return port::Status(port::error::UNIMPLEMENTED, "");
|
||||
}
|
||||
|
||||
port::Status WaitForEvent(Stream *stream, Event *event) override {
|
||||
return port::Status(port::error::UNIMPLEMENTED, "");
|
||||
}
|
||||
|
||||
Event::Status PollForEventStatus(Event *event) override {
|
||||
return Event::Status::kError;
|
||||
}
|
||||
|
||||
bool AllocateStream(Stream *stream) override;
|
||||
void DeallocateStream(Stream *stream) override;
|
||||
bool CreateStreamDependency(Stream *dependent, Stream *other) override;
|
||||
|
||||
// No special initialization is necessary for host timers.
|
||||
bool AllocateTimer(Timer *timer) override { return true; }
|
||||
|
||||
void DeallocateTimer(Timer *timer) override {}
|
||||
|
||||
bool StartTimer(Stream *stream, Timer *timer) override;
|
||||
|
||||
bool StopTimer(Stream *stream, Timer *timer) override;
|
||||
|
||||
port::Status BlockHostUntilDone(Stream *stream) override;
|
||||
|
||||
int PlatformDeviceCount() override { return 1; }
|
||||
|
||||
bool DeviceMemoryUsage(int64 *free, int64 *total) const override {
|
||||
return false;
|
||||
}
|
||||
|
||||
DeviceDescription *PopulateDeviceDescription() const override;
|
||||
|
||||
port::Status EnablePeerAccessTo(StreamExecutorInterface *other) override {
|
||||
return port::Status::OK();
|
||||
}
|
||||
|
||||
bool CanEnablePeerAccessTo(StreamExecutorInterface *other) override {
|
||||
return true;
|
||||
}
|
||||
|
||||
SharedMemoryConfig GetDeviceSharedMemoryConfig() override {
|
||||
LOG(INFO) << "Shared memory configuration is unsupported for host "
|
||||
<< "executors.";
|
||||
return SharedMemoryConfig::kDefault;
|
||||
}
|
||||
|
||||
port::Status SetDeviceSharedMemoryConfig(SharedMemoryConfig config) override {
|
||||
string error_msg{
|
||||
"Shared memory configuration is unsupported for host "
|
||||
"executors."};
|
||||
LOG(INFO) << error_msg;
|
||||
return port::Status(port::error::UNIMPLEMENTED, error_msg);
|
||||
}
|
||||
|
||||
bool SupportsBlas() const override;
|
||||
blas::BlasSupport *CreateBlas() override;
|
||||
|
||||
bool SupportsDnn() const override { return false; }
|
||||
dnn::DnnSupport *CreateDnn() override { return nullptr; }
|
||||
|
||||
bool SupportsFft() const override;
|
||||
fft::FftSupport *CreateFft() override;
|
||||
|
||||
bool SupportsRng() const override;
|
||||
rng::RngSupport *CreateRng() override;
|
||||
|
||||
std::unique_ptr<internal::EventInterface> CreateEventImplementation()
|
||||
override {
|
||||
LOG(WARNING) << "Events not currently supported by HostExecutor.";
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::unique_ptr<internal::KernelInterface> CreateKernelImplementation()
|
||||
override {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::unique_ptr<internal::StreamInterface> GetStreamImplementation()
|
||||
override {
|
||||
return std::unique_ptr<internal::StreamInterface>(new HostStream());
|
||||
}
|
||||
|
||||
std::unique_ptr<internal::TimerInterface> GetTimerImplementation() override {
|
||||
return std::unique_ptr<internal::TimerInterface>(new HostTimer());
|
||||
}
|
||||
|
||||
void *GpuContextHack() override { return nullptr; }
|
||||
|
||||
private:
|
||||
const PluginConfig plugin_config_;
|
||||
};
|
||||
|
||||
} // namespace host
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_GPU_EXECUTOR_H_
|
||||
@@ -0,0 +1,83 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Declares the "host" platform, which is a CPU-only implementation of the
|
||||
// StreamExecutor. The host platform only supports memory operations and plugin
|
||||
// routines, and is primarily used for testing.
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_PLATFORM_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_PLATFORM_H_
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/stream_executor/executor_cache.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/multi_platform_manager.h"
|
||||
#include "tensorflow/stream_executor/platform.h"
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/platform/thread_annotations.h"
|
||||
#include "tensorflow/stream_executor/stream_executor_pimpl.h"
|
||||
#include "tensorflow/stream_executor/trace_listener.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace host {
|
||||
|
||||
// Host (CPU) platform plugin, registered as a singleton value via module
|
||||
// initializer.
|
||||
class HostPlatform : public Platform {
|
||||
public:
|
||||
HostPlatform();
|
||||
~HostPlatform() override;
|
||||
|
||||
Platform::Id id() const override;
|
||||
|
||||
// Device count is less clear-cut for CPUs than accelerators. This call
|
||||
// currently returns the number of thread units in the host, as reported by
|
||||
// base::NumCPUs().
|
||||
int VisibleDeviceCount() const override;
|
||||
|
||||
const string& Name() const override;
|
||||
|
||||
port::StatusOr<StreamExecutor*> ExecutorForDevice(int ordinal) override;
|
||||
|
||||
port::StatusOr<StreamExecutor*> ExecutorForDeviceWithPluginConfig(
|
||||
int ordinal, const PluginConfig& config) override;
|
||||
|
||||
port::StatusOr<StreamExecutor*> GetExecutor(
|
||||
const StreamExecutorConfig& config) override;
|
||||
|
||||
port::StatusOr<std::unique_ptr<StreamExecutor>> GetUncachedExecutor(
|
||||
const StreamExecutorConfig& config) override;
|
||||
|
||||
void RegisterTraceListener(std::unique_ptr<TraceListener> listener) override;
|
||||
|
||||
void UnregisterTraceListener(TraceListener* listener) override;
|
||||
|
||||
private:
|
||||
// This platform's name.
|
||||
string name_;
|
||||
|
||||
// Cache of created StreamExecutors.
|
||||
ExecutorCache executor_cache_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(HostPlatform);
|
||||
};
|
||||
|
||||
} // namespace host
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_PLATFORM_H_
|
||||
@@ -0,0 +1,34 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_PLATFORM_ID_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_PLATFORM_ID_H_
|
||||
|
||||
#include "tensorflow/stream_executor/platform.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace host {
|
||||
|
||||
// Opaque and unique identifier for the host platform.
|
||||
// This is needed so that plugins can refer to/identify this platform without
|
||||
// instantiating a HostPlatform object.
|
||||
// This is broken out here to avoid a circular dependency between HostPlatform
|
||||
// and HostStreamExecutor.
|
||||
extern const Platform::Id kHostPlatformId;
|
||||
|
||||
} // namespace host
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_PLATFORM_ID_H_
|
||||
@@ -0,0 +1,56 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Class declaration for Stream type that enqueues tasks onto a host/CPU-based
|
||||
// execution context (as opposed to a GPU device), HostExecutor.
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_STREAM_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_STREAM_H_
|
||||
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
|
||||
#include "tensorflow/stream_executor/lib/threadpool.h"
|
||||
#include "tensorflow/stream_executor/stream_executor_internal.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace host {
|
||||
|
||||
class HostStream : public internal::StreamInterface {
|
||||
public:
|
||||
HostStream();
|
||||
~HostStream() override;
|
||||
|
||||
bool EnqueueTask(std::function<void()> task);
|
||||
|
||||
void *GpuStreamHack() override { return nullptr; }
|
||||
void **GpuStreamMemberHack() override { return nullptr; }
|
||||
|
||||
void BlockUntilDone();
|
||||
|
||||
private:
|
||||
// Use only one thread and own task queue to preserve FIFO ordering
|
||||
// for the operations enqueued by any given stream.
|
||||
static const int kExecutorThreads = 1;
|
||||
std::unique_ptr<port::ThreadPool> host_executor_;
|
||||
|
||||
mutex mu_;
|
||||
int pending_tasks_ GUARDED_BY(mu_) = 0;
|
||||
condition_variable completion_condition_;
|
||||
};
|
||||
|
||||
} // namespace host
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_STREAM_H_
|
||||
@@ -0,0 +1,61 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_TIMER_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_TIMER_H_
|
||||
|
||||
#include <chrono>
|
||||
|
||||
#include "tensorflow/stream_executor/stream_executor_internal.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace host {
|
||||
|
||||
class HostTimer : public internal::TimerInterface {
|
||||
public:
|
||||
HostTimer() {}
|
||||
~HostTimer() override {}
|
||||
|
||||
// Begins the timer at the present point in the stream.
|
||||
bool Start(Stream *stream);
|
||||
|
||||
// Stops the timer at the present point in the stream.
|
||||
bool Stop(Stream *stream);
|
||||
|
||||
// Returns the most recent value recorded for a start/stopcycle, in
|
||||
// microseconds.
|
||||
uint64 Microseconds() const override;
|
||||
|
||||
// Returns the most recent value recorded for a start/stopcycle, in
|
||||
// nanoseconds.
|
||||
uint64 Nanoseconds() const override;
|
||||
|
||||
private:
|
||||
using clock = std::chrono::high_resolution_clock;
|
||||
|
||||
clock::time_point start_time_;
|
||||
clock::duration duration_;
|
||||
|
||||
// Actually starts (rather than enqueues starting) the timer.
|
||||
void StartNow();
|
||||
|
||||
// Actually stops (rather than enqueues stopping) the timer.
|
||||
void StopNow();
|
||||
};
|
||||
|
||||
} // namespace host
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_TIMER_H_
|
||||
@@ -0,0 +1,46 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_HOST_BUFFER_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_HOST_BUFFER_H_
|
||||
|
||||
#include "tensorflow/stream_executor/dnn.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
// A HostBuffer is a block of memory in host memory containing the data for a
|
||||
// dnn::BatchDescriptor using a device-dependent memory layout.
|
||||
// Derived classes provide methods to construct a HostBuffer for a specific
|
||||
// device, and to copy data in and out of the buffer.
|
||||
class HostBuffer {
|
||||
public:
|
||||
const dnn::BatchDescriptor& descriptor() const { return descriptor_; }
|
||||
|
||||
// Returns a string describing the HostBuffer.
|
||||
virtual string AsString() const = 0;
|
||||
|
||||
protected:
|
||||
// Construct a HostBuffer from the supplied dnn::BatchDescriptor.
|
||||
explicit HostBuffer(const dnn::BatchDescriptor& descriptor)
|
||||
: descriptor_(descriptor) {}
|
||||
virtual ~HostBuffer() {}
|
||||
|
||||
private:
|
||||
const dnn::BatchDescriptor descriptor_;
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_HOST_BUFFER_H_
|
||||
@@ -0,0 +1,56 @@
|
||||
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_HOST_OR_DEVICE_SCALAR_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_HOST_OR_DEVICE_SCALAR_H_
|
||||
|
||||
#include "tensorflow/stream_executor/device_memory.h"
|
||||
#include "tensorflow/stream_executor/platform/logging.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
// Allows to represent a value that is either a host scalar or a scalar stored
|
||||
// on the GPU device.
|
||||
template <typename ElemT>
|
||||
class HostOrDeviceScalar {
|
||||
public:
|
||||
// Not marked as explicit because when using this constructor, we usually want
|
||||
// to set this to a compile-time constant.
|
||||
HostOrDeviceScalar(ElemT value) : value_(value), is_pointer_(false) {}
|
||||
explicit HostOrDeviceScalar(const DeviceMemory<ElemT>& pointer)
|
||||
: pointer_(pointer), is_pointer_(true) {
|
||||
CHECK_EQ(1, pointer.ElementCount());
|
||||
}
|
||||
|
||||
bool is_pointer() const { return is_pointer_; }
|
||||
const DeviceMemory<ElemT>& pointer() const {
|
||||
CHECK(is_pointer());
|
||||
return pointer_;
|
||||
}
|
||||
const ElemT& value() const {
|
||||
CHECK(!is_pointer());
|
||||
return value_;
|
||||
}
|
||||
|
||||
private:
|
||||
union {
|
||||
ElemT value_;
|
||||
DeviceMemory<ElemT> pointer_;
|
||||
};
|
||||
bool is_pointer_;
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_HOST_OR_DEVICE_SCALAR_H_
|
||||
@@ -0,0 +1,715 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Suite of datatypes to represent data-parallel kernel objects (code entities).
|
||||
// Kernel is the untyped variant, whereas TypedKernel takes a type signature
|
||||
// to do some template-based helper generation and give compile-time type
|
||||
// checking for kernel launch parameters.
|
||||
//
|
||||
// Users typically don't see KernelBase, they see typed kernels, analogous to a
|
||||
// typed function pointer. TypedKernels express their argument types via
|
||||
// template parameters like so:
|
||||
//
|
||||
// TypedKernel<DeviceMemory<int>*, int>
|
||||
//
|
||||
// Which expresses a data parallel kernel signature for:
|
||||
//
|
||||
// void(int*, int);
|
||||
//
|
||||
// And for a const memory region:
|
||||
//
|
||||
// TypedKernel<const DeviceMemory<int>&, int>
|
||||
//
|
||||
// Corresponds to a data parallel kernel signature for:
|
||||
//
|
||||
// void(const int*, int)
|
||||
//
|
||||
// Note that kernels always have a void return type, so results typically must
|
||||
// be memcpy'ied from device memory to the host.
|
||||
//
|
||||
// Also note that a scalar integer residing in device memory and an array of
|
||||
// integers residing in device memory have the same signature: DeviceMemory<T>.
|
||||
// However, in the future, checks may be added for additional safety that arrays
|
||||
// of minimum sizes are passed when those minimum sizes are contractually
|
||||
// expected by the kernel.
|
||||
//
|
||||
// For user-defined types whose definitions are appropriately shared between the
|
||||
// host code doing the launching and the kernel code being launched, the user
|
||||
// defined types are similarly permitted to be expressed as residing in device
|
||||
// memory:
|
||||
//
|
||||
// TypedKernel<DeviceMemory<MyUserDefinedStructure>>
|
||||
//
|
||||
// And, when the alignment and padding are agreed upon, POD types will also be
|
||||
// able to be passed by value; for example, it is a common idiom to specify a
|
||||
// bunch of options simultaneously with a structure:
|
||||
//
|
||||
// TypedKernel<MyOptionsStructurePassedByValue, DeviceMemory<float>>
|
||||
//
|
||||
// Which corresponds to a data parallel kernel signature like:
|
||||
//
|
||||
// void(MyOptionsStructurePassedByValue value, float *result);
|
||||
//
|
||||
// Users typically won't need to type out the TypedKernel signature in full, it
|
||||
// will be typedef'd by automatically generated code; for example, see
|
||||
// stream_executor::executor_sample::VecReduceAddKernel.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_KERNEL_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_KERNEL_H_
|
||||
|
||||
#include <array>
|
||||
#include <memory>
|
||||
#include <tuple>
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/stream_executor/device_memory.h"
|
||||
#include "tensorflow/stream_executor/kernel_cache_config.h"
|
||||
#include "tensorflow/stream_executor/lib/array_slice.h"
|
||||
#include "tensorflow/stream_executor/lib/inlined_vector.h"
|
||||
#include "tensorflow/stream_executor/lib/stringpiece.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class DeviceMemoryBase;
|
||||
template <typename ElemT>
|
||||
class DeviceMemory;
|
||||
class StreamExecutor;
|
||||
|
||||
namespace internal {
|
||||
class KernelInterface;
|
||||
} // namespace internal
|
||||
|
||||
// KernelMetadata holds runtime-queryable attributes of a loaded kernel, such as
|
||||
// registers allocated, shared memory used, etc.
|
||||
// Not all platforms support reporting of all information, so each accessor
|
||||
// returns false if the associated field is not populated in the underlying
|
||||
// platform.
|
||||
class KernelMetadata {
|
||||
public:
|
||||
KernelMetadata()
|
||||
: has_registers_per_thread_(false), has_shared_memory_bytes_(false) {}
|
||||
|
||||
// Returns the number of registers used per thread executing this kernel.
|
||||
bool registers_per_thread(int *registers_per_thread) const;
|
||||
|
||||
// Sets the number of registers used per thread executing this kernel.
|
||||
void set_registers_per_thread(int registers_per_thread);
|
||||
|
||||
// Returns the amount of [static] shared memory used per block executing this
|
||||
// kernel. Note that dynamic shared memory allocations are not (and can not)
|
||||
// be reported here (since they're not specified until kernel launch time).
|
||||
bool shared_memory_bytes(int *shared_memory_bytes) const;
|
||||
|
||||
// Sets the amount of [static] shared memory used per block executing this
|
||||
// kernel.
|
||||
void set_shared_memory_bytes(int shared_memory_bytes);
|
||||
|
||||
private:
|
||||
// Holds the value returned by registers_per_thread above.
|
||||
bool has_registers_per_thread_;
|
||||
int registers_per_thread_;
|
||||
|
||||
// Holds the value returned by shared_memory_bytes above.
|
||||
bool has_shared_memory_bytes_;
|
||||
int64 shared_memory_bytes_;
|
||||
};
|
||||
|
||||
// A data-parallel kernel (code entity) for launching via the StreamExecutor,
|
||||
// analogous to a void* device function pointer. See TypedKernel for the typed
|
||||
// variant.
|
||||
//
|
||||
// Thread-compatible.
|
||||
class KernelBase {
|
||||
public:
|
||||
KernelBase(KernelBase &&from);
|
||||
|
||||
// Constructs an "empty" (not-yet-loaded) kernel instance.
|
||||
//
|
||||
// parent is the StreamExecutor that will be responsible for loading the
|
||||
// implementation of this kernel. It must not be null.
|
||||
explicit KernelBase(StreamExecutor *parent);
|
||||
|
||||
// Test-only constructor that can take a mock KernelInterface implementation.
|
||||
KernelBase(StreamExecutor *parent, internal::KernelInterface *implementation);
|
||||
|
||||
// Releases resources associated with the kernel instance (i.e.
|
||||
// platform-specific implementation).
|
||||
~KernelBase();
|
||||
|
||||
// Returns the number of parameters that this kernel accepts. (Arity refers to
|
||||
// nullary, unary, ...).
|
||||
unsigned Arity() const;
|
||||
|
||||
// Returns the StreamExecutor that represents the platform this kernel
|
||||
// executes upon.
|
||||
StreamExecutor *parent() const { return parent_; }
|
||||
|
||||
// Returns a const pointer to the (opaque) platform-dependent implementation.
|
||||
const internal::KernelInterface *implementation() const {
|
||||
return implementation_.get();
|
||||
}
|
||||
|
||||
// Returns a non-const pointer to the (opaque) platform-dependent
|
||||
// implementation.
|
||||
internal::KernelInterface *implementation() { return implementation_.get(); }
|
||||
|
||||
void set_metadata(const KernelMetadata &metadata) { metadata_ = metadata; }
|
||||
|
||||
const KernelMetadata &metadata() const { return metadata_; }
|
||||
|
||||
// Sets the preferred cache configuration for a kernel. This is just a
|
||||
// suggestion to the runtime, and may not be honored during execution.
|
||||
void SetPreferredCacheConfig(KernelCacheConfig config);
|
||||
|
||||
// Gets the preferred cache configuration for a kernel.
|
||||
KernelCacheConfig GetPreferredCacheConfig() const;
|
||||
|
||||
void set_name(port::StringPiece name);
|
||||
const string &name() const { return name_; }
|
||||
const string &demangled_name() const { return demangled_name_; }
|
||||
|
||||
private:
|
||||
// The StreamExecutor that loads this kernel object.
|
||||
StreamExecutor *parent_;
|
||||
|
||||
// Implementation delegated to for platform-specific functionality.
|
||||
std::unique_ptr<internal::KernelInterface> implementation_;
|
||||
|
||||
string name_;
|
||||
string demangled_name_;
|
||||
|
||||
KernelMetadata metadata_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(KernelBase);
|
||||
};
|
||||
|
||||
// Whether T is a DeviceMemory-family pointer.
|
||||
template <typename T>
|
||||
struct IsDeviceMemoryPointer {
|
||||
static constexpr bool value = false;
|
||||
};
|
||||
|
||||
template <typename U>
|
||||
struct IsDeviceMemoryPointer<DeviceMemory<U> *> {
|
||||
static constexpr bool value = true;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct IsDeviceMemoryPointer<DeviceMemoryBase *> {
|
||||
static constexpr bool value = true;
|
||||
};
|
||||
|
||||
// Whether T is a DeviceMemory-family value-like thing (which includes a
|
||||
// reference). This trait is useful because we pack values in the same manner as
|
||||
// references.
|
||||
template <typename T>
|
||||
struct IsDeviceMemoryValueLike {
|
||||
static constexpr bool value = false;
|
||||
};
|
||||
|
||||
template <typename U>
|
||||
struct IsDeviceMemoryValueLike<DeviceMemory<U> &> {
|
||||
static constexpr bool value = true;
|
||||
};
|
||||
|
||||
// We need to treat SharedDeviceMemory types differently than other DeviceMemory
|
||||
// types (since they maintain no allocations), hence these specializations.
|
||||
template <typename U>
|
||||
struct IsDeviceMemoryValueLike<SharedDeviceMemory<U> &> {
|
||||
static constexpr bool value = false;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct IsDeviceMemoryValueLike<DeviceMemoryBase &> {
|
||||
static constexpr bool value = true;
|
||||
};
|
||||
|
||||
template <typename U>
|
||||
struct IsDeviceMemoryValueLike<DeviceMemory<U>> {
|
||||
static constexpr bool value = true;
|
||||
};
|
||||
|
||||
template <typename U>
|
||||
struct IsDeviceMemoryValueLike<SharedDeviceMemory<U>> {
|
||||
static constexpr bool value = false;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct IsDeviceMemoryValueLike<DeviceMemoryBase> {
|
||||
static constexpr bool value = true;
|
||||
};
|
||||
|
||||
template <typename U>
|
||||
struct IsSharedDeviceMemory {
|
||||
static constexpr bool value = false;
|
||||
};
|
||||
|
||||
template <typename U>
|
||||
struct IsSharedDeviceMemory<SharedDeviceMemory<U> &> {
|
||||
static constexpr bool value = true;
|
||||
};
|
||||
|
||||
template <typename U>
|
||||
struct IsSharedDeviceMemory<SharedDeviceMemory<U>> {
|
||||
static constexpr bool value = true;
|
||||
};
|
||||
|
||||
// Basic data about a kernel argument.
|
||||
struct KernelArg {
|
||||
bool is_shared;
|
||||
const void *address;
|
||||
size_t size;
|
||||
};
|
||||
|
||||
// An iterator for traversing all the arguments of a KernelArgsArray.
|
||||
class KernelArgIterator {
|
||||
public:
|
||||
KernelArgIterator(int number_of_argument_addresses,
|
||||
int number_of_shared_memory_arguments,
|
||||
const void *const *arg_addresses_data,
|
||||
const size_t *arg_sizes_data,
|
||||
const size_t *shmem_bytes_data,
|
||||
const size_t *shmem_indices_data)
|
||||
: arg_index_(0),
|
||||
number_of_arguments_(number_of_argument_addresses +
|
||||
number_of_shared_memory_arguments),
|
||||
arg_address_iter_(arg_addresses_data),
|
||||
arg_size_iter_(arg_sizes_data),
|
||||
shmem_bytes_iter_(shmem_bytes_data),
|
||||
shmem_indices_iter_(shmem_indices_data),
|
||||
shmem_indices_end_(shmem_indices_data +
|
||||
number_of_shared_memory_arguments) {}
|
||||
|
||||
// Returns true if another argument is present in the iterator.
|
||||
bool has_next() { return arg_index_ < number_of_arguments_; }
|
||||
|
||||
// Returns the next argument in the iterator.
|
||||
//
|
||||
// Returns a default-constructed KernelArg if there is no next argument.
|
||||
KernelArg next() {
|
||||
KernelArg result = {};
|
||||
if (!has_next()) {
|
||||
return result;
|
||||
} else if ((shmem_indices_iter_ != shmem_indices_end_) &&
|
||||
(arg_index_ == *shmem_indices_iter_)) {
|
||||
result.is_shared = true;
|
||||
result.address = nullptr;
|
||||
result.size = *shmem_bytes_iter_;
|
||||
++shmem_indices_iter_;
|
||||
++shmem_bytes_iter_;
|
||||
} else {
|
||||
result.is_shared = false;
|
||||
result.address = *arg_address_iter_;
|
||||
result.size = *arg_size_iter_;
|
||||
++arg_address_iter_;
|
||||
++arg_size_iter_;
|
||||
}
|
||||
++arg_index_;
|
||||
return result;
|
||||
}
|
||||
|
||||
private:
|
||||
size_t arg_index_;
|
||||
size_t number_of_arguments_;
|
||||
const void *const *arg_address_iter_;
|
||||
const size_t *arg_size_iter_;
|
||||
const size_t *shmem_bytes_iter_;
|
||||
const size_t *shmem_indices_iter_;
|
||||
const size_t *const shmem_indices_end_;
|
||||
};
|
||||
|
||||
// Base class for KernelArgsArray.
|
||||
//
|
||||
// Supports all the getter methods that do not depend on the compile-time number
|
||||
// of arguments template parameter.
|
||||
//
|
||||
// This class exists as a way to pass kernel arguments to
|
||||
// StreamExecutorInterface::Launch. That Launch method is virtual, so it can't
|
||||
// be templated to accept any KernelArgsArray type, therefore a reference to
|
||||
// this base type is passed instead.
|
||||
//
|
||||
// Performance is not a concern here because each of these methods will be
|
||||
// called at most once per kernel launch. Past performance concerns with
|
||||
// KernelArgsArray have been in reference to the argument packing routines which
|
||||
// are called once per kernel argument. Those packing routines are now handled
|
||||
// by the templated KernelArgsArray subclass of this class where they can take
|
||||
// advantage of compile-time knowledge of the number of arguments in order to be
|
||||
// very efficient.
|
||||
class KernelArgsArrayBase {
|
||||
public:
|
||||
virtual ~KernelArgsArrayBase() = default;
|
||||
|
||||
// Gets the number of arguments added so far, including shared memory
|
||||
// arguments.
|
||||
virtual size_t number_of_arguments() const = 0;
|
||||
|
||||
// Gets the total number of shared memory bytes added so far.
|
||||
virtual uint64 number_of_shared_bytes() const = 0;
|
||||
|
||||
// Gets the list of argument addresses.
|
||||
virtual port::ArraySlice<const void *> argument_addresses() const = 0;
|
||||
|
||||
// Gets an iterator to the arguments in the array.
|
||||
virtual KernelArgIterator arg_iterator() const = 0;
|
||||
};
|
||||
|
||||
// A list of arguments for a kernel call.
|
||||
//
|
||||
// The template parameter kNumArgs is the maximum number of arguments which can
|
||||
// be stored in the list.
|
||||
//
|
||||
// Contains a list of addresses for non-shared-memory arguments and a list of
|
||||
// sizes for shared-memory arguments. Since the shared-memory arguments may be
|
||||
// interspersed with the non-shared-memory arguments, it also stores a list of
|
||||
// the indices at which the shared-memory arguments appeared.
|
||||
//
|
||||
// For example, if the argument address list contains {a, b, c, d, e}, the
|
||||
// shared-memory arguments list contains the sizes of {A, B, C}, and the
|
||||
// shared-memory indices list contains {0, 3, 5}, then the original list of
|
||||
// arguments was {A, a, b, B, c, C, d, e}.
|
||||
//
|
||||
// This way of storing the arguments makes CUDA kernel calls efficient because
|
||||
// they only require the argument address list and the total number of shared
|
||||
// bytes, but it also makes it possible for OpenCL kernel calls because they
|
||||
// depend on the location of each shared-memory argument and its size.
|
||||
//
|
||||
// Note that the code for adding arguments has been identified as a performance
|
||||
// hotspot in some real-world applications so this structure has been optimized
|
||||
// for the performance of argument adding.
|
||||
template <size_t kNumArgs>
|
||||
class KernelArgsArray : public KernelArgsArrayBase {
|
||||
public:
|
||||
explicit KernelArgsArray()
|
||||
: total_shared_memory_bytes_(0),
|
||||
number_of_argument_addresses_(0),
|
||||
number_of_shared_memory_arguments_(0) {}
|
||||
|
||||
// Adds an argument to the list.
|
||||
//
|
||||
// Note that the address of the argument is stored, so the input must not go
|
||||
// out of scope before the instance of this class that calls this method does.
|
||||
template <typename T>
|
||||
void add_argument(const T &arg) {
|
||||
argument_addresses_[number_of_argument_addresses_] =
|
||||
static_cast<const void *>(&arg);
|
||||
argument_sizes_[number_of_argument_addresses_] = sizeof(arg);
|
||||
++number_of_argument_addresses_;
|
||||
}
|
||||
|
||||
// Adds a device memory argument to the list.
|
||||
void add_device_memory_argument(const DeviceMemoryBase &arg) {
|
||||
const void **copy_ptr =
|
||||
&device_memory_opaque_pointers_[number_of_argument_addresses_];
|
||||
*copy_ptr = arg.opaque();
|
||||
argument_addresses_[number_of_argument_addresses_] = copy_ptr;
|
||||
argument_sizes_[number_of_argument_addresses_] = sizeof(void *);
|
||||
++number_of_argument_addresses_;
|
||||
}
|
||||
|
||||
// Adds a shared memory argument to the list.
|
||||
//
|
||||
// The only significant information about a shared argument is its size, so
|
||||
// that is the only parameter in this function.
|
||||
void add_shared_bytes(size_t number_of_bytes) {
|
||||
shared_memory_indices_[number_of_shared_memory_arguments_] =
|
||||
number_of_argument_addresses_ + number_of_shared_memory_arguments_;
|
||||
shared_memory_bytes_[number_of_shared_memory_arguments_] = number_of_bytes;
|
||||
++number_of_shared_memory_arguments_;
|
||||
total_shared_memory_bytes_ += number_of_bytes;
|
||||
}
|
||||
|
||||
// Gets the number of arguments added so far, including shared memory
|
||||
// arguments.
|
||||
size_t number_of_arguments() const override {
|
||||
return number_of_argument_addresses_ + number_of_shared_memory_arguments_;
|
||||
}
|
||||
|
||||
// Gets the total number of shared memory bytes added so far.
|
||||
uint64 number_of_shared_bytes() const override {
|
||||
return total_shared_memory_bytes_;
|
||||
}
|
||||
|
||||
// Gets the list of argument addresses.
|
||||
port::ArraySlice<const void *> argument_addresses() const override {
|
||||
return port::ArraySlice<const void *>(argument_addresses_.data(),
|
||||
number_of_argument_addresses_);
|
||||
}
|
||||
|
||||
// Gets an iterator to the arguments in the array.
|
||||
KernelArgIterator arg_iterator() const override {
|
||||
return KernelArgIterator(
|
||||
number_of_argument_addresses_, number_of_shared_memory_arguments_,
|
||||
argument_addresses_.data(), argument_sizes_.data(),
|
||||
shared_memory_bytes_.data(), shared_memory_indices_.data());
|
||||
}
|
||||
|
||||
private:
|
||||
// A place to store copies of opaque pointers from device memory arguments.
|
||||
std::array<const void *, kNumArgs> device_memory_opaque_pointers_;
|
||||
|
||||
// Addresses for non-shared-memory arguments.
|
||||
std::array<const void *, kNumArgs> argument_addresses_;
|
||||
|
||||
// Sizes for non-shared-memory arguments.
|
||||
std::array<size_t, kNumArgs> argument_sizes_;
|
||||
|
||||
// Size in bytes for each shared memory argument.
|
||||
std::array<size_t, kNumArgs> shared_memory_bytes_;
|
||||
|
||||
// Indices in the arguments array for shared memory arguments.
|
||||
std::array<size_t, kNumArgs> shared_memory_indices_;
|
||||
|
||||
// Total of all shared memory sizes.
|
||||
size_t total_shared_memory_bytes_;
|
||||
|
||||
// Number of significant entries in argument_addresses_ and argument_sizes_.
|
||||
size_t number_of_argument_addresses_;
|
||||
|
||||
// Number of significant entries in shared_memory_bytes_ and
|
||||
// shared_memory_indices_.
|
||||
size_t number_of_shared_memory_arguments_;
|
||||
};
|
||||
|
||||
// Typed variant of KernelBase, like a typed device function pointer. See the
|
||||
// file comment for details and example usage.
|
||||
//
|
||||
// This class contains template metaprogramming magic to type check the
|
||||
// parameters passed to a kernel launch are acceptable, and subsequently pack
|
||||
// them into a form which can be used by the StreamExecutorInterface
|
||||
// implementation. (i.e. CUDA and OpenCL both bind void*s with associated
|
||||
// sizes as kernel arguments.)
|
||||
//
|
||||
// Thread-compatible.
|
||||
template <typename... Params>
|
||||
class TypedKernel : public KernelBase {
|
||||
public:
|
||||
static constexpr size_t kNumberOfParameters = sizeof...(Params);
|
||||
|
||||
// Delegates to KernelBase::KernelBase(), see that constructor.
|
||||
explicit TypedKernel(StreamExecutor *parent) : KernelBase(parent) {}
|
||||
|
||||
// Test-only constructor that can take a mock KernelInterface implementation.
|
||||
// Takes ownership of implementation, it should not be null.
|
||||
TypedKernel(StreamExecutor *parent, internal::KernelInterface *implementation)
|
||||
: KernelBase(parent, implementation) {}
|
||||
|
||||
private:
|
||||
// Stream needs access to the specific parameter-packing functionality that
|
||||
// the TypedKernel provides for its corresponding type signature (and no other
|
||||
// type signatures).
|
||||
friend class Stream;
|
||||
|
||||
// This is the main entry point into the magic. Packs the parameters (which
|
||||
// must type check against the class template) into the args and sizes
|
||||
// arrays.
|
||||
//
|
||||
// Const refs are taken as parameters on all of the handlers to avoid
|
||||
// implicit type promotion of integers.
|
||||
//
|
||||
// WARNING: as a performance optimization this method may store pointers to
|
||||
// some of the input parameters in the kernel args structure, so any params
|
||||
// passed into this method must live at least as long as the kernel args
|
||||
// structure.
|
||||
void PackParams(KernelArgsArray<kNumberOfParameters> *args,
|
||||
Params &... params) const {
|
||||
PackOneParam(args, params...);
|
||||
}
|
||||
|
||||
template <typename T, typename... RestOfParams>
|
||||
void PackOneParam(KernelArgsArray<kNumberOfParameters> *args, const T &arg,
|
||||
const RestOfParams &... rest) const {
|
||||
PackOneParam(args, arg);
|
||||
PackOneParam(args, rest...);
|
||||
}
|
||||
|
||||
// Packs one (non-DeviceMemoryBase) parameter into the arg and sizes array.
|
||||
// The enable_if<> is for excluding DeviceMemoryBase args, which have a
|
||||
// separate implementation below.
|
||||
template <typename T>
|
||||
void PackOneParam(
|
||||
KernelArgsArray<kNumberOfParameters> *args, const T &arg,
|
||||
typename std::enable_if<!IsDeviceMemoryValueLike<T>::value &&
|
||||
!IsDeviceMemoryPointer<T>::value &&
|
||||
!IsSharedDeviceMemory<T>::value>::type * =
|
||||
nullptr) const {
|
||||
static_assert(!std::is_pointer<T>::value,
|
||||
"cannot pass raw pointer to the device");
|
||||
static_assert(!std::is_convertible<T, DeviceMemoryBase>::value,
|
||||
"cannot pass device memory as a normal value");
|
||||
args->add_argument(arg);
|
||||
}
|
||||
|
||||
// DeviceMemoryBase family reference override.
|
||||
template <typename T>
|
||||
void PackOneParam(
|
||||
KernelArgsArray<kNumberOfParameters> *args, const T &arg,
|
||||
typename std::enable_if<IsDeviceMemoryValueLike<T>::value>::type * =
|
||||
nullptr) const {
|
||||
args->add_device_memory_argument(arg);
|
||||
}
|
||||
|
||||
// DeviceMemoryBase family pointer override.
|
||||
template <typename T>
|
||||
void PackOneParam(
|
||||
KernelArgsArray<kNumberOfParameters> *args, T arg,
|
||||
typename std::enable_if<IsDeviceMemoryPointer<T>::value>::type * =
|
||||
nullptr) const {
|
||||
DeviceMemoryBase *ptr = static_cast<DeviceMemoryBase *>(arg);
|
||||
args->add_device_memory_argument(*ptr);
|
||||
}
|
||||
|
||||
// Dynamic shared device memory has a size, but no associated allocation on
|
||||
// the host; internally, the device will allocate storage.
|
||||
template <typename T>
|
||||
void PackOneParam(
|
||||
KernelArgsArray<kNumberOfParameters> *args, T arg,
|
||||
typename std::enable_if<IsSharedDeviceMemory<T>::value>::type * =
|
||||
nullptr) const {
|
||||
args->add_shared_bytes(arg.size());
|
||||
}
|
||||
|
||||
// Base case for variadic template expansion - nothing to do!
|
||||
void PackOneParam(KernelArgsArray<kNumberOfParameters> *args) const {}
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(TypedKernel);
|
||||
};
|
||||
|
||||
// Template metaprogramming helper type that helps us produce better error
|
||||
// messages at compile time when the are mismatches between the parameter
|
||||
// type list and the argument type list.
|
||||
template <typename ParamTuple, typename ArgTuple>
|
||||
struct KernelInvocationChecker {
|
||||
// Whether the parameter tuple and argument tuple match in length.
|
||||
static constexpr bool kLengthMatches =
|
||||
std::tuple_size<ParamTuple>::value == std::tuple_size<ArgTuple>::value;
|
||||
|
||||
// The (matching) length of the parameters and arguments type lists.
|
||||
static constexpr int kTupleLength =
|
||||
static_cast<int>(std::tuple_size<ArgTuple>::value);
|
||||
|
||||
// Helper trait to say whether the parameter wants a DeviceMemory-reference
|
||||
// compatible type. This is for inexact type matches, so that it doesn't have
|
||||
// to be precisely a const DeviceMemory<T>&, but can also be a value that
|
||||
// represents the same.
|
||||
template <typename ParamType, typename ArgType>
|
||||
struct IsCompatibleDeviceMemoryRef {
|
||||
static constexpr bool value = false;
|
||||
};
|
||||
|
||||
// See type trait definition above.
|
||||
template <typename U>
|
||||
struct IsCompatibleDeviceMemoryRef<const DeviceMemory<U> &, DeviceMemory<U>> {
|
||||
static constexpr bool value = true;
|
||||
};
|
||||
|
||||
// See type trait definition above.
|
||||
template <typename U>
|
||||
struct IsCompatibleDeviceMemoryRef<const SharedDeviceMemory<U> &,
|
||||
SharedDeviceMemory<U>> {
|
||||
static constexpr bool value = true;
|
||||
};
|
||||
|
||||
// Returns whether ParamT and ArgT are compatible for data parallel kernel
|
||||
// parameter packing without any assert functionality.
|
||||
template <typename ParamT, typename ArgT>
|
||||
static constexpr bool CompatibleNoAssert() {
|
||||
return std::is_same<typename std::remove_const<ParamT>::type,
|
||||
ArgT>::value ||
|
||||
IsCompatibleDeviceMemoryRef<ParamT, ArgT>::value;
|
||||
}
|
||||
|
||||
// Checks whether ParamT and ArgT are compatible for data parallel kernel
|
||||
// parameter packing. kArgumentNumber is unused, it just for error display.
|
||||
//
|
||||
// NOTE: if you encounter an error here, you can see the mismatch by looking
|
||||
// at the end of the last error message, which will be of the form:
|
||||
//
|
||||
// ...::Compatible<const stream_executor::DeviceMemory<OneThing> &,
|
||||
// stream_executor::DeviceMemory<AnotherThing>, true,
|
||||
// 0>'
|
||||
// requested here
|
||||
//
|
||||
// This means that the 0th argument you passed to the kernel invocation should
|
||||
// have been DeviceMemory<OneThing> but was observed to be
|
||||
// DeviceMemory<AnotherThing>.
|
||||
template <typename ParamT, typename ArgT, bool kShouldStaticAssert,
|
||||
int kArgumentNumber>
|
||||
static constexpr bool Compatible() {
|
||||
static_assert(
|
||||
kShouldStaticAssert ? CompatibleNoAssert<ParamT, ArgT>() : true,
|
||||
"parameter type (LHS) is not compatible with argument type (RHS)");
|
||||
return CompatibleNoAssert<ParamT, ArgT>();
|
||||
}
|
||||
|
||||
// Checks the parameter/argument match at kArgumentNumber for an out of bounds
|
||||
// argument number.
|
||||
//
|
||||
// This is the base case: we've run out of argument to check, so we're all
|
||||
// good.
|
||||
template <int kArgumentNumber, bool kShouldStaticAssert>
|
||||
static constexpr bool CheckParam(
|
||||
typename std::enable_if<(kArgumentNumber < 0)>::type *dummy = nullptr) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Checks the parameter/argument match at kArgumentNumber.
|
||||
// kShouldStaticAssert determines whether to assert out on a mismatch, or just
|
||||
// yield the constexpr boolean value.
|
||||
template <int kArgumentNumber, bool kShouldStaticAssert>
|
||||
static constexpr bool CheckParam(
|
||||
typename std::enable_if<kArgumentNumber >= 0>::type *dummy = nullptr) {
|
||||
typedef typename std::tuple_element<kArgumentNumber, ParamTuple>::type
|
||||
ParamT;
|
||||
typedef typename std::tuple_element<kArgumentNumber, ArgTuple>::type ArgT;
|
||||
return Compatible<ParamT, ArgT, kShouldStaticAssert, kArgumentNumber>() &&
|
||||
CheckParam<kArgumentNumber - 1, kShouldStaticAssert>();
|
||||
}
|
||||
|
||||
// Checks the parameters/arguments for match, but doesn't static assert out.
|
||||
// This is useful for testing/inspecting whether a set of parameters match in
|
||||
// things like tests.
|
||||
static constexpr bool CheckAllNoStaticAssert() {
|
||||
return kLengthMatches && CheckParam<kTupleLength - 1, false>();
|
||||
}
|
||||
|
||||
// Checks the parameters and static asserts out with a helpful error message
|
||||
// (and useful template parameters in the instantiation stack) if there is an
|
||||
// error.
|
||||
static constexpr bool CheckAllStaticAssert() {
|
||||
static_assert(kLengthMatches,
|
||||
"argument length mismatched against typed kernel parameters");
|
||||
return kLengthMatches && CheckParam<kTupleLength - 1, true>();
|
||||
}
|
||||
};
|
||||
|
||||
// This is a convenience type for checking whether a typed kernel matches
|
||||
// against a type list.
|
||||
template <typename KernelT, typename... Params>
|
||||
struct KernelParamsOk {
|
||||
static constexpr bool kResult = false;
|
||||
};
|
||||
|
||||
// See above.
|
||||
template <typename... Params, typename... Args>
|
||||
struct KernelParamsOk<TypedKernel<Params...>, Args...> {
|
||||
static constexpr bool kResult = KernelInvocationChecker<
|
||||
std::tuple<Params...>, std::tuple<Args...>>::CheckAllNoStaticAssert();
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_KERNEL_H_
|
||||
@@ -0,0 +1,42 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// This file contains declarations relating to kernel cache configuration
|
||||
// parameters recognized by the StreamExecutor.
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_KERNEL_CACHE_CONFIG_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_KERNEL_CACHE_CONFIG_H_
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
// This enum represents potential configurations of L1/shared memory when
|
||||
// running a particular kernel. These values represent user preference, and
|
||||
// the runtime is not required to respect these choices.
|
||||
enum class KernelCacheConfig {
|
||||
// Indicates no preference for device L1/shared memory configuration.
|
||||
kNoPreference,
|
||||
|
||||
// Indicates a preference for more shared memory than L1 cache.
|
||||
kPreferShared,
|
||||
|
||||
// Indicates a preference for more L1 cache than shared memory.
|
||||
kPreferL1,
|
||||
|
||||
// Indicates a preference for equal amounts of L1 cache and shared memory.
|
||||
kPreferEqual,
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_KERNEL_CACHE_CONFIG_H_
|
||||
@@ -0,0 +1,378 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Kernel-loader specs are structures that describe how to load a data-parallel
|
||||
// kernel on a given platform for subsequent launching. Headers that instantiate
|
||||
// these data structures will typically be auto-generated. However, users can
|
||||
// also instantiate them by hand.
|
||||
//
|
||||
// A kernel with the same exact functionality and type signature may be
|
||||
// implemented on several different platforms. Typical usage is to create a
|
||||
// singleton that describes how to load a kernel on the various supported
|
||||
// platforms:
|
||||
//
|
||||
// static const MultiKernelLoaderSpec &SaxpySpec() {
|
||||
// static auto *mkls =
|
||||
// (new MultiKernelLoaderSpec{4 /* = arity */})
|
||||
// ->AddCudaPtxOnDisk(ptx_file_path, ptx_kernelname)
|
||||
// ->AddOpenCLTextOnDisk(opencl_text_file_path, ocl_kernelname);
|
||||
// };
|
||||
//
|
||||
// return *mkls;
|
||||
// }
|
||||
//
|
||||
// This lazily instantiates an object that describes how to load CUDA PTX
|
||||
// present on disk that implements saxpy for the for the CUDA platform, or
|
||||
// OpenCL text present on disk that implements saxpy for an OpenCL-based
|
||||
// platform. The CudaPtxOnDisk and OpenCLTextOnDisk objects are subtypes of
|
||||
// KernelLoaderSpec -- KernelLoaderSpec describes how to load a kernel for
|
||||
// subsequent launching on a single platform.
|
||||
//
|
||||
// For the loader functionality that accepts these KernelLoaderSpecs in order
|
||||
// to grab the kernel appropriately, see StreamExecutor::GetKernel().
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_KERNEL_SPEC_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_KERNEL_SPEC_H_
|
||||
|
||||
#include <stddef.h>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
#include "tensorflow/stream_executor/lib/stringpiece.h"
|
||||
#include "tensorflow/stream_executor/platform/logging.h"
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
// Describes how to load a kernel on a target platform.
|
||||
//
|
||||
// This is an abstract base class, subclassed for specific platforms.
|
||||
// The filename_or_text field represents the program location (i.e. PTX or
|
||||
// OpenCL loadable translation unit path) and is simply stored; whether it is a
|
||||
// filename or text is exposed via more specifically named accessors in
|
||||
// subclasses.
|
||||
//
|
||||
// These kernel loader specifications are typically auto-generated into header
|
||||
// files at build time, but can also be specified manually.
|
||||
class KernelLoaderSpec {
|
||||
public:
|
||||
virtual ~KernelLoaderSpec() {}
|
||||
|
||||
// Returns the kernel name to load out of the program.
|
||||
const string &kernelname() const { return kernelname_; }
|
||||
|
||||
protected:
|
||||
explicit KernelLoaderSpec(port::StringPiece kernelname);
|
||||
|
||||
private:
|
||||
// The kernel name that should be loaded out of the program description given
|
||||
// above.
|
||||
string kernelname_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(KernelLoaderSpec);
|
||||
};
|
||||
|
||||
// An abstract kernel loader spec that has an associated file path, where
|
||||
// there's a canonical suffix for the filename; e.g. see CudaPtxOnDisk whose
|
||||
// canonical filename suffix is ".ptx".
|
||||
class OnDiskKernelLoaderSpec : public KernelLoaderSpec {
|
||||
public:
|
||||
~OnDiskKernelLoaderSpec() override {}
|
||||
|
||||
// Returns the path to the on-disk loadable kernel file.
|
||||
const string &filename() const { return filename_; }
|
||||
|
||||
// Returns the canonical suffix for this on-disk kernel loader spec format;
|
||||
// e.g. PTX files on disk have a canonical suffix of ".ptx".
|
||||
virtual const char *CanonicalSuffix() const = 0;
|
||||
|
||||
protected:
|
||||
OnDiskKernelLoaderSpec(port::StringPiece filename,
|
||||
port::StringPiece kernelname);
|
||||
|
||||
string filename_;
|
||||
|
||||
private:
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(OnDiskKernelLoaderSpec);
|
||||
};
|
||||
|
||||
// Kernel loader specification for PTX text that resides on disk.
|
||||
class CudaPtxOnDisk : public OnDiskKernelLoaderSpec {
|
||||
public:
|
||||
CudaPtxOnDisk(port::StringPiece filename, port::StringPiece kernelname);
|
||||
~CudaPtxOnDisk() override {}
|
||||
|
||||
const char *CanonicalSuffix() const override { return ".ptx"; }
|
||||
|
||||
private:
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CudaPtxOnDisk);
|
||||
};
|
||||
|
||||
// Kernel loader specification for CUBIN binary that resides on disk.
|
||||
class CudaCubinOnDisk : public OnDiskKernelLoaderSpec {
|
||||
public:
|
||||
CudaCubinOnDisk(port::StringPiece filename, port::StringPiece kernelname);
|
||||
~CudaCubinOnDisk() override {}
|
||||
|
||||
const string &filename() const { return filename_; }
|
||||
|
||||
const char *CanonicalSuffix() const override { return ".cubin"; }
|
||||
|
||||
private:
|
||||
string filename_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CudaCubinOnDisk);
|
||||
};
|
||||
|
||||
// Kernel loader specification for PTX text that resides in memory.
|
||||
class CudaPtxInMemory : public KernelLoaderSpec {
|
||||
public:
|
||||
// Components: compute capability major number, compute capability minor
|
||||
// number, and PTX source.
|
||||
typedef std::tuple<int, int, port::StringPiece> PtxSpec;
|
||||
|
||||
// Single-PTX constructor. Adds the provided PTX version with an unknown
|
||||
// compute capability. Since the CC is unknown, the PTX is assumed to be very
|
||||
// generally usable - in other words, PTX specified in this manner is VERY
|
||||
// likely to be used as the default! Note that the PTX can be compressed,
|
||||
// which is indicated by the argument ptx_compressed.
|
||||
//
|
||||
// Warning: the string backing the provided port::StringPiece ptx must outlive this
|
||||
// instance.
|
||||
CudaPtxInMemory(port::StringPiece ptx, port::StringPiece kernelname,
|
||||
bool ptx_compressed = false);
|
||||
|
||||
// Multiple-PTX-version constructor. Adds each item in spec_list to this
|
||||
// object. Note that the PTX can be compressed, which is indicated by the
|
||||
// argument ptx_compressed.
|
||||
CudaPtxInMemory(const std::initializer_list<PtxSpec> &spec_list,
|
||||
port::StringPiece kernel_name, bool ptx_compressed = false);
|
||||
~CudaPtxInMemory() override {}
|
||||
|
||||
// Add the PTX implementation described by ptx_spec to this object. On
|
||||
// collision (i.e., if a version with the same compute_capability already
|
||||
// exists), the existing implementation will be overwritten.
|
||||
void AddSpec(PtxSpec ptx_spec);
|
||||
|
||||
// Returns pointer to the ptx of available implementation with the
|
||||
// lowest-valued compute capability. For example, if PTX written to CC2.0,
|
||||
// 3.0, and 3.5 are all available, the version for CC2.0 will be set. Returns
|
||||
// nullptr on failed lookup (if any version is not available).
|
||||
// When the ptx is compressed, returns the decompressed ptx.
|
||||
const char *default_text() const;
|
||||
|
||||
// Similar to default_text().
|
||||
// When the ptx is compressed, returns the decompressed ptx.
|
||||
const char *original_default_text() const;
|
||||
|
||||
// Returns pointer to the ptx for the requested compute capability.
|
||||
// Returns nullptr on failed lookup (if the requested version is not
|
||||
// available).
|
||||
// When the ptx is compressed, returns the decompressed ptx.
|
||||
const char *text(int compute_capability_major,
|
||||
int compute_capability_minor) const;
|
||||
|
||||
// Similar to text().
|
||||
// When the ptx is compressed, returns the original compressed ptx.
|
||||
const char *original_text(int compute_capability_major,
|
||||
int compute_capability_minor) const;
|
||||
|
||||
// Decompresses the PTX string using bzip2.
|
||||
static string DecompressPtx(const char *ptx);
|
||||
|
||||
private:
|
||||
// PTX translation unit text contents in memory. The key is of as a tuple
|
||||
// "<cc_major>,<cc_minor>", i.e., "2,0", "3,0", "3,5". Because CC's
|
||||
// represented in this way have a clear sorting order, map::begin() will give
|
||||
// the lowest-numbered version available, i.e. the default.
|
||||
std::map<std::tuple<int, int>, const char *,
|
||||
bool (*)(const std::tuple<int, int> &, const std::tuple<int, int> &)>
|
||||
ptx_by_compute_capability_;
|
||||
|
||||
// Stores all decompressed ptx strings, with original ptx string as keys.
|
||||
// It is marked as mutable for lazy decompression.
|
||||
mutable std::map<const char *, string> decompressed_ptx_;
|
||||
mutable mutex mu_;
|
||||
|
||||
// Defines the minimum compute capability possible. Used when PTX has no
|
||||
// compute capability specified (in the single-PTX constructor).
|
||||
static const std::tuple<int, int> kMinimumCapability;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CudaPtxInMemory);
|
||||
};
|
||||
|
||||
// Kernel loader specification for OpenCL text that resides on disk.
|
||||
class OpenCLTextOnDisk : public OnDiskKernelLoaderSpec {
|
||||
public:
|
||||
OpenCLTextOnDisk(port::StringPiece filename, port::StringPiece kernelname);
|
||||
~OpenCLTextOnDisk() override {}
|
||||
|
||||
const char *CanonicalSuffix() const override { return ".ocl"; }
|
||||
|
||||
private:
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(OpenCLTextOnDisk);
|
||||
};
|
||||
|
||||
// Kernel loader specification for OpenCL binary that resides on disk.
|
||||
class OpenCLBinaryOnDisk : public OnDiskKernelLoaderSpec {
|
||||
public:
|
||||
OpenCLBinaryOnDisk(port::StringPiece filename, port::StringPiece kernelname);
|
||||
~OpenCLBinaryOnDisk() override {}
|
||||
|
||||
const char *CanonicalSuffix() const override { return ".aocx"; }
|
||||
|
||||
private:
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(OpenCLBinaryOnDisk);
|
||||
};
|
||||
|
||||
// Kernel loader specification for OpenCL text that resides in memory.
|
||||
class OpenCLTextInMemory : public KernelLoaderSpec {
|
||||
public:
|
||||
OpenCLTextInMemory(port::StringPiece text, port::StringPiece kernelname);
|
||||
~OpenCLTextInMemory() override {}
|
||||
|
||||
// Returns the OpenCL text contents.
|
||||
const string &text() const { return text_; }
|
||||
|
||||
private:
|
||||
// OpenCL translation unit text contents in memory.
|
||||
string text_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(OpenCLTextInMemory);
|
||||
};
|
||||
|
||||
// Kernel loader specification for a CUBIN blob that resides in memory.
|
||||
class CudaCubinInMemory : public KernelLoaderSpec {
|
||||
public:
|
||||
CudaCubinInMemory(const char *bytes, port::StringPiece kernelname);
|
||||
~CudaCubinInMemory() override {}
|
||||
|
||||
const char *bytes() const { return bytes_; }
|
||||
|
||||
private:
|
||||
const char *bytes_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CudaCubinInMemory);
|
||||
};
|
||||
|
||||
// Describes how to load a kernel on any subset of a number of target platforms.
|
||||
class MultiKernelLoaderSpec {
|
||||
public:
|
||||
explicit MultiKernelLoaderSpec(size_t arity);
|
||||
|
||||
// Returns the number of arguments that this kernel accepts.
|
||||
size_t arity() const { return arity_; }
|
||||
|
||||
// Convenience getters for testing whether these platform variants have
|
||||
// kernel loader specifications available.
|
||||
bool has_cuda_ptx_on_disk() const { return cuda_ptx_on_disk_ != nullptr; }
|
||||
bool has_cuda_cubin_on_disk() const { return cuda_cubin_on_disk_ != nullptr; }
|
||||
bool has_cuda_cubin_in_memory() const {
|
||||
return cuda_cubin_in_memory_ != nullptr;
|
||||
}
|
||||
bool has_cuda_ptx_in_memory() const { return cuda_ptx_in_memory_ != nullptr; }
|
||||
bool has_ocl_text_on_disk() const { return ocl_text_on_disk_ != nullptr; }
|
||||
bool has_ocl_binary_on_disk() const { return ocl_binary_on_disk_ != nullptr; }
|
||||
bool has_ocl_text_in_memory() const { return ocl_text_in_memory_ != nullptr; }
|
||||
|
||||
// Accessors for platform variant kernel load specifications.
|
||||
// Precondition: corresponding has_* is true.
|
||||
const CudaPtxOnDisk &cuda_ptx_on_disk() const {
|
||||
CHECK(has_cuda_ptx_on_disk());
|
||||
return *cuda_ptx_on_disk_;
|
||||
}
|
||||
const CudaCubinOnDisk &cuda_cubin_on_disk() const {
|
||||
CHECK(has_cuda_cubin_on_disk());
|
||||
return *cuda_cubin_on_disk_;
|
||||
}
|
||||
const CudaCubinInMemory &cuda_cubin_in_memory() const {
|
||||
CHECK(has_cuda_cubin_in_memory());
|
||||
return *cuda_cubin_in_memory_;
|
||||
}
|
||||
const CudaPtxInMemory &cuda_ptx_in_memory() const {
|
||||
CHECK(has_cuda_ptx_in_memory());
|
||||
return *cuda_ptx_in_memory_;
|
||||
}
|
||||
const OpenCLTextOnDisk &ocl_text_on_disk() const {
|
||||
CHECK(has_ocl_text_on_disk());
|
||||
return *ocl_text_on_disk_;
|
||||
}
|
||||
const OpenCLBinaryOnDisk &ocl_binary_on_disk() const {
|
||||
CHECK(has_ocl_binary_on_disk());
|
||||
return *ocl_binary_on_disk_;
|
||||
}
|
||||
const OpenCLTextInMemory &ocl_text_in_memory() const {
|
||||
CHECK(has_ocl_text_in_memory());
|
||||
return *ocl_text_in_memory_;
|
||||
}
|
||||
|
||||
// Builder-pattern-like methods for use in initializing a
|
||||
// MultiKernelLoaderSpec. Each of these should be used at most once for a
|
||||
// single MultiKernelLoaderSpec object. See file comment for example usage.
|
||||
//
|
||||
// Note that the kernelname parameter must be consistent with the kernel in
|
||||
// the PTX or OpenCL being loaded. Also be aware that in CUDA C++ the kernel
|
||||
// name may be mangled by the compiler if it is not declared in an
|
||||
// extern "C" scope.
|
||||
MultiKernelLoaderSpec *AddOpenCLTextOnDisk(port::StringPiece filename,
|
||||
port::StringPiece kernelname);
|
||||
MultiKernelLoaderSpec *AddOpenCLBinaryOnDisk(port::StringPiece filename,
|
||||
port::StringPiece kernelname);
|
||||
MultiKernelLoaderSpec *AddOpenCLTextInMemory(port::StringPiece ocl_text,
|
||||
port::StringPiece kernelname);
|
||||
MultiKernelLoaderSpec *AddCudaPtxOnDisk(port::StringPiece filename,
|
||||
port::StringPiece kernelname);
|
||||
MultiKernelLoaderSpec *AddCudaCubinOnDisk(port::StringPiece filename,
|
||||
port::StringPiece kernelname);
|
||||
MultiKernelLoaderSpec *AddCudaCubinInMemory(const char *cubin_bytes,
|
||||
port::StringPiece kernelname);
|
||||
MultiKernelLoaderSpec *AddCudaPtxInMemory(port::StringPiece ptx,
|
||||
port::StringPiece kernelname);
|
||||
MultiKernelLoaderSpec *AddCudaCompressedPtxInMemory(
|
||||
port::StringPiece ptx, port::StringPiece kernelname);
|
||||
MultiKernelLoaderSpec *AddCudaPtxInMemory(
|
||||
std::initializer_list<CudaPtxInMemory::PtxSpec> spec_list,
|
||||
port::StringPiece kernelname);
|
||||
MultiKernelLoaderSpec *AddCudaCompressedPtxInMemory(
|
||||
std::initializer_list<CudaPtxInMemory::PtxSpec> spec_list,
|
||||
port::StringPiece kernelname);
|
||||
|
||||
private:
|
||||
std::unique_ptr<CudaPtxOnDisk>
|
||||
cuda_ptx_on_disk_; // PTX text that resides in a file.
|
||||
std::unique_ptr<CudaCubinOnDisk>
|
||||
cuda_cubin_on_disk_; // Binary CUDA program in a file.
|
||||
std::unique_ptr<CudaCubinInMemory>
|
||||
cuda_cubin_in_memory_; // Binary CUDA program in memory.
|
||||
std::unique_ptr<CudaPtxInMemory>
|
||||
cuda_ptx_in_memory_; // PTX text that resides in memory.
|
||||
std::unique_ptr<OpenCLTextOnDisk>
|
||||
ocl_text_on_disk_; // OpenCL text that resides on disk.
|
||||
std::unique_ptr<OpenCLBinaryOnDisk>
|
||||
ocl_binary_on_disk_; // OpenCL binary that resides on disk.
|
||||
std::unique_ptr<OpenCLTextInMemory>
|
||||
ocl_text_in_memory_; // OpenCL text that resides in memory.
|
||||
|
||||
// Number of parameters that the kernel takes. (This is nicer to have in a
|
||||
// constexpr than having to determine it from the types via template
|
||||
// metaprogramming).
|
||||
size_t arity_;
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_KERNEL_SPEC_H_
|
||||
@@ -0,0 +1,78 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Types to express dimensionality of a kernel launch. Blocks and threads
|
||||
// are (up to) 3-dimensional.
|
||||
//
|
||||
// A thread is conceptually like a SIMD lane. Some number, typically 32
|
||||
// (though that fact should not be relied on) SIMD lanes are tied together with
|
||||
// a single PC in a unit called a warp. There is a maximum number of threads
|
||||
// that can execute in a shared-context entity called a block. Presently, that
|
||||
// number is 1024 -- again, something that should not be relied on from this
|
||||
// comment, but checked via stream_executor::DeviceDescription.
|
||||
//
|
||||
// For additional information, see
|
||||
// http://docs.nvidia.com/cuda/kepler-tuning-guide/#device-utilization-and-occupancy
|
||||
//
|
||||
// Because of that modest thread-per-block limit, a kernel can be launched with
|
||||
// multiple blocks. Each block is indivisibly scheduled onto a single core.
|
||||
// Blocks can also be used in a multi-dimensional configuration, and the block
|
||||
// count has much less modest limits -- typically they're similar to the maximum
|
||||
// amount of addressable memory.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LAUNCH_DIM_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LAUNCH_DIM_H_
|
||||
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
#include "tensorflow/stream_executor/lib/strcat.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
// Basic type that represents a 3-dimensional index space.
|
||||
struct Dim3D {
|
||||
uint64 x, y, z;
|
||||
|
||||
Dim3D(uint64 x, uint64 y, uint64 z) : x(x), y(y), z(z) {}
|
||||
};
|
||||
|
||||
// Thread dimensionality for use in a kernel launch. See file comment for
|
||||
// details.
|
||||
struct ThreadDim : public Dim3D {
|
||||
explicit ThreadDim(uint64 x = 1, uint64 y = 1, uint64 z = 1)
|
||||
: Dim3D(x, y, z) {}
|
||||
|
||||
// Returns a string representation of the thread dimensionality.
|
||||
string ToString() const {
|
||||
return port::StrCat("ThreadDim{", x, ", ", y, ", ", z, "}");
|
||||
}
|
||||
};
|
||||
|
||||
// Block dimensionality for use in a kernel launch. See file comment for
|
||||
// details.
|
||||
struct BlockDim : public Dim3D {
|
||||
explicit BlockDim(uint64 x = 1, uint64 y = 1, uint64 z = 1)
|
||||
: Dim3D(x, y, z) {}
|
||||
|
||||
// Returns a string representation of the block dimensionality.
|
||||
string ToString() const {
|
||||
return port::StrCat("BlockDim{", x, ", ", y, ", ", z, "}");
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LAUNCH_DIM_H_
|
||||
@@ -0,0 +1,43 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_ARRAY_SLICE_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_ARRAY_SLICE_H_
|
||||
|
||||
#include "absl/types/span.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
template <typename T>
|
||||
using ArraySlice = absl::Span<const T>;
|
||||
template <typename T>
|
||||
using MutableArraySlice = absl::Span<T>;
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
namespace perftools {
|
||||
namespace gputools {
|
||||
|
||||
// Temporarily pull stream_executor into perftools::gputools while we migrate
|
||||
// code to the new namespace. TODO(b/77980417): Remove this once we've
|
||||
// completed the migration.
|
||||
using namespace stream_executor; // NOLINT[build/namespaces]
|
||||
|
||||
} // namespace gputools
|
||||
} // namespace perftools
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_ARRAY_SLICE_H_
|
||||
@@ -0,0 +1,100 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// IWYU pragma: private, include "third_party/tensorflow/stream_executor/stream_executor.h"
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_CASTS_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_CASTS_H_
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
// port::bit_cast<Dest,Source> is a template function that implements the
|
||||
// equivalent of "*reinterpret_cast<Dest*>(&source)". We need this in
|
||||
// very low-level functions like the protobuf library and fast math
|
||||
// support.
|
||||
//
|
||||
// float f = 3.14159265358979;
|
||||
// int i = port::bit_cast<int32>(f);
|
||||
// // i = 0x40490fdb
|
||||
//
|
||||
// The classical address-casting method is:
|
||||
//
|
||||
// // WRONG
|
||||
// float f = 3.14159265358979; // WRONG
|
||||
// int i = * reinterpret_cast<int*>(&f); // WRONG
|
||||
//
|
||||
// The address-casting method actually produces undefined behavior
|
||||
// according to ISO C++ specification section 3.10 -15 -. Roughly, this
|
||||
// section says: if an object in memory has one type, and a program
|
||||
// accesses it with a different type, then the result is undefined
|
||||
// behavior for most values of "different type".
|
||||
//
|
||||
// This is true for any cast syntax, either *(int*)&f or
|
||||
// *reinterpret_cast<int*>(&f). And it is particularly true for
|
||||
// conversions between integral lvalues and floating-point lvalues.
|
||||
//
|
||||
// The purpose of 3.10 -15- is to allow optimizing compilers to assume
|
||||
// that expressions with different types refer to different memory. gcc
|
||||
// 4.0.1 has an optimizer that takes advantage of this. So a
|
||||
// non-conforming program quietly produces wildly incorrect output.
|
||||
//
|
||||
// The problem is not the use of reinterpret_cast. The problem is type
|
||||
// punning: holding an object in memory of one type and reading its bits
|
||||
// back using a different type.
|
||||
//
|
||||
// The C++ standard is more subtle and complex than this, but that
|
||||
// is the basic idea.
|
||||
//
|
||||
// Anyways ...
|
||||
//
|
||||
// port::bit_cast<> calls memcpy() which is blessed by the standard,
|
||||
// especially by the example in section 3.9 . Also, of course,
|
||||
// port::bit_cast<> wraps up the nasty logic in one place.
|
||||
//
|
||||
// Fortunately memcpy() is very fast. In optimized mode, with a
|
||||
// constant size, gcc 2.95.3, gcc 4.0.1, and msvc 7.1 produce inline
|
||||
// code with the minimal amount of data movement. On a 32-bit system,
|
||||
// memcpy(d,s,4) compiles to one load and one store, and memcpy(d,s,8)
|
||||
// compiles to two loads and two stores.
|
||||
//
|
||||
// I tested this code with gcc 2.95.3, gcc 4.0.1, icc 8.1, and msvc 7.1.
|
||||
//
|
||||
// WARNING: if Dest or Source is a non-POD type, the result of the memcpy
|
||||
// is likely to surprise you.
|
||||
//
|
||||
// Props to Bill Gibbons for the compile time assertion technique and
|
||||
// Art Komninos and Igor Tandetnik for the msvc experiments.
|
||||
//
|
||||
// -- mec 2005-10-17
|
||||
|
||||
template <class Dest, class Source>
|
||||
inline Dest bit_cast(const Source& source) {
|
||||
// Compile time assertion: sizeof(Dest) == sizeof(Source)
|
||||
// A compile error here means your Dest and Source have different sizes.
|
||||
static_assert(sizeof(Dest) == sizeof(Source),
|
||||
"src and dst types must have equal sizes");
|
||||
|
||||
Dest dest;
|
||||
memcpy(&dest, &source, sizeof(dest));
|
||||
return dest;
|
||||
}
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_CASTS_H_
|
||||
@@ -0,0 +1,29 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_DEMANGLE_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_DEMANGLE_H_
|
||||
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
string Demangle(const char* mangled);
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_DEMANGLE_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_ENV_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_ENV_H_
|
||||
|
||||
#include "tensorflow/core/platform/env.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/lib/stringpiece.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
using tensorflow::Env;
|
||||
using tensorflow::Thread;
|
||||
|
||||
inline Status FileExists(const string& filename) {
|
||||
return Env::Default()->FileExists(filename);
|
||||
}
|
||||
|
||||
inline Status FileExists(const port::StringPiece& filename) {
|
||||
return Env::Default()->FileExists(string(filename));
|
||||
}
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_ENV_H_
|
||||
@@ -0,0 +1,31 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// IWYU pragma: private, include "third_party/tensorflow/stream_executor/stream_executor.h"
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_ERROR_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_ERROR_H_
|
||||
|
||||
#include "tensorflow/core/lib/core/error_codes.pb.h" // IWYU pragma: export
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
namespace error = tensorflow::error;
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_ERROR_H_
|
||||
@@ -0,0 +1,71 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_HUMAN_READABLE_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_HUMAN_READABLE_H_
|
||||
|
||||
#include <assert.h>
|
||||
#include <limits>
|
||||
|
||||
#include "tensorflow/stream_executor/lib/stringprintf.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
class HumanReadableNumBytes {
|
||||
public:
|
||||
static string ToString(int64 num_bytes) {
|
||||
if (num_bytes == std::numeric_limits<int64>::min()) {
|
||||
// Special case for number with not representable nagation.
|
||||
return "-8E";
|
||||
}
|
||||
|
||||
const char* neg_str = GetNegStr(&num_bytes);
|
||||
|
||||
// Special case for bytes.
|
||||
if (num_bytes < 1024LL) {
|
||||
// No fractions for bytes.
|
||||
return port::Printf("%s%lldB", neg_str, num_bytes);
|
||||
}
|
||||
|
||||
static const char units[] = "KMGTPE"; // int64 only goes up to E.
|
||||
const char* unit = units;
|
||||
while (num_bytes >= (1024LL) * (1024LL)) {
|
||||
num_bytes /= (1024LL);
|
||||
++unit;
|
||||
assert(unit < units + sizeof(units));
|
||||
}
|
||||
|
||||
return port::Printf(((*unit == 'K') ? "%s%.1f%c" : "%s%.2f%c"), neg_str,
|
||||
num_bytes / 1024.0, *unit);
|
||||
}
|
||||
|
||||
private:
|
||||
template <typename T>
|
||||
static const char* GetNegStr(T* value) {
|
||||
if (*value < 0) {
|
||||
*value = -(*value);
|
||||
return "-";
|
||||
} else {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_HUMAN_READABLE_H_
|
||||
@@ -0,0 +1,70 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_INITIALIZE_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_INITIALIZE_H_
|
||||
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
#if defined(PLATFORM_GOOGLE)
|
||||
#include "tensorflow/stream_executor/platform/google/initialize.h"
|
||||
#else
|
||||
|
||||
#undef REGISTER_MODULE_INITIALIZER
|
||||
#undef DECLARE_MODULE_INITIALIZER
|
||||
#undef REGISTER_MODULE_INITIALIZER_SEQUENCE
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
class Initializer {
|
||||
public:
|
||||
typedef void (*InitializerFunc)();
|
||||
explicit Initializer(InitializerFunc func) { func(); }
|
||||
|
||||
struct Dependency {
|
||||
Dependency(const char *n, Initializer *i) : name(n), initializer(i) {}
|
||||
const char *const name;
|
||||
Initializer *const initializer;
|
||||
};
|
||||
|
||||
struct DependencyRegisterer {
|
||||
DependencyRegisterer(const char *type, const char *name,
|
||||
Initializer *initializer,
|
||||
const Dependency &dependency);
|
||||
};
|
||||
};
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#define REGISTER_INITIALIZER(type, name, body) \
|
||||
static void google_init_##type##_##name() { body; } \
|
||||
::stream_executor::port::Initializer google_initializer_##type##_##name( \
|
||||
google_init_##type##_##name)
|
||||
|
||||
#define REGISTER_MODULE_INITIALIZER(name, body) \
|
||||
REGISTER_INITIALIZER(module, name, body)
|
||||
|
||||
#define DECLARE_INITIALIZER(type, name) \
|
||||
extern ::stream_executor::port::Initializer google_initializer_##type##_##name
|
||||
|
||||
#define DECLARE_MODULE_INITIALIZER(name) DECLARE_INITIALIZER(module, name)
|
||||
|
||||
#define REGISTER_MODULE_INITIALIZER_SEQUENCE(name1, name2)
|
||||
|
||||
#endif // !defined(PLATFORM_GOOGLE)
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_INITIALIZE_H_
|
||||
@@ -0,0 +1,29 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_INLINED_VECTOR_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_INLINED_VECTOR_H_
|
||||
|
||||
#include "absl/container/inlined_vector.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
using absl::InlinedVector;
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_INLINED_VECTOR_H_
|
||||
@@ -0,0 +1,101 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_MATHUTIL_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_MATHUTIL_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <limits>
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/stream_executor/platform/logging.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
class MathUtil {
|
||||
public:
|
||||
template <typename IntegralType>
|
||||
static IntegralType CeilOfRatio(IntegralType numerator,
|
||||
IntegralType denominator) {
|
||||
return CeilOrFloorOfRatio<IntegralType, true>(numerator, denominator);
|
||||
}
|
||||
template <typename IntegralType>
|
||||
static IntegralType FloorOfRatio(IntegralType numerator,
|
||||
IntegralType denominator) {
|
||||
return CeilOrFloorOfRatio<IntegralType, false>(numerator, denominator);
|
||||
}
|
||||
template <typename IntegralType, bool ceil>
|
||||
static IntegralType CeilOrFloorOfRatio(IntegralType numerator,
|
||||
IntegralType denominator);
|
||||
};
|
||||
|
||||
// ---- CeilOrFloorOfRatio ----
|
||||
// This is a branching-free, cast-to-double-free implementation.
|
||||
//
|
||||
// Casting to double is in general incorrect because of loss of precision
|
||||
// when casting an int64 into a double.
|
||||
//
|
||||
// There's a bunch of 'recipes' to compute a integer ceil (or floor) on the web,
|
||||
// and most of them are incorrect.
|
||||
template<typename IntegralType, bool ceil>
|
||||
IntegralType MathUtil::CeilOrFloorOfRatio(IntegralType numerator,
|
||||
IntegralType denominator) {
|
||||
static_assert(std::is_integral<IntegralType>::value,
|
||||
"CeilOfRatio_is_only_defined_for_integral_types");
|
||||
assert(denominator != 0);
|
||||
// Dividing the smallest signed integer by -1 is not supported: it would
|
||||
// SIGFPE
|
||||
assert(!std::is_signed<IntegralType>::value ||
|
||||
numerator != std::numeric_limits<IntegralType>::min() ||
|
||||
denominator != -1);
|
||||
|
||||
const IntegralType rounded_toward_zero = numerator / denominator;
|
||||
const IntegralType intermediate_product = rounded_toward_zero * denominator;
|
||||
|
||||
if (ceil) { // Compile-time condition: not an actual branching
|
||||
// When rounded_toward_zero is negative, then an adjustment is never needed:
|
||||
// the real ratio is negative, and so rounded toward zero is the ceil.
|
||||
// When rounded_toward_zero is non-negative, an adjustment is needed if the
|
||||
// sign of the difference numerator - intermediate_product is the same as
|
||||
// the sign of the denominator.
|
||||
//
|
||||
// Using a bool and then a static_cast to IntegralType is not strictly
|
||||
// necessary, but it makes the code clear, and anyway the compiler should
|
||||
// get rid of it.
|
||||
const bool needs_adjustment = (rounded_toward_zero >= 0) &&
|
||||
((denominator > 0 && numerator > intermediate_product) ||
|
||||
(denominator < 0 && numerator < intermediate_product));
|
||||
const IntegralType adjustment = static_cast<IntegralType>(needs_adjustment);
|
||||
const IntegralType ceil_of_ratio = rounded_toward_zero + adjustment;
|
||||
return ceil_of_ratio;
|
||||
} else {
|
||||
// Floor case: symmetrical to the previous one
|
||||
const bool needs_adjustment = (rounded_toward_zero <= 0) &&
|
||||
((denominator > 0 && numerator < intermediate_product) ||
|
||||
(denominator < 0 && numerator > intermediate_product));
|
||||
const IntegralType adjustment = static_cast<IntegralType>(needs_adjustment);
|
||||
const IntegralType floor_of_ratio = rounded_toward_zero - adjustment;
|
||||
return floor_of_ratio;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_MATHUTIL_H_
|
||||
@@ -0,0 +1,29 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_NOTIFICATION_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_NOTIFICATION_H_
|
||||
|
||||
#include "tensorflow/core/platform/notification.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
using tensorflow::Notification;
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_NOTIFICATION_H_
|
||||
@@ -0,0 +1,32 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_NUMBERS_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_NUMBERS_H_
|
||||
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
// Convert strings to floating point values.
|
||||
// Leading and trailing spaces are allowed.
|
||||
// Values may be rounded on over- and underflow.
|
||||
bool safe_strto32(const string& str, int32* value);
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_NUMBERS_H_
|
||||
@@ -0,0 +1,60 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_PATH_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_PATH_H_
|
||||
|
||||
#include "tensorflow/core/lib/io/path.h"
|
||||
#include "tensorflow/stream_executor/lib/stringpiece.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
using tensorflow::io::Dirname;
|
||||
|
||||
namespace internal {
|
||||
// TODO(rspringer): Move to cc/implementation file.
|
||||
// Not part of the public API.
|
||||
string JoinPathImpl(std::initializer_list<port::StringPiece> paths);
|
||||
} // namespace internal
|
||||
|
||||
// Join multiple paths together.
|
||||
// JoinPath unconditionally joins all paths together. For example:
|
||||
//
|
||||
// Arguments | JoinPath
|
||||
// ---------------------------+---------------------
|
||||
// '/foo', 'bar' | /foo/bar
|
||||
// '/foo/', 'bar' | /foo/bar
|
||||
// '/foo', '/bar' | /foo/bar
|
||||
// '/foo', '/bar', '/baz' | /foo/bar/baz
|
||||
//
|
||||
// All paths will be treated as relative paths, regardless of whether or not
|
||||
// they start with a leading '/'. That is, all paths will be concatenated
|
||||
// together, with the appropriate path separator inserted in between.
|
||||
// Arguments must be convertible to port::StringPiece.
|
||||
//
|
||||
// Usage:
|
||||
// string path = file::JoinPath("/var/log", dirname, filename);
|
||||
// string path = file::JoinPath(FLAGS_test_srcdir, filename);
|
||||
template <typename... T>
|
||||
inline string JoinPath(const T&... args) {
|
||||
return internal::JoinPathImpl({args...});
|
||||
}
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_PATH_H_
|
||||
@@ -0,0 +1,30 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_PROCESS_STATE_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_PROCESS_STATE_H_
|
||||
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
string Hostname();
|
||||
bool GetCurrentDirectory(string* dir);
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_PROCESS_STATE_H_
|
||||
@@ -0,0 +1,38 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_PTR_UTIL_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_PTR_UTIL_H_
|
||||
|
||||
#include <memory>
|
||||
#include "tensorflow/core/util/ptr_util.h"
|
||||
|
||||
namespace stream_executor {
|
||||
using tensorflow::MakeUnique;
|
||||
using tensorflow::WrapUnique;
|
||||
} // namespace stream_executor
|
||||
|
||||
namespace perftools {
|
||||
namespace gputools {
|
||||
|
||||
// Temporarily pull stream_executor into perftools::gputools while we migrate
|
||||
// code to the new namespace. TODO(jlebar): Remove this once we've completed
|
||||
// the migration.
|
||||
using namespace stream_executor; // NOLINT[build/namespaces]
|
||||
|
||||
} // namespace gputools
|
||||
} // namespace perftools
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_PTR_UTIL_H_
|
||||
@@ -0,0 +1,30 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STACKTRACE_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STACKTRACE_H_
|
||||
|
||||
#include "tensorflow/core/platform/stacktrace.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
using tensorflow::CurrentStackTrace;
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STACKTRACE_H_
|
||||
@@ -0,0 +1,46 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STATIC_THREADLOCAL_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STATIC_THREADLOCAL_H_
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define __thread __declspec(thread)
|
||||
#endif
|
||||
|
||||
// For POD types in TLS mode, s_obj_VAR is the thread-local variable.
|
||||
#define SE_STATIC_THREAD_LOCAL_POD(_Type_, _var_) \
|
||||
static __thread _Type_ s_obj_##_var_; \
|
||||
namespace { \
|
||||
class ThreadLocal_##_var_ { \
|
||||
public: \
|
||||
ThreadLocal_##_var_() {} \
|
||||
void Init() {} \
|
||||
inline _Type_ *pointer() const { \
|
||||
return &s_obj_##_var_; \
|
||||
} \
|
||||
inline _Type_ *safe_pointer() const { \
|
||||
return &s_obj_##_var_; \
|
||||
} \
|
||||
_Type_ &get() const { \
|
||||
return s_obj_##_var_; \
|
||||
} \
|
||||
bool is_native_tls() const { return true; } \
|
||||
private: \
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(ThreadLocal_##_var_); \
|
||||
} _var_; \
|
||||
}
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STATIC_THREADLOCAL_H_
|
||||
@@ -0,0 +1,60 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// IWYU pragma: private, include "third_party/tensorflow/stream_executor/stream_executor.h"
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STATUS_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STATUS_H_
|
||||
|
||||
#include "tensorflow/core/lib/core/status.h"
|
||||
#include "tensorflow/stream_executor/lib/error.h" // IWYU pragma: export
|
||||
#include "tensorflow/stream_executor/lib/stringpiece.h"
|
||||
#include "tensorflow/stream_executor/platform/logging.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
using Status = tensorflow::Status;
|
||||
|
||||
#define SE_CHECK_OK(val) TF_CHECK_OK(val)
|
||||
#define SE_ASSERT_OK(val) \
|
||||
ASSERT_EQ(::stream_executor::port::Status::OK(), (val))
|
||||
|
||||
// Define some canonical error helpers.
|
||||
inline Status UnimplementedError(StringPiece message) {
|
||||
return Status(error::UNIMPLEMENTED, message);
|
||||
}
|
||||
inline Status InternalError(StringPiece message) {
|
||||
return Status(error::INTERNAL, message);
|
||||
}
|
||||
inline Status FailedPreconditionError(StringPiece message) {
|
||||
return Status(error::FAILED_PRECONDITION, message);
|
||||
}
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
namespace perftools {
|
||||
namespace gputools {
|
||||
|
||||
// Temporarily pull stream_executor into perftools::gputools while we migrate
|
||||
// code to the new namespace. TODO(b/77980417): Remove this once we've
|
||||
// completed the migration.
|
||||
using namespace stream_executor; // NOLINT[build/namespaces]
|
||||
|
||||
} // namespace gputools
|
||||
} // namespace perftools
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STATUS_H_
|
||||
@@ -0,0 +1,53 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Helper macros for dealing with the port::Status datatype.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STATUS_MACROS_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STATUS_MACROS_H_
|
||||
|
||||
// Early-returns the status if it is in error; otherwise, proceeds.
|
||||
//
|
||||
// The argument expression is guaranteed to be evaluated exactly once.
|
||||
#define SE_RETURN_IF_ERROR(__status) \
|
||||
do { \
|
||||
auto status = __status; \
|
||||
if (!status.ok()) { \
|
||||
return status; \
|
||||
} \
|
||||
} while (false)
|
||||
|
||||
// Identifier concatenation helper macros.
|
||||
#define SE_MACRO_CONCAT_INNER(__x, __y) __x##__y
|
||||
#define SE_MACRO_CONCAT(__x, __y) SE_MACRO_CONCAT_INNER(__x, __y)
|
||||
|
||||
// Implementation of SE_ASSIGN_OR_RETURN that uses a unique temporary identifier
|
||||
// for avoiding collision in the enclosing scope.
|
||||
#define SE_ASSIGN_OR_RETURN_IMPL(__lhs, __rhs, __name) \
|
||||
auto __name = (__rhs); \
|
||||
if (!__name.ok()) { \
|
||||
return __name.status(); \
|
||||
} \
|
||||
__lhs = std::move(__name.ValueOrDie());
|
||||
|
||||
// Early-returns the status if it is in error; otherwise, assigns the
|
||||
// right-hand-side expression to the left-hand-side expression.
|
||||
//
|
||||
// The right-hand-side expression is guaranteed to be evaluated exactly once.
|
||||
#define SE_ASSIGN_OR_RETURN(__lhs, __rhs) \
|
||||
SE_ASSIGN_OR_RETURN_IMPL(__lhs, __rhs, \
|
||||
SE_MACRO_CONCAT(__status_or_value, __COUNTER__))
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STATUS_MACROS_H_
|
||||
@@ -0,0 +1,311 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// StatusOr<T> is the union of a Status object and a T object. StatusOr models
|
||||
// the concept of an object that is either a value, or an error Status
|
||||
// explaining why such a value is not present. To this end, StatusOr<T> does not
|
||||
// allow its Status value to be Status::OK.
|
||||
//
|
||||
// The primary use-case for StatusOr<T> is as the return value of a
|
||||
// function which may fail.
|
||||
//
|
||||
// Example client usage for a StatusOr<T>, where T is not a pointer:
|
||||
//
|
||||
// StatusOr<float> result = DoBigCalculationThatCouldFail();
|
||||
// if (result.ok()) {
|
||||
// float answer = result.ValueOrDie();
|
||||
// printf("Big calculation yielded: %f", answer);
|
||||
// } else {
|
||||
// LOG(ERROR) << result.status();
|
||||
// }
|
||||
//
|
||||
// Example client usage for a StatusOr<T*>:
|
||||
//
|
||||
// StatusOr<Foo*> result = FooFactory::MakeNewFoo(arg);
|
||||
// if (result.ok()) {
|
||||
// std::unique_ptr<Foo> foo(result.ValueOrDie());
|
||||
// foo->DoSomethingCool();
|
||||
// } else {
|
||||
// LOG(ERROR) << result.status();
|
||||
// }
|
||||
//
|
||||
// Example client usage for a StatusOr<std::unique_ptr<T>>:
|
||||
//
|
||||
// StatusOr<std::unique_ptr<Foo>> result = FooFactory::MakeNewFoo(arg);
|
||||
// if (result.ok()) {
|
||||
// std::unique_ptr<Foo> foo = std::move(result.ValueOrDie());
|
||||
// foo->DoSomethingCool();
|
||||
// } else {
|
||||
// LOG(ERROR) << result.status();
|
||||
// }
|
||||
//
|
||||
// Example factory implementation returning StatusOr<T*>:
|
||||
//
|
||||
// StatusOr<Foo*> FooFactory::MakeNewFoo(int arg) {
|
||||
// if (arg <= 0) {
|
||||
// return tensorflow::InvalidArgument("Arg must be positive");
|
||||
// } else {
|
||||
// return new Foo(arg);
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// Note that the assignment operators require that destroying the currently
|
||||
// stored value cannot invalidate the argument; in other words, the argument
|
||||
// cannot be an alias for the current value, or anything owned by the current
|
||||
// value.
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_H_
|
||||
|
||||
#include "tensorflow/core/platform/macros.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor_internals.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
#if defined(__clang__)
|
||||
// Only clang supports warn_unused_result as a type annotation.
|
||||
template <typename T>
|
||||
class TF_MUST_USE_RESULT StatusOr;
|
||||
#endif
|
||||
|
||||
template <typename T>
|
||||
class StatusOr : private internal_statusor::StatusOrData<T>,
|
||||
private internal_statusor::TraitsBase<
|
||||
std::is_copy_constructible<T>::value,
|
||||
std::is_move_constructible<T>::value> {
|
||||
template <typename U>
|
||||
friend class StatusOr;
|
||||
|
||||
typedef internal_statusor::StatusOrData<T> Base;
|
||||
|
||||
public:
|
||||
typedef T element_type;
|
||||
|
||||
// Constructs a new StatusOr with Status::UNKNOWN status. This is marked
|
||||
// 'explicit' to try to catch cases like 'return {};', where people think
|
||||
// StatusOr<std::vector<int>> will be initialized with an empty vector,
|
||||
// instead of a Status::UNKNOWN status.
|
||||
explicit StatusOr();
|
||||
|
||||
// StatusOr<T> will be copy constructible/assignable if T is copy
|
||||
// constructible.
|
||||
StatusOr(const StatusOr&) = default;
|
||||
StatusOr& operator=(const StatusOr&) = default;
|
||||
|
||||
// StatusOr<T> will be move constructible/assignable if T is move
|
||||
// constructible.
|
||||
StatusOr(StatusOr&&) = default;
|
||||
StatusOr& operator=(StatusOr&&) = default;
|
||||
|
||||
// Conversion copy/move constructor, T must be convertible from U.
|
||||
template <typename U, typename std::enable_if<
|
||||
std::is_convertible<U, T>::value>::type* = nullptr>
|
||||
StatusOr(const StatusOr<U>& other);
|
||||
template <typename U, typename std::enable_if<
|
||||
std::is_convertible<U, T>::value>::type* = nullptr>
|
||||
StatusOr(StatusOr<U>&& other);
|
||||
|
||||
// Conversion copy/move assignment operator, T must be convertible from U.
|
||||
template <typename U, typename std::enable_if<
|
||||
std::is_convertible<U, T>::value>::type* = nullptr>
|
||||
StatusOr& operator=(const StatusOr<U>& other);
|
||||
template <typename U, typename std::enable_if<
|
||||
std::is_convertible<U, T>::value>::type* = nullptr>
|
||||
StatusOr& operator=(StatusOr<U>&& other);
|
||||
|
||||
// Constructs a new StatusOr with the given value. After calling this
|
||||
// constructor, calls to ValueOrDie() will succeed, and calls to status() will
|
||||
// return OK.
|
||||
//
|
||||
// NOTE: Not explicit - we want to use StatusOr<T> as a return type
|
||||
// so it is convenient and sensible to be able to do 'return T()'
|
||||
// when the return type is StatusOr<T>.
|
||||
//
|
||||
// REQUIRES: T is copy constructible.
|
||||
StatusOr(const T& value);
|
||||
|
||||
// Constructs a new StatusOr with the given non-ok status. After calling
|
||||
// this constructor, calls to ValueOrDie() will CHECK-fail.
|
||||
//
|
||||
// NOTE: Not explicit - we want to use StatusOr<T> as a return
|
||||
// value, so it is convenient and sensible to be able to do 'return
|
||||
// Status()' when the return type is StatusOr<T>.
|
||||
//
|
||||
// REQUIRES: !status.ok(). This requirement is DCHECKed.
|
||||
// In optimized builds, passing Status::OK() here will have the effect
|
||||
// of passing tensorflow::error::INTERNAL as a fallback.
|
||||
StatusOr(const Status& status);
|
||||
StatusOr& operator=(const Status& status);
|
||||
|
||||
// TODO(b/62186997): Add operator=(T) overloads.
|
||||
|
||||
// Similar to the `const T&` overload.
|
||||
//
|
||||
// REQUIRES: T is move constructible.
|
||||
StatusOr(T&& value);
|
||||
|
||||
// RValue versions of the operations declared above.
|
||||
StatusOr(Status&& status);
|
||||
StatusOr& operator=(Status&& status);
|
||||
|
||||
// Returns this->status().ok()
|
||||
bool ok() const { return this->status_.ok(); }
|
||||
|
||||
// Returns a reference to our status. If this contains a T, then
|
||||
// returns Status::OK().
|
||||
const Status& status() const &;
|
||||
Status status() &&;
|
||||
|
||||
// Returns a reference to our current value, or CHECK-fails if !this->ok().
|
||||
//
|
||||
// Note: for value types that are cheap to copy, prefer simple code:
|
||||
//
|
||||
// T value = statusor.ValueOrDie();
|
||||
//
|
||||
// Otherwise, if the value type is expensive to copy, but can be left
|
||||
// in the StatusOr, simply assign to a reference:
|
||||
//
|
||||
// T& value = statusor.ValueOrDie(); // or `const T&`
|
||||
//
|
||||
// Otherwise, if the value type supports an efficient move, it can be
|
||||
// used as follows:
|
||||
//
|
||||
// T value = std::move(statusor).ValueOrDie();
|
||||
//
|
||||
// The std::move on statusor instead of on the whole expression enables
|
||||
// warnings about possible uses of the statusor object after the move.
|
||||
// C++ style guide waiver for ref-qualified overloads granted in cl/143176389
|
||||
// See go/ref-qualifiers for more details on such overloads.
|
||||
const T& ValueOrDie() const &;
|
||||
T& ValueOrDie() &;
|
||||
const T&& ValueOrDie() const &&;
|
||||
T&& ValueOrDie() &&;
|
||||
|
||||
T ConsumeValueOrDie() { return std::move(ValueOrDie()); }
|
||||
|
||||
// Ignores any errors. This method does nothing except potentially suppress
|
||||
// complaints from any tools that are checking that errors are not dropped on
|
||||
// the floor.
|
||||
void IgnoreError() const;
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Implementation details for StatusOr<T>
|
||||
|
||||
template <typename T>
|
||||
StatusOr<T>::StatusOr() : Base(Status(tensorflow::error::UNKNOWN, "")) {}
|
||||
|
||||
template <typename T>
|
||||
StatusOr<T>::StatusOr(const T& value) : Base(value) {}
|
||||
|
||||
template <typename T>
|
||||
StatusOr<T>::StatusOr(const Status& status) : Base(status) {}
|
||||
|
||||
template <typename T>
|
||||
StatusOr<T>& StatusOr<T>::operator=(const Status& status) {
|
||||
this->Assign(status);
|
||||
return *this;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
StatusOr<T>::StatusOr(T&& value) : Base(std::move(value)) {}
|
||||
|
||||
template <typename T>
|
||||
StatusOr<T>::StatusOr(Status&& status) : Base(std::move(status)) {}
|
||||
|
||||
template <typename T>
|
||||
StatusOr<T>& StatusOr<T>::operator=(Status&& status) {
|
||||
this->Assign(std::move(status));
|
||||
return *this;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <typename U,
|
||||
typename std::enable_if<std::is_convertible<U, T>::value>::type*>
|
||||
inline StatusOr<T>::StatusOr(const StatusOr<U>& other)
|
||||
: Base(static_cast<const typename StatusOr<U>::Base&>(other)) {}
|
||||
|
||||
template <typename T>
|
||||
template <typename U,
|
||||
typename std::enable_if<std::is_convertible<U, T>::value>::type*>
|
||||
inline StatusOr<T>& StatusOr<T>::operator=(const StatusOr<U>& other) {
|
||||
if (other.ok())
|
||||
this->Assign(other.ValueOrDie());
|
||||
else
|
||||
this->Assign(other.status());
|
||||
return *this;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <typename U,
|
||||
typename std::enable_if<std::is_convertible<U, T>::value>::type*>
|
||||
inline StatusOr<T>::StatusOr(StatusOr<U>&& other)
|
||||
: Base(static_cast<typename StatusOr<U>::Base&&>(other)) {}
|
||||
|
||||
template <typename T>
|
||||
template <typename U,
|
||||
typename std::enable_if<std::is_convertible<U, T>::value>::type*>
|
||||
inline StatusOr<T>& StatusOr<T>::operator=(StatusOr<U>&& other) {
|
||||
if (other.ok()) {
|
||||
this->Assign(std::move(other).ValueOrDie());
|
||||
} else {
|
||||
this->Assign(std::move(other).status());
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
const Status& StatusOr<T>::status() const & {
|
||||
return this->status_;
|
||||
}
|
||||
template <typename T>
|
||||
Status StatusOr<T>::status() && {
|
||||
return ok() ? Status::OK() : std::move(this->status_);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
const T& StatusOr<T>::ValueOrDie() const & {
|
||||
this->EnsureOk();
|
||||
return this->data_;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
T& StatusOr<T>::ValueOrDie() & {
|
||||
this->EnsureOk();
|
||||
return this->data_;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
const T&& StatusOr<T>::ValueOrDie() const && {
|
||||
this->EnsureOk();
|
||||
return std::move(this->data_);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
T&& StatusOr<T>::ValueOrDie() && {
|
||||
this->EnsureOk();
|
||||
return std::move(this->data_);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void StatusOr<T>::IgnoreError() const {
|
||||
// no-op
|
||||
}
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_H_
|
||||
@@ -0,0 +1,247 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_INTERNALS_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_INTERNALS_H_
|
||||
|
||||
#include "tensorflow/core/platform/macros.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
namespace internal_statusor {
|
||||
|
||||
class Helper {
|
||||
public:
|
||||
// Move type-agnostic error handling to the .cc.
|
||||
static void HandleInvalidStatusCtorArg(Status*);
|
||||
TF_ATTRIBUTE_NORETURN static void Crash(const Status& status);
|
||||
};
|
||||
|
||||
// Construct an instance of T in `p` through placement new, passing Args... to
|
||||
// the constructor.
|
||||
// This abstraction is here mostly for the gcc performance fix.
|
||||
template <typename T, typename... Args>
|
||||
void PlacementNew(void* p, Args&&... args) {
|
||||
#if defined(__GNUC__) && !defined(__clang__)
|
||||
// Teach gcc that 'p' cannot be null, fixing code size issues.
|
||||
if (p == nullptr) __builtin_unreachable();
|
||||
#endif
|
||||
new (p) T(std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
// Helper base class to hold the data and all operations.
|
||||
// We move all this to a base class to allow mixing with the appropriate
|
||||
// TraitsBase specialization.
|
||||
template <typename T>
|
||||
class StatusOrData {
|
||||
template <typename U>
|
||||
friend class StatusOrData;
|
||||
|
||||
public:
|
||||
StatusOrData() = delete;
|
||||
|
||||
StatusOrData(const StatusOrData& other) {
|
||||
if (other.ok()) {
|
||||
MakeValue(other.data_);
|
||||
MakeStatus();
|
||||
} else {
|
||||
MakeStatus(other.status_);
|
||||
}
|
||||
}
|
||||
|
||||
StatusOrData(StatusOrData&& other) noexcept {
|
||||
if (other.ok()) {
|
||||
MakeValue(std::move(other.data_));
|
||||
MakeStatus();
|
||||
} else {
|
||||
MakeStatus(std::move(other.status_));
|
||||
}
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
StatusOrData(const StatusOrData<U>& other) {
|
||||
if (other.ok()) {
|
||||
MakeValue(other.data_);
|
||||
MakeStatus();
|
||||
} else {
|
||||
MakeStatus(other.status_);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
StatusOrData(StatusOrData<U>&& other) {
|
||||
if (other.ok()) {
|
||||
MakeValue(std::move(other.data_));
|
||||
MakeStatus();
|
||||
} else {
|
||||
MakeStatus(std::move(other.status_));
|
||||
}
|
||||
}
|
||||
|
||||
explicit StatusOrData(const T& value) : data_(value) { MakeStatus(); }
|
||||
explicit StatusOrData(T&& value) : data_(std::move(value)) { MakeStatus(); }
|
||||
|
||||
explicit StatusOrData(const Status& status) : status_(status) {
|
||||
EnsureNotOk();
|
||||
}
|
||||
explicit StatusOrData(Status&& status) : status_(std::move(status)) {
|
||||
EnsureNotOk();
|
||||
}
|
||||
|
||||
StatusOrData& operator=(const StatusOrData& other) {
|
||||
if (this == &other) return *this;
|
||||
if (other.ok())
|
||||
Assign(other.data_);
|
||||
else
|
||||
Assign(other.status_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
StatusOrData& operator=(StatusOrData&& other) {
|
||||
if (this == &other) return *this;
|
||||
if (other.ok())
|
||||
Assign(std::move(other.data_));
|
||||
else
|
||||
Assign(std::move(other.status_));
|
||||
return *this;
|
||||
}
|
||||
|
||||
~StatusOrData() {
|
||||
if (ok()) {
|
||||
status_.~Status();
|
||||
data_.~T();
|
||||
} else {
|
||||
status_.~Status();
|
||||
}
|
||||
}
|
||||
|
||||
void Assign(const T& value) {
|
||||
if (ok()) {
|
||||
data_.~T();
|
||||
MakeValue(value);
|
||||
} else {
|
||||
MakeValue(value);
|
||||
status_ = Status::OK();
|
||||
}
|
||||
}
|
||||
|
||||
void Assign(T&& value) {
|
||||
if (ok()) {
|
||||
data_.~T();
|
||||
MakeValue(std::move(value));
|
||||
} else {
|
||||
MakeValue(std::move(value));
|
||||
status_ = Status::OK();
|
||||
}
|
||||
}
|
||||
|
||||
void Assign(const Status& status) {
|
||||
Clear();
|
||||
status_ = status;
|
||||
EnsureNotOk();
|
||||
}
|
||||
|
||||
void Assign(Status&& status) {
|
||||
Clear();
|
||||
status_ = std::move(status);
|
||||
EnsureNotOk();
|
||||
}
|
||||
|
||||
bool ok() const { return status_.ok(); }
|
||||
|
||||
protected:
|
||||
// status_ will always be active after the constructor.
|
||||
// We make it a union to be able to initialize exactly how we need without
|
||||
// waste.
|
||||
// Eg. in the copy constructor we use the default constructor of Status in
|
||||
// the ok() path to avoid an extra Ref call.
|
||||
union {
|
||||
Status status_;
|
||||
};
|
||||
|
||||
// data_ is active iff status_.ok()==true
|
||||
struct Dummy {};
|
||||
union {
|
||||
// When T is const, we need some non-const object we can cast to void* for
|
||||
// the placement new. dummy_ is that object.
|
||||
Dummy dummy_;
|
||||
T data_;
|
||||
};
|
||||
|
||||
void Clear() {
|
||||
if (ok()) data_.~T();
|
||||
}
|
||||
|
||||
void EnsureOk() const {
|
||||
if (!ok()) Helper::Crash(status_);
|
||||
}
|
||||
|
||||
void EnsureNotOk() {
|
||||
if (ok()) Helper::HandleInvalidStatusCtorArg(&status_);
|
||||
}
|
||||
|
||||
// Construct the value (ie. data_) through placement new with the passed
|
||||
// argument.
|
||||
template <typename Arg>
|
||||
void MakeValue(Arg&& arg) {
|
||||
internal_statusor::PlacementNew<T>(&dummy_, std::forward<Arg>(arg));
|
||||
}
|
||||
|
||||
// Construct the status (ie. status_) through placement new with the passed
|
||||
// argument.
|
||||
template <typename... Args>
|
||||
void MakeStatus(Args&&... args) {
|
||||
internal_statusor::PlacementNew<Status>(&status_,
|
||||
std::forward<Args>(args)...);
|
||||
}
|
||||
};
|
||||
|
||||
// Helper base class to allow implicitly deleted constructors and assignment
|
||||
// operations in StatusOr.
|
||||
// TraitsBase will explicitly delete what it can't support and StatusOr will
|
||||
// inherit that behavior implicitly.
|
||||
template <bool Copy, bool Move>
|
||||
struct TraitsBase {
|
||||
TraitsBase() = default;
|
||||
TraitsBase(const TraitsBase&) = default;
|
||||
TraitsBase(TraitsBase&&) = default;
|
||||
TraitsBase& operator=(const TraitsBase&) = default;
|
||||
TraitsBase& operator=(TraitsBase&&) = default;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct TraitsBase<false, true> {
|
||||
TraitsBase() = default;
|
||||
TraitsBase(const TraitsBase&) = delete;
|
||||
TraitsBase(TraitsBase&&) = default;
|
||||
TraitsBase& operator=(const TraitsBase&) = delete;
|
||||
TraitsBase& operator=(TraitsBase&&) = default;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct TraitsBase<false, false> {
|
||||
TraitsBase() = default;
|
||||
TraitsBase(const TraitsBase&) = delete;
|
||||
TraitsBase(TraitsBase&&) = delete;
|
||||
TraitsBase& operator=(const TraitsBase&) = delete;
|
||||
TraitsBase& operator=(TraitsBase&&) = delete;
|
||||
};
|
||||
|
||||
} // namespace internal_statusor
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_INTERNALS_H_
|
||||
@@ -0,0 +1,42 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STR_UTIL_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STR_UTIL_H_
|
||||
|
||||
#include "tensorflow/core/lib/strings/str_util.h"
|
||||
#include "tensorflow/stream_executor/lib/stringpiece.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
using tensorflow::str_util::Join;
|
||||
using tensorflow::str_util::Split;
|
||||
|
||||
// Returns a copy of the input string 'str' with the given 'suffix'
|
||||
// removed. If the suffix doesn't match, returns a copy of the original string.
|
||||
inline string StripSuffixString(port::StringPiece str, port::StringPiece suffix) {
|
||||
if (tensorflow::str_util::EndsWith(str, suffix)) {
|
||||
str.remove_suffix(suffix.size());
|
||||
}
|
||||
return string(str);
|
||||
}
|
||||
|
||||
using tensorflow::str_util::Lowercase;
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STR_UTIL_H_
|
||||
@@ -0,0 +1,32 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// IWYU pragma: private, include "third_party/tensorflow/stream_executor/stream_executor.h"
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STRCAT_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STRCAT_H_
|
||||
|
||||
#include "absl/strings/str_cat.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
using absl::StrAppend;
|
||||
using absl::StrCat;
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STRCAT_H_
|
||||
@@ -0,0 +1,29 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STRINGPIECE_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STRINGPIECE_H_
|
||||
|
||||
#include "absl/strings/string_view.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
using StringPiece = absl::string_view;
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STRINGPIECE_H_
|
||||
@@ -0,0 +1,30 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STRINGPRINTF_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STRINGPRINTF_H_
|
||||
|
||||
#include "tensorflow/core/lib/strings/stringprintf.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
using tensorflow::strings::Printf;
|
||||
using tensorflow::strings::Appendf;
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STRINGPRINTF_H_
|
||||
@@ -0,0 +1,29 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_THREAD_OPTIONS_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_THREAD_OPTIONS_H_
|
||||
|
||||
#include "tensorflow/core/platform/env.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
using tensorflow::ThreadOptions;
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_THREAD_OPTIONS_H_
|
||||
@@ -0,0 +1,32 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_THREADPOOL_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_LIB_THREADPOOL_H_
|
||||
|
||||
#include "tensorflow/core/lib/core/threadpool.h"
|
||||
#include "tensorflow/stream_executor/lib/env.h"
|
||||
#include "tensorflow/stream_executor/lib/notification.h"
|
||||
#include "tensorflow/stream_executor/lib/thread_options.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace port {
|
||||
|
||||
using tensorflow::thread::ThreadPool;
|
||||
|
||||
} // namespace port
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_THREADPOOL_H_
|
||||
@@ -0,0 +1,66 @@
|
||||
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_MODULE_SPEC_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_MODULE_SPEC_H_
|
||||
|
||||
#include "tensorflow/stream_executor/lib/array_slice.h"
|
||||
#include "tensorflow/stream_executor/lib/stringpiece.h"
|
||||
#include "tensorflow/stream_executor/platform/logging.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
// Describes how to load a module on a target platform.
|
||||
//
|
||||
// The exact meaning of a "module" may differ from platform to platform but
|
||||
// loosely speaking a module a collection of kernels and global variables. It
|
||||
// corresponds to CUmodule when running on CUDA.
|
||||
class MultiModuleLoaderSpec {
|
||||
public:
|
||||
bool has_cuda_cubin_in_memory() const { return has_cuda_cubin_in_memory_; }
|
||||
port::ArraySlice<const uint8> cuda_cubin_in_memory() const {
|
||||
CHECK(has_cuda_cubin_in_memory());
|
||||
return {cuda_cubin_in_memory_.data(), cuda_cubin_in_memory_.size()};
|
||||
}
|
||||
|
||||
bool has_cuda_ptx_in_memory() const { return has_cuda_ptx_in_memory_; }
|
||||
const char* cuda_ptx_in_memory() const {
|
||||
CHECK(has_cuda_ptx_in_memory());
|
||||
return cuda_ptx_in_memory_;
|
||||
}
|
||||
|
||||
void AddCudaCubinInMemory(port::ArraySlice<const uint8> cubin_bytes) {
|
||||
CHECK(!cubin_bytes.empty());
|
||||
has_cuda_cubin_in_memory_ = true;
|
||||
cuda_cubin_in_memory_ = cubin_bytes;
|
||||
}
|
||||
|
||||
void AddCudaPtxInMemory(const char* ptx) {
|
||||
has_cuda_ptx_in_memory_ = true;
|
||||
// The CUDA driver does not like getting an empty string as PTX.
|
||||
cuda_ptx_in_memory_ = *ptx ? ptx : nullptr;
|
||||
}
|
||||
|
||||
private:
|
||||
port::ArraySlice<const uint8> cuda_cubin_in_memory_;
|
||||
bool has_cuda_cubin_in_memory_ = false;
|
||||
const char* cuda_ptx_in_memory_;
|
||||
bool has_cuda_ptx_in_memory_ = false;
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_MODULE_SPEC_H_
|
||||
@@ -0,0 +1,191 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// This is a registration-oriented interface for multiple platforms. It will
|
||||
// replace the MachineManager singleton interface, as MachineManager does not
|
||||
// currently support simultaneous use of multiple platforms.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// In your BUILD rule, add a dependency on a platform plugin that you'd like
|
||||
// to use, such as:
|
||||
//
|
||||
// //third_party/tensorflow/stream_executor/cuda:cuda_platform
|
||||
// //third_party/tensorflow/stream_executor/opencl:opencl_platform
|
||||
//
|
||||
// This will register platform plugins that can be discovered via this
|
||||
// interface. Sample API usage:
|
||||
//
|
||||
// port::StatusOr<Platform*> platform_status =
|
||||
// se::MultiPlatformManager::PlatformWithName("OpenCL");
|
||||
// if (!platform_status.ok()) { ... }
|
||||
// Platform* platform = platform_status.ValueOrDie();
|
||||
// LOG(INFO) << platform->VisibleDeviceCount() << " devices visible";
|
||||
// if (platform->VisibleDeviceCount() <= 0) { return; }
|
||||
//
|
||||
// for (int i = 0; i < platform->VisibleDeviceCount(); ++i) {
|
||||
// port::StatusOr<StreamExecutor*> executor_status =
|
||||
// platform->ExecutorForDevice(i);
|
||||
// if (!executor_status.ok()) {
|
||||
// LOG(INFO) << "could not retrieve executor for device ordinal " << i
|
||||
// << ": " << executor_status.status();
|
||||
// continue;
|
||||
// }
|
||||
// LOG(INFO) << "found usable executor: " << executor_status.ValueOrDie();
|
||||
// }
|
||||
//
|
||||
// A few things to note:
|
||||
// - There is no standard formatting/practice for identifying the name of a
|
||||
// platform. Ideally, a platform will list its registered name in its header
|
||||
// or in other associated documentation.
|
||||
// - Platform name lookup is case-insensitive. "OpenCL" or "opencl" (or even
|
||||
// ("OpEnCl") would work correctly in the above example.
|
||||
//
|
||||
// And similarly, for standard interfaces (BLAS, RNG, etc.) you can add
|
||||
// dependencies on support libraries, e.g.:
|
||||
//
|
||||
// //third_party/tensorflow/stream_executor/cuda:pluton_blas_plugin
|
||||
// //third_party/tensorflow/stream_executor/cuda:cudnn_plugin
|
||||
// //third_party/tensorflow/stream_executor/cuda:cublas_plugin
|
||||
// //third_party/tensorflow/stream_executor/cuda:curand_plugin
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_MULTI_PLATFORM_MANAGER_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_MULTI_PLATFORM_MANAGER_H_
|
||||
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
|
||||
#include "tensorflow/stream_executor/lib/initialize.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/platform.h"
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/platform/thread_annotations.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
// Manages multiple platforms that may be present on the current machine.
|
||||
class MultiPlatformManager {
|
||||
public:
|
||||
// Registers a platform object, returns an error status if the platform is
|
||||
// already registered. The associated listener, if not null, will be used to
|
||||
// trace events for ALL executors for that platform.
|
||||
// Takes ownership of listener.
|
||||
static port::Status RegisterPlatform(std::unique_ptr<Platform> platform)
|
||||
LOCKS_EXCLUDED(platforms_mutex_);
|
||||
|
||||
// Retrieves the platform registered with the given platform name (e.g.
|
||||
// "CUDA", "OpenCL", ...) or id (an opaque, comparable value provided by the
|
||||
// Platform's Id() method).
|
||||
//
|
||||
// If the platform has not already been initialized, it will be initialized
|
||||
// with a default set of parameters.
|
||||
//
|
||||
// If the requested platform is not registered, an error status is returned.
|
||||
// Ownership of the platform is NOT transferred to the caller --
|
||||
// the MultiPlatformManager owns the platforms in a singleton-like fashion.
|
||||
static port::StatusOr<Platform*> PlatformWithName(const string& target)
|
||||
LOCKS_EXCLUDED(platforms_mutex_);
|
||||
static port::StatusOr<Platform*> PlatformWithId(const Platform::Id& id)
|
||||
LOCKS_EXCLUDED(platforms_mutex_);
|
||||
|
||||
// Retrieves the platform registered with the given platform name (e.g.
|
||||
// "CUDA", "OpenCL", ...) or id (an opaque, comparable value provided by the
|
||||
// Platform's Id() method).
|
||||
//
|
||||
// The platform will be initialized with the given options. If the platform
|
||||
// was already initialized, an error will be returned.
|
||||
//
|
||||
// If the requested platform is not registered, an error status is returned.
|
||||
// Ownership of the platform is NOT transferred to the caller --
|
||||
// the MultiPlatformManager owns the platforms in a singleton-like fashion.
|
||||
static port::StatusOr<Platform*> InitializePlatformWithName(
|
||||
const string& target, const std::map<string, string>& options)
|
||||
LOCKS_EXCLUDED(platforms_mutex_);
|
||||
static port::StatusOr<Platform*> InitializePlatformWithId(
|
||||
const Platform::Id& id, const std::map<string, string>& options)
|
||||
LOCKS_EXCLUDED(platforms_mutex_);
|
||||
|
||||
// Clears the set of registered platforms, primarily used for testing.
|
||||
static void ClearPlatformRegistry() LOCKS_EXCLUDED(platforms_mutex_);
|
||||
|
||||
// Although the MultiPlatformManager "owns" its platforms, it holds them as
|
||||
// undecorated pointers to prevent races during program exit (between this
|
||||
// object's data and the underlying platforms (e.g., CUDA, OpenCL).
|
||||
// Because certain platforms have unpredictable deinitialization
|
||||
// times/sequences, it is not possible to strucure a safe deinitialization
|
||||
// sequence. Thus, we intentionally "leak" allocated platforms to defer
|
||||
// cleanup to the OS. This should be acceptable, as these are one-time
|
||||
// allocations per program invocation.
|
||||
// The MultiPlatformManager should be considered the owner
|
||||
// of any platforms registered with it, and leak checking should be disabled
|
||||
// during allocation of such Platforms, to avoid spurious reporting at program
|
||||
// exit.
|
||||
using PlatformMap = std::map<string, Platform*>;
|
||||
|
||||
// Provides access to the available set of platforms under a lock.
|
||||
static port::Status WithPlatforms(
|
||||
std::function<port::Status(PlatformMap*)> callback)
|
||||
LOCKS_EXCLUDED(platforms_mutex_) {
|
||||
mutex_lock lock(platforms_mutex_);
|
||||
return callback(GetPlatformMap());
|
||||
}
|
||||
|
||||
private:
|
||||
using PlatformIdMap = std::map<Platform::Id, Platform*>;
|
||||
|
||||
static mutex platforms_mutex_;
|
||||
|
||||
// TODO(b/22689637): Clean up these two maps; make sure they coexist nicely.
|
||||
// TODO(b/22689637): Move this (whatever the final/"official" map is) to
|
||||
// plugin_regstry.h, along with the associated functionality.
|
||||
// Platform-name-to-object mapping. These platforms are registered via module
|
||||
// initializers, and linkage determines which platforms are available to a
|
||||
// given target.
|
||||
static PlatformMap* GetPlatformMap() {
|
||||
static PlatformMap* instance = new PlatformMap;
|
||||
return instance;
|
||||
}
|
||||
|
||||
// Holds a Platform::Id-to-object mapping.
|
||||
// Unlike platforms_ above, this map does not own its contents.
|
||||
static PlatformIdMap* GetPlatformByIdMap() {
|
||||
static PlatformIdMap* instance = new PlatformIdMap;
|
||||
return instance;
|
||||
}
|
||||
|
||||
// Looks up the platform object with the given name. Assumes the Platforms
|
||||
// mutex is held.
|
||||
static port::StatusOr<Platform*> LookupByNameLocked(const string& target)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(platforms_mutex_);
|
||||
|
||||
// Looks up the platform object with the given id. Assumes the Platforms
|
||||
// mutex is held.
|
||||
static port::StatusOr<Platform*> LookupByIdLocked(const Platform::Id& id)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(platforms_mutex_);
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(MultiPlatformManager);
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
// multi_platform_manager.cc will define this instance. Includers of this header
|
||||
// should use
|
||||
// REGISTER_MODULE_INITIALIZER_SEQUENCE(my_platform, multi_platform_manager);
|
||||
DECLARE_MODULE_INITIALIZER(multi_platform_manager);
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_MULTI_PLATFORM_MANAGER_H_
|
||||
@@ -0,0 +1,209 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Defines types and declares functions for identifying and extracting
|
||||
// information about the types of platforms and supporting libraries for which
|
||||
// StreamExecutor implementations exist.
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_PLATFORM_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_PLATFORM_H_
|
||||
|
||||
#include <map>
|
||||
|
||||
#include "tensorflow/stream_executor/device_options.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/lib/status_macros.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/plugin.h"
|
||||
#include "tensorflow/stream_executor/trace_listener.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class StreamExecutor;
|
||||
|
||||
// Describes the platform for a StreamExecutor instantiation to act upon.
|
||||
//
|
||||
// Implementors: if you add a value here be sure to update PlatformKindString
|
||||
// and CheckPlatformKindIsValid.
|
||||
enum class PlatformKind {
|
||||
kInvalid,
|
||||
kCuda,
|
||||
kOpenCL,
|
||||
kHost,
|
||||
kMock,
|
||||
kSize,
|
||||
};
|
||||
|
||||
// Returns true if kind represents a valid platform capable of enqueuing items
|
||||
// on a stream, but not necessarily on an accelerator device.
|
||||
// Returns false for kMock and any invalid PlatformKind values.
|
||||
bool PlatformIsRunnable(PlatformKind kind);
|
||||
|
||||
// Returns true if kind represents a valid platform capable of running kernels
|
||||
// on an accelerator device. Returns false for kHost*, kMock and any invalid
|
||||
// PlatformKind values.
|
||||
bool PlatformIsRunnableOnDevice(PlatformKind kind);
|
||||
|
||||
// Returns a printable description of a PlatformKind.
|
||||
string PlatformKindString(PlatformKind kind);
|
||||
|
||||
// Returns the PlatformKind corresponding to the input string; returns kInvalid
|
||||
// in the case of no match.
|
||||
PlatformKind PlatformKindFromString(string platform_string);
|
||||
|
||||
// Checks that kind takes on a valid value.
|
||||
void CheckPlatformKindIsValid(PlatformKind kind);
|
||||
|
||||
// StreamExecutorConfig encapsulates the set of options for constructing a
|
||||
// StreamExecutor for a given platform.
|
||||
struct StreamExecutorConfig {
|
||||
// Sets members to defaults: -1 for ordinal (must be changed), and default
|
||||
// PluginConfig and DeviceOptions.
|
||||
StreamExecutorConfig();
|
||||
|
||||
// Simple ordinal-setting constructor.
|
||||
explicit StreamExecutorConfig(int ordinal);
|
||||
|
||||
// The ordinal of the device to be managed by the returned StreamExecutor.
|
||||
int ordinal;
|
||||
|
||||
// The PluginConfig for the returned StreamExecutor.
|
||||
PluginConfig plugin_config;
|
||||
|
||||
// The DeviceOptions for the returned StreamExecutor.
|
||||
DeviceOptions device_options;
|
||||
};
|
||||
|
||||
// Abstract base class for a platform registered with the MultiPlatformManager.
|
||||
class Platform {
|
||||
public:
|
||||
virtual ~Platform();
|
||||
|
||||
// A platform ID is a unique identifier for each registered platform type -
|
||||
// each platform is required to expose an ID to ensure unique registration and
|
||||
// as a target against which plugins can register.
|
||||
//
|
||||
// The macro below is provided to help generate a [process-unique] identifier.
|
||||
using Id = void*;
|
||||
|
||||
// Helper macro to define a plugin ID. To be used only inside plugin
|
||||
// implementation files. Works by "reserving" an address/value (guaranteed to be
|
||||
// unique) inside a process space.
|
||||
#define PLATFORM_DEFINE_ID(ID_VAR_NAME) \
|
||||
namespace { \
|
||||
int plugin_id_value; \
|
||||
} \
|
||||
const ::stream_executor::Platform::Id ID_VAR_NAME = &plugin_id_value;
|
||||
|
||||
// Returns a key uniquely identifying this platform.
|
||||
virtual Id id() const = 0;
|
||||
|
||||
// Name of this platform.
|
||||
virtual const string& Name() const = 0;
|
||||
|
||||
// Returns the number of devices accessible on this platform.
|
||||
//
|
||||
// Note that, though these devices are visible, if there is only one userspace
|
||||
// context allowed for the device at a time and another process is using this
|
||||
// device, a call to ExecutorForDevice may return an error status.
|
||||
virtual int VisibleDeviceCount() const = 0;
|
||||
|
||||
// Returns true iff the platform has been initialized.
|
||||
virtual bool Initialized() const;
|
||||
|
||||
// Initializes the platform with a custom set of options. The platform must be
|
||||
// initialized before obtaining StreamExecutor objects. The interpretation of
|
||||
// the platform_options argument is implementation specific. This method may
|
||||
// return an error if unrecognized options are provided. If using
|
||||
// MultiPlatformManager, this method will be called automatically by
|
||||
// InitializePlatformWithId/InitializePlatformWithName.
|
||||
virtual port::Status Initialize(
|
||||
const std::map<string, string>& platform_options);
|
||||
|
||||
// Returns a device with the given ordinal on this platform with a default
|
||||
// plugin configuration or, if none can be found with the given ordinal or
|
||||
// there is an error in opening a context to communicate with the device, an
|
||||
// error status is returned.
|
||||
//
|
||||
// Ownership of the executor is NOT transferred to the caller --
|
||||
// the Platform owns the executors in a singleton-like fashion.
|
||||
virtual port::StatusOr<StreamExecutor*> ExecutorForDevice(int ordinal) = 0;
|
||||
|
||||
// Returns a device or error, as above, with the specified plugins.
|
||||
//
|
||||
// Ownership of the executor is NOT transferred to the caller.
|
||||
virtual port::StatusOr<StreamExecutor*> ExecutorForDeviceWithPluginConfig(
|
||||
int ordinal, const PluginConfig& plugin_config) = 0;
|
||||
|
||||
// Returns a device constructed with the options specified in "config".
|
||||
// Ownership of the executor is NOT transferred to the caller.
|
||||
virtual port::StatusOr<StreamExecutor*> GetExecutor(
|
||||
const StreamExecutorConfig& config) = 0;
|
||||
|
||||
// Returns a device constructed with the options specified in "config" without
|
||||
// looking in or storing to the Platform's executor cache.
|
||||
// Ownership IS transferred to the caller.
|
||||
virtual port::StatusOr<std::unique_ptr<StreamExecutor>> GetUncachedExecutor(
|
||||
const StreamExecutorConfig& config) = 0;
|
||||
|
||||
// Warning: this is a dangerous API and should be used with caution.
|
||||
//
|
||||
// Forces the platform to delete executor instances, releasing their
|
||||
// associated device contexts. There must be no held instances of the executor
|
||||
// and there must be no outstanding activity on the devices for this platform.
|
||||
//
|
||||
// This is only useful on platforms which bind a device to a single process
|
||||
// that has obtained the device context. May return UNIMPLEMENTED on platforms
|
||||
// that have no reason to destroy device contexts.
|
||||
//
|
||||
// The platform must be reinitialized after this is called.
|
||||
virtual port::Status ForceExecutorShutdown();
|
||||
|
||||
// Registers a TraceListener to listen to all StreamExecutors for this
|
||||
// platform.
|
||||
// Takes ownership of listener.
|
||||
virtual void RegisterTraceListener(
|
||||
std::unique_ptr<TraceListener> listener) = 0;
|
||||
|
||||
// Removes the specified TraceListener from all StreamExecutors.
|
||||
virtual void UnregisterTraceListener(TraceListener* listener) = 0;
|
||||
|
||||
// Map of executor-to-executor coordinate and boolean, indicating if the first
|
||||
// executor can access the second's memory.
|
||||
using PeerAccessMap = std::map<std::pair<int, int>, bool>;
|
||||
|
||||
// Returns a matrix indicating which executors can access which other
|
||||
// executors' memory.
|
||||
virtual std::unique_ptr<PeerAccessMap> GetPeerAccessMap();
|
||||
|
||||
// Attempts to enable all peer-to-peer access links described by the result of
|
||||
// GetPeerAccessMap(). Note that calling this routine will force the creation
|
||||
// of a default-argument (see StreamExecutorConfig) StreamExecutor object for
|
||||
// each device ordinal in the system, should any not yet exist.
|
||||
virtual port::Status EnablePeerAccess();
|
||||
|
||||
protected:
|
||||
// SE_DISALLOW_COPY_AND_ASSIGN declares a constructor, which suppresses the
|
||||
// presence of the default constructor. This statement re-enables it, which
|
||||
// simplifies subclassing.
|
||||
Platform() = default;
|
||||
|
||||
private:
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(Platform);
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_PLATFORM_H_
|
||||
@@ -0,0 +1,39 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_PLATFORM_DEFAULT_MUTEX_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_PLATFORM_DEFAULT_MUTEX_H_
|
||||
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
#undef mutex_lock
|
||||
#undef tf_shared_lock
|
||||
|
||||
using tensorflow::ConditionResult;
|
||||
using tensorflow::WaitForMilliseconds;
|
||||
using tensorflow::condition_variable;
|
||||
using tensorflow::mutex;
|
||||
using tensorflow::mutex_lock;
|
||||
using tensorflow::tf_shared_lock;
|
||||
|
||||
#define mutex_lock(x) static_assert(0, "mutex_lock_decl_missing_var_name");
|
||||
#define tf_shared_lock(x) \
|
||||
static_assert(0, "tf_shared_lock_decl_missing_var_name");
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_PLATFORM_DEFAULT_MUTEX_H_
|
||||
@@ -0,0 +1,36 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_PLATFORM_LOGGING_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_PLATFORM_LOGGING_H_
|
||||
|
||||
#include "tensorflow/core/platform/logging.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
#if !defined(PLATFORM_GOOGLE)
|
||||
|
||||
// A CHECK() macro that lets you assert the success of a function that
|
||||
// returns -1 and sets errno in case of an error. E.g.
|
||||
//
|
||||
// CHECK_ERR(mkdir(path, 0700));
|
||||
//
|
||||
// or
|
||||
//
|
||||
// int fd = open(filename, flags); CHECK_ERR(fd) << ": open " << filename;
|
||||
#define CHECK_ERR(invocation) CHECK((invocation) != -1) << #invocation
|
||||
|
||||
#endif
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_PLATFORM_LOGGING_H_
|
||||
@@ -0,0 +1,28 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_PLATFORM_MUTEX_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_PLATFORM_MUTEX_H_
|
||||
|
||||
#include "tensorflow/core/platform/mutex.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
#if defined(PLATFORM_GOOGLE)
|
||||
#include "tensorflow/stream_executor/platform/google/mutex.h"
|
||||
#else
|
||||
#include "tensorflow/stream_executor/platform/default/mutex.h"
|
||||
#endif
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_PLATFORM_MUTEX_H_
|
||||
@@ -0,0 +1,53 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// IWYU pragma: private, include "third_party/tensorflow/stream_executor/stream_executor.h"
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_PLATFORM_PORT_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_PLATFORM_PORT_H_
|
||||
|
||||
#include "tensorflow/core/platform/macros.h"
|
||||
#include "tensorflow/core/platform/mutex.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
using tensorflow::int8;
|
||||
using tensorflow::int16;
|
||||
using tensorflow::int32;
|
||||
using tensorflow::int64;
|
||||
|
||||
using tensorflow::uint8;
|
||||
using tensorflow::uint16;
|
||||
using tensorflow::uint32;
|
||||
using tensorflow::uint64;
|
||||
|
||||
#if !defined(PLATFORM_GOOGLE)
|
||||
using std::string;
|
||||
#endif
|
||||
|
||||
using tensorflow::LinkerInitialized;
|
||||
using tensorflow::LINKER_INITIALIZED;
|
||||
|
||||
#define SE_FALLTHROUGH_INTENDED TF_FALLTHROUGH_INTENDED
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#define SE_DISALLOW_COPY_AND_ASSIGN TF_DISALLOW_COPY_AND_ASSIGN
|
||||
#define SE_MUST_USE_RESULT TF_MUST_USE_RESULT
|
||||
#define SE_PREDICT_TRUE TF_PREDICT_TRUE
|
||||
#define SE_PREDICT_FALSE TF_PREDICT_FALSE
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_PLATFORM_PORT_H_
|
||||
@@ -0,0 +1,21 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_PLATFORM_THREAD_ANNOTATIONS_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_PLATFORM_THREAD_ANNOTATIONS_H_
|
||||
|
||||
#include "tensorflow/core/platform/thread_annotations.h"
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_PLATFORM_THREAD_ANNOTATIONS_H_
|
||||
@@ -0,0 +1,87 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_PLUGIN_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_PLUGIN_H_
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
// A plugin ID is a unique identifier for each registered plugin type.
|
||||
typedef void* PluginId;
|
||||
|
||||
// Helper macro to define a plugin ID. To be used only inside plugin
|
||||
// implementation files. Works by "reserving" an address/value (guaranteed to be
|
||||
// unique) inside a process space.
|
||||
#define PLUGIN_REGISTRY_DEFINE_PLUGIN_ID(ID_VAR_NAME) \
|
||||
namespace { \
|
||||
int plugin_id_value; \
|
||||
} \
|
||||
const PluginId ID_VAR_NAME = &plugin_id_value;
|
||||
|
||||
// kNullPlugin denotes an invalid plugin identifier.
|
||||
extern const PluginId kNullPlugin;
|
||||
|
||||
// Enumeration to list the supported types of plugins / support libraries.
|
||||
enum class PluginKind {
|
||||
kInvalid,
|
||||
kBlas,
|
||||
kDnn,
|
||||
kFft,
|
||||
kRng,
|
||||
};
|
||||
|
||||
// A PluginConfig describes the set of plugins to be used by a StreamExecutor
|
||||
// instance. Each plugin is defined by an arbitrary identifier, usually best set
|
||||
// to the address static member in the implementation (to avoid conflicts).
|
||||
//
|
||||
// A PluginConfig may be passed to the StreamExecutor constructor - the plugins
|
||||
// described therein will be used to provide BLAS, DNN, FFT, and RNG
|
||||
// functionality. Platform-appropriate defaults will be used for any un-set
|
||||
// libraries. If a platform does not support a specified plugin (ex. cuBLAS on
|
||||
// an OpenCL executor), then an error will be logged and no plugin operations
|
||||
// will succeed.
|
||||
//
|
||||
// The StreamExecutor BUILD target does not link ANY plugin libraries - even
|
||||
// common host fallbacks! Any plugins must be explicitly linked by dependent
|
||||
// targets. See the cuda, opencl and host BUILD files for implemented plugin
|
||||
// support (search for "plugin").
|
||||
class PluginConfig {
|
||||
public:
|
||||
// Value specifying the platform's default option for that plugin.
|
||||
static const PluginId kDefault;
|
||||
|
||||
// Initializes all members to the default options.
|
||||
PluginConfig();
|
||||
|
||||
bool operator==(const PluginConfig& rhs) const;
|
||||
|
||||
// Sets the appropriate library kind to that passed in.
|
||||
PluginConfig& SetBlas(PluginId blas);
|
||||
PluginConfig& SetDnn(PluginId dnn);
|
||||
PluginConfig& SetFft(PluginId fft);
|
||||
PluginConfig& SetRng(PluginId rng);
|
||||
|
||||
PluginId blas() const { return blas_; }
|
||||
PluginId dnn() const { return dnn_; }
|
||||
PluginId fft() const { return fft_; }
|
||||
PluginId rng() const { return rng_; }
|
||||
|
||||
private:
|
||||
PluginId blas_, dnn_, fft_, rng_;
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_PLUGIN_H_
|
||||
@@ -0,0 +1,166 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_PLUGIN_REGISTRY_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_PLUGIN_REGISTRY_H_
|
||||
|
||||
#include <map>
|
||||
|
||||
#include "absl/base/macros.h"
|
||||
#include "tensorflow/stream_executor/blas.h"
|
||||
#include "tensorflow/stream_executor/dnn.h"
|
||||
#include "tensorflow/stream_executor/fft.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/platform.h"
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/plugin.h"
|
||||
#include "tensorflow/stream_executor/rng.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
namespace internal {
|
||||
class StreamExecutorInterface;
|
||||
}
|
||||
|
||||
// The PluginRegistry is a singleton that maintains the set of registered
|
||||
// "support library" plugins. Currently, there are four kinds of plugins:
|
||||
// BLAS, DNN, FFT, and RNG. Each interface is defined in the corresponding
|
||||
// gpu_{kind}.h header.
|
||||
//
|
||||
// At runtime, a StreamExecutor object will query the singleton registry to
|
||||
// retrieve the plugin kind that StreamExecutor was configured with (refer to
|
||||
// the StreamExecutor and PluginConfig declarations).
|
||||
//
|
||||
// Plugin libraries are best registered using REGISTER_MODULE_INITIALIZER,
|
||||
// but can be registered at any time. When registering a DSO-backed plugin, it
|
||||
// is usually a good idea to load the DSO at registration time, to prevent
|
||||
// late-loading from distorting performance/benchmarks as much as possible.
|
||||
class PluginRegistry {
|
||||
public:
|
||||
typedef blas::BlasSupport* (*BlasFactory)(internal::StreamExecutorInterface*);
|
||||
typedef dnn::DnnSupport* (*DnnFactory)(internal::StreamExecutorInterface*);
|
||||
typedef fft::FftSupport* (*FftFactory)(internal::StreamExecutorInterface*);
|
||||
typedef rng::RngSupport* (*RngFactory)(internal::StreamExecutorInterface*);
|
||||
|
||||
// Gets (and creates, if necessary) the singleton PluginRegistry instance.
|
||||
static PluginRegistry* Instance();
|
||||
|
||||
// Registers the specified factory with the specified platform.
|
||||
// Returns a non-successful status if the factory has already been registered
|
||||
// with that platform (but execution should be otherwise unaffected).
|
||||
template <typename FactoryT>
|
||||
port::Status RegisterFactory(Platform::Id platform_id, PluginId plugin_id,
|
||||
const string& name, FactoryT factory);
|
||||
|
||||
// Registers the specified factory as usable by _all_ platform types.
|
||||
// Reports errors just as RegisterFactory.
|
||||
template <typename FactoryT>
|
||||
port::Status RegisterFactoryForAllPlatforms(PluginId plugin_id,
|
||||
const string& name,
|
||||
FactoryT factory);
|
||||
|
||||
// TODO(b/22689637): Setter for temporary mapping until all users are using
|
||||
// MultiPlatformManager / PlatformId.
|
||||
void MapPlatformKindToId(PlatformKind platform_kind,
|
||||
Platform::Id platform_id);
|
||||
|
||||
// Potentially sets the plugin identified by plugin_id to be the default
|
||||
// for the specified platform and plugin kind. If this routine is called
|
||||
// multiple types for the same PluginKind, the PluginId given in the last call
|
||||
// will be used.
|
||||
bool SetDefaultFactory(Platform::Id platform_id, PluginKind plugin_kind,
|
||||
PluginId plugin_id);
|
||||
|
||||
// Return true if the factory/id has been registered for the
|
||||
// specified platform and plugin kind and false otherwise.
|
||||
bool HasFactory(Platform::Id platform_id, PluginKind plugin_kind,
|
||||
PluginId plugin) const;
|
||||
|
||||
// Retrieves the factory registered for the specified kind,
|
||||
// or a port::Status on error.
|
||||
template <typename FactoryT>
|
||||
port::StatusOr<FactoryT> GetFactory(Platform::Id platform_id,
|
||||
PluginId plugin_id);
|
||||
|
||||
// TODO(b/22689637): Deprecated/temporary. Will be deleted once all users are
|
||||
// on MultiPlatformManager / PlatformId.
|
||||
template <typename FactoryT>
|
||||
ABSL_DEPRECATED("Use MultiPlatformManager / PlatformId instead.")
|
||||
port::StatusOr<FactoryT> GetFactory(PlatformKind platform_kind,
|
||||
PluginId plugin_id);
|
||||
|
||||
private:
|
||||
// Containers for the sets of registered factories, by plugin kind.
|
||||
struct PluginFactories {
|
||||
std::map<PluginId, BlasFactory> blas;
|
||||
std::map<PluginId, DnnFactory> dnn;
|
||||
std::map<PluginId, FftFactory> fft;
|
||||
std::map<PluginId, RngFactory> rng;
|
||||
};
|
||||
|
||||
// Simple structure to hold the currently configured default plugins (for a
|
||||
// particular Platform).
|
||||
struct DefaultFactories {
|
||||
DefaultFactories();
|
||||
PluginId blas, dnn, fft, rng;
|
||||
};
|
||||
|
||||
PluginRegistry();
|
||||
|
||||
// Actually performs the work of registration.
|
||||
template <typename FactoryT>
|
||||
port::Status RegisterFactoryInternal(PluginId plugin_id,
|
||||
const string& plugin_name,
|
||||
FactoryT factory,
|
||||
std::map<PluginId, FactoryT>* factories);
|
||||
|
||||
// Actually performs the work of factory retrieval.
|
||||
template <typename FactoryT>
|
||||
port::StatusOr<FactoryT> GetFactoryInternal(
|
||||
PluginId plugin_id, const std::map<PluginId, FactoryT>& factories,
|
||||
const std::map<PluginId, FactoryT>& generic_factories) const;
|
||||
|
||||
// Returns true if the specified plugin has been registered with the specified
|
||||
// platform factories. Unlike the other overload of this method, this does
|
||||
// not implicitly examine the default factory lists.
|
||||
bool HasFactory(const PluginFactories& factories, PluginKind plugin_kind,
|
||||
PluginId plugin) const;
|
||||
|
||||
// The singleton itself.
|
||||
static PluginRegistry* instance_;
|
||||
|
||||
// TODO(b/22689637): Temporary mapping until all users are using
|
||||
// MultiPlatformManager / PlatformId.
|
||||
std::map<PlatformKind, Platform::Id> platform_id_by_kind_;
|
||||
|
||||
// The set of registered factories, keyed by platform ID.
|
||||
std::map<Platform::Id, PluginFactories> factories_;
|
||||
|
||||
// Plugins supported for all platform kinds.
|
||||
PluginFactories generic_factories_;
|
||||
|
||||
// The sets of default factories, keyed by platform ID.
|
||||
std::map<Platform::Id, DefaultFactories> default_factories_;
|
||||
|
||||
// Lookup table for plugin names.
|
||||
std::map<PluginId, string> plugin_names_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(PluginRegistry);
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_PLUGIN_REGISTRY_H_
|
||||
@@ -0,0 +1,93 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_RNG_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_RNG_H_
|
||||
|
||||
#include <limits.h>
|
||||
#include <complex>
|
||||
|
||||
#include "tensorflow/stream_executor/platform/logging.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class Stream;
|
||||
template <typename ElemT>
|
||||
class DeviceMemory;
|
||||
|
||||
namespace rng {
|
||||
|
||||
// Random-number-generation support interface -- this can be derived from a GPU
|
||||
// executor when the underlying platform has an RNG library implementation
|
||||
// available. See StreamExecutor::AsRng().
|
||||
// When a seed is not specified, the backing RNG will be initialized with the
|
||||
// default seed for that implementation.
|
||||
//
|
||||
// Thread-hostile: see StreamExecutor class comment for details on
|
||||
// thread-hostility.
|
||||
class RngSupport {
|
||||
public:
|
||||
static const int kMinSeedBytes = 16;
|
||||
static const int kMaxSeedBytes = INT_MAX;
|
||||
|
||||
// Releases any random-number-generation resources associated with this
|
||||
// support object in the underlying platform implementation.
|
||||
virtual ~RngSupport() {}
|
||||
|
||||
// Populates a GPU memory allocation with random values appropriate for the
|
||||
// DeviceMemory element type; i.e. populates DeviceMemory<float> with random
|
||||
// float values.
|
||||
virtual bool DoPopulateRandUniform(Stream *stream,
|
||||
DeviceMemory<float> *v) = 0;
|
||||
virtual bool DoPopulateRandUniform(Stream *stream,
|
||||
DeviceMemory<double> *v) = 0;
|
||||
virtual bool DoPopulateRandUniform(Stream *stream,
|
||||
DeviceMemory<std::complex<float>> *v) = 0;
|
||||
virtual bool DoPopulateRandUniform(Stream *stream,
|
||||
DeviceMemory<std::complex<double>> *v) = 0;
|
||||
|
||||
// Populates a GPU memory allocation with random values sampled from a
|
||||
// Gaussian distribution with the given mean and standard deviation.
|
||||
virtual bool DoPopulateRandGaussian(Stream *stream, float mean, float stddev,
|
||||
DeviceMemory<float> *v) {
|
||||
LOG(ERROR)
|
||||
<< "platform's random number generator does not support gaussian";
|
||||
return false;
|
||||
}
|
||||
virtual bool DoPopulateRandGaussian(Stream *stream, double mean,
|
||||
double stddev, DeviceMemory<double> *v) {
|
||||
LOG(ERROR)
|
||||
<< "platform's random number generator does not support gaussian";
|
||||
return false;
|
||||
}
|
||||
|
||||
// Specifies the seed used to initialize the RNG.
|
||||
// This call does not transfer ownership of the buffer seed; its data should
|
||||
// not be altered for the lifetime of this call. At least 16 bytes of seed
|
||||
// data must be provided, but not all seed data will necessarily be used.
|
||||
// seed: Pointer to seed data. Must not be null.
|
||||
// seed_bytes: Size of seed buffer in bytes. Must be >= 16.
|
||||
virtual bool SetSeed(Stream *stream, const uint8 *seed,
|
||||
uint64 seed_bytes) = 0;
|
||||
|
||||
protected:
|
||||
static bool CheckSeed(const uint8 *seed, uint64 seed_bytes);
|
||||
};
|
||||
|
||||
} // namespace rng
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_RNG_H_
|
||||
@@ -0,0 +1,81 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_SCRATCH_ALLOCATOR_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_SCRATCH_ALLOCATOR_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "tensorflow/stream_executor/device_memory.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/temporary_device_memory.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class Stream;
|
||||
|
||||
// Interface that allows stream operations (e.g.
|
||||
// Stream::ThenConvolveWithScratch) to optionally request scratch space be
|
||||
// allocated in order to speed up the operation being enqueued.
|
||||
//
|
||||
// Note that the caller is responsible for deallocating the scratch space at a
|
||||
// known-safe point, when all scratch-memory-consuming kernels are known for
|
||||
// sure to have finished; e.g. at stream synchronization time. This is different
|
||||
// from a traditional C++ object allocator, where the client is responsible for
|
||||
// releasing. (Conceptually, scratch memory is a form of "temporary" device
|
||||
// memory allocation.)
|
||||
class ScratchAllocator {
|
||||
public:
|
||||
virtual ~ScratchAllocator();
|
||||
|
||||
// Returns a limit of memory this scratch allocator wants to produce, in
|
||||
// bytes. This information may be used to help select an algorithm.
|
||||
//
|
||||
// Returns values < 0 to indicate that there is no recommended limit.
|
||||
virtual int64 GetMemoryLimitInBytes(Stream* stream) = 0;
|
||||
|
||||
// Returns an allocation on byte_size bytes for use in an operation on stream.
|
||||
//
|
||||
// This is a temporary allocation, and the caller is responsible for
|
||||
// deallocating at some known-safe point. See the class comment above.
|
||||
virtual port::StatusOr<DeviceMemory<uint8>> AllocateBytes(
|
||||
Stream* stream, int64 byte_size) = 0;
|
||||
};
|
||||
|
||||
// Allocates a single temporary memory allocation -- this memory is deallocated
|
||||
// at the next stream synchronization point after this object has gone out of
|
||||
// scope. This satisfies the lifetime and deallocation properties given in the
|
||||
// class comment above.
|
||||
//
|
||||
// Thread-compatible, but not thread-safe (use in scenarios where only one
|
||||
// thread will request the scratch allocation).
|
||||
class OneTimeScratchAllocator : public ScratchAllocator {
|
||||
public:
|
||||
OneTimeScratchAllocator();
|
||||
~OneTimeScratchAllocator() override;
|
||||
int64 GetMemoryLimitInBytes(Stream* stream) override;
|
||||
port::StatusOr<DeviceMemory<uint8>> AllocateBytes(Stream* stream,
|
||||
int64 byte_size) override;
|
||||
|
||||
private:
|
||||
std::unique_ptr<TemporaryDeviceMemory<uint8>> temporary_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(OneTimeScratchAllocator);
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_SCRATCH_ALLOCATOR_H_
|
||||
@@ -0,0 +1,34 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// This file defines a uniform interface to configuration options for shared
|
||||
// memory for supported devices. As with many StreamExecutor-supported features,
|
||||
// support for the options defined herein is device-dependent.
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_SHARED_MEMORY_CONFIG_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_SHARED_MEMORY_CONFIG_H_
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
// SharedMemoryConfig enum describes potential widths of shared memory banks for
|
||||
// a device or kernel.
|
||||
enum class SharedMemoryConfig {
|
||||
kDefault, // Use the device default configuration.
|
||||
kFourByte, // Sets shared memory banks to be four bytes wide.
|
||||
kEightByte, // Sets shared memory banks to be eight bytes wide.
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_SHARED_MEMORY_CONFIG_H_
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,49 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// The StreamExecutor is a single-device abstraction for:
|
||||
//
|
||||
// * Loading/launching data-parallel-kernels
|
||||
// * Invoking pre-canned high-performance library routines (like matrix
|
||||
// multiply)
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_H_
|
||||
|
||||
#include "tensorflow/stream_executor/device_description.h" // IWYU pragma: export
|
||||
#include "tensorflow/stream_executor/device_memory.h" // IWYU pragma: export
|
||||
#include "tensorflow/stream_executor/device_options.h" // IWYU pragma: export
|
||||
#include "tensorflow/stream_executor/event.h" // IWYU pragma: export
|
||||
#include "tensorflow/stream_executor/kernel.h" // IWYU pragma: export
|
||||
#include "tensorflow/stream_executor/kernel_spec.h" // IWYU pragma: export
|
||||
#include "tensorflow/stream_executor/launch_dim.h" // IWYU pragma: export
|
||||
#include "tensorflow/stream_executor/multi_platform_manager.h" // IWYU pragma: export
|
||||
#include "tensorflow/stream_executor/platform.h" // IWYU pragma: export
|
||||
#include "tensorflow/stream_executor/stream.h" // IWYU pragma: export
|
||||
#include "tensorflow/stream_executor/stream_executor_pimpl.h" // IWYU pragma: export
|
||||
#include "tensorflow/stream_executor/timer.h" // IWYU pragma: export
|
||||
|
||||
namespace perftools {
|
||||
namespace gputools {
|
||||
|
||||
// Temporarily pull stream_executor into perftools::gputools while we migrate
|
||||
// code to the new namespace. TODO(b/77980417): Remove this once we've
|
||||
// completed the migration.
|
||||
using namespace stream_executor; // NOLINT[build/namespaces]
|
||||
|
||||
} // namespace gputools
|
||||
} // namespace perftools
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_H_
|
||||
@@ -0,0 +1,388 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Interfaces for platform-dependent implementations to satisfy. This are
|
||||
// delegated to from the StreamExecutor in pointer-to-implementation style; i.e.
|
||||
// the StreamExecutor is just a husk that delegates calls to the
|
||||
// platform-specific objects which implement the interfaces defined here.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_INTERNAL_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_INTERNAL_H_
|
||||
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/stream_executor/device_description.h"
|
||||
#include "tensorflow/stream_executor/device_memory.h"
|
||||
#include "tensorflow/stream_executor/device_options.h"
|
||||
#include "tensorflow/stream_executor/dnn.h"
|
||||
#include "tensorflow/stream_executor/event.h"
|
||||
#include "tensorflow/stream_executor/kernel.h"
|
||||
#include "tensorflow/stream_executor/kernel_cache_config.h"
|
||||
#include "tensorflow/stream_executor/kernel_spec.h"
|
||||
#include "tensorflow/stream_executor/launch_dim.h"
|
||||
#include "tensorflow/stream_executor/lib/inlined_vector.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/module_spec.h"
|
||||
#include "tensorflow/stream_executor/platform.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/plugin_registry.h"
|
||||
#include "tensorflow/stream_executor/shared_memory_config.h"
|
||||
#include "tensorflow/stream_executor/trace_listener.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class Stream;
|
||||
class Timer;
|
||||
|
||||
// An opaque handle to a loaded module.
|
||||
//
|
||||
// An instance of this is returned from StreamExecutor::GetModule.
|
||||
class ModuleHandle {
|
||||
public:
|
||||
/*implicit*/ ModuleHandle(void *id = nullptr) : id_(id) {}
|
||||
|
||||
// A ModuleHandle with id() == nullptr is an invalid module handle, akin to a
|
||||
// null pointer.
|
||||
void *id() const { return id_; }
|
||||
|
||||
explicit operator bool() const { return id() != nullptr; }
|
||||
|
||||
private:
|
||||
void *id_;
|
||||
};
|
||||
|
||||
namespace internal {
|
||||
|
||||
// Platform-dependent interface class for the generic Events interface, in
|
||||
// the PIMPL style.
|
||||
class EventInterface {
|
||||
public:
|
||||
EventInterface() {}
|
||||
virtual ~EventInterface() {}
|
||||
|
||||
private:
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(EventInterface);
|
||||
};
|
||||
|
||||
// Pointer-to-implementation object type (i.e. the KernelBase class delegates to
|
||||
// this interface) with virtual destruction. This class exists for the
|
||||
// platform-dependent code to hang any kernel data/resource info/functionality
|
||||
// off of.
|
||||
class KernelInterface {
|
||||
public:
|
||||
// Default constructor for the abstract interface.
|
||||
KernelInterface() {}
|
||||
|
||||
// Default destructor for the abstract interface.
|
||||
virtual ~KernelInterface() {}
|
||||
|
||||
// Returns the number of formal parameters that this kernel accepts.
|
||||
virtual unsigned Arity() const = 0;
|
||||
|
||||
// Sets the preferred cache configuration.
|
||||
virtual void SetPreferredCacheConfig(KernelCacheConfig config) = 0;
|
||||
|
||||
// Gets the preferred cache configuration.
|
||||
virtual KernelCacheConfig GetPreferredCacheConfig() const = 0;
|
||||
|
||||
private:
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(KernelInterface);
|
||||
};
|
||||
|
||||
// Pointer-to-implementation object type (i.e. the Stream class delegates to
|
||||
// this interface) with virtual destruction. This class exists for the
|
||||
// platform-dependent code to hang any kernel data/resource info/functionality
|
||||
// off of.
|
||||
class StreamInterface {
|
||||
public:
|
||||
// Default constructor for the abstract interface.
|
||||
StreamInterface() {}
|
||||
|
||||
// Default destructor for the abstract interface.
|
||||
virtual ~StreamInterface() {}
|
||||
|
||||
// Returns the GPU stream associated with this platform's stream
|
||||
// implementation.
|
||||
//
|
||||
// WARNING: checks that the underlying platform is, in fact, CUDA or ROCm,
|
||||
// causing a fatal error if it is not. This hack is made available solely for
|
||||
// use from distbelief code, which temporarily has strong ties to CUDA or
|
||||
// ROCm as a platform.
|
||||
virtual void *GpuStreamHack() { return nullptr; }
|
||||
|
||||
// See the above comment on GpuStreamHack -- this further breaks abstraction
|
||||
// for Eigen within distbelief, which has strong ties to CUDA or ROCm as a
|
||||
// platform, and a historical attachment to a programming model which takes a
|
||||
// stream-slot rather than a stream-value.
|
||||
virtual void **GpuStreamMemberHack() { return nullptr; }
|
||||
|
||||
private:
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(StreamInterface);
|
||||
};
|
||||
|
||||
// Pointer-to-implementation object type (i.e. the Timer class delegates to
|
||||
// this interface) with virtual destruction. This class exists for the
|
||||
// platform-dependent code to hang any timer data/resource info/functionality
|
||||
// off of.
|
||||
class TimerInterface {
|
||||
public:
|
||||
// Default constructor for the abstract interface.
|
||||
TimerInterface() {}
|
||||
|
||||
// Default destructor for the abstract interface.
|
||||
virtual ~TimerInterface() {}
|
||||
|
||||
// Returns the number of microseconds elapsed in a completed timer.
|
||||
virtual uint64 Microseconds() const = 0;
|
||||
|
||||
// Returns the number of nanoseconds elapsed in a completed timer.
|
||||
virtual uint64 Nanoseconds() const = 0;
|
||||
|
||||
private:
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(TimerInterface);
|
||||
};
|
||||
|
||||
// Interface for the different StreamExecutor platforms (i.e. CUDA, OpenCL).
|
||||
//
|
||||
// Various platforms will provide an implementation that satisfy this interface.
|
||||
class StreamExecutorInterface {
|
||||
public:
|
||||
// Default constructor for the abstract interface.
|
||||
StreamExecutorInterface() {}
|
||||
|
||||
// Default destructor for the abstract interface.
|
||||
virtual ~StreamExecutorInterface() {}
|
||||
|
||||
// Returns the (transitively) wrapped executor if this executor is
|
||||
// wrapping another executor; otherwise, returns this.
|
||||
virtual StreamExecutorInterface *GetUnderlyingExecutor() { return this; }
|
||||
|
||||
// See the StreamExecutor interface for comments on the same-named methods.
|
||||
virtual port::Status Init(int device_ordinal,
|
||||
DeviceOptions device_options) = 0;
|
||||
|
||||
virtual bool GetKernel(const MultiKernelLoaderSpec &spec,
|
||||
KernelBase *kernel) {
|
||||
return false;
|
||||
}
|
||||
virtual bool LoadModule(const MultiModuleLoaderSpec &spec,
|
||||
ModuleHandle *module_handle) {
|
||||
return false;
|
||||
}
|
||||
virtual bool UnloadModule(ModuleHandle module_handle) { return false; }
|
||||
virtual bool Launch(Stream *stream, const ThreadDim &thread_dims,
|
||||
const BlockDim &block_dims, const KernelBase &k,
|
||||
const KernelArgsArrayBase &args) {
|
||||
return false;
|
||||
}
|
||||
// Releases any state associated with the kernel.
|
||||
virtual void UnloadKernel(const KernelBase *kernel) {}
|
||||
virtual void *Allocate(uint64 size) = 0;
|
||||
virtual void *AllocateSubBuffer(DeviceMemoryBase *parent, uint64 offset,
|
||||
uint64 size) = 0;
|
||||
virtual void Deallocate(DeviceMemoryBase *mem) = 0;
|
||||
// Allocates unified memory space of the given size, if supported.
|
||||
// See
|
||||
// https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#um-unified-memory-programming-hd
|
||||
// for more details on unified memory.
|
||||
virtual void *UnifiedMemoryAllocate(uint64 size) { return nullptr; }
|
||||
|
||||
// Deallocates unified memory space previously allocated with
|
||||
// UnifiedMemoryAllocate.
|
||||
virtual void UnifiedMemoryDeallocate(void *mem) {}
|
||||
virtual void *HostMemoryAllocate(uint64 size) = 0;
|
||||
virtual void HostMemoryDeallocate(void *mem) = 0;
|
||||
virtual bool HostMemoryRegister(void *mem, uint64 size) = 0;
|
||||
virtual bool HostMemoryUnregister(void *mem) = 0;
|
||||
virtual bool SynchronizeAllActivity() = 0;
|
||||
virtual bool SynchronousMemZero(DeviceMemoryBase *location, uint64 size) = 0;
|
||||
virtual bool SynchronousMemSet(DeviceMemoryBase *location, int value,
|
||||
uint64 size) = 0;
|
||||
virtual port::Status SynchronousMemcpy(DeviceMemoryBase *gpu_dst,
|
||||
const void *host_src, uint64 size) = 0;
|
||||
virtual port::Status SynchronousMemcpy(void *host_dst,
|
||||
const DeviceMemoryBase &gpu_src,
|
||||
uint64 size) = 0;
|
||||
virtual port::Status SynchronousMemcpyDeviceToDevice(
|
||||
DeviceMemoryBase *gpu_dst, const DeviceMemoryBase &gpu_src,
|
||||
uint64 size) = 0;
|
||||
virtual bool MemZero(Stream *stream, DeviceMemoryBase *location,
|
||||
uint64 size) = 0;
|
||||
virtual bool Memset(Stream *stream, DeviceMemoryBase *location, uint8 pattern,
|
||||
uint64 size) {
|
||||
return false;
|
||||
}
|
||||
virtual bool Memset32(Stream *stream, DeviceMemoryBase *location,
|
||||
uint32 pattern, uint64 size) = 0;
|
||||
virtual bool Memcpy(Stream *stream, void *host_dst,
|
||||
const DeviceMemoryBase &gpu_src, uint64 size) = 0;
|
||||
virtual bool Memcpy(Stream *stream, DeviceMemoryBase *gpu_dst,
|
||||
const void *host_src, uint64 size) = 0;
|
||||
virtual bool MemcpyDeviceToDevice(Stream *stream, DeviceMemoryBase *gpu_dst,
|
||||
const DeviceMemoryBase &gpu_src,
|
||||
uint64 size) = 0;
|
||||
virtual bool HostCallback(Stream *stream, std::function<void()> callback) = 0;
|
||||
virtual bool HostCallback(Stream *stream,
|
||||
std::function<port::Status()> callback);
|
||||
virtual port::Status AllocateEvent(Event *event) = 0;
|
||||
virtual port::Status DeallocateEvent(Event *event) = 0;
|
||||
virtual port::Status RecordEvent(Stream *stream, Event *event) = 0;
|
||||
virtual port::Status WaitForEvent(Stream *stream, Event *event) = 0;
|
||||
virtual Event::Status PollForEventStatus(Event *event) = 0;
|
||||
virtual bool AllocateStream(Stream *stream) = 0;
|
||||
virtual void DeallocateStream(Stream *stream) = 0;
|
||||
virtual bool CreateStreamDependency(Stream *dependent, Stream *other) = 0;
|
||||
virtual bool AllocateTimer(Timer *timer) = 0;
|
||||
virtual void DeallocateTimer(Timer *timer) = 0;
|
||||
virtual bool StartTimer(Stream *stream, Timer *timer) = 0;
|
||||
virtual bool StopTimer(Stream *stream, Timer *timer) = 0;
|
||||
virtual port::Status BlockHostUntilDone(Stream *stream) = 0;
|
||||
virtual int PlatformDeviceCount() = 0;
|
||||
virtual port::Status EnablePeerAccessTo(StreamExecutorInterface *other) = 0;
|
||||
virtual bool CanEnablePeerAccessTo(StreamExecutorInterface *other) = 0;
|
||||
virtual SharedMemoryConfig GetDeviceSharedMemoryConfig() = 0;
|
||||
virtual port::Status SetDeviceSharedMemoryConfig(
|
||||
SharedMemoryConfig config) = 0;
|
||||
|
||||
virtual int64 GetDeviceLoad() { return -1; }
|
||||
|
||||
virtual bool DeviceMemoryUsage(int64 *free, int64 *total) const {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Retrieves device pointer and size for a symbol. The device pointer is
|
||||
// stored at mem, and the size is stored at size. Either mem or bytes can be
|
||||
// null, however, both of them cannot be null at the same time. To use
|
||||
// constant memory in CUDA, GetSymbol has to be used. Returns true if symbol
|
||||
// is found.
|
||||
//
|
||||
// If ModuleHandle is set then we search for `symbol_name` only within the
|
||||
// module corresponding to `module_handle`. Otherwise all loaded modules are
|
||||
// searched.
|
||||
virtual bool GetSymbol(const string &symbol_name, ModuleHandle module_handle,
|
||||
void **mem, size_t *bytes) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Creates a new DeviceDescription object. Ownership is transferred to the
|
||||
// caller.
|
||||
virtual DeviceDescription *PopulateDeviceDescription() const = 0;
|
||||
|
||||
// Attempts to register the provided TraceListener with the device-specific
|
||||
// Executor implementation. When this is called, the PIMPL interface has
|
||||
// already taken ownership of the object and is managing the generic tracing
|
||||
// events. The device-specific implementation must determine if the passed
|
||||
// listener is of a type appropriate for it to trace during registration (and
|
||||
// before dispatching events to it).
|
||||
// Returns true if the listener was successfully registered, false otherwise.
|
||||
// Does not take ownership of listener.
|
||||
virtual bool RegisterTraceListener(TraceListener* listener) { return false; }
|
||||
|
||||
// Unregisters the specified listener from the device-specific Executor.
|
||||
// Returns true if the listener was successfully registered, false otherwise.
|
||||
virtual bool UnregisterTraceListener(TraceListener* listener) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Returns whether this StreamExecutor has BLAS support for its underlying
|
||||
// platform.
|
||||
virtual bool SupportsBlas() const { return false; }
|
||||
|
||||
// Creates a new BlasSupport object, ownership is transferred to the caller.
|
||||
// If SupportsBlas() is false, this will always return null.
|
||||
//
|
||||
// If SupportsBlas() is true, this may return null, for example, if the BLAS
|
||||
// initialization fails.
|
||||
virtual blas::BlasSupport *CreateBlas() { return nullptr; }
|
||||
|
||||
// Returns whether this StreamExecutor has FFT support for its underlying
|
||||
// platform.
|
||||
virtual bool SupportsFft() const { return false; }
|
||||
|
||||
// Creates a new fft::FftSupport object, ownership is transferred to the
|
||||
// caller.
|
||||
// If SupportsFft() is false, this will always return null.
|
||||
//
|
||||
// If SupportsFft() is true, this may return null, for example, if the FFT
|
||||
// initialization fails.
|
||||
virtual fft::FftSupport *CreateFft() { return nullptr; }
|
||||
|
||||
// Returns whether this StreamExecutor has Random Number Generation support
|
||||
// for
|
||||
// its underlying platform.
|
||||
virtual bool SupportsRng() const { return false; }
|
||||
|
||||
// Returns whether this StreamExecutor has neural net support for its
|
||||
// underlying
|
||||
// platform.
|
||||
virtual bool SupportsDnn() const { return false; }
|
||||
|
||||
// Creates a new RngSupport object, ownership is transferred to the caller.
|
||||
// If SupportsRng() is false, this will always return null.
|
||||
//
|
||||
// If SupportsRng() is true, this may return null, for example, if the RNG
|
||||
// initialization fails.
|
||||
virtual rng::RngSupport *CreateRng() { return nullptr; }
|
||||
|
||||
// Creates a new DnnSupport object, ownership is transferred to the caller.
|
||||
// If SupportsDnn() is false, this will always return null.
|
||||
//
|
||||
// If SupportsDnn() is true, this may return null, for example, if the DNN
|
||||
// initialization fails.
|
||||
virtual dnn::DnnSupport *CreateDnn() { return nullptr; }
|
||||
|
||||
// Each call creates a new instance of the platform-specific implementation of
|
||||
// the corresponding interface type.
|
||||
virtual std::unique_ptr<EventInterface> CreateEventImplementation() = 0;
|
||||
virtual std::unique_ptr<KernelInterface> CreateKernelImplementation() = 0;
|
||||
virtual std::unique_ptr<StreamInterface> GetStreamImplementation() = 0;
|
||||
virtual std::unique_ptr<TimerInterface> GetTimerImplementation() = 0;
|
||||
|
||||
// Returns the CUDA or ROCm context associated with this StreamExecutor
|
||||
// platform implementation.
|
||||
//
|
||||
// WARNING: checks that the underlying platform is, in fact, CUDA or ROCm,
|
||||
// causing a fatal error if it is not. This hack is made available solely for
|
||||
// use from distbelief code, which temporarily has strong ties to CUDA or ROCm
|
||||
// as a platform.
|
||||
virtual void *GpuContextHack() { return nullptr; }
|
||||
|
||||
private:
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(StreamExecutorInterface);
|
||||
};
|
||||
|
||||
using StreamExecutorFactory =
|
||||
std::function<StreamExecutorInterface *(const PluginConfig &)>;
|
||||
using EventFactory = std::function<EventInterface *(StreamExecutor *)>;
|
||||
using StreamFactory = std::function<StreamInterface *(StreamExecutor *)>;
|
||||
using TimerFactory = std::function<TimerInterface *(StreamExecutor *)>;
|
||||
using KernelFactory = std::function<KernelInterface*()>;
|
||||
|
||||
StreamExecutorFactory* MakeCUDAExecutorImplementation();
|
||||
|
||||
StreamExecutorFactory* MakeOpenCLExecutorImplementation();
|
||||
|
||||
extern StreamExecutorFactory MakeHostExecutorImplementation;
|
||||
|
||||
|
||||
} // namespace internal
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_INTERNAL_H_
|
||||
@@ -0,0 +1,881 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_PIMPL_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_PIMPL_H_
|
||||
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <tuple>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/base/macros.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/lib/strcat.h"
|
||||
#include "tensorflow/stream_executor/lib/threadpool.h"
|
||||
#include "tensorflow/stream_executor/platform.h"
|
||||
#include "tensorflow/stream_executor/platform/logging.h"
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/platform/thread_annotations.h"
|
||||
#include "tensorflow/stream_executor/rng.h"
|
||||
#include "tensorflow/stream_executor/shared_memory_config.h"
|
||||
#include "tensorflow/stream_executor/stream.h"
|
||||
#include "tensorflow/stream_executor/stream_executor_internal.h"
|
||||
#include "tensorflow/stream_executor/trace_listener.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
// Structure used for device memory leak checking.
|
||||
struct AllocRecord {
|
||||
// The requested allocation size of the buffer.
|
||||
uint64 bytes;
|
||||
|
||||
// Holds a representation of the stack at the time the associated buffer was
|
||||
// allocated. Produced in a form described in
|
||||
// //util/symbolize/symbolized_stacktrace.h.
|
||||
string stack_trace;
|
||||
};
|
||||
|
||||
// Forward declaration of private friend class.
|
||||
template <typename BeginCallT, typename CompleteCallT,
|
||||
typename ReturnT, typename... BeginArgsT>
|
||||
class ScopedTracer;
|
||||
|
||||
// A StreamExecutor manages a single device, in terms of executing work (kernel
|
||||
// launches) and memory management (allocation/deallocation, memory copies to
|
||||
// and from the device). It is conceptually the "handle" for a device -- Stream
|
||||
// objects, which are used to enqueue work to run on the
|
||||
// coprocessor have a StreamExecutor instance as their "parent" object.
|
||||
//
|
||||
// StreamExecutor objects have an underlying platform that is specified up
|
||||
// front;
|
||||
// e.g. either it is a CUDA or OpenCL executor.
|
||||
//
|
||||
// Thread-safe after initialization.
|
||||
// StreamExecutor interface should not be invoked from a signal handler.
|
||||
class StreamExecutor {
|
||||
public:
|
||||
explicit StreamExecutor(PlatformKind kind,
|
||||
const PluginConfig &plugin_config = PluginConfig());
|
||||
|
||||
StreamExecutor(
|
||||
const Platform *platform,
|
||||
std::unique_ptr<internal::StreamExecutorInterface> implementation);
|
||||
|
||||
~StreamExecutor();
|
||||
|
||||
port::Status Init();
|
||||
port::Status Init(int device_ordinal, DeviceOptions device_options);
|
||||
|
||||
// Returns the platform that this StreamExecutor is acting upon.
|
||||
ABSL_DEPRECATED("Use platform() instead.")
|
||||
PlatformKind platform_kind() const { return platform_kind_; }
|
||||
|
||||
// Returns a reference to the platform that created this executor.
|
||||
const Platform *platform() const { return platform_; }
|
||||
|
||||
// Retrieves (loads) a kernel for the platform this StreamExecutor is acting
|
||||
// upon, if one exists.
|
||||
//
|
||||
// Parameters:
|
||||
// spec: The MultiKernelLoaderSpec is usually generated as a compile-time
|
||||
// constant into an appropriate namespace. For example, see
|
||||
// stream_executor::executor_sample::kKernelLoaderSpecs, from which a
|
||||
// MultiKernelLoaderSpec is selected.
|
||||
// kernel: Outparam that the kernel is loaded into. A given Kernel
|
||||
// instantiation should not be loaded into more than once.
|
||||
//
|
||||
// If an error occurs, or there is no kernel available for the StreamExecutor
|
||||
// platform, false is returned.
|
||||
bool GetKernel(const MultiKernelLoaderSpec &spec, KernelBase *kernel);
|
||||
|
||||
// Releases any state associated with the previously loaded kernel.
|
||||
void UnloadKernel(const KernelBase *kernel);
|
||||
|
||||
// Loads a module for the platform this StreamExecutor is acting upon.
|
||||
//
|
||||
// `spec` describes the module to be loaded. On success writes the handle for
|
||||
// the loaded module to `module_handle` and returns true. Else returns false.
|
||||
bool LoadModule(const MultiModuleLoaderSpec &spec,
|
||||
ModuleHandle *module_handle);
|
||||
|
||||
// Unloads the module with handle `module_handle`.
|
||||
bool UnloadModule(ModuleHandle module_handle);
|
||||
|
||||
// Synchronously allocates an array on the device of type T with element_count
|
||||
// elements.
|
||||
template <typename T>
|
||||
DeviceMemory<T> AllocateArray(uint64 element_count);
|
||||
|
||||
// As AllocateArray(), but returns a ScopedDeviceMemory<T>.
|
||||
template <typename T>
|
||||
ScopedDeviceMemory<T> AllocateOwnedArray(uint64 element_count) {
|
||||
return ScopedDeviceMemory<T>(this, AllocateArray<T>(element_count));
|
||||
}
|
||||
|
||||
// Convenience wrapper that allocates space for a single element of type T in
|
||||
// device memory.
|
||||
template <typename T>
|
||||
DeviceMemory<T> AllocateScalar() {
|
||||
return AllocateArray<T>(1);
|
||||
}
|
||||
|
||||
// As AllocateScalar(), but returns a ScopedDeviceMemory<T>.
|
||||
template <typename T>
|
||||
ScopedDeviceMemory<T> AllocateOwnedScalar() {
|
||||
return AllocateOwnedArray<T>(1);
|
||||
}
|
||||
|
||||
// Synchronously allocates a scalar of type T on the device that is (POD)
|
||||
// zero-byte initialized.
|
||||
template <typename T>
|
||||
DeviceMemory<T> AllocateZeroed();
|
||||
|
||||
// As AllocateZeroed(), but returns a ScopedDeviceMemory<T>.
|
||||
template <typename T>
|
||||
ScopedDeviceMemory<T> AllocateOwnedZeroed() {
|
||||
return ScopedDeviceMemory<T>(this, AllocateZeroed<T>());
|
||||
}
|
||||
|
||||
// Allocate a memory region inside another allocated memory region.
|
||||
// Offset and size are specified in terms of T elements.
|
||||
// Warning: Do not free a parent buffer before its sub-buffers; this may cause
|
||||
// use-after-free issues (the specific behavior is not consistent across
|
||||
// platforms).
|
||||
// - Note: OpenCL uses refcounting to manage buffer lifetimes, so use of a
|
||||
// sub-buffer after parent deallocation is expected to be safe. This will
|
||||
// render your code non-platform-portable, however.
|
||||
template <typename T>
|
||||
DeviceMemory<T> AllocateSubBuffer(DeviceMemory<T> *parent,
|
||||
uint64 element_offset,
|
||||
uint64 element_count);
|
||||
|
||||
// As AllocateSubBuffer(), but returns a ScopedDeviceMemory<T>.
|
||||
template <typename T>
|
||||
ScopedDeviceMemory<T> AllocateOwnedSubBuffer(DeviceMemory<T> *parent,
|
||||
uint64 element_offset,
|
||||
uint64 element_count) {
|
||||
return ScopedDeviceMemory<T>(
|
||||
this, AllocateSubBuffer<T>(parent, element_offset, element_count));
|
||||
}
|
||||
|
||||
// Finds a symbol and returns device memory allocated to the symbol. The
|
||||
// symbol is searched in any kernels that were previously loaded through
|
||||
// GetKernel() before the GetSymbol() call. The user has to make sure that the
|
||||
// type of symbol and T match.
|
||||
// - Note: symbol_name should include its namespace as well. For example,
|
||||
// pass "nms0::symbol" if referring to nms0::symbol.
|
||||
//
|
||||
// If `module_handle` is set then searches only within the module
|
||||
// corresponding to `module_handle`.
|
||||
template <typename T>
|
||||
port::StatusOr<DeviceMemory<T>> GetSymbol(const string &symbol_name,
|
||||
ModuleHandle module_handle = {});
|
||||
|
||||
// An untyped version of GetSymbol.
|
||||
port::StatusOr<DeviceMemoryBase> GetUntypedSymbol(
|
||||
const string &symbol_name, ModuleHandle module_handle = {});
|
||||
|
||||
// Deallocate the DeviceMemory previously allocated via this interface.
|
||||
// Deallocation of a nullptr-representative value is permitted.
|
||||
//
|
||||
// Resets the internal contents of mem to be null-representative, but this
|
||||
// null-out effect should not be relied upon in client code.
|
||||
//
|
||||
// TODO(jlebar): Change this to accept a DeviceMemoryBase by value, see
|
||||
// discussion in cl/195744342.
|
||||
void Deallocate(DeviceMemoryBase *mem);
|
||||
|
||||
// Retrieves a mapping of active opaque device memory pointer to a string
|
||||
// representation of the [allocating thread's] stack at the time the pointer
|
||||
// was allocated. Useful for tracking device memory leaks.
|
||||
//
|
||||
// Note: this will only be populated if --check_device_leaks flag is
|
||||
// activated.
|
||||
void GetMemAllocs(std::map<void *, AllocRecord> *records_out);
|
||||
|
||||
// Allocates unified memory space of the given size, if supported.
|
||||
// See
|
||||
// https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#um-unified-memory-programming-hd
|
||||
// for more details on unified memory.
|
||||
void *UnifiedMemoryAllocate(uint64 bytes);
|
||||
|
||||
// Deallocates unified memory space previously allocated with
|
||||
// UnifiedMemoryAllocate.
|
||||
void UnifiedMemoryDeallocate(void *location);
|
||||
|
||||
// Allocates a region of host memory and registers it with the platform API.
|
||||
// Memory allocated in this manner (or allocated and registered with
|
||||
// HostMemoryRegister() is required for use in asynchronous memcpy operations,
|
||||
// such as Stream::ThenMemcpy.
|
||||
void *HostMemoryAllocate(uint64 bytes);
|
||||
|
||||
// Deallocates a region of host memory allocated by HostMemoryAllocate().
|
||||
void HostMemoryDeallocate(void *location);
|
||||
|
||||
// Registers a region of host memory with the platform API. Registered memory
|
||||
// (or memory allocated with HostMemoryAllocate) is required for use with
|
||||
// asynchronous memcpy operations, such as Stream::ThenMemcpy. This method
|
||||
// is used to register memory allocated outside the StreamExecutor;
|
||||
// HostMemoryAllocate implicitly registers its allocations and
|
||||
// HostMemoryDeallocate implicitly deregisters on deallocation.
|
||||
bool HostMemoryRegister(void *location, uint64 size) SE_MUST_USE_RESULT;
|
||||
|
||||
// Unregisters a region of host memory registered with HostMemoryRegister.
|
||||
// This should be done before deallocating the region with delete[]/free/etc.
|
||||
bool HostMemoryUnregister(void *location) SE_MUST_USE_RESULT;
|
||||
|
||||
// Synchronizes all activity occurring in the StreamExecutor's context (most
|
||||
// likely a whole device).
|
||||
bool SynchronizeAllActivity() SE_MUST_USE_RESULT;
|
||||
|
||||
// Blocks the caller while "size" bytes are zeroed out (in POD fashion) at the
|
||||
// given location in device memory.
|
||||
bool SynchronousMemZero(DeviceMemoryBase *location,
|
||||
uint64 size) SE_MUST_USE_RESULT;
|
||||
|
||||
// Blocks the caller while "size" bytes are initialized to "value" (in POD
|
||||
// fashion) at the given location in device memory.
|
||||
bool SynchronousMemSet(DeviceMemoryBase *location, int value,
|
||||
uint64 size) SE_MUST_USE_RESULT;
|
||||
|
||||
// [deprecated] Blocks the caller while a data segment of the given size is
|
||||
// copied from the host source to the device destination.
|
||||
ABSL_DEPRECATED(
|
||||
"Prefer SynchronousMemcpyH2D, to avoid error-prone API usage.")
|
||||
bool SynchronousMemcpy(DeviceMemoryBase *device_dst, const void *host_src,
|
||||
uint64 size) SE_MUST_USE_RESULT;
|
||||
|
||||
// [deprecated] Blocks the caller while a data segment of the given size is
|
||||
// copied from the device source to the host destination.
|
||||
ABSL_DEPRECATED(
|
||||
"Prefer SynchronousMemcpyD2H, to avoid error-prone API usage.")
|
||||
bool SynchronousMemcpy(void *host_dst, const DeviceMemoryBase &device_src,
|
||||
uint64 size) SE_MUST_USE_RESULT;
|
||||
|
||||
// Same as SynchronousMemcpy(DeviceMemoryBase*, ...) above.
|
||||
port::Status SynchronousMemcpyH2D(const void *host_src, int64 size,
|
||||
DeviceMemoryBase *device_dst);
|
||||
|
||||
// Alternative interface for memcpying from host to device that takes an
|
||||
// array slice. Checks that the destination size can accommodate the host
|
||||
// slice size.
|
||||
template <class T>
|
||||
port::Status SynchronousMemcpyH2D(port::ArraySlice<T> host_src,
|
||||
DeviceMemoryBase *device_dst) {
|
||||
auto host_size = host_src.size() * sizeof(T);
|
||||
CHECK(device_dst->size() == 0 || device_dst->size() >= host_size);
|
||||
return SynchronousMemcpyH2D(host_src.begin(), host_size, device_dst);
|
||||
}
|
||||
|
||||
// Same as SynchronousMemcpy(void*, ...) above.
|
||||
port::Status SynchronousMemcpyD2H(const DeviceMemoryBase &device_src,
|
||||
int64 size, void *host_dst);
|
||||
|
||||
// Alternative interface for memcpying from device to host that takes an
|
||||
// array slice. Checks that the destination size can accommodate the host
|
||||
// slice size.
|
||||
template <typename T>
|
||||
port::Status SynchronousMemcpyD2H(const DeviceMemory<T> &device_src,
|
||||
port::MutableArraySlice<T> host_dst) {
|
||||
auto host_size = host_dst.size() * sizeof(T);
|
||||
CHECK(device_src.size() == 0 || host_size >= device_src.size());
|
||||
return SynchronousMemcpyD2H(device_src, host_size, host_dst.begin());
|
||||
}
|
||||
|
||||
// Blocks the caller while a data segment of the given size is copied from the
|
||||
// device source to the device destination.
|
||||
bool SynchronousMemcpy(DeviceMemoryBase *device_dst,
|
||||
const DeviceMemoryBase &device_src,
|
||||
uint64 size) SE_MUST_USE_RESULT;
|
||||
|
||||
// Enqueues an operation onto stream to zero out size bytes at the given
|
||||
// device memory location. Neither stream nor location may be null. Returns
|
||||
// whether the operation was successfully enqueued onto the stream.
|
||||
bool MemZero(Stream *stream, DeviceMemoryBase *location,
|
||||
uint64 size) SE_MUST_USE_RESULT;
|
||||
|
||||
// Enqueues an operation onto stream to set 32-bit patterns starting at
|
||||
// location, for byte count given by size. size must be 32-bit quantified
|
||||
// (i.e. evently divisible by 4). Returns whether the operation was
|
||||
// successfully enqueued onto the stream.
|
||||
bool Memset32(Stream *stream, DeviceMemoryBase *location, uint32 pattern,
|
||||
uint64 size) SE_MUST_USE_RESULT;
|
||||
|
||||
// Enables peer access from this StreamExecutor to memory
|
||||
// allocated by other, such that launched device code, memcpies, etc may
|
||||
// access it directly.
|
||||
//
|
||||
// Both this StreamExecutor and other must be backed by the same platform (as
|
||||
// in
|
||||
// CUDA vs OpenCL) implementation.
|
||||
port::Status EnablePeerAccessTo(StreamExecutor *other);
|
||||
|
||||
// Returns whether it's possible to enable peer access from this
|
||||
// StreamExecutor
|
||||
// to memory allocated by another.
|
||||
//
|
||||
// Even when this returns true, EnablePeerAccessTo may fail for other reasons;
|
||||
// this is more an up-front test as to whether it's expressly forbidden.
|
||||
bool CanEnablePeerAccessTo(StreamExecutor *other);
|
||||
|
||||
// Gets the preferred shared memory configuration for the device to which this
|
||||
// executor is bound.
|
||||
SharedMemoryConfig GetDeviceSharedMemoryConfig();
|
||||
|
||||
// Sets the preferred shared memory configuration for the device to which this
|
||||
// executor is bound.
|
||||
port::Status SetDeviceSharedMemoryConfig(SharedMemoryConfig config);
|
||||
|
||||
// Obtains metadata about the underlying device.
|
||||
// The value is cached on first use.
|
||||
const DeviceDescription &GetDeviceDescription() const;
|
||||
|
||||
// If implemented, returns device specific measurement of load
|
||||
// (e.g. pending requests).
|
||||
int64 GetDeviceLoad() const;
|
||||
|
||||
// Returns the underlying device memory usage information, if it is available.
|
||||
// If it is not available (false is returned), free/total may not be
|
||||
// initialized.
|
||||
//
|
||||
// Note: "Free" reflects the amount of free memory on the underlying device,
|
||||
// so allocations via other StreamExecutors that have the same underlying
|
||||
// device
|
||||
// will be reflected in "free".
|
||||
bool DeviceMemoryUsage(int64 *free, int64 *total) const;
|
||||
|
||||
// The device count reported by this StreamExecutor's platform.
|
||||
// Note: on OpenCL we implicitly select platform zero at the moment.
|
||||
int PlatformDeviceCount() const;
|
||||
|
||||
// Returns whether the StreamExecutor supports BLAS routines for the platform
|
||||
// that underlies this interface.
|
||||
bool SupportsBlas() const;
|
||||
|
||||
// Returns whether the StreamExecutor supports FFT routines for the platform
|
||||
// that underlies this interface.
|
||||
bool SupportsFft() const;
|
||||
|
||||
// Returns whether the StreamExecutor supports RNG routines for the platform
|
||||
// that underlies this interface.
|
||||
bool SupportsRng() const;
|
||||
|
||||
// Returns whether the StreamExecutor support neural net routines for the
|
||||
// platform that underlies this interface.
|
||||
bool SupportsDnn() const;
|
||||
|
||||
// Returns the list of supported algorithms for the forward convolution
|
||||
// operation.
|
||||
bool GetConvolveAlgorithms(bool with_winograd_nonfused,
|
||||
std::vector<dnn::AlgorithmDesc> *out_algorithms);
|
||||
|
||||
// Returns the list of supported algorithms for rnn operation.
|
||||
bool GetRnnAlgorithms(std::vector<dnn::AlgorithmDesc> *out_algorithms);
|
||||
|
||||
// Get the list of supported algorithms for the backward convolution on data.
|
||||
bool GetConvolveBackwardDataAlgorithms(
|
||||
bool with_winograd_nonfused,
|
||||
std::vector<dnn::AlgorithmDesc> *out_algorithms);
|
||||
|
||||
// Get the list of supported algorithms for the backward convolution on the
|
||||
// filter.
|
||||
bool GetConvolveBackwardFilterAlgorithms(
|
||||
bool with_winograd_nonfused,
|
||||
std::vector<dnn::AlgorithmDesc> *out_algorithms);
|
||||
|
||||
// Get the list of supported algorithms for BLAS gemm.
|
||||
bool GetBlasGemmAlgorithms(std::vector<blas::AlgorithmType> *out_algorithms);
|
||||
|
||||
// Create an RNN descriptor based on model shapes and configurations.
|
||||
// The caller retains the ownership of the descriptor.
|
||||
port::StatusOr<std::unique_ptr<dnn::RnnDescriptor>> createRnnDescriptor(
|
||||
int num_layers, int hidden_size, int input_size, int batch_size,
|
||||
dnn::RnnInputMode input_mode, dnn::RnnDirectionMode direction_mode,
|
||||
dnn::RnnMode rnn_mode, dnn::DataType data_type,
|
||||
const dnn::AlgorithmConfig &algorithm_config, float dropout, uint64 seed,
|
||||
ScratchAllocator *state_allocator);
|
||||
|
||||
// Create a RNN sequence descriptor that specifies either the input or output
|
||||
// sequence. The caller retains the ownership of the returned descriptor.
|
||||
port::StatusOr<std::unique_ptr<dnn::RnnSequenceTensorDescriptor>>
|
||||
createRnnSequenceTensorDescriptor(int seq_length, int batch_size,
|
||||
int data_size, dnn::DataType data_type);
|
||||
|
||||
// Create an RNN state descriptor that specifies the input or hidden state.
|
||||
// The caller retains the ownership of the returned descriptor.
|
||||
port::StatusOr<std::unique_ptr<dnn::RnnStateTensorDescriptor>>
|
||||
createRnnStateTensorDescriptor(int num_layer, int batch_size, int data_size,
|
||||
dnn::DataType data_type);
|
||||
|
||||
// Returns the device ordinal that this StreamExecutor was initialized with.
|
||||
// Meaningless before initialization.
|
||||
int device_ordinal() const { return device_ordinal_; }
|
||||
|
||||
// Returns a borrowed pointer to the underlying StreamExecutor implementation.
|
||||
internal::StreamExecutorInterface *implementation();
|
||||
|
||||
// Warning: use Stream::ThenLaunch instead, this method is not for general
|
||||
// consumption. However, this is the only way to launch a kernel for which
|
||||
// the type signature is only known at runtime; say, if an application
|
||||
// supports loading/launching kernels with arbitrary type signatures.
|
||||
// In this case, the application is expected to know how to do parameter
|
||||
// packing that obeys the contract of the underlying platform implementation.
|
||||
//
|
||||
// Launches a data parallel kernel with the given thread/block
|
||||
// dimensionality and already-packed args/sizes to pass to the underlying
|
||||
// platform driver.
|
||||
//
|
||||
// This is called by Stream::Launch() to delegate to the platform's launch
|
||||
// implementation in StreamExecutorInterface::Launch().
|
||||
bool Launch(Stream *stream, const ThreadDim &thread_dims,
|
||||
const BlockDim &block_dims, const KernelBase &kernel,
|
||||
const KernelArgsArrayBase &args);
|
||||
|
||||
// Gets-or-creates (creates with memoization) a FftSupport datatype that can
|
||||
// be used to execute FFT routines on the current platform.
|
||||
//
|
||||
// Ownership and user-facing is the same as AsBlas() below.
|
||||
//
|
||||
// Returns null if there was an error initializing the FFT support for the
|
||||
// underlying platform.
|
||||
fft::FftSupport *AsFft();
|
||||
|
||||
// Gets-or-creates (creates with memoization) a DnnSupport datatype that can
|
||||
// be used for neural network routines on the current platform.
|
||||
//
|
||||
// Ownership and user-facing is the same as AsBlas() below.
|
||||
//
|
||||
// Returns null if there was an error initializing the DNN support for the
|
||||
// underlying platform.
|
||||
dnn::DnnSupport *AsDnn();
|
||||
|
||||
// Turns StreamExecutor operation tracing on or off.
|
||||
void EnableTracing(bool enable);
|
||||
|
||||
// Registers a trace listener to receive callbacks for only a single
|
||||
// StreamExecutor instance.
|
||||
// To register a listener for all executors for a given platform, see
|
||||
// Platform::RegisterTraceListener().
|
||||
// Does not take ownership of listener.
|
||||
void RegisterTraceListener(TraceListener* listener);
|
||||
|
||||
// Removes a TraceListener from this StreamExecutor instance.
|
||||
// Returns false (and logs) in cases where the argument listener was not
|
||||
// previously registered.
|
||||
bool UnregisterTraceListener(TraceListener* listener);
|
||||
|
||||
private:
|
||||
template <typename BeginCallT, typename CompleteCallT,
|
||||
typename ReturnT, typename... BeginArgsT>
|
||||
friend class ScopedTracer;
|
||||
friend class Event;
|
||||
friend class Stream;
|
||||
friend class Timer;
|
||||
template <typename... Params>
|
||||
friend class TypedKernel;
|
||||
template <typename... Args>
|
||||
friend struct ThenBlasImpl;
|
||||
|
||||
// Gets-or-creates (creates with memoization) a BlasSupport datatype that can
|
||||
// be used to execute BLAS routines on the current platform. This is typically
|
||||
// not user-facing, as users will use the Stream::ThenBlas* family of routines
|
||||
// to entrain BLAS operations. See blas.h for additional details.
|
||||
//
|
||||
// Ownership is not transferred to the caller -- ownership is retained by this
|
||||
// object for memoization. This BLAS interface is also only expected to be
|
||||
// used by a Stream for entraining calls to BLAS functionality.
|
||||
//
|
||||
// Returns null if there was an error initializing the BLAS support for the
|
||||
// underlying platform.
|
||||
blas::BlasSupport *AsBlas();
|
||||
|
||||
// Gets-or-creates (creates with memoization) an RngSupport datatype that can
|
||||
// be used for random-number-generation routines on the current platform.
|
||||
//
|
||||
// Ownership and user-facing is the same as AsBlas() above.
|
||||
//
|
||||
// Returns null if there was an error initializing the RNG support for the
|
||||
// underlying platform.
|
||||
rng::RngSupport *AsRng();
|
||||
|
||||
// Causes the host code to synchronously wait for operations entrained onto
|
||||
// stream to complete. Effectively a join on the asynchronous device
|
||||
// operations enqueued on the stream before this program point.
|
||||
port::Status BlockHostUntilDone(Stream *stream);
|
||||
|
||||
// Synchronously allocates size bytes on the underlying platform and returns
|
||||
// an opaque void* representing that allocation. In the case of failure,
|
||||
// nullptr is returned.
|
||||
void *Allocate(uint64 size);
|
||||
|
||||
// Finds and retrieves device memory for the symbol on the underlying
|
||||
// platform.
|
||||
bool GetSymbol(const string &symbol_name, ModuleHandle module_handle,
|
||||
void **mem, size_t *bytes);
|
||||
|
||||
// Entrains a memcpy operation onto stream, with a host destination location
|
||||
// host_dst and a device memory source, with target size size.
|
||||
bool Memcpy(Stream *stream, void *host_dst,
|
||||
const DeviceMemoryBase &device_src, uint64 size);
|
||||
|
||||
// Entrains a memcpy operation onto stream, with a device destination location
|
||||
// and a host memory source, with target size size.
|
||||
bool Memcpy(Stream *stream, DeviceMemoryBase *device_dst,
|
||||
const void *host_src, uint64 size);
|
||||
|
||||
// Entrains a memcpy operation onto stream, with a device destination location
|
||||
// and a device source location, with target size size. Peer access should
|
||||
// have been enabled between the StreamExecutors owning the device memory
|
||||
// regions.
|
||||
bool MemcpyDeviceToDevice(Stream *stream, DeviceMemoryBase *device_dst,
|
||||
const DeviceMemoryBase &device_src, uint64 size);
|
||||
|
||||
// Entrains on a stream a user-specified function to be run on the host.
|
||||
// See Stream::ThenDoHostCallback for full details.
|
||||
bool HostCallback(Stream *stream, std::function<void()> callback);
|
||||
|
||||
// Entrains on a stream a user-specified function to be run on the host.
|
||||
// See Stream::ThenDoHostCallback for full details.
|
||||
// This is the preferred form for a callback that may return an error.
|
||||
bool HostCallback(Stream *stream, std::function<port::Status()> callback);
|
||||
|
||||
// Performs platform-specific allocation and initialization of an event.
|
||||
port::Status AllocateEvent(Event *event);
|
||||
|
||||
// Performs platform-specific deallocation and cleanup of an event.
|
||||
port::Status DeallocateEvent(Event *event);
|
||||
|
||||
// Inserts the specified event at the end of the specified stream.
|
||||
port::Status RecordEvent(Stream *stream, Event *event);
|
||||
|
||||
// Wait for the specified event at the end of the specified stream.
|
||||
port::Status WaitForEvent(Stream *stream, Event *event);
|
||||
|
||||
// Requests the current status of the event from the underlying platform.
|
||||
Event::Status PollForEventStatus(Event *event);
|
||||
|
||||
// Allocates stream resources on the underlying platform for subject and
|
||||
// initializes its internals.
|
||||
bool AllocateStream(Stream *subject);
|
||||
|
||||
// Deallocates stream resources on the underlying platform.
|
||||
void DeallocateStream(Stream *subject);
|
||||
|
||||
// Causes dependent to not begin execution until other has finished its
|
||||
// last-enqueued work.
|
||||
bool CreateStreamDependency(Stream *dependent, Stream *other);
|
||||
|
||||
// Allocates timer resources on the underlying platform for subject and
|
||||
// initializes its internals.
|
||||
bool AllocateTimer(Timer *subject);
|
||||
|
||||
// Deallocates timer resources on the underlying platform.
|
||||
void DeallocateTimer(Timer *subject);
|
||||
|
||||
// Records a start event for an interval timer.
|
||||
bool StartTimer(Stream *stream, Timer *timer);
|
||||
|
||||
// Records a stop event for an interval timer.
|
||||
bool StopTimer(Stream *stream, Timer *timer);
|
||||
|
||||
// Allocates a new metadata object, appropriately populated, on the heap, with
|
||||
// ownership transfer to caller.
|
||||
DeviceDescription *PopulateDeviceDescription() const;
|
||||
|
||||
// Adds a task to the port::ThreadPool work queue. These tasks must be
|
||||
// fire-and-forget and have no external data or timing dependencies; their
|
||||
// execution order and completion time have no guarantees.
|
||||
// For an example of an appropriate task, see HostBlas::DoBlasGemmInternal;
|
||||
// there, temporary internal buffers are freed using this method.
|
||||
void EnqueueOnBackgroundThread(std::function<void()> task);
|
||||
|
||||
// Adds an AllocRecord for 'opaque' of size 'bytes' to the record map, for
|
||||
// leak checking. NULL buffer pointers and buffer sizes of 0 will not be
|
||||
// tracked.
|
||||
void CreateAllocRecord(void *opaque, uint64 size);
|
||||
|
||||
// Removes the AllocRecord keyed by 'opaque' from the record map. NULL
|
||||
// pointers will not be erased (as they're not tracked, per above).
|
||||
void EraseAllocRecord(void *opaque);
|
||||
|
||||
// Calls the relevant TraceListener routine to begin tracing for the specified
|
||||
// asynchronous method.
|
||||
template <typename TraceCallT, typename... ArgsT>
|
||||
void SubmitTrace(TraceCallT trace_call, ArgsT&&... args);
|
||||
|
||||
// Reader/writer lock for class-static StreamExecutor members.
|
||||
static mutex static_mu_;
|
||||
|
||||
// Reader/writer lock for mutable data structures on this StreamExecutor.
|
||||
//
|
||||
// Mutable so that caching functions (like DeviceDescription, AsBlas, etc.)
|
||||
// can acquire the lock on their first (mutating) call as well.
|
||||
mutable mutex mu_;
|
||||
|
||||
// Reference to the platform that created this executor.
|
||||
const Platform *platform_;
|
||||
|
||||
// Pointer to the platform-specific-interface implementation. This is
|
||||
// delegated to by the interface routines in pointer-to-implementation
|
||||
// fashion.
|
||||
std::unique_ptr<internal::StreamExecutorInterface> implementation_;
|
||||
|
||||
// A mapping of pointer (to device memory) to string representation of the
|
||||
// stack (of the allocating thread) at the time at which the pointer was
|
||||
// allocated.
|
||||
std::map<void *, AllocRecord> mem_allocs_ GUARDED_BY(mu_);
|
||||
|
||||
// Memoized BLAS support object -- we only want to create this once when asked
|
||||
// for a BLAS interface.
|
||||
std::unique_ptr<blas::BlasSupport> blas_ GUARDED_BY(mu_);
|
||||
|
||||
// Memoized DNN support object -- we only want to create this once when asked
|
||||
// for an DNN interface.
|
||||
std::unique_ptr<dnn::DnnSupport> dnn_ GUARDED_BY(mu_);
|
||||
|
||||
// Memoized FFT support object -- we only want to create this once when asked
|
||||
// for a FFT interface.
|
||||
std::unique_ptr<fft::FftSupport> fft_;
|
||||
|
||||
// Memoized RNG support object -- we only want to create this once when asked
|
||||
// for an RNG interface.
|
||||
std::unique_ptr<rng::RngSupport> rng_ GUARDED_BY(mu_);
|
||||
|
||||
// Slot to cache the owned DeviceDescription for the underlying device
|
||||
// once it has been quieried from DeviceDescription().
|
||||
mutable std::unique_ptr<DeviceDescription> device_description_
|
||||
GUARDED_BY(mu_);
|
||||
|
||||
// The kind of the underlying platform that is being targeted, as passed
|
||||
// during construction.
|
||||
//
|
||||
// Immutable post-initialization.
|
||||
PlatformKind platform_kind_;
|
||||
|
||||
// The device ordinal that this object was initialized with.
|
||||
//
|
||||
// Immutable post-initialization.
|
||||
int device_ordinal_;
|
||||
|
||||
// Executor for handling host callback work that cannot be performed
|
||||
// by a host callback thread - for example, cleanup after a host BLAS routine
|
||||
// (which may make device API calls). This work cannot block the host
|
||||
// callback thread, will be completed asynchronously, and should be treated
|
||||
// as fire-and-forget. Assume no ordering guarantees WRT the tasks enqueued
|
||||
// here.
|
||||
//
|
||||
// Immutable post-initialization. Object is thread-safe.
|
||||
std::unique_ptr<port::ThreadPool> background_threads_;
|
||||
|
||||
// Counter for the current number of live streams. This is used to check
|
||||
// for accidentally-outstanding streams at StreamExecutor teardown time, as
|
||||
// well
|
||||
// as to indicate leaks (via a large outstanding count being logged) in the
|
||||
// case we can't allocate more streams.
|
||||
std::atomic_int_fast32_t live_stream_count_;
|
||||
|
||||
// Only one worker thread is needed; little work will be done by the
|
||||
// executor.
|
||||
static const int kNumBackgroundThreads = 1;
|
||||
|
||||
// Indicates if StreamExecutor operation tracing should be performed.
|
||||
bool tracing_enabled_;
|
||||
|
||||
// The set of TraceListeners registered for this StreamExecutor.
|
||||
std::set<TraceListener*> listeners_ GUARDED_BY(mu_);
|
||||
|
||||
// Allocated memory in bytes.
|
||||
int64 mem_alloc_bytes_;
|
||||
|
||||
// Memory limit in bytes. Value less or equal to 0 indicates there is no
|
||||
// limit.
|
||||
int64 memory_limit_bytes_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(StreamExecutor);
|
||||
};
|
||||
|
||||
// A wrapper around ModuleHandle that uses RAII to manage its lifetime.
|
||||
class ScopedModuleHandle {
|
||||
public:
|
||||
explicit ScopedModuleHandle(StreamExecutor *executor,
|
||||
ModuleHandle module_handle)
|
||||
: executor_(executor), module_handle_(module_handle) {}
|
||||
|
||||
ScopedModuleHandle(ScopedModuleHandle &&other) {
|
||||
executor_ = other.executor_;
|
||||
module_handle_ = other.module_handle_;
|
||||
other.executor_ = nullptr;
|
||||
other.module_handle_ = ModuleHandle();
|
||||
}
|
||||
|
||||
ScopedModuleHandle &operator=(ScopedModuleHandle &&other) {
|
||||
executor_ = other.executor_;
|
||||
module_handle_ = other.module_handle_;
|
||||
other.executor_ = nullptr;
|
||||
other.module_handle_ = ModuleHandle();
|
||||
return *this;
|
||||
}
|
||||
|
||||
~ScopedModuleHandle() {
|
||||
if (static_cast<bool>(module_handle_)) {
|
||||
CHECK(executor_->UnloadModule(module_handle_));
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
StreamExecutor *executor_;
|
||||
ModuleHandle module_handle_;
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(ScopedModuleHandle);
|
||||
};
|
||||
|
||||
////////////
|
||||
// Inlines
|
||||
|
||||
template <typename T>
|
||||
inline DeviceMemory<T> StreamExecutor::AllocateArray(uint64 element_count) {
|
||||
uint64 bytes = sizeof(T) * element_count;
|
||||
void *opaque = Allocate(bytes);
|
||||
return DeviceMemory<T>::MakeFromByteSize(opaque, bytes);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline port::StatusOr<DeviceMemory<T>> StreamExecutor::GetSymbol(
|
||||
const string &symbol_name, ModuleHandle module_handle) {
|
||||
port::StatusOr<DeviceMemoryBase> untyped_symbol =
|
||||
GetUntypedSymbol(symbol_name, module_handle);
|
||||
if (!untyped_symbol.ok()) {
|
||||
return untyped_symbol.status();
|
||||
}
|
||||
return DeviceMemory<T>(untyped_symbol.ValueOrDie());
|
||||
}
|
||||
|
||||
template <typename ElemT>
|
||||
ScopedDeviceMemory<ElemT>::ScopedDeviceMemory()
|
||||
: wrapped_(DeviceMemoryBase()), parent_(nullptr) {}
|
||||
|
||||
template <typename ElemT>
|
||||
ScopedDeviceMemory<ElemT>::ScopedDeviceMemory(StreamExecutor *parent,
|
||||
DeviceMemoryBase value)
|
||||
: wrapped_(value), parent_(parent) {}
|
||||
|
||||
template <typename ElemT>
|
||||
ScopedDeviceMemory<ElemT>::ScopedDeviceMemory(
|
||||
StreamExecutor *parent, std::initializer_list<ElemT> values)
|
||||
: ScopedDeviceMemory(parent, parent->AllocateArray<ElemT>(values.size())) {
|
||||
if (ptr() != nullptr) {
|
||||
std::vector<ElemT> local(values);
|
||||
if (!parent->SynchronousMemcpy(ptr(), const_cast<const ElemT *>(&local[0]),
|
||||
ptr()->size())) {
|
||||
Reset(nullptr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename ElemT>
|
||||
ScopedDeviceMemory<ElemT>::~ScopedDeviceMemory() {
|
||||
if (wrapped_ == nullptr) return;
|
||||
DCHECK(parent_ != nullptr);
|
||||
parent_->Deallocate(&wrapped_);
|
||||
}
|
||||
|
||||
template <typename ElemT>
|
||||
void ScopedDeviceMemory<ElemT>::Reset(DeviceMemory<ElemT> updated) {
|
||||
if (wrapped_ != nullptr) {
|
||||
DCHECK(parent_ != nullptr);
|
||||
parent_->Deallocate(&wrapped_);
|
||||
}
|
||||
wrapped_ = updated;
|
||||
}
|
||||
|
||||
template <typename ElemT>
|
||||
void ScopedDeviceMemory<ElemT>::Reset(std::nullptr_t) {
|
||||
if (wrapped_ != nullptr) {
|
||||
DCHECK(parent_ != nullptr);
|
||||
parent_->Deallocate(&wrapped_);
|
||||
}
|
||||
wrapped_ = DeviceMemory<ElemT>{};
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
DeviceMemory<T> StreamExecutor::AllocateZeroed() {
|
||||
void *opaque = Allocate(sizeof(T));
|
||||
if (opaque == nullptr) {
|
||||
return DeviceMemory<T>{};
|
||||
}
|
||||
|
||||
DeviceMemory<T> result = DeviceMemory<T>::MakeFromByteSize(opaque, sizeof(T));
|
||||
bool ok = SynchronousMemZero(&result, sizeof(T));
|
||||
if (!ok) {
|
||||
Deallocate(&result);
|
||||
return DeviceMemory<T>{};
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
DeviceMemory<T> StreamExecutor::AllocateSubBuffer(DeviceMemory<T> *parent,
|
||||
uint64 element_offset,
|
||||
uint64 element_count) {
|
||||
if (element_offset + element_count > parent->ElementCount()) {
|
||||
LOG(ERROR) << "requested sub-buffer allocation (offset + size) is greater "
|
||||
<< "than parent allocation size: (" << element_offset << " + "
|
||||
<< element_count << ") vs. (" << parent->ElementCount() << ")";
|
||||
return DeviceMemory<T>{};
|
||||
}
|
||||
|
||||
void *opaque = implementation_->AllocateSubBuffer(
|
||||
parent, sizeof(T) * element_offset, sizeof(T) * element_count);
|
||||
if (opaque == nullptr) {
|
||||
return DeviceMemory<T>{};
|
||||
}
|
||||
CreateAllocRecord(opaque, sizeof(T) * element_count);
|
||||
return DeviceMemory<T>(DeviceMemoryBase(opaque, sizeof(T) * element_count,
|
||||
true /* = is_sub_buffer */));
|
||||
}
|
||||
|
||||
template <typename... Params, typename... Args>
|
||||
inline Stream &Stream::ThenLaunch(ThreadDim thread_dims, BlockDim block_dims,
|
||||
const TypedKernel<Params...> &kernel,
|
||||
Args... args) {
|
||||
KernelInvocationChecker<std::tuple<Params...>,
|
||||
std::tuple<Args...>>::CheckAllStaticAssert();
|
||||
if (ok()) {
|
||||
// This is the core that allows type-safe kernel launching.
|
||||
// Since the platforms take kernel arguments as tuples of (void *, size),
|
||||
// we pack the variadic parameters passed as ...args into the desired
|
||||
// tuple form and pass that packed form to the StreamExecutor::Launch()
|
||||
// implementation.
|
||||
KernelArgsArray<sizeof...(args)> kernel_args;
|
||||
kernel.PackParams(&kernel_args, args...);
|
||||
DCHECK(parent_ != nullptr);
|
||||
bool ok =
|
||||
parent_->Launch(this, thread_dims, block_dims, kernel, kernel_args);
|
||||
if (!ok) {
|
||||
SetError();
|
||||
LOG(WARNING) << "parent failed to launch kernel: " << &kernel;
|
||||
}
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_PIMPL_H_
|
||||
@@ -0,0 +1,136 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Temporary memories are used to allocate scratch space required by an
|
||||
// operation about to be enqueued onto a stream.
|
||||
//
|
||||
// std::unique_ptr<TemporaryDeviceMemory<float>> temporary_memory =
|
||||
// stream.AllocateTemporaryArray<float>(1024).ConsumeValueOrDie();
|
||||
// // ... enqueue stuff onto the stream using the temporary memory ...
|
||||
// // Note that the memory is accessible via
|
||||
// // temporary_memory->device_memory() and similar.
|
||||
//
|
||||
// // Finalize the temporary memory. The underlying device memory may
|
||||
// // be released any time after this program point, as another thread may
|
||||
// // call Stream::BlockHostUntilDone, causing synchronization. This
|
||||
// // finalization also happens automatically for the user if the unique_ptr
|
||||
// // goes out of scope.
|
||||
// temporary_memory.Finalize();
|
||||
//
|
||||
// WARNING: do NOT hold onto the device memory associated with temporary_memory
|
||||
// after finalization. If temporary_memory->device_memory() is used after the
|
||||
// temporary memory is finalized, it will cause a DCHECK failure.
|
||||
//
|
||||
// Note that standard usage takes advantage of the type-safe wrapper,
|
||||
// TemporaryDeviceMemory<T>, defined below.
|
||||
//
|
||||
// Also see tests for executable sample usage.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_TEMPORARY_DEVICE_MEMORY_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_TEMPORARY_DEVICE_MEMORY_H_
|
||||
|
||||
#include "tensorflow/stream_executor/device_memory.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class Stream;
|
||||
namespace internal {
|
||||
class TemporaryMemoryManager;
|
||||
}
|
||||
|
||||
// Untyped base class (analogous to a void*) for temporary device memory
|
||||
// allocations associated with a stream.
|
||||
class TemporaryDeviceMemoryBase {
|
||||
public:
|
||||
// Marks the temporary memory as finalized if it is not already marked as
|
||||
// such.
|
||||
~TemporaryDeviceMemoryBase();
|
||||
|
||||
// Precondition: !IsFinalized()
|
||||
DeviceMemoryBase* mutable_device_memory();
|
||||
|
||||
// Precondition: !IsFinalized()
|
||||
const DeviceMemoryBase& device_memory() const;
|
||||
|
||||
// "Finalizes" this temporary memory, making it acceptable to release at the
|
||||
// next stream synchronization point -- the device memory can be reclaimed at
|
||||
// any time after the temporary memory is marked as finalized (e.g. if a
|
||||
// separate thread is calls Stream::BlockHostUntilDone). This may only be
|
||||
// called once -- see the precondition below.
|
||||
//
|
||||
// Precondition: !IsFinalized()
|
||||
void Finalize();
|
||||
|
||||
// Returns true iff the temporary memory is finalized (that is, the user is
|
||||
// done referring to the temporary device memory, and thus it can be released
|
||||
// at the next stream synchronization point).
|
||||
bool IsFinalized() const;
|
||||
|
||||
// Returns true iff the temporary memory is still allocated.
|
||||
//
|
||||
// Note: this is a polling call, no guarantee is made that the temporary
|
||||
// memory is still allocated after the call has completed.
|
||||
bool IsAllocated() const;
|
||||
|
||||
private:
|
||||
friend class internal::TemporaryMemoryManager;
|
||||
friend class TemporaryDeviceMemoryTest;
|
||||
|
||||
// Note: construction DCHECKs that the memory is known-allocated in the
|
||||
// stream's temporary-allocation-manager.
|
||||
TemporaryDeviceMemoryBase(Stream* parent, DeviceMemoryBase device_memory,
|
||||
uint64 allocation_generation);
|
||||
|
||||
// The device memory region that has allocated.
|
||||
DeviceMemoryBase device_memory_;
|
||||
|
||||
// The generation counter value for the temporary memory record in the
|
||||
// temporary memory manager.
|
||||
uint64 allocation_generation_;
|
||||
|
||||
// The stream that this temporary memory was allocated for.
|
||||
Stream* parent_;
|
||||
};
|
||||
|
||||
// Type-safe wrapper around the base type (which is analogous to a void*).
|
||||
template <typename T>
|
||||
class TemporaryDeviceMemory : public TemporaryDeviceMemoryBase {
|
||||
public:
|
||||
// Type-safe wrapper around TemporaryDeviceMemoryBase::mutable_device_memory.
|
||||
DeviceMemory<T>* mutable_device_memory() {
|
||||
StaticSlicingAssertionDummy();
|
||||
return reinterpret_cast<DeviceMemory<T>*>(
|
||||
TemporaryDeviceMemoryBase::mutable_device_memory());
|
||||
}
|
||||
|
||||
// Type-safe wrapper around TemporaryDeviceMemoryBase::device_memory.
|
||||
const DeviceMemory<T>& device_memory() const {
|
||||
StaticSlicingAssertionDummy();
|
||||
return reinterpret_cast<const DeviceMemory<T>&>(
|
||||
TemporaryDeviceMemoryBase::device_memory());
|
||||
}
|
||||
|
||||
private:
|
||||
static void StaticSlicingAssertionDummy() {
|
||||
static_assert(
|
||||
sizeof(TemporaryDeviceMemory) == sizeof(TemporaryDeviceMemoryBase),
|
||||
"derived class is simply a wrapper, no members may be added due to "
|
||||
"slicing");
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_TEMPORARY_DEVICE_MEMORY_H_
|
||||
@@ -0,0 +1,151 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// The temporary-memory-manager is a helper class for a Stream to keep track of
|
||||
// temporary allocations. These allocations defer their deallocation to the next
|
||||
// Stream::BlockHostUntilDone call for efficiency purposes (as deallocation
|
||||
// itself generally forces synchronization to occur).
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_TEMPORARY_MEMORY_MANAGER_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_TEMPORARY_MEMORY_MANAGER_H_
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
|
||||
#include "tensorflow/stream_executor/device_memory.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/platform/thread_annotations.h"
|
||||
#include "tensorflow/stream_executor/temporary_device_memory.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace internal {
|
||||
|
||||
// Record used inside the TemporaryMemoryManager as metadata for a given device
|
||||
// memory region.
|
||||
struct TemporaryMemoryRecord {
|
||||
// What "generation" this record was allocated in.
|
||||
//
|
||||
// Currently the generation counter is bumped for every allocation, but this
|
||||
// could be made coarser if necessary.
|
||||
uint64 allocation_generation;
|
||||
|
||||
// Notes whether the temporary memory has been marked as finalized, such that
|
||||
// we can release the DeviceMemory associated with this record at
|
||||
// synchronization time.
|
||||
bool finalized;
|
||||
};
|
||||
|
||||
// Manages temporary memories associated with a stream -- keeps records of
|
||||
// outstanding temporaries and their state, and can deallocate them
|
||||
// appropriately at points in the Stream lifecycle (e.g. BlockHostUntilDone,
|
||||
// destruction).
|
||||
class TemporaryMemoryManager {
|
||||
public:
|
||||
explicit TemporaryMemoryManager(Stream* stream) : stream_(stream) {}
|
||||
|
||||
// Allocates a temporary array that is then managed by this object.
|
||||
template <typename T>
|
||||
port::StatusOr<std::unique_ptr<TemporaryDeviceMemory<T>>> AllocateArray(
|
||||
uint64 element_count);
|
||||
|
||||
// Forces deallocation of all managed temporary memory regions.
|
||||
//
|
||||
// Called, for example, when the Stream owning this temporary memory manager
|
||||
// is destroyed.
|
||||
//
|
||||
// Note: These calls to Deallocate will likely force synchronization.
|
||||
void ForceDeallocateAll();
|
||||
|
||||
// Marks the given memory region as finalized.
|
||||
//
|
||||
// If must_exist is set, this will check-fail if the temporary memory record
|
||||
// is not found.
|
||||
void MarkFinalized(const DeviceMemoryBase& device_memory, uint64 generation,
|
||||
bool must_exist);
|
||||
|
||||
// Deallocates temporary memories that have been finalized.
|
||||
//
|
||||
// Note: These calls to Deallocate will likely force synchronization, so it is
|
||||
// meant to be called before a "BlockHostUntilDone" is about to be performed.
|
||||
void DeallocateFinalizedTemporaries();
|
||||
|
||||
// Returns whether the provided device_memory is finalized.
|
||||
//
|
||||
// In the vacuous case where the device memory doesn't appear in the temporary
|
||||
// memory records, it is either not a temporary at all, or has already been
|
||||
// deallocated, and thus returns true.
|
||||
bool IsFinalized(const DeviceMemoryBase& device_memory,
|
||||
uint64 allocation_generation) const;
|
||||
|
||||
// Returns whether the manager has a live allocation record for the given
|
||||
// device memory pointer with the given generation counter.
|
||||
//
|
||||
// Note: this is a polling call -- there is no guarantee that the region is
|
||||
// still allocated once the call has completed.
|
||||
bool HasAllocated(const DeviceMemoryBase& device_memory,
|
||||
uint64 generation) const;
|
||||
|
||||
private:
|
||||
// Allocates an array without type parameterization, so that the
|
||||
// implementation can live in the source file. Without this base allocation
|
||||
// method, we incur a circular dependency between the StreamExecutor
|
||||
// definition and this class' definition.
|
||||
port::StatusOr<std::unique_ptr<TemporaryDeviceMemoryBase>> AllocateArrayBase(
|
||||
uint64 element_count, uint64 element_size);
|
||||
|
||||
// Mutex to guard temporary record state.
|
||||
mutable mutex mutex_;
|
||||
|
||||
// Mapping from device memory to the current (live) temporary memory record.
|
||||
//
|
||||
// If a device memory is not in this mapping, it is not a temporary currently
|
||||
// allocated and owned by this temporary memory manager.
|
||||
std::map<DeviceMemoryBase, TemporaryMemoryRecord> records_ GUARDED_BY(mutex_);
|
||||
|
||||
// Allocation generation -- we bump this counter to distinguish temporary
|
||||
// memory handles that have been deallocated and later reallocated at the same
|
||||
// device memory address.
|
||||
uint64 generation_ GUARDED_BY(mutex_);
|
||||
|
||||
// The stream (parent object) for this temporary memory manager -- allocations
|
||||
// are performed through this stream handle.
|
||||
Stream* stream_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(TemporaryMemoryManager);
|
||||
};
|
||||
|
||||
////////////
|
||||
// Inlines
|
||||
|
||||
template <typename T>
|
||||
port::StatusOr<std::unique_ptr<TemporaryDeviceMemory<T>>>
|
||||
TemporaryMemoryManager::AllocateArray(uint64 element_count) {
|
||||
port::StatusOr<std::unique_ptr<TemporaryDeviceMemoryBase>> temporary_memory =
|
||||
AllocateArrayBase(element_count, sizeof(T));
|
||||
if (!temporary_memory.ok()) {
|
||||
return temporary_memory.status();
|
||||
}
|
||||
|
||||
return std::unique_ptr<TemporaryDeviceMemory<T>>(
|
||||
reinterpret_cast<TemporaryDeviceMemory<T>*>(
|
||||
temporary_memory.ConsumeValueOrDie().release()));
|
||||
}
|
||||
|
||||
} // namespace internal
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_TEMPORARY_MEMORY_MANAGER_H_
|
||||
@@ -0,0 +1,73 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_TIMER_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_TIMER_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
namespace internal {
|
||||
class TimerInterface;
|
||||
} // namespace internal
|
||||
|
||||
class StreamExecutor;
|
||||
|
||||
// An interval timer, suitable for use in timing the operations which occur in
|
||||
// streams.
|
||||
//
|
||||
// Thread-hostile: CUDA associates a CUDA-context with a particular thread in
|
||||
// the system. Any operation that a user attempts to perform by using a Timer
|
||||
// on a thread not-associated with the CUDA-context has unknown behavior at the
|
||||
// current time; see b/13176597
|
||||
class Timer {
|
||||
public:
|
||||
// Instantiate a timer tied to parent as a platform executor.
|
||||
explicit Timer(StreamExecutor *parent);
|
||||
|
||||
// Deallocates any timer resources that the parent StreamExecutor has bestowed
|
||||
// upon this object.
|
||||
~Timer();
|
||||
|
||||
// Returns the elapsed number of microseconds for a completed timer.
|
||||
// Completed means has been through a start/stop lifecycle.
|
||||
uint64 Microseconds() const;
|
||||
|
||||
// Returns the elapsed number of nanoseconds for a completed timer.
|
||||
// Completed means has been through a start/stop lifecycle.
|
||||
uint64 Nanoseconds() const;
|
||||
|
||||
// Returns the (opaque) backing platform ITimer instance. Ownership is
|
||||
// not transferred to the caller.
|
||||
internal::TimerInterface *implementation() { return implementation_.get(); }
|
||||
|
||||
private:
|
||||
// The StreamExecutor that manages the platform-specific internals for this
|
||||
// timer.
|
||||
StreamExecutor *parent_;
|
||||
|
||||
// Platform-dependent implementation of the timer internals for the underlying
|
||||
// platform. This class just delegates to this opaque instance.
|
||||
std::unique_ptr<internal::TimerInterface> implementation_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(Timer);
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_TIMER_H_
|
||||
@@ -0,0 +1,73 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// This file defines the StreamExecutor trace listener, used for inserting
|
||||
// non-device-specific instrumentation into the StreamExecutor.
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_TRACE_LISTENER_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_TRACE_LISTENER_H_
|
||||
|
||||
#include "tensorflow/stream_executor/device_memory.h"
|
||||
#include "tensorflow/stream_executor/kernel.h"
|
||||
#include "tensorflow/stream_executor/launch_dim.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class Stream;
|
||||
|
||||
// Traces StreamExecutor PIMPL-level events.
|
||||
// The few StreamExecutor interfaces that are synchronous have both Begin and
|
||||
// Complete versions of their trace calls. Asynchronous operations only have
|
||||
// Submit calls, as execution of the underlying operations is device-specific.
|
||||
// As all tracing calls mirror StreamExecutor routines, documentation here is
|
||||
// minimal.
|
||||
//
|
||||
// All calls have default implementations that perform no work; subclasses
|
||||
// should override functionality of interest. Keep in mind that these routines
|
||||
// are not called on a dedicated thread, so callbacks should execute quickly.
|
||||
//
|
||||
// Note: This API is constructed on an as-needed basis. Users should add
|
||||
// support for further StreamExecutor operations as required. By enforced
|
||||
// convention (see SCOPED_TRACE in stream_executor_pimpl.cc), synchronous
|
||||
// tracepoints should be named NameBegin and NameComplete.
|
||||
class TraceListener {
|
||||
public:
|
||||
virtual ~TraceListener() {}
|
||||
|
||||
virtual void LaunchSubmit(Stream* stream, const ThreadDim& thread_dims,
|
||||
const BlockDim& block_dims,
|
||||
const KernelBase& kernel,
|
||||
const KernelArgsArrayBase& args) {}
|
||||
|
||||
virtual void SynchronousMemcpyH2DBegin(int64 correlation_id,
|
||||
const void* host_src, int64 size,
|
||||
DeviceMemoryBase* gpu_dst) {}
|
||||
virtual void SynchronousMemcpyH2DComplete(int64 correlation_id,
|
||||
const port::Status* result) {}
|
||||
|
||||
virtual void SynchronousMemcpyD2HBegin(int64 correlation_id,
|
||||
const DeviceMemoryBase& gpu_src,
|
||||
int64 size, void* host_dst) {}
|
||||
virtual void SynchronousMemcpyD2HComplete(int64 correlation_id,
|
||||
const port::Status* result) {}
|
||||
|
||||
virtual void BlockHostUntilDoneBegin(int64 correlation_id, Stream* stream) {}
|
||||
virtual void BlockHostUntilDoneComplete(int64 correlation_id,
|
||||
const port::Status* result) {}
|
||||
};
|
||||
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_TRACE_LISTENER_H_
|
||||
Reference in new issue
Block a user