This commit is contained in:
Kevin Darby committed 2019-01-02 17:04:10 -05:00
1 parent eb1b42a5e8
commit 458eb2aef1
1283 files changed
+323373

No files matched your search

@@ -0,0 +1,61 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This file contains APIs that assume a StreamExecutor is backed by CUDA.
// It reaches into the CUDA implementation to activate an underlying CUDA
// context.
//
// Having this file separate from cuda_gpu_executor.h means that dependent
// code does not also have to depend on cuda.h.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_ACTIVATION_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_ACTIVATION_H_
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
class StreamExecutor;
namespace cuda {
class CUDAExecutor;
class ScopedActivateContext;
// Activates a CUDA context within an enclosing scope.
class ScopedActivateExecutorContext {
public:
// Form that takes a CUDA executor implementation.
explicit ScopedActivateExecutorContext(CUDAExecutor* cuda_exec);
// Form that takes a pImpl executor and extracts a CUDA implementation --
// fatal failure if it is not CUDA inside.
explicit ScopedActivateExecutorContext(StreamExecutor* stream_exec);
ScopedActivateExecutorContext(ScopedActivateExecutorContext&& other);
~ScopedActivateExecutorContext();
private:
// The cuda.h-using datatype that we wrap.
ScopedActivateContext* driver_scoped_activate_context_;
SE_DISALLOW_COPY_AND_ASSIGN(ScopedActivateExecutorContext);
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_ACTIVATION_H_
@@ -0,0 +1,162 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// CUDA-specific support for BLAS functionality -- this wraps the cuBLAS library
// capabilities, and is only included into CUDA implementation code -- it will
// not introduce cuda headers into other code.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_BLAS_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_BLAS_H_
#include "tensorflow/stream_executor/blas.h"
#include "tensorflow/stream_executor/host_or_device_scalar.h"
#include "tensorflow/stream_executor/lib/stringpiece.h"
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/platform/thread_annotations.h"
#include "tensorflow/stream_executor/plugin_registry.h"
typedef struct cublasContext *cublasHandle_t;
namespace stream_executor {
class Stream;
namespace cuda {
// Opaque and unique identifier for the cuBLAS plugin.
extern const PluginId kCuBlasPlugin;
class CUDAExecutor;
// BLAS plugin for CUDA platform via cuBLAS library.
//
// This satisfies the platform-agnostic BlasSupport interface.
//
// Note that the cuBLAS handle that this encapsulates is implicitly tied to the
// context (and, as a result, the device) that the parent CUDAExecutor is tied
// to. This simply happens as an artifact of creating the cuBLAS handle when a
// CUDA context is active.
//
// Thread-safe post-initialization.
class CUDABlas : public blas::BlasSupport {
public:
explicit CUDABlas(CUDAExecutor *parent);
// Allocates a cuBLAS handle.
bool Init();
// Releases the cuBLAS handle, if present.
~CUDABlas() override;
TENSORFLOW_STREAM_EXECUTOR_GPU_BLAS_SUPPORT_OVERRIDES
private:
// Tells cuBLAS to enqueue the BLAS operation onto a particular Stream.
//
// cuBLAS is stateful, and only be associated with one stream (in order to
// enqueue dispatch) at a given time. As a result, this generally must be
// invoked before calling into cuBLAS.
bool SetStream(Stream *stream) EXCLUSIVE_LOCKS_REQUIRED(mu_);
// A helper function that calls the real cuBLAS function together with error
// handling.
//
// cublas_func: cuBLAS function pointer.
// cublas_name: cuBLAS function name.
// stream: Stream to enqueue the BLAS operation onto.
// pointer_mode_host: Indicate if the pointer to a scalar value is from host
// (true) or device (false).
// err_on_failure: Whether to print an error if the cublas function fails.
// args: Arguments of cuBLAS function.
template <typename FuncT, typename... Args>
bool DoBlasInternalImpl(FuncT cublas_func, Stream *stream,
bool pointer_mode_host, bool err_on_failure,
bool use_tensor_op_math, Args... args);
// Convenience functions that call DoBlasInternalImpl with different values
// for err_on_failure.
template <typename FuncT, typename... Args>
bool DoBlasInternal(FuncT cublas_func, Stream *stream, bool pointer_mode_host,
Args... args) {
return DoBlasInternalImpl(cublas_func, stream, pointer_mode_host,
/*err_on_failure=*/true, /*use_tensor_ops=*/false,
args...);
}
template <typename FuncT, typename... Args>
bool DoBlasInternalFailureOK(FuncT cublas_func, Stream *stream,
bool pointer_mode_host, Args... args) {
// Tensor ops are hard-coded off in this path, but can still be enabled with
// a specific algorithm choice as in DoBlasGemmWithAlgorithmImpl().
return DoBlasInternalImpl(cublas_func, stream, pointer_mode_host,
/*err_on_failure=*/false,
/*use_tensor_ops=*/false, args...);
}
// A helper function to implement DoBlasGemmBatched interfaces for generic
// types.
template <typename T, typename Scalar, typename FuncT>
port::Status DoBlasGemmBatchedInternal(
FuncT cublas_func, Stream *stream, blas::Transpose transa,
blas::Transpose transb, uint64 m, uint64 n, uint64 k, Scalar alpha,
const port::ArraySlice<DeviceMemory<T> *> &a_array, int lda,
const port::ArraySlice<DeviceMemory<T> *> &b_array, int ldb, Scalar beta,
const port::ArraySlice<DeviceMemory<T> *> &c_array, int ldc,
int batch_count, ScratchAllocator *scratch_allocator);
// Helper function for implementing DoBlasGemmWithAlgorithm.
template <typename InT, typename OutT, typename CompT>
bool DoBlasGemmWithAlgorithmImpl(
Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m,
uint64 n, uint64 k, const HostOrDeviceScalar<CompT> &alpha,
const DeviceMemory<InT> &a, int lda, const DeviceMemory<InT> &b, int ldb,
const HostOrDeviceScalar<CompT> &beta, DeviceMemory<OutT> *c, int ldc,
blas::ComputationType computation_type, blas::AlgorithmType algorithm,
blas::ProfileResult *output_profile_result);
// Helper function for implementing DoBlasGemmWithProfiling.
template <typename T, typename ParamType>
bool DoBlasGemmWithProfilingImpl(
Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m,
uint64 n, uint64 k, const ParamType &alpha, const DeviceMemory<T> &a,
int lda, const DeviceMemory<T> &b, int ldb, const ParamType &beta,
DeviceMemory<T> *c, int ldc, blas::ProfileResult *output_profile_result);
// Helper function for implementing DoBlasGemvWithProfiling.
template <typename T>
bool DoBlasGemvWithProfilingImpl(Stream *stream, blas::Transpose trans,
uint64 m, uint64 n, const T &alpha,
const DeviceMemory<T> &a, int lda,
const DeviceMemory<T> &x, int incx,
const T &beta, DeviceMemory<T> *y, int incy,
blas::ProfileResult *output_profile_result);
// mutex that guards the cuBLAS handle for this device.
mutex mu_;
// CUDAExecutor which instantiated this CUDABlas.
// Immutable post-initialization.
CUDAExecutor *parent_;
// cuBLAS library handle on the device.
cublasHandle_t blas_ GUARDED_BY(mu_);
SE_DISALLOW_COPY_AND_ASSIGN(CUDABlas);
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_BLAS_H_
@@ -0,0 +1,97 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DIAGNOSTICS_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DIAGNOSTICS_H_
#include "tensorflow/stream_executor/platform/port.h"
#include <tuple>
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
namespace cuda {
// e.g. DriverVersion{346, 3, 4}
using DriverVersion = std::tuple<int, int, int>;
// Converts a parsed driver version to string form.
string DriverVersionToString(DriverVersion version);
// Converts a parsed driver version or status value to natural string form.
string DriverVersionStatusToString(port::StatusOr<DriverVersion> version);
// Converts a string of a form like "331.79" to a DriverVersion{331, 79}.
port::StatusOr<DriverVersion> StringToDriverVersion(const string &value);
class Diagnostician {
public:
// Logs diagnostic information when CUDA appears to be misconfigured (e.g. is
// not initializing).
//
// Note: if we're running on a machine that has no GPUs, we don't want to
// produce very much log spew beyond saying, "looks like there's no CUDA
// kernel
// module running".
//
// Note: we use non-Google-File:: API here because we may be called before
// InitGoogle has completed.
static void LogDiagnosticInformation();
// Given the driver version file contents, finds the kernel module version and
// returns it as a string.
//
// This is solely used for more informative log messages when the user is
// running on a machine that happens to have a libcuda/kernel driver mismatch.
static port::StatusOr<DriverVersion> FindKernelModuleVersion(
const string &driver_version_file_contents);
// Extracts the kernel driver version from the current host.
static port::StatusOr<DriverVersion> FindKernelDriverVersion();
// Iterates through loaded DSOs with DlIteratePhdrCallback to find the
// driver-interfacing DSO version number. Returns it as a string.
static port::StatusOr<DriverVersion> FindDsoVersion();
// Logs information about the kernel driver version and userspace driver
// library version.
static void LogDriverVersionInformation();
private:
// Given the DSO version number and the driver version file contents, extracts
// the driver version and compares, warning the user in the case of
// incompatibility.
//
// This is solely used for more informative log messages when the user is
// running on a machine that happens to have a libcuda/kernel driver mismatch.
static void WarnOnDsoKernelMismatch(
port::StatusOr<DriverVersion> dso_version,
port::StatusOr<DriverVersion> kernel_version);
// Logs information about the dev nodes present on this machine: their
// existence, permissions, accessibility from this uid/gid.
static void LogDevNodeDiagnosticInformation();
static string GetDevNodePath(int dev_node_ordinal);
SE_DISALLOW_COPY_AND_ASSIGN(Diagnostician);
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DIAGNOSTICS_H_
@@ -0,0 +1,778 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// The CUDA-specific DNN library support, implementing the general DnnSupport
// interface.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DNN_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DNN_H_
#include "tensorflow/stream_executor/cuda/cuda_activation.h"
#include "tensorflow/stream_executor/dnn.h"
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/platform/thread_annotations.h"
#include "tensorflow/stream_executor/plugin_registry.h"
#include "tensorflow/stream_executor/temporary_device_memory.h"
namespace stream_executor {
namespace cuda {
class CUDAExecutor;
class CudnnRnnDescriptor;
class CudnnRnnSequenceTensorDescriptor;
class CudnnRnnStateTensorDescriptor;
// Opaque and unique identifier for the cuDNN plugin.
extern const PluginId kCuDnnPlugin;
// cudnn-library based DNN support. For details on overridden interface
// functions, see dnn.h.
class CudnnSupport : public dnn::DnnSupport {
public:
explicit CudnnSupport(CUDAExecutor* parent);
port::Status Init() override;
port::StatusOr<perftools::gputools::dnn::VersionInfo> GetVersion() override;
port::StatusOr<std::unique_ptr<dnn::RnnDescriptor>> createRnnDescriptor(
int num_layers, int hidden_size, int input_size, int batch_size,
dnn::RnnInputMode input_mode, dnn::RnnDirectionMode direction_mode,
dnn::RnnMode rnn_mode, dnn::DataType data_type,
const dnn::AlgorithmConfig& algorithm_config, float dropout, uint64 seed,
ScratchAllocator* state_allocator) override;
port::StatusOr<std::unique_ptr<dnn::RnnSequenceTensorDescriptor>>
createRnnSequenceTensorDescriptor(int seq_length, int batch_size,
int data_size,
dnn::DataType data_type) override;
port::StatusOr<std::unique_ptr<dnn::RnnStateTensorDescriptor>>
createRnnStateTensorDescriptor(int num_layer, int batch_size, int data_size,
dnn::DataType data_type) override;
bool DoRnnForward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
const dnn::RnnSequenceTensorDescriptor& input_desc,
const DeviceMemory<Eigen::half>& input_data,
const dnn::RnnStateTensorDescriptor& input_h_desc,
const DeviceMemory<Eigen::half>& input_h_data,
const dnn::RnnStateTensorDescriptor& input_c_desc,
const DeviceMemory<Eigen::half>& input_c_data,
const DeviceMemory<Eigen::half>& params,
const dnn::RnnSequenceTensorDescriptor& output_desc,
DeviceMemory<Eigen::half>* output_data,
const dnn::RnnStateTensorDescriptor& output_h_desc,
DeviceMemory<Eigen::half>* output_h_data,
const dnn::RnnStateTensorDescriptor& output_c_desc,
DeviceMemory<Eigen::half>* output_c_data, bool is_training,
ScratchAllocator* reserve_space_allocator,
ScratchAllocator* workspace_allocator,
dnn::ProfileResult* output_profile_result) override;
bool DoRnnForward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
const dnn::RnnSequenceTensorDescriptor& input_desc,
const DeviceMemory<float>& input_data,
const dnn::RnnStateTensorDescriptor& input_h_desc,
const DeviceMemory<float>& input_h_data,
const dnn::RnnStateTensorDescriptor& input_c_desc,
const DeviceMemory<float>& input_c_data,
const DeviceMemory<float>& params,
const dnn::RnnSequenceTensorDescriptor& output_desc,
DeviceMemory<float>* output_data,
const dnn::RnnStateTensorDescriptor& output_h_desc,
DeviceMemory<float>* output_h_data,
const dnn::RnnStateTensorDescriptor& output_c_desc,
DeviceMemory<float>* output_c_data, bool is_training,
ScratchAllocator* reserve_space_allocator,
ScratchAllocator* workspace_allocator,
dnn::ProfileResult* output_profile_result) override;
bool DoRnnForward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
const dnn::RnnSequenceTensorDescriptor& input_desc,
const DeviceMemory<double>& input_data,
const dnn::RnnStateTensorDescriptor& input_h_desc,
const DeviceMemory<double>& input_h_data,
const dnn::RnnStateTensorDescriptor& input_c_desc,
const DeviceMemory<double>& input_c_data,
const DeviceMemory<double>& params,
const dnn::RnnSequenceTensorDescriptor& output_desc,
DeviceMemory<double>* output_data,
const dnn::RnnStateTensorDescriptor& output_h_desc,
DeviceMemory<double>* output_h_data,
const dnn::RnnStateTensorDescriptor& output_c_desc,
DeviceMemory<double>* output_c_data, bool is_training,
ScratchAllocator* reserve_space_allocator,
ScratchAllocator* workspace_allocator,
dnn::ProfileResult* output_profile_result) override;
bool DoRnnBackward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
const dnn::RnnSequenceTensorDescriptor& input_desc,
const DeviceMemory<Eigen::half>& input_data,
const dnn::RnnStateTensorDescriptor& input_h_desc,
const DeviceMemory<Eigen::half>& input_h_data,
const dnn::RnnStateTensorDescriptor& input_c_desc,
const DeviceMemory<Eigen::half>& input_c_data,
const DeviceMemory<Eigen::half>& params,
const dnn::RnnSequenceTensorDescriptor& output_desc,
const DeviceMemory<Eigen::half>& output_data,
const dnn::RnnStateTensorDescriptor& output_h_desc,
const DeviceMemory<Eigen::half>& output_h_data,
const dnn::RnnStateTensorDescriptor& output_c_desc,
const DeviceMemory<Eigen::half>& output_c_data,
const DeviceMemory<Eigen::half>& output_backprop_data,
const DeviceMemory<Eigen::half>& output_h_backprop_data,
const DeviceMemory<Eigen::half>& output_c_backprop_data,
DeviceMemory<Eigen::half>* input_backprop_data,
DeviceMemory<Eigen::half>* input_h_backprop_data,
DeviceMemory<Eigen::half>* input_c_backprop_data,
DeviceMemory<Eigen::half>* params_backprop_data,
DeviceMemory<uint8>* reserve_space_data,
ScratchAllocator* workspace_allocator,
dnn::ProfileResult* output_profile_result) override;
bool DoRnnBackward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
const dnn::RnnSequenceTensorDescriptor& input_desc,
const DeviceMemory<float>& input_data,
const dnn::RnnStateTensorDescriptor& input_h_desc,
const DeviceMemory<float>& input_h_data,
const dnn::RnnStateTensorDescriptor& input_c_desc,
const DeviceMemory<float>& input_c_data,
const DeviceMemory<float>& params,
const dnn::RnnSequenceTensorDescriptor& output_desc,
const DeviceMemory<float>& output_data,
const dnn::RnnStateTensorDescriptor& output_h_desc,
const DeviceMemory<float>& output_h_data,
const dnn::RnnStateTensorDescriptor& output_c_desc,
const DeviceMemory<float>& output_c_data,
const DeviceMemory<float>& output_backprop_data,
const DeviceMemory<float>& output_h_backprop_data,
const DeviceMemory<float>& output_c_backprop_data,
DeviceMemory<float>* input_backprop_data,
DeviceMemory<float>* input_h_backprop_data,
DeviceMemory<float>* input_c_backprop_data,
DeviceMemory<float>* params_backprop_data,
DeviceMemory<uint8>* reserve_space_data,
ScratchAllocator* workspace_allocator,
dnn::ProfileResult* output_profile_result) override;
bool DoRnnBackward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
const dnn::RnnSequenceTensorDescriptor& input_desc,
const DeviceMemory<double>& input_data,
const dnn::RnnStateTensorDescriptor& input_h_desc,
const DeviceMemory<double>& input_h_data,
const dnn::RnnStateTensorDescriptor& input_c_desc,
const DeviceMemory<double>& input_c_data,
const DeviceMemory<double>& params,
const dnn::RnnSequenceTensorDescriptor& output_desc,
const DeviceMemory<double>& output_data,
const dnn::RnnStateTensorDescriptor& output_h_desc,
const DeviceMemory<double>& output_h_data,
const dnn::RnnStateTensorDescriptor& output_c_desc,
const DeviceMemory<double>& output_c_data,
const DeviceMemory<double>& output_backprop_data,
const DeviceMemory<double>& output_h_backprop_data,
const DeviceMemory<double>& output_c_backprop_data,
DeviceMemory<double>* input_backprop_data,
DeviceMemory<double>* input_h_backprop_data,
DeviceMemory<double>* input_c_backprop_data,
DeviceMemory<double>* params_backprop_data,
DeviceMemory<uint8>* reserve_space_data,
ScratchAllocator* workspace_allocator,
dnn::ProfileResult* output_profile_result) override;
bool GetConvolveAlgorithms(
bool with_winograd_nonfused, int cc_major, int cc_minor,
std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
bool GetRnnAlgorithms(
std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
bool GetConvolveBackwardDataAlgorithms(
bool with_winograd_nonfused, int cc_major, int cc_minor,
std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
bool GetConvolveBackwardFilterAlgorithms(
bool with_winograd_nonfused, int cc_major, int cc_minor,
std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
bool DoBatchNormalizationForward(
Stream* stream, const DeviceMemory<float>& x,
const DeviceMemory<float>& scale, const DeviceMemory<float>& offset,
const DeviceMemory<float>& estimated_mean,
const DeviceMemory<float>& estimated_variance,
const dnn::BatchDescriptor& x_desc,
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
DeviceMemory<float>* y, DeviceMemory<float>* batch_mean,
DeviceMemory<float>* batch_var, DeviceMemory<float>* saved_mean,
DeviceMemory<float>* saved_inv_var, bool is_training,
std::function<const DeviceMemory<float>&()> var_to_inv_var,
std::function<void()> inv_var_to_var) override;
bool DoBatchNormalizationForward(
Stream* stream, const DeviceMemory<Eigen::half>& x,
const DeviceMemory<float>& scale, const DeviceMemory<float>& offset,
const DeviceMemory<float>& estimated_mean,
const DeviceMemory<float>& estimated_variance,
const dnn::BatchDescriptor& x_desc,
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
DeviceMemory<Eigen::half>* y, DeviceMemory<float>* batch_mean,
DeviceMemory<float>* batch_var, DeviceMemory<float>* saved_mean,
DeviceMemory<float>* saved_inv_var, bool is_training,
std::function<const DeviceMemory<float>&()> var_to_inv_var,
std::function<void()> inv_var_to_var) override;
bool DoBatchNormalizationBackward(
Stream* stream, const DeviceMemory<float>& y_backprop,
const DeviceMemory<float>& x, const DeviceMemory<float>& scale,
const DeviceMemory<float>& mean, const DeviceMemory<float>& inv_var,
const dnn::BatchDescriptor& x_desc,
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
DeviceMemory<float>* x_backprop, DeviceMemory<float>* scale_backprop,
DeviceMemory<float>* offset_backprop) override;
bool DoBatchNormalizationBackward(
Stream* stream, const DeviceMemory<Eigen::half>& y_backprop,
const DeviceMemory<Eigen::half>& x, const DeviceMemory<float>& scale,
const DeviceMemory<float>& mean, const DeviceMemory<float>& inv_var,
const dnn::BatchDescriptor& x_desc,
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
DeviceMemory<Eigen::half>* x_backprop,
DeviceMemory<float>* scale_backprop,
DeviceMemory<float>* offset_backprop) override;
bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
const DeviceMemory<float>& input_data,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<float>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<float>* output_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
const DeviceMemory<double>& input_data,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<double>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<double>* output_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
const DeviceMemory<Eigen::half>& input_data,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<Eigen::half>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<Eigen::half>* output_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoFusedConvolve(
Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
const DeviceMemory<double>& conv_input_data, double conv_input_scale,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<double>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const DeviceMemory<double>& side_input_data, double side_input_scale,
const dnn::BatchDescriptor& bias_descriptor,
const DeviceMemory<double>& biases, dnn::ActivationMode activation_mode,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoFusedConvolve(
Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
const DeviceMemory<float>& conv_input_data, float conv_input_scale,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<float>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const DeviceMemory<float>& side_input_data, float side_input_scale,
const dnn::BatchDescriptor& bias_descriptor,
const DeviceMemory<float>& biases, dnn::ActivationMode activation_mode,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoFusedConvolve(Stream* stream,
const dnn::BatchDescriptor& conv_input_descriptor,
const DeviceMemory<Eigen::half>& conv_input_data,
float conv_input_scale,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<Eigen::half>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const DeviceMemory<Eigen::half>& side_input_data,
float side_input_scale,
const dnn::BatchDescriptor& bias_descriptor,
const DeviceMemory<Eigen::half>& biases,
dnn::ActivationMode activation_mode,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<Eigen::half>* output_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoFusedConvolve(
Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
const DeviceMemory<int8>& conv_input_data, float conv_input_scale,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<int8>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const DeviceMemory<int8>& side_input_data, float side_input_scale,
const dnn::BatchDescriptor& bias_descriptor,
const DeviceMemory<float>& biases, dnn::ActivationMode activation_mode,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<int8>* output_data, ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolveQuantized(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<float>& input_data,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<int8>& filter_coefficients,
const DeviceMemory<float>& coefficient_scales,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<float>* output_data) override {
LOG(ERROR) << "DoConvolveQuantized not supported by cuDNN";
return false;
}
bool DoConvolveQuantized(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<float>& input_data,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<int16>& filter_coefficients,
const DeviceMemory<float>& coefficient_scales,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<float>* output_data) override {
LOG(ERROR) << "DoConvolveQuantized not supported by cuDNN";
return false;
}
bool DoSeparableConvolve(
Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
const DeviceMemory<float>& input_data,
const dnn::FilterDescriptor& filter_descriptor, int depth_multiplier,
const DeviceMemory<float>& first_weights,
const DeviceMemory<float>& second_weights,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<float>* output_data) override {
LOG(ERROR) << "separable convolution not supported by CUDNN";
return false;
}
bool DoConvolveBackwardData(
Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<double>& filter_data,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<double> backward_output_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& input_descriptor,
DeviceMemory<double>* backward_input_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolveBackwardData(
Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<float>& filter_data,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<float> backward_output_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& input_descriptor,
DeviceMemory<float>* backward_input_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolveBackwardData(
Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<Eigen::half>& filter_data,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<Eigen::half> backward_output_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& input_descriptor,
DeviceMemory<Eigen::half>* backward_input_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolveBackwardFilter(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<double>& input_data,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<double> backward_output_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::FilterDescriptor& filter_descriptor,
DeviceMemory<double>* backward_filter_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolveBackwardFilter(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<float>& input_data,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<float> backward_output_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::FilterDescriptor& filter_descriptor,
DeviceMemory<float>* backward_filter_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolveBackwardFilter(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<Eigen::half>& input_data,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<Eigen::half> backward_output_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::FilterDescriptor& filter_descriptor,
DeviceMemory<Eigen::half>* backward_filter_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolveBackwardBias(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<double>& input_data,
const dnn::BatchDescriptor& bias_descriptor,
DeviceMemory<double>* backward_bias_data) override;
bool DoConvolveBackwardBias(Stream* stream,
const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<float>& input_data,
const dnn::BatchDescriptor& bias_descriptor,
DeviceMemory<float>* backward_bias_data) override;
bool DoConvolveBackwardBias(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<Eigen::half>& input_data,
const dnn::BatchDescriptor& bias_descriptor,
DeviceMemory<Eigen::half>* backward_bias_data) override;
bool DoMatMul(Stream* stream, const DeviceMemory<float>& input_data,
const DeviceMemory<float>& weights,
const dnn::BatchDescriptor& input_dimensions,
const dnn::BatchDescriptor& output_dimensions,
DeviceMemory<float>* output_data) override;
bool DoMatMulQuantized(Stream* stream, const DeviceMemory<float>& input_data,
const DeviceMemory<int8>& quantized_weights,
const DeviceMemory<float>& weight_scales,
const dnn::BatchDescriptor& input_dimensions,
const dnn::BatchDescriptor& output_dimensions,
DeviceMemory<float>* output_data) override {
LOG(ERROR) << "DNN MatMulQuantized not supported by CUDNN";
return false;
}
bool DoMatMulQuantized(Stream* stream, const DeviceMemory<float>& input_data,
const DeviceMemory<int16>& quantized_weights,
const DeviceMemory<float>& weight_scales,
const dnn::BatchDescriptor& input_dimensions,
const dnn::BatchDescriptor& output_dimensions,
DeviceMemory<float>* output_data) override {
LOG(ERROR) << "DNN MatMulQuantized not supported by CUDNN";
return false;
}
bool DoBiasAdd(Stream* stream, const DeviceMemory<float>& input_data,
const DeviceMemory<float>& biases,
const dnn::BatchDescriptor& dimensions,
DeviceMemory<float>* output_data) override;
bool DoActivate(Stream* stream, dnn::ActivationMode activation_mode,
const dnn::BatchDescriptor& dimensions,
const DeviceMemory<float>& input_data,
DeviceMemory<float>* output_data, uint64 options) override;
bool DoPoolForward(Stream* stream,
const dnn::PoolingDescriptor& pooling_dimensions,
const dnn::BatchDescriptor& input_dimensions,
const DeviceMemory<double>& input_data,
const dnn::BatchDescriptor& output_dimensions,
DeviceMemory<double>* output_data,
ScratchAllocator* workspace_allocator) override;
bool DoPoolForward(Stream* stream,
const dnn::PoolingDescriptor& pooling_dimensions,
const dnn::BatchDescriptor& input_dimensions,
const DeviceMemory<float>& input_data,
const dnn::BatchDescriptor& output_dimensions,
DeviceMemory<float>* output_data,
ScratchAllocator* workspace_allocator) override;
bool DoPoolForward(Stream* stream,
const dnn::PoolingDescriptor& pooling_dimensions,
const dnn::BatchDescriptor& input_dimensions,
const DeviceMemory<Eigen::half>& input_data,
const dnn::BatchDescriptor& output_dimensions,
DeviceMemory<Eigen::half>* output_data,
ScratchAllocator* workspace_allocator) override;
bool DoPoolBackward(Stream* stream,
const dnn::PoolingDescriptor& pooling_dimensions,
const dnn::BatchDescriptor& input_dimensions,
const DeviceMemory<double>& input_data,
const dnn::BatchDescriptor& output_dimensions,
const DeviceMemory<double>& output_data,
const DeviceMemory<double>& input_diff_data,
DeviceMemory<double>* output_diff_data,
ScratchAllocator* workspace_allocator) override;
bool DoPoolBackward(Stream* stream,
const dnn::PoolingDescriptor& pooling_dimensions,
const dnn::BatchDescriptor& input_dimensions,
const DeviceMemory<float>& input_data,
const dnn::BatchDescriptor& output_dimensions,
const DeviceMemory<float>& output_data,
const DeviceMemory<float>& input_diff_data,
DeviceMemory<float>* output_diff_data,
ScratchAllocator* workspace_allocator) override;
bool DoPoolBackward(Stream* stream,
const dnn::PoolingDescriptor& pooling_dimensions,
const dnn::BatchDescriptor& input_dimensions,
const DeviceMemory<Eigen::half>& input_data,
const dnn::BatchDescriptor& output_dimensions,
const DeviceMemory<Eigen::half>& output_data,
const DeviceMemory<Eigen::half>& input_diff_data,
DeviceMemory<Eigen::half>* output_diff_data,
ScratchAllocator* workspace_allocator) override;
bool DoNormalize(Stream* stream,
const dnn::NormalizeDescriptor& normalize_descriptor,
const DeviceMemory<float>& input_data,
DeviceMemory<float>* output_data) override;
bool DoNormalizeWithDimensions(
Stream* stream, const dnn::NormalizeDescriptor& normalize_descriptor,
const dnn::BatchDescriptor& dimensions,
const DeviceMemory<float>& input_data,
DeviceMemory<float>* output_data) override;
bool DoNormalizeBackwardWithDimensions(
Stream* stream, const dnn::NormalizeDescriptor& normalize_descriptor,
const dnn::BatchDescriptor& dimensions,
const DeviceMemory<float>& raw_data,
const DeviceMemory<float>& normalized_data,
const DeviceMemory<float>& normalized_variable_gradient,
DeviceMemory<float>* raw_variable_gradient,
ScratchAllocator* workspace_allocator) override;
bool DoDepthConcatenate(
Stream* stream, port::ArraySlice<dnn::BatchDescriptor> input_dimensions,
port::ArraySlice<const DeviceMemory<float>*> input_data,
DeviceMemory<float>* output_data) override;
bool DoElementwiseOperate(
Stream* stream, dnn::ElementwiseOperation operation,
port::ArraySlice<dnn::BatchDescriptor> input_dimensions,
port::ArraySlice<const DeviceMemory<float>*> input_data,
const dnn::BatchDescriptor& output_dimensions,
DeviceMemory<float>* output_data) override;
bool DoXYPad(Stream* stream, const dnn::BatchDescriptor &dimensions,
const DeviceMemory<float> &input_data,
int64 left_pad, int64 right_pad, int64 top_pad,
int64 bottom_pad, DeviceMemory<float> *output_data) override;
bool DoXYSlice(Stream* stream, const dnn::BatchDescriptor &dimensions,
const DeviceMemory<float> &input_data,
int64 left_trim, int64 right_trim, int64 top_trim,
int64 bottom_trim, DeviceMemory<float> *output_data) override;
bool DoMemcpyD2HQuantized(Stream* stream,
const DeviceMemory<float>& device_unquantized_src,
dnn::QuantizedActivationMode mode, void* host_dst,
int64 size) override;
bool DoMemcpyH2DQuantized(
Stream* stream, const void* host_src, int64 size,
dnn::QuantizedActivationMode mode,
DeviceMemory<float>* device_unquantized_dst) override;
// Derives an output batch descriptor from an input batch and convolution
// descriptors.
bool DeriveOutputBatchDescriptor(
const dnn::BatchDescriptor& batch_descriptor,
const dnn::FilterDescriptor& filter_descriptor,
const dnn::ConvolutionDescriptor& convolution_descriptor,
dnn::BatchDescriptor* output_batch_descriptor);
bool DoTransformTensor(Stream* stream, const dnn::BatchDescriptor& input_desc,
dnn::DataType input_type,
const DeviceMemoryBase& input_data,
const dnn::BatchDescriptor& output_desc,
dnn::DataType output_type, float scale,
DeviceMemoryBase* output_data) override;
private:
CUDAExecutor* parent_; // Parent executor object. Not owned.
// Provides access to the cuDNN handle.
std::unique_ptr<class CudnnAccess> cudnn_;
template <class T, class U>
port::Status DoBatchNormalizationForwardImpl(
Stream* stream, dnn::DataType input_data_type,
dnn::DataType scale_data_type, const DeviceMemory<T>& x,
const DeviceMemory<U>& scale, const DeviceMemory<U>& offset,
const DeviceMemory<U>& estimated_mean,
const DeviceMemory<U>& estimated_variance,
const dnn::BatchDescriptor& x_desc,
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
DeviceMemory<T>* y, DeviceMemory<U>* batch_mean,
DeviceMemory<U>* batch_var, DeviceMemory<U>* saved_mean,
DeviceMemory<U>* saved_inv_var, bool is_training,
std::function<const DeviceMemory<U>&()> var_to_inv_var,
std::function<void()> inv_var_to_var);
template <class T, class U>
port::Status DoBatchNormalizationBackwardImpl(
Stream* stream, int cudnn_input_type, int cudnn_scale_type,
const DeviceMemory<T>& y_backprop, const DeviceMemory<T>& x,
const DeviceMemory<U>& scale, const DeviceMemory<U>& mean,
const DeviceMemory<U>& inv_var, const dnn::BatchDescriptor& x_desc,
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
DeviceMemory<T>* x_backprop, DeviceMemory<U>* scale_backprop,
DeviceMemory<U>* offset_backprop);
template <class T>
port::Status DoConvolveImpl(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<T>& input_data,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<T>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<T>* output_data, ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result);
template <typename AccumulatorType, typename ElementType, typename BiasType,
typename ScaleType>
port::Status DoFusedConvolveImpl(
Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
const DeviceMemory<ElementType>& conv_input_data,
ScaleType conv_input_scale,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<ElementType>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const DeviceMemory<ElementType>& side_input_data,
ScaleType side_input_scale, const dnn::BatchDescriptor& bias_descriptor,
const DeviceMemory<BiasType>& biases, dnn::ActivationMode activation_mode,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<ElementType>* output_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result);
template <class T>
port::Status DoConvolveBackwardDataImpl(
Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<T>& filter_data,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<T> backward_output_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& input_descriptor,
DeviceMemory<T>* backward_input_data, ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result);
template <class T>
port::Status DoConvolveBackwardFilterImpl(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<T>& input_data,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<T> backward_output_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::FilterDescriptor& filter_descriptor,
DeviceMemory<T>* backward_filter_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result);
template <class T>
port::Status DoConvolveBackwardBiasImpl(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<T>& input_data,
const dnn::BatchDescriptor& bias_descriptor,
DeviceMemory<T>* backward_bias_data);
template <class T>
port::Status DoRnnForwardImpl(
Stream* stream, const CudnnRnnDescriptor& rnn_desc,
const CudnnRnnSequenceTensorDescriptor& input_desc,
const DeviceMemory<T>& input_data,
const CudnnRnnStateTensorDescriptor& input_h_desc,
const DeviceMemory<T>& input_h_data,
const CudnnRnnStateTensorDescriptor& input_c_desc,
const DeviceMemory<T>& input_c_data, const DeviceMemory<T>& params,
const CudnnRnnSequenceTensorDescriptor& output_desc,
DeviceMemory<T>* output_data,
const CudnnRnnStateTensorDescriptor& output_h_desc,
DeviceMemory<T>* output_h_data,
const CudnnRnnStateTensorDescriptor& output_c_desc,
DeviceMemory<T>* output_c_data, bool is_training,
ScratchAllocator* reserve_space_allocator,
ScratchAllocator* workspace_allocator,
dnn::ProfileResult* output_profile_result);
template <class T>
port::Status DoRnnBackwardImpl(
Stream* stream, const CudnnRnnDescriptor& rnn_desc,
const CudnnRnnSequenceTensorDescriptor& input_desc,
const DeviceMemory<T>& input_data,
const CudnnRnnStateTensorDescriptor& input_h_desc,
const DeviceMemory<T>& input_h_data,
const CudnnRnnStateTensorDescriptor& input_c_desc,
const DeviceMemory<T>& input_c_data, const DeviceMemory<T>& params,
const CudnnRnnSequenceTensorDescriptor& output_desc,
const DeviceMemory<T>& output_data,
const CudnnRnnStateTensorDescriptor& output_h_desc,
const DeviceMemory<T>& output_h_data,
const CudnnRnnStateTensorDescriptor& output_c_desc,
const DeviceMemory<T>& output_c_data,
const DeviceMemory<T>& output_backprop_data,
const DeviceMemory<T>& output_h_backprop_data,
const DeviceMemory<T>& output_c_backprop_data,
DeviceMemory<T>* input_backprop_data,
DeviceMemory<T>* input_h_backprop_data,
DeviceMemory<T>* input_c_backprop_data,
DeviceMemory<T>* params_backprop_data,
DeviceMemory<uint8>* reserve_space_data,
ScratchAllocator* workspace_allocator,
dnn::ProfileResult* output_profile_result);
SE_DISALLOW_COPY_AND_ASSIGN(CudnnSupport);
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DNN_H_
@@ -0,0 +1,520 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// CUDA userspace driver library wrapper functionality.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_H_
#include <stddef.h>
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/device_options.h"
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "cuda/include/cuda.h"
namespace stream_executor {
namespace cuda {
// Identifies the memory space where an allocation resides. See
// CUDADriver::GetPointerMemorySpace().
enum class MemorySpace { kHost, kDevice };
// Returns a casual string, such as "host" for the provided memory space.
string MemorySpaceString(MemorySpace memory_space);
class CudaContext;
// CUDADriver contains wrappers for calls to the userspace library driver. It's
// useful to isolate these calls and put basic wrappers around them to separate
// userspace library driver behaviors from the rest of the program.
//
// At the moment it's simply used as a namespace.
//
// The calls log any specific errors internally and return whether the operation
// was successful to the caller.
//
// The order of parameters is generally kept symmetric with the underlying CUDA
// driver API.
//
// Links on functions are to specific documentation under
// http://docs.nvidia.com/cuda/cuda-driver-api/
//
// Thread safety: these functions should not be used from signal handlers.
class CUDADriver {
public:
// Wraps a call to cuInit with logging to help indicate what has gone wrong in
// the case of failure. Safe to call multiple times; will be fast on all calls
// after the first.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__INITIALIZE.html#group__CUDA__INITIALIZE_1g0a2f1517e1bd8502c7194c3a8c134bc3
static port::Status Init();
// Returns the device associated with the given context.
// device is an outparam owned by the caller, must not be null.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g4e84b109eba36cdaaade167f34ae881e
static port::StatusOr<CUdevice> DeviceFromContext(CudaContext* context);
// Creates a new CUDA stream associated with the given context via
// cuStreamCreate.
// stream is an outparam owned by the caller, must not be null.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1ga581f0c5833e21ded8b5a56594e243f4
static bool CreateStream(CudaContext* context, CUstream *stream);
// Destroys a CUDA stream associated with the given context.
// stream is owned by the caller, must not be null, and *stream is set to null
// if the stream is successfully destroyed.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g244c8833de4596bcd31a06cdf21ee758
static void DestroyStream(CudaContext* context, CUstream *stream);
// CUDA events can explicitly disable event TSC retrieval for some presumed
// performance improvement if timing is unnecessary.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g450687e75f3ff992fe01662a43d9d3db
enum class EventFlags { kDefault, kDisableTiming };
// Creates a new event associated with the given context.
// result is an outparam owned by the caller and must not be null.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g450687e75f3ff992fe01662a43d9d3db
static port::Status CreateEvent(CudaContext* context, CUevent *result,
EventFlags flags);
// Destroys *event and turns it into a nullptr. event may not be null, but
// *event may be, via cuEventDestroy
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g593ec73a8ec5a5fc031311d3e4dca1ef
static port::Status DestroyEvent(CudaContext* context, CUevent *event);
// Allocates a GPU memory space of size bytes associated with the given
// context via cuMemAlloc.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gb82d2a09844a58dd9e744dc31e8aa467
static void *DeviceAllocate(CudaContext* context, uint64 bytes);
// Deallocates a GPU memory space of size bytes associated with the given
// context via cuMemFree.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g89b3f154e17cc89b6eea277dbdf5c93a
static void DeviceDeallocate(CudaContext* context, void *location);
// Allocates a unified memory space of size bytes associated with the given
// context via cuMemAllocManaged.
// https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gb347ded34dc326af404aa02af5388a32
static void* UnifiedMemoryAllocate(CudaContext* context, uint64 bytes);
// Deallocates a unified memory space of size bytes associated with the given
// context via cuMemFree.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g89b3f154e17cc89b6eea277dbdf5c93a
static void UnifiedMemoryDeallocate(CudaContext* context, void* location);
// Allocates page-locked and CUDA-registered memory on the host via
// cuMemAllocHost.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gdd8311286d2c2691605362c689bc64e0
static void *HostAllocate(CudaContext* context, uint64 bytes);
// Deallocates a location created by HostAllocate, via cuMemFreeHost.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g62e0fdbe181dab6b1c90fa1a51c7b92c
static void HostDeallocate(CudaContext* context, void *location);
// Registers a memory region at location of size bytes via cuMemHostRegister.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gf0a9fe11544326dabd743b7aa6b54223
static bool HostRegister(CudaContext* context, void *location, uint64 bytes);
// Unregisters a memory region that was previously registered at location via
// cuMemHostUnregister.
//
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g63f450c8125359be87b7623b1c0b2a14
//
// TODO(leary) verify an error will be returned if the location wasn't
// previously registered.
static bool HostUnregister(CudaContext* context, void *location);
// Given a device ordinal, returns a device handle into the device outparam,
// which must not be null.
//
// N.B. these device handles do not have a corresponding destroy function in
// the CUDA driver API.
static port::Status GetDevice(int device_ordinal, CUdevice *device);
// Given a device handle, returns the name reported by the driver for the
// device.
static bool GetDeviceName(CUdevice device, string *name_out);
// Given a device to create a context for, returns a context handle into the
// context outparam, which must not be null.
//
// N.B. CUDA contexts are weird. They are implicitly associated with the
// calling thread. Current documentation on contexts and their influence on
// userspace processes is given here:
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g65dc0012348bc84810e2103a40d8e2cf
static port::Status CreateContext(CUdevice device,
const DeviceOptions& device_options,
CudaContext** context);
// Destroys the provided context via cuCtxDestroy.
// Don't do this while clients could still be using the context, per the docs
// bad things will happen.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g27a365aebb0eb548166309f58a1e8b8e
static void DestroyContext(CudaContext* context);
// Queries the runtime for the specified attribute of the specified function.
// cuFuncGetAttribute (the underlying CUDA driver API routine) only operates
// in terms of integer-sized values, so there's no potential for overrun (as
// of CUDA 5.5).
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1g5e92a1b0d8d1b82cb00dcfb2de15961b
static bool FuncGetAttribute(CUfunction_attribute attribute,
CUfunction function, int *attribute_value);
// Sets the preferred cache configuration for the specified function.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1g40f8c11e81def95dc0072a375f965681
static bool FuncSetCacheConfig(CUfunction function,
CUfunc_cache cache_config);
// Gets the preferred shared memory bank configuration for the specified
// CONTEXT (not function!), either default or four- or eight-byte bank size.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g17153a1b8b8c756f7ab8505686a4ad74
static port::StatusOr<CUsharedconfig> ContextGetSharedMemConfig(
CudaContext* context);
// Sets the preferred shared memory bank configuration for the specified
// CONTEXT (not function!), either default or four- or eight-byte bank size.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g2574235fa643f8f251bf7bc28fac3692
static port::Status ContextSetSharedMemConfig(
CudaContext* context, CUsharedconfig shared_mem_config);
// Launches a CUDA kernel via cuLaunchKernel.
// TODO(leary) describe the structure of kernel_params and extra in a readable
// way.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1gb8f3dc3031b40da29d5f9a7139e52e15
static bool LaunchKernel(CudaContext* context, CUfunction function,
unsigned int grid_dim_x, unsigned int grid_dim_y,
unsigned int grid_dim_z, unsigned int block_dim_x,
unsigned int block_dim_y, unsigned int block_dim_z,
unsigned int shared_mem_bytes, CUstream stream,
void **kernel_params, void **extra);
// Loads ptx_contents with the CUDA driver's PTX JIT and stores the resulting
// handle in "module". Any error logs that are produced are logged internally.
static bool LoadPtx(CudaContext* context, const char *ptx_contents,
CUmodule *module);
// Loads cubin_bytes with the CUDA driver's blob loading interface and stores
// the resulting handle in "module".
static port::Status LoadCubin(CudaContext* context, const char *cubin_bytes,
CUmodule *module);
// Retrieves a named kernel from a loaded module, and places the resulting
// handle into function (outparam) on success. Neither kernel_name nor
// function may be null. No ownership is taken of kernel_name.
static bool GetModuleFunction(CudaContext* context, CUmodule module,
const char *kernel_name, CUfunction *function);
// Retrieves a named global/constant symbol from a loaded module, and returns
// a device pointer and size of the symbol on success. symbol_name may not be
// null. At least one of dptr or bytes should not be null. No ownership is
// taken of symbol_name.
static bool GetModuleSymbol(CudaContext* context, CUmodule module,
const char *symbol_name, CUdeviceptr *dptr,
size_t *bytes);
// Unloads module from the current context via cuModuleUnload.
// TODO(leary) the documentation doesn't say what kind of disasters happen
// if you try to unload a module while its CUfunctions are in use.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MODULE.html#group__CUDA__MODULE_1g8ea3d716524369de3763104ced4ea57b
static void UnloadModule(CudaContext* context, CUmodule module);
// Performs a synchronous memset of the device memory segment via cuMemsetD8.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g6e582bf866e9e2fb014297bfaf354d7b
static bool SynchronousMemsetUint8(CudaContext* context, CUdeviceptr location,
uint8 value, size_t size);
// Performs a synchronous memset of the device memory segment via cuMemsetD32.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g983e8d8759acd1b64326317481fbf132
static bool SynchronousMemsetUint32(CudaContext* context,
CUdeviceptr location, uint32 value,
size_t uint32_count);
// Performs an asynchronous memset of the device memory segment via
// cuMemsetD8Async.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gaef08a7ccd61112f94e82f2b30d43627
static bool AsynchronousMemsetUint8(CudaContext* context, CUdeviceptr location,
uint8 value, size_t uint32_count,
CUstream stream);
// Performs an asynchronous memset of the device memory segment via
// cuMemsetD32Async.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g58229da5d30f1c0cdf667b320ec2c0f5
static bool AsynchronousMemsetUint32(CudaContext* context,
CUdeviceptr location, uint32 value,
size_t uint32_count, CUstream stream);
// -- Synchronous memcopies.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g4d32266788c440b0220b1a9ba5795169
static port::Status SynchronousMemcpyD2H(CudaContext* context, void* host_dst,
CUdeviceptr gpu_src, uint64 size);
static port::Status SynchronousMemcpyH2D(CudaContext* context,
CUdeviceptr gpu_dst,
const void* host_src, uint64 size);
static port::Status SynchronousMemcpyD2D(CudaContext* context,
CUdeviceptr gpu_dst,
CUdeviceptr gpu_src, uint64 size);
// -- Asynchronous memcopies.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g56f30236c7c5247f8e061b59d3268362
static bool AsynchronousMemcpyD2H(CudaContext* context, void *host_dst,
CUdeviceptr gpu_src, uint64 size,
CUstream stream);
static bool AsynchronousMemcpyH2D(CudaContext* context, CUdeviceptr gpu_dst,
const void *host_src, uint64 size,
CUstream stream);
static bool AsynchronousMemcpyD2D(CudaContext* context, CUdeviceptr gpu_dst,
CUdeviceptr gpu_src, uint64 size,
CUstream stream);
// The CUDA stream callback type signature.
// The data passed to AddStreamCallback is subsequently passed to this
// callback when it fires.
//
// Some notable things:
// * Callbacks must not make any CUDA API calls.
// * Callbacks from independent streams execute in an undefined order and may
// be serialized.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g613d97a277d7640f4cb1c03bd51c2483
typedef void (*StreamCallback)(CUstream stream, CUresult status, void *data);
// Enqueues a callback operation into stream.
// See StreamCallback above and the NVIDIA documentation for additional
// details.
static bool AddStreamCallback(CudaContext* context, CUstream stream,
StreamCallback callback, void *data);
// Causes stream to wait for event to trigger before proceeding via
// cuStreamWaitEvent.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#axzz334nAXAhM
static bool WaitStreamOnEvent(CudaContext* context, CUstream stream,
CUevent event);
// Blocks the calling thread until the operations enqueued onto stream have
// been completed, via cuStreamSynchronize.
//
// TODO(leary) if a pathological thread enqueues operations onto the stream
// while another thread blocks like this, can you wind up waiting an unbounded
// amount of time?
//
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g15e49dd91ec15991eb7c0a741beb7dad
static port::Status SynchronizeStream(CudaContext* context, CUstream stream);
// Blocks the calling thread until the operations associated with the context
// have been completed, via cuCtxSynchronize.
//
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g7a54725f28d34b8c6299f0c6ca579616
static bool SynchronizeContext(CudaContext* context);
// Returns true if all stream tasks have completed at time of the call. Note
// the potential for races around this call (if another thread adds work to
// the stream immediately after this returns).
static bool IsStreamIdle(CudaContext* context, CUstream stream);
// Returns whether code in the from context can access memory in the to
// context via cuDeviceCanAccessPeer.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__PEER__ACCESS.html#group__CUDA__PEER__ACCESS_1g496bdaae1f632ebfb695b99d2c40f19e
static bool CanEnablePeerAccess(CudaContext* from, CudaContext* to);
// Enables peer access per CanEnablePeerAccess, via cuCtxEnablePeerAccess.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__PEER__ACCESS.html#group__CUDA__PEER__ACCESS_1g0889ec6728e61c05ed359551d67b3f5a
static port::Status EnablePeerAccess(CudaContext* from, CudaContext* to);
// Returns the elapsed milliseconds between start and stop via
// cuEventElapsedTime.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1gdfb1178807353bbcaa9e245da497cf97
static bool GetEventElapsedTime(CudaContext* context,
float *elapsed_milliseconds, CUevent start,
CUevent stop);
// Records that an event occurred when execution reaches the current point in
// thestream via cuEventRecord.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g95424d3be52c4eb95d83861b70fb89d1
static port::Status RecordEvent(CudaContext* context, CUevent event,
CUstream stream);
// Polls (without blocking) to determine the status of an event - pending or
// complete (or an error status).
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g6f0704d755066b0ee705749ae911deef
static port::StatusOr<CUresult> QueryEvent(CudaContext* context,
CUevent event);
// -- Pointer-specific calls.
// Returns the context in which pointer was allocated or registered.
static port::StatusOr<CudaContext*> GetPointerContext(CUdeviceptr pointer);
// Returns the device associated with the context from GetPointerContext().
static port::StatusOr<CUdevice> GetPointerDevice(CUdeviceptr pointer);
// Returns the memory space addressed by pointer.
static port::StatusOr<MemorySpace> GetPointerMemorySpace(CUdeviceptr pointer);
// Returns the base address and size of the device pointer dptr.
static port::Status GetPointerAddressRange(CUdeviceptr dptr,
CUdeviceptr *base, size_t *size);
// -- Device-specific calls.
// Returns the compute capability for the device; i.e (3, 5).
// This is currently done via the deprecated device API.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE__DEPRECATED.html#group__CUDA__DEVICE__DEPRECATED_1ge2091bbac7e1fb18c2821612115607ea
static port::Status GetComputeCapability(int *cc_major, int *cc_minor,
CUdevice device);
// Returns the number of multiprocessors on the device (note that the device
// may be multi-GPU-per-board).
static port::StatusOr<int> GetMultiprocessorCount(CUdevice device);
// Returns the limit on number of threads that can be resident in a single
// multiprocessor.
static port::StatusOr<int64> GetMaxThreadsPerMultiprocessor(CUdevice device);
// Returns the limit on number of threads which may be resident for a single
// block (cooperative thread array).
static port::StatusOr<int64> GetMaxThreadsPerBlock(CUdevice device);
// Returns the amount of shared memory available on a single GPU core (i.e.
// SM on NVIDIA devices).
static port::StatusOr<int64> GetMaxSharedMemoryPerCore(CUdevice device);
// Returns the amount of shared memory available for a single block
// (cooperative thread array).
static port::StatusOr<int64> GetMaxSharedMemoryPerBlock(CUdevice device);
// Returns the maximum supported number of registers per block.
static port::StatusOr<int64> GetMaxRegistersPerBlock(CUdevice device);
// Returns the number of threads per warp.
static port::StatusOr<int64> GetThreadsPerWarp(CUdevice device);
// Queries the grid limits for device with cuDeviceGetAttribute calls.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
static bool GetGridLimits(int *x, int *y, int *z, CUdevice device);
// Returns a grab-bag of device properties in a caller-owned device_properties
// structure for device_ordinal via cuDeviceGetProperties.
//
// This call is deprecated in the NVIDIA driver API; its replacement is
// GetDeviceAttribute
//
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE__DEPRECATED.html#group__CUDA__DEVICE__DEPRECATED_1g65a5b4e25186bd257df80b98c98cffe6
static bool GetDeviceProperties(CUdevprop *device_properties,
int device_ordinal);
// Gets a specific integer-valued property about the given device.
//
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
static port::StatusOr<int> GetDeviceAttribute(CUdevice_attribute attribute,
CUdevice device);
// Returns whether ECC is enabled for the given CUdevice via
// cuDeviceGetattribute with CU_DEVICE_ATTRIBUTE_ECC_ENABLED.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
static bool IsEccEnabled(CUdevice device, bool *result);
// Returns the total amount of memory available for allocation by the CUDA
// context, in bytes, via cuDeviceTotalMem.
static bool GetDeviceTotalMemory(CUdevice device, uint64 *result);
// Returns the free amount of memory and total amount of memory, as reported
// by cuMemGetInfo.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g808f555540d0143a331cc42aa98835c0
static bool GetDeviceMemoryInfo(CudaContext* context, int64* free,
int64* total);
// Returns a PCI bus id string for the device.
// [domain]:[bus]:[device].[function]
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g85295e7d9745ab8f0aa80dd1e172acfc
static string GetPCIBusID(CUdevice device);
// -- Context- and device-independent calls.
// Returns the number of visible CUDA device via cuDeviceGetCount.
// This should correspond to the set of device ordinals available.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g52b5ce05cb8c5fb6831b2c0ff2887c74
static int GetDeviceCount();
// Returns the driver version number via cuDriverGetVersion.
// This is, surprisingly, NOT the actual driver version (e.g. 331.79) but,
// instead, the CUDA toolkit release number that this driver is compatible
// with; e.g. 6000 (for a CUDA 6.0 compatible driver) or 6050 (for a CUDA 6.5
// compatible driver).
//
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__VERSION.html#group__CUDA__VERSION_1g8b7a10395392e049006e61bcdc8ebe71
static bool GetDriverVersion(int *driver_version);
// -- Other calls
// Returns the maximum number of blocks (per multiprocessor) occupied by the
// specified kernel/CUfunction when launched with the specified parameters.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__OCCUPANCY.html#group__CUDA__OCCUPANCY_1gcc6e1094d05cba2cee17fe33ddd04a98
static port::StatusOr<int> GetMaxOccupiedBlocksPerCore(
CudaContext* context, CUfunction kernel, int threads_per_block,
size_t dynamic_shared_memory_bytes);
// Returns the current context set in CUDA. This is done by calling the cuda
// driver (e.g., this value is not our cached view of the current context).
static CUcontext CurrentContextOrDie();
// Seam for injecting an error at CUDA initialization time for testing
// purposes.
static bool driver_inject_init_error_;
};
// Ensures a context is activated within a scope.
class ScopedActivateContext {
public:
// Activates the context via cuCtxSetCurrent, if it is not the currently
// active context (a la cuCtxGetCurrent). Note the alternative push/pop
// mechanism is said by NVIDIA to be relatively slow and deprecated.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1gbe562ee6258b4fcc272ca6478ca2a2f7
explicit ScopedActivateContext(CudaContext* context);
// Checks that the context has remained activated for the duration of the
// scope.
~ScopedActivateContext();
private:
CudaContext* to_restore_ = nullptr;
};
// CudaContext wraps a cuda CUcontext handle, and includes a unique id. The
// unique id is positive, and ids are not repeated within the process.
class CudaContext {
public:
CudaContext(CUcontext context, int64 id) : context_(context), id_(id) { }
CUcontext context() const { return context_; }
int64 id() const { return id_; }
// Disallow copying and moving.
CudaContext(CudaContext&&) = delete;
CudaContext(const CudaContext&) = delete;
CudaContext& operator=(CudaContext&&) = delete;
CudaContext& operator=(const CudaContext&) = delete;
private:
CUcontext const context_;
const int64 id_;
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_H_
@@ -0,0 +1,62 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_EVENT_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_EVENT_H_
#include "tensorflow/stream_executor/cuda/cuda_driver.h"
#include "tensorflow/stream_executor/cuda/cuda_stream.h"
#include "tensorflow/stream_executor/event.h"
#include "tensorflow/stream_executor/lib/status.h"
namespace stream_executor {
namespace cuda {
// CUDAEvent wraps a CUevent in the platform-independent EventInterface
// interface.
class CUDAEvent : public internal::EventInterface {
public:
explicit CUDAEvent(CUDAExecutor* parent);
~CUDAEvent() override;
// Populates the CUDA-platform-specific elements of this object.
port::Status Init();
// Deallocates any platform-specific elements of this object. This is broken
// out (not part of the destructor) to allow for error reporting.
port::Status Destroy();
// Inserts the event at the current position into the specified stream.
port::Status Record(CUDAStream* stream);
// Polls the CUDA platform for the event's current status.
Event::Status PollForStatus();
// The underlying CUDA event element.
const CUevent& cuda_event();
private:
// The Executor used to which this object and CUevent are bound.
CUDAExecutor* parent_;
// The underlying CUDA event element.
CUevent cuda_event_;
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_EVENT_H_
@@ -0,0 +1,137 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// CUDA-specific support for FFT functionality -- this wraps the cuFFT library
// capabilities, and is only included into CUDA implementation code -- it will
// not introduce cuda headers into other code.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_FFT_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_FFT_H_
#include "cuda/include/cufft.h"
#include "tensorflow/stream_executor/fft.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/plugin_registry.h"
#include "tensorflow/stream_executor/scratch_allocator.h"
namespace stream_executor {
class Stream;
namespace cuda {
class CUDAExecutor;
// Opaque and unique indentifier for the cuFFT plugin.
extern const PluginId kCuFftPlugin;
// CUDAFftPlan uses deferred initialization. Only a single call of
// Initialize() is allowed to properly create cufft plan and set member
// variable is_initialized_ to true. Newly added interface that uses member
// variables should first check is_initialized_ to make sure that the values of
// member variables are valid.
class CUDAFftPlan : public fft::Plan {
public:
CUDAFftPlan()
: parent_(nullptr),
plan_(-1),
fft_type_(fft::Type::kInvalid),
scratch_(nullptr),
scratch_size_bytes_(0),
is_initialized_(false) {}
~CUDAFftPlan() override;
// Get FFT direction in cuFFT based on FFT type.
int GetFftDirection() const;
cufftHandle GetPlan() const {
if (IsInitialized()) {
return plan_;
} else {
LOG(FATAL) << "Try to get cufftHandle value before initialization.";
}
}
// Initialize function for batched plan
port::Status Initialize(CUDAExecutor *parent, Stream *stream, int rank,
uint64 *elem_count, uint64 *input_embed,
uint64 input_stride, uint64 input_distance,
uint64 *output_embed, uint64 output_stride,
uint64 output_distance, fft::Type type,
int batch_count, ScratchAllocator *scratch_allocator);
// Initialize function for 1d,2d, and 3d plan
port::Status Initialize(CUDAExecutor *parent, Stream *stream, int rank,
uint64 *elem_count, fft::Type type,
ScratchAllocator *scratch_allocator);
port::Status UpdateScratchAllocator(Stream *stream,
ScratchAllocator *scratch_allocator);
protected:
bool IsInitialized() const { return is_initialized_; }
private:
CUDAExecutor *parent_;
cufftHandle plan_;
fft::Type fft_type_;
DeviceMemory<uint8> scratch_;
size_t scratch_size_bytes_;
bool is_initialized_;
};
// FFT support for CUDA platform via cuFFT library.
//
// This satisfies the platform-agnostic FftSupport interface.
//
// Note that the cuFFT handle that this encapsulates is implicitly tied to the
// context (and, as a result, the device) that the parent CUDAExecutor is tied
// to. This simply happens as an artifact of creating the cuFFT handle when a
// CUDA context is active.
//
// Thread-safe. The CUDA context associated with all operations is the CUDA
// context of parent_, so all context is explicit.
class CUDAFft : public fft::FftSupport {
public:
explicit CUDAFft(CUDAExecutor *parent) : parent_(parent) {}
~CUDAFft() override {}
TENSORFLOW_STREAM_EXECUTOR_GPU_FFT_SUPPORT_OVERRIDES
private:
CUDAExecutor *parent_;
// Two helper functions that execute dynload::cufftExec?2?.
// This is for complex to complex FFT, when the direction is required.
template <typename FuncT, typename InputT, typename OutputT>
bool DoFftWithDirectionInternal(Stream *stream, fft::Plan *plan,
FuncT cufft_exec,
const DeviceMemory<InputT> &input,
DeviceMemory<OutputT> *output);
// This is for complex to real or real to complex FFT, when the direction
// is implied.
template <typename FuncT, typename InputT, typename OutputT>
bool DoFftInternal(Stream *stream, fft::Plan *plan, FuncT cufft_exec,
const DeviceMemory<InputT> &input,
DeviceMemory<OutputT> *output);
SE_DISALLOW_COPY_AND_ASSIGN(CUDAFft);
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_FFT_H_
@@ -0,0 +1,299 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// The CUDA implementation of the StreamExecutorInterface functionality.
// CUDA inclusions are ideally confined to this implementation file.
//
// The notions from the StreamExecutor basically correspond to the CUDA streams
// programming model provided by the libcuda.so driver APIs, so we don't have
// to do much more than wrap the calls to the libraries appropriately.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_GPU_EXECUTOR_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_GPU_EXECUTOR_H_
#include <set>
#include <unordered_map>
#include "tensorflow/stream_executor/cuda/cuda_kernel.h"
#include "tensorflow/stream_executor/event.h"
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/platform.h"
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/platform/thread_annotations.h"
#include "tensorflow/stream_executor/stream_executor_internal.h"
namespace stream_executor {
namespace cuda {
// CUDA-platform implementation of the platform-agnostic
// StreamExecutorInferface.
class CUDAExecutor : public internal::StreamExecutorInterface {
public:
// sub_platform indicates the subplatform used in this executor; it must
// be a CUDA type.
explicit CUDAExecutor(const PluginConfig &plugin_config)
: device_(0),
context_(nullptr),
device_ordinal_(0),
cc_major_(0),
cc_minor_(0),
plugin_config_(plugin_config) {}
// See the corresponding StreamExecutor methods for method comments on the
// following overrides.
~CUDAExecutor() override;
port::Status Init(int device_ordinal, DeviceOptions device_options) override;
bool GetKernel(const MultiKernelLoaderSpec &spec,
KernelBase *kernel) override;
void UnloadKernel(const KernelBase *kernel) override;
bool LoadModule(const MultiModuleLoaderSpec &spec,
ModuleHandle *module_handle) override;
bool UnloadModule(ModuleHandle module_handle) override;
bool Launch(Stream *stream, const ThreadDim &thread_dims,
const BlockDim &block_dims, const KernelBase &k,
const KernelArgsArrayBase &args) override;
void *Allocate(uint64 size) override;
void *AllocateSubBuffer(DeviceMemoryBase *mem, uint64 offset_bytes,
uint64 size_bytes) override;
void Deallocate(DeviceMemoryBase *mem) override;
void *UnifiedMemoryAllocate(uint64 size) override {
return CUDADriver::UnifiedMemoryAllocate(context_, size);
}
void UnifiedMemoryDeallocate(void *location) override {
return CUDADriver::UnifiedMemoryDeallocate(context_, location);
}
// CUDA allocation/registration functions are necessary because the driver
// internally sets up buffers for DMA operations (and page locks them).
// There's no external interface for us to otherwise control these DMA
// settings.
void *HostMemoryAllocate(uint64 size) override {
return CUDADriver::HostAllocate(context_, size);
}
void HostMemoryDeallocate(void *location) override {
return CUDADriver::HostDeallocate(context_, location);
}
bool HostMemoryRegister(void *location, uint64 size) override;
bool HostMemoryUnregister(void *location) override;
bool SynchronizeAllActivity() override;
bool SynchronousMemZero(DeviceMemoryBase *location, uint64 size) override;
bool SynchronousMemSet(DeviceMemoryBase *location, int value,
uint64 size) override;
port::Status SynchronousMemcpy(DeviceMemoryBase *gpu_dst,
const void *host_src, uint64 size) override;
port::Status SynchronousMemcpy(void *host_dst,
const DeviceMemoryBase &gpu_src,
uint64 size) override;
port::Status SynchronousMemcpyDeviceToDevice(DeviceMemoryBase *gpu_dst,
const DeviceMemoryBase &gpu_src,
uint64 size) override;
bool MemZero(Stream *stream, DeviceMemoryBase *location,
uint64 size) override;
bool Memset(Stream *stream, DeviceMemoryBase *location, uint8 pattern,
uint64 size) override;
bool Memset32(Stream *stream, DeviceMemoryBase *location, uint32 pattern,
uint64 size) override;
bool Memcpy(Stream *stream, void *host_dst, const DeviceMemoryBase &gpu_src,
uint64 size) override;
bool Memcpy(Stream *stream, DeviceMemoryBase *gpu_dst, const void *host_src,
uint64 size) override;
bool MemcpyDeviceToDevice(Stream *stream, DeviceMemoryBase *gpu_dst,
const DeviceMemoryBase &gpu_src,
uint64 size) override;
bool HostCallback(Stream *stream, std::function<void()> callback) override;
bool AllocateStream(Stream *stream) override;
void DeallocateStream(Stream *stream) override;
bool CreateStreamDependency(Stream *dependent, Stream *other) override;
bool AllocateTimer(Timer *timer) override;
void DeallocateTimer(Timer *timer) override;
bool StartTimer(Stream *stream, Timer *timer) override;
bool StopTimer(Stream *stream, Timer *timer) override;
port::Status AllocateEvent(Event *event) override;
port::Status DeallocateEvent(Event *event) override;
port::Status RecordEvent(Stream *stream, Event *event) override;
port::Status WaitForEvent(Stream *stream, Event *event) override;
Event::Status PollForEventStatus(Event *event) override;
port::Status BlockHostUntilDone(Stream *stream) override;
int PlatformDeviceCount() override { return CUDADriver::GetDeviceCount(); }
port::Status EnablePeerAccessTo(StreamExecutorInterface *other) override;
bool CanEnablePeerAccessTo(StreamExecutorInterface *other) override;
SharedMemoryConfig GetDeviceSharedMemoryConfig() override;
port::Status SetDeviceSharedMemoryConfig(SharedMemoryConfig config) override;
bool DeviceMemoryUsage(int64 *free, int64 *total) const override;
// Search for the symbol and returns a device pointer and size.
// Returns false if symbol does not exist.
bool GetSymbol(const string &symbol_name, ModuleHandle module_handle,
void **mem, size_t *bytes) override;
DeviceDescription *PopulateDeviceDescription() const override;
// Populates the block_dim_limit by querying the device driver API. If an
// error occurs at any point while asking the driver for block dim limits, it
// will be only partially populated as a result, and an error will be logged.
bool FillBlockDimLimit(BlockDim *block_dim_limit) const;
bool SupportsBlas() const override;
blas::BlasSupport *CreateBlas() override;
bool SupportsFft() const override;
fft::FftSupport *CreateFft() override;
bool SupportsRng() const override;
rng::RngSupport *CreateRng() override;
bool SupportsDnn() const override;
dnn::DnnSupport *CreateDnn() override;
std::unique_ptr<internal::EventInterface> CreateEventImplementation()
override;
std::unique_ptr<internal::KernelInterface> CreateKernelImplementation()
override;
std::unique_ptr<internal::StreamInterface> GetStreamImplementation() override;
std::unique_ptr<internal::TimerInterface> GetTimerImplementation() override;
void *GpuContextHack() override;
CudaContext* cuda_context();
private:
// Attempts to find a more specific version of the file indicated by
// filename by looking for compute-capability-specific suffixed versions; i.e.
// looking for "foo.ptx" will check to see if "foo.ptx.cc30.ptx" is present if
// we're on a compute capability 3.0 machine.
bool FindOnDiskForComputeCapability(port::StringPiece filename,
port::StringPiece canonical_suffix,
string *found_filename) const;
// Host callback landing routine invoked by CUDA.
// data: User-provided callback provided to HostCallback() above, captured
// as a std::function<void()>. Allocated/initialized inside
// HostCallback() and owned and deleted by this call.
static void InternalHostCallback(CUstream stream, CUresult status,
void *data);
// Collects metadata for the specified kernel.
bool GetKernelMetadata(CUDAKernel *cuda_kernel,
KernelMetadata *kernel_metadata);
// Prints to VLOG(2) information about the kernel's occupancy and how it might
// be improved.
void VlogOccupancyInfo(const KernelBase &kernel, const ThreadDim &thread_dims,
const BlockDim &block_dims);
bool LoadModuleFromCuBin(const char *cubin, CUmodule *module)
EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
// Loads the PTX text `ptx` as a CUDA module. `ptx` must be null terminated.
bool LoadModuleFromPtx(const char *ptx, CUmodule *module)
EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
bool UnloadGpuBinary(const void *gpu_binary)
EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
// Guards the in-memory-module mapping.
mutex in_memory_modules_mu_;
// Kernel -> loaded GPU binary. Many kernels may load the same binary.
std::unordered_map<const KernelBase *, const void *> kernel_to_gpu_binary_
GUARDED_BY(in_memory_modules_mu_);
// GPU binary (PTX or CUBIN) -> {CUDA module, reference count}.
std::unordered_map<const void *, std::pair<CUmodule, uint64>>
gpu_binary_to_module_ GUARDED_BY(in_memory_modules_mu_);
// Guards the launched kernel set.
mutex launched_kernels_mu_;
// Keeps track of the set of launched kernels. Currently used to suppress the
// occupancy check on subsequent launches.
std::set<CUfunction> launched_kernels_ GUARDED_BY(launched_kernels_mu_);
// Handle for the CUDA device being operated on. Immutable
// post-initialization.
CUdevice device_;
// Handle for session with the library/driver. Immutable post-initialization.
CudaContext* context_;
// The device ordinal value that this executor was initialized with; recorded
// for use in getting device metadata. Immutable post-initialization.
int device_ordinal_;
// The major verion of the compute capability for device_.
int cc_major_;
// The minor verion of the compute capability for device_.
int cc_minor_;
// The plugin configuration associated with this instance.
PluginConfig plugin_config_;
SE_DISALLOW_COPY_AND_ASSIGN(CUDAExecutor);
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_GPU_EXECUTOR_H_
@@ -0,0 +1,105 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Common helper functions used for dealing with CUDA API datatypes.
//
// These are typically placed here for use by multiple source components (for
// example, BLAS and executor components).
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_HELPERS_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_HELPERS_H_
#include <stddef.h>
#include <complex>
#include "cuda/include/cuComplex.h"
#include "cuda/include/cuda.h"
namespace stream_executor {
template <typename ElemT>
class DeviceMemory;
namespace cuda {
// Converts a const DeviceMemory reference to its underlying typed pointer in
// CUDA
// device memory.
template <typename T>
const T *CUDAMemory(const DeviceMemory<T> &mem) {
return static_cast<const T *>(mem.opaque());
}
// Converts a (non-const) DeviceMemory pointer reference to its underlying typed
// pointer in CUDA device memory.
template <typename T>
T *CUDAMemoryMutable(DeviceMemory<T> *mem) {
return static_cast<T *>(mem->opaque());
}
static_assert(sizeof(std::complex<float>) == sizeof(cuComplex),
"std::complex<float> and cuComplex should have the same size");
static_assert(offsetof(cuComplex, x) == 0,
"The real part of cuComplex should appear first.");
static_assert(sizeof(std::complex<double>) == sizeof(cuDoubleComplex),
"std::complex<double> and cuDoubleComplex should have the same "
"size");
static_assert(offsetof(cuDoubleComplex, x) == 0,
"The real part of cuDoubleComplex should appear first.");
// Type traits to get CUDA complex types from std::complex<>.
template <typename T>
struct CUDAComplexT {
typedef T type;
};
template <>
struct CUDAComplexT<std::complex<float>> {
typedef cuComplex type;
};
template <>
struct CUDAComplexT<std::complex<double>> {
typedef cuDoubleComplex type;
};
// Converts pointers of std::complex<> to pointers of
// cuComplex/cuDoubleComplex. No type conversion for non-complex types.
template <typename T>
inline const typename CUDAComplexT<T>::type *CUDAComplex(const T *p) {
return reinterpret_cast<const typename CUDAComplexT<T>::type *>(p);
}
template <typename T>
inline typename CUDAComplexT<T>::type *CUDAComplex(T *p) {
return reinterpret_cast<typename CUDAComplexT<T>::type *>(p);
}
// Converts values of std::complex<float/double> to values of
// cuComplex/cuDoubleComplex.
inline cuComplex CUDAComplexValue(std::complex<float> val) {
return {val.real(), val.imag()};
}
inline cuDoubleComplex CUDAComplexValue(std::complex<double> val) {
return {val.real(), val.imag()};
}
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_HELPERS_H_
@@ -0,0 +1,128 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// The CUDA implementation of the StreamExecutorInterface functionality.
// CUDA inclusions are ideally confined to this implementation file.
//
// The notions from the StreamExecutor basically correspond to the CUDA streams
// programming model provided by the libcuda.so driver APIs, so we don't have
// to do much more than wrap the calls to the libraries appropriately.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_KERNEL_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_KERNEL_H_
#include "tensorflow/stream_executor/kernel_cache_config.h"
#include "tensorflow/stream_executor/stream_executor_internal.h"
#include "tensorflow/stream_executor/cuda/cuda_driver.h"
#include "tensorflow/stream_executor/lib/casts.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/platform/logging.h"
#include "cuda/include/cuda.h"
#ifdef PLATFORMS_GPUS_CUDA_DYNAMIC_LIBCUDA_DYNAMIC_LIBCUDA_H_
#error \
"No driver calls in this file, wrap driver functionality in cuda_driver.cc."
#endif
#ifdef __CUDA_RUNTIME_H__
#error \
"CUDA runtime being included into CUDA GPU executor; should be driver only."
#endif
namespace stream_executor {
namespace cuda {
// Wraps a CUfunction to implement the platform-independent KernelInterface.
class CUDAKernel : public internal::KernelInterface {
public:
CUDAKernel() : cuda_function_(nullptr), arity_(0),
preferred_cache_config_(KernelCacheConfig::kNoPreference) {}
// Note that the function is unloaded when the module is unloaded, and the
// module that the function is contained in is owned by the CUDAExecutor.
~CUDAKernel() override {}
// As arity cannot be reflected upon using the CUDA API, the arity is
// explicitly set during the CUDAExecutor::GetKernel initialization process.
void set_arity(unsigned arity) { arity_ = arity; }
unsigned Arity() const override { return arity_; }
// Returns the CUfunction value for passing to the CUDA API.
CUfunction AsCUDAFunctionValue() const {
DCHECK(cuda_function_ != nullptr);
return const_cast<CUfunction>(cuda_function_);
}
// Returns the slot that the CUfunction is stored within for this object,
// for the CUDA API which wants to load into a CUfunction*.
CUfunction *cuda_function_ptr() { return &cuda_function_; }
// CUDA supports setting the preferred cache configuration of a CUfunction
// (more-or-less equivalent to a CUDAKernel). We support this via the below
// functions; users can set a preference, and that is applied when the kernel
// is [lazy-]loaded (in CUDAExecutor::Launch). The alternative would be to
// load the kernel & set the preference when the user calls the setter below;
// either approach is valid.
// Sets the current kernel cache configuration preference.
void SetPreferredCacheConfig(KernelCacheConfig config) override {
preferred_cache_config_ = config;
}
// Returns the current kernel cache configuration preference.
KernelCacheConfig GetPreferredCacheConfig() const override {
return preferred_cache_config_;
}
// Returns the current kernel cache configuration preference as a
// CUfunc_cache.
CUfunc_cache GetCUDACacheConfig() const {
switch (preferred_cache_config_) {
case KernelCacheConfig::kNoPreference:
return CU_FUNC_CACHE_PREFER_NONE;
case KernelCacheConfig::kPreferShared:
return CU_FUNC_CACHE_PREFER_SHARED;
case KernelCacheConfig::kPreferL1:
return CU_FUNC_CACHE_PREFER_L1;
case KernelCacheConfig::kPreferEqual:
return CU_FUNC_CACHE_PREFER_EQUAL;
default:
LOG(FATAL) << "Unknown KernelCacheConfig"
<< static_cast<int32>(preferred_cache_config_);
}
}
private:
CUfunction cuda_function_; // Wrapped CUDA kernel handle.
unsigned arity_; // Number of formal parameters the kernel takes.
// Preferred (but not required) cache configuration for this kernel.
KernelCacheConfig preferred_cache_config_;
};
// Given a platform-independent kernel datatype, returns the (const) internal
// CUDA platform implementation pointer.
inline const CUDAKernel *AsCUDAKernel(const KernelBase *kernel) {
return static_cast<const CUDAKernel *>(kernel->implementation());
}
// Given a platform-independent kernel datatype, returns the (non-const)
// internal CUDA platform implementation pointer.
inline CUDAKernel *AsCUDAKernel(KernelBase *kernel) {
return static_cast<CUDAKernel *>(kernel->implementation());
}
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_KERNEL_H_
@@ -0,0 +1,108 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_H_
#include <memory>
#include "tensorflow/stream_executor/platform/port.h"
#include <vector>
#include "tensorflow/stream_executor/executor_cache.h"
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/multi_platform_manager.h"
#include "tensorflow/stream_executor/platform.h"
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/platform/thread_annotations.h"
#include "tensorflow/stream_executor/stream_executor_internal.h"
#include "tensorflow/stream_executor/stream_executor_pimpl.h"
#include "tensorflow/stream_executor/trace_listener.h"
namespace stream_executor {
namespace cuda {
// Opaque and unique identifier for the CUDA platform plugin.
// This is needed so that plugins can refer to/identify this platform without
// instantiating a CudaPlatform object.
extern const Platform::Id kCudaPlatformId;
// Cuda-specific platform plugin, registered as a singleton value via module
// initializer.
class CudaPlatform : public Platform {
public:
CudaPlatform();
~CudaPlatform() override;
// CudaPlatform-specific functionality
// Returns the number of distinct buses / NUMA nodes on the machine.
int BusCount();
// Returns the bus/NUMA node for the specified device ordinal.
int DeviceToBus(int device_ordinal);
// Returns the lowest-ordinal-number StreamExecutor on the specified bus.
port::StatusOr<StreamExecutor*> FirstExecutorForBus(int bus_ordinal);
// Platform interface implementation:
// Returns the same value as kCudaPlatform above.
Platform::Id id() const override;
// Returns -1 as a sentinel on internal failure (and logs the error).
int VisibleDeviceCount() const override;
const string& Name() const override;
port::StatusOr<StreamExecutor*> ExecutorForDevice(int ordinal) override;
port::StatusOr<StreamExecutor*> ExecutorForDeviceWithPluginConfig(
int ordinal, const PluginConfig& config) override;
port::StatusOr<StreamExecutor*> GetExecutor(
const StreamExecutorConfig& config) override;
port::StatusOr<std::unique_ptr<StreamExecutor>> GetUncachedExecutor(
const StreamExecutorConfig& config) override;
void RegisterTraceListener(std::unique_ptr<TraceListener> listener) override;
void UnregisterTraceListener(TraceListener* listener) override;
private:
// Determines the number of NUMA nodes and the assignment of executor to each.
void InspectNumaNodes();
// This platform's name.
string name_;
// Cache of created executors.
ExecutorCache executor_cache_;
// The smallest NUMA node value for any device managed by this machine
// manager. Used, along with limit_numa_node_, to convert NUMA nodes into bus
// ordinals. The NUMA node space occupied by GPUs is assumed to be dense./
int min_numa_node_;
// Larger than the NUMA node value for any device managed by this machine
// manager.
int limit_numa_node_;
SE_DISALLOW_COPY_AND_ASSIGN(CudaPlatform);
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_H_
@@ -0,0 +1,34 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_ID_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_ID_H_
#include "tensorflow/stream_executor/platform.h"
namespace stream_executor {
namespace cuda {
// Opaque and unique identifier for the cuda platform.
// This is needed so that plugins can refer to/identify this platform without
// instantiating a CudaPlatform object.
// This is broken out here to avoid a circular dependency between CudaPlatform
// and CudaExecutor.
extern const Platform::Id kCudaPlatformId;
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_ID_H_
@@ -0,0 +1,102 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_RNG_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_RNG_H_
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/platform/thread_annotations.h"
#include "tensorflow/stream_executor/plugin_registry.h"
#include "tensorflow/stream_executor/rng.h"
typedef struct curandGenerator_st *curandGenerator_t;
namespace stream_executor {
class Stream;
template <typename ElemT>
class DeviceMemory;
namespace cuda {
// Opaque and unique identifier for the cuRAND plugin.
extern const PluginId kCuRandPlugin;
class CUDAExecutor;
// CUDA-platform implementation of the random number generation support
// interface.
//
// Thread-safe post-initialization.
class CUDARng : public rng::RngSupport {
public:
explicit CUDARng(CUDAExecutor *parent);
// Retrieves a curand library generator handle. This is necessary for
// enqueuing random number generation work onto the device.
// TODO(leary) provide a way for users to select the RNG algorithm.
bool Init();
// Releases a curand library generator handle, if one was acquired.
~CUDARng() override;
// See rng::RngSupport for details on the following overrides.
bool DoPopulateRandUniform(Stream *stream, DeviceMemory<float> *v) override;
bool DoPopulateRandUniform(Stream *stream, DeviceMemory<double> *v) override;
bool DoPopulateRandUniform(Stream *stream,
DeviceMemory<std::complex<float>> *v) override;
bool DoPopulateRandUniform(Stream *stream,
DeviceMemory<std::complex<double>> *v) override;
bool DoPopulateRandGaussian(Stream *stream, float mean, float stddev,
DeviceMemory<float> *v) override;
bool DoPopulateRandGaussian(Stream *stream, double mean, double stddev,
DeviceMemory<double> *v) override;
bool SetSeed(Stream *stream, const uint8 *seed, uint64 seed_bytes) override;
private:
// Actually performs the work of generating random numbers - the public
// methods are thin wrappers to this interface.
template <typename T>
bool DoPopulateRandUniformInternal(Stream *stream, DeviceMemory<T> *v);
template <typename ElemT, typename FuncT>
bool DoPopulateRandGaussianInternal(Stream *stream, ElemT mean, ElemT stddev,
DeviceMemory<ElemT> *v, FuncT func);
// Sets the stream for the internal curand generator.
//
// This is a stateful operation, as the handle can only have one stream set at
// a given time, so it is usually performed right before enqueuing work to do
// with random number generation.
bool SetStream(Stream *stream) EXCLUSIVE_LOCKS_REQUIRED(mu_);
// mutex that guards the cuRAND handle for this device.
mutex mu_;
// CUDAExecutor which instantiated this CUDARng.
// Immutable post-initialization.
CUDAExecutor *parent_;
// cuRANDalibrary handle on the device.
curandGenerator_t rng_ GUARDED_BY(mu_);
SE_DISALLOW_COPY_AND_ASSIGN(CUDARng);
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_RNG_H_
@@ -0,0 +1,93 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Defines the CUDAStream type - the CUDA-specific implementation of the generic
// StreamExecutor Stream interface.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_STREAM_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_STREAM_H_
#include "tensorflow/stream_executor/cuda/cuda_driver.h"
#include "tensorflow/stream_executor/platform/thread_annotations.h"
#include "tensorflow/stream_executor/stream_executor_internal.h"
namespace stream_executor {
namespace cuda {
class CUDAExecutor;
// Wraps a CUstream in order to satisfy the platform-independent
// StreamInterface.
//
// Thread-safe post-initialization.
class CUDAStream : public internal::StreamInterface {
public:
explicit CUDAStream(CUDAExecutor *parent)
: parent_(parent), cuda_stream_(nullptr), completed_event_(nullptr) {}
// Note: teardown is handled by a parent's call to DeallocateStream.
~CUDAStream() override {}
void *GpuStreamHack() override { return cuda_stream_; }
void **GpuStreamMemberHack() override {
return reinterpret_cast<void **>(&cuda_stream_);
}
// Explicitly initialize the CUDA resources associated with this stream, used
// by StreamExecutor::AllocateStream().
bool Init();
// Explicitly destroy the CUDA resources associated with this stream, used by
// StreamExecutor::DeallocateStream().
void Destroy();
// Returns true if no work is pending or executing on the stream.
bool IsIdle() const;
// Retrieves an event which indicates that all work enqueued into the stream
// has completed. Ownership of the event is not transferred to the caller, the
// event is owned by this stream.
CUevent* completed_event() { return &completed_event_; }
// Returns the CUstream value for passing to the CUDA API.
//
// Precond: this CUDAStream has been allocated (otherwise passing a nullptr
// into the NVIDIA library causes difficult-to-understand faults).
CUstream cuda_stream() const {
DCHECK(cuda_stream_ != nullptr);
return const_cast<CUstream>(cuda_stream_);
}
CUDAExecutor *parent() const { return parent_; }
private:
CUDAExecutor *parent_; // Executor that spawned this stream.
CUstream cuda_stream_; // Wrapped CUDA stream handle.
// Event that indicates this stream has completed.
CUevent completed_event_ = nullptr;
};
// Helper functions to simplify extremely common flows.
// Converts a Stream to the underlying CUDAStream implementation.
CUDAStream *AsCUDAStream(Stream *stream);
// Extracts a CUstream from a CUDAStream-backed Stream object.
CUstream AsCUDAStreamValue(Stream *stream);
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_STREAM_H_
@@ -0,0 +1,90 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Defines the CUDATimer type - the CUDA-specific implementation of the generic
// StreamExecutor Timer interface.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_TIMER_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_TIMER_H_
#include "tensorflow/stream_executor/stream_executor_internal.h"
#include "tensorflow/stream_executor/cuda/cuda_driver.h"
#include "tensorflow/stream_executor/cuda/cuda_gpu_executor.h"
namespace stream_executor {
namespace cuda {
class CUDAExecutor;
class CUDAStream;
// Wraps a pair of CUevents in order to satisfy the platform-independent
// TimerInferface -- both a start and a stop event are present which may be
// recorded in a stream.
class CUDATimer : public internal::TimerInterface {
public:
explicit CUDATimer(CUDAExecutor *parent)
: parent_(parent), start_event_(nullptr), stop_event_(nullptr) {}
// Note: teardown needs to be explicitly handled in this API by a call to
// StreamExecutor::DeallocateTimer(), which invokes Destroy().
// TODO(csigg): Change to RAII.
~CUDATimer() override {}
// Allocates the platform-specific pieces of the timer, called as part of
// StreamExecutor::AllocateTimer().
bool Init();
// Deallocates the platform-specific pieces of the timer, called as part of
// StreamExecutor::DeallocateTimer().
void Destroy();
// Records the "timer start" event at the current point in the stream.
bool Start(CUDAStream *stream);
// Records the "timer stop" event at the current point in the stream.
bool Stop(CUDAStream *stream);
// Returns the elapsed time, in milliseconds, between the start and stop
// events.
float GetElapsedMilliseconds() const;
// See Timer::Microseconds().
// TODO(leary) make this into an error code interface...
uint64 Microseconds() const override {
return GetElapsedMilliseconds() * 1e3;
}
// See Timer::Nanoseconds().
uint64 Nanoseconds() const override { return GetElapsedMilliseconds() * 1e6; }
private:
CUDAExecutor *parent_;
CUevent start_event_; // Event recorded to indicate the "start" timestamp
// executing in a stream.
CUevent stop_event_; // Event recorded to indicate the "stop" timestamp
// executing in a stream.
};
struct TimerDeleter {
void operator()(CUDATimer *t) {
t->Destroy();
delete t;
}
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_TIMER_H_
@@ -0,0 +1,50 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDNN_VERSION_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDNN_VERSION_H_
#include <string>
#include "tensorflow/core/lib/strings/strcat.h"
namespace stream_executor {
namespace cuda {
struct CudnnVersion {
CudnnVersion() = default;
CudnnVersion(int major, int minor, int patch)
: major_version(major), minor_version(minor), patch_level(patch) {}
tensorflow::string ToString() const {
return tensorflow::strings::StrCat(major_version, ".", minor_version, ".",
patch_level);
}
int major_version;
int minor_version;
int patch_level;
};
// Returns true if the given source CuDNN version is compatible with the given
// loaded version.
bool IsSourceCompatibleWithCudnnLibrary(CudnnVersion source_version,
CudnnVersion loaded_version);
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDNN_VERSION_H_