tf
This commit is contained in:
1 parent
eb1b42a5e8
commit
458eb2aef1
1283 files changed
+323373
No files matched your search
@@ -0,0 +1,61 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// This file contains APIs that assume a StreamExecutor is backed by CUDA.
|
||||
// It reaches into the CUDA implementation to activate an underlying CUDA
|
||||
// context.
|
||||
//
|
||||
// Having this file separate from cuda_gpu_executor.h means that dependent
|
||||
// code does not also have to depend on cuda.h.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_ACTIVATION_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_ACTIVATION_H_
|
||||
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class StreamExecutor;
|
||||
|
||||
namespace cuda {
|
||||
|
||||
class CUDAExecutor;
|
||||
class ScopedActivateContext;
|
||||
|
||||
// Activates a CUDA context within an enclosing scope.
|
||||
class ScopedActivateExecutorContext {
|
||||
public:
|
||||
// Form that takes a CUDA executor implementation.
|
||||
explicit ScopedActivateExecutorContext(CUDAExecutor* cuda_exec);
|
||||
|
||||
// Form that takes a pImpl executor and extracts a CUDA implementation --
|
||||
// fatal failure if it is not CUDA inside.
|
||||
explicit ScopedActivateExecutorContext(StreamExecutor* stream_exec);
|
||||
|
||||
ScopedActivateExecutorContext(ScopedActivateExecutorContext&& other);
|
||||
|
||||
~ScopedActivateExecutorContext();
|
||||
|
||||
private:
|
||||
// The cuda.h-using datatype that we wrap.
|
||||
ScopedActivateContext* driver_scoped_activate_context_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(ScopedActivateExecutorContext);
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_ACTIVATION_H_
|
||||
@@ -0,0 +1,162 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// CUDA-specific support for BLAS functionality -- this wraps the cuBLAS library
|
||||
// capabilities, and is only included into CUDA implementation code -- it will
|
||||
// not introduce cuda headers into other code.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_BLAS_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_BLAS_H_
|
||||
|
||||
#include "tensorflow/stream_executor/blas.h"
|
||||
#include "tensorflow/stream_executor/host_or_device_scalar.h"
|
||||
#include "tensorflow/stream_executor/lib/stringpiece.h"
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/platform/thread_annotations.h"
|
||||
#include "tensorflow/stream_executor/plugin_registry.h"
|
||||
|
||||
typedef struct cublasContext *cublasHandle_t;
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class Stream;
|
||||
|
||||
namespace cuda {
|
||||
|
||||
// Opaque and unique identifier for the cuBLAS plugin.
|
||||
extern const PluginId kCuBlasPlugin;
|
||||
|
||||
class CUDAExecutor;
|
||||
|
||||
// BLAS plugin for CUDA platform via cuBLAS library.
|
||||
//
|
||||
// This satisfies the platform-agnostic BlasSupport interface.
|
||||
//
|
||||
// Note that the cuBLAS handle that this encapsulates is implicitly tied to the
|
||||
// context (and, as a result, the device) that the parent CUDAExecutor is tied
|
||||
// to. This simply happens as an artifact of creating the cuBLAS handle when a
|
||||
// CUDA context is active.
|
||||
//
|
||||
// Thread-safe post-initialization.
|
||||
class CUDABlas : public blas::BlasSupport {
|
||||
public:
|
||||
explicit CUDABlas(CUDAExecutor *parent);
|
||||
|
||||
// Allocates a cuBLAS handle.
|
||||
bool Init();
|
||||
|
||||
// Releases the cuBLAS handle, if present.
|
||||
~CUDABlas() override;
|
||||
|
||||
TENSORFLOW_STREAM_EXECUTOR_GPU_BLAS_SUPPORT_OVERRIDES
|
||||
|
||||
private:
|
||||
// Tells cuBLAS to enqueue the BLAS operation onto a particular Stream.
|
||||
//
|
||||
// cuBLAS is stateful, and only be associated with one stream (in order to
|
||||
// enqueue dispatch) at a given time. As a result, this generally must be
|
||||
// invoked before calling into cuBLAS.
|
||||
bool SetStream(Stream *stream) EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
||||
|
||||
// A helper function that calls the real cuBLAS function together with error
|
||||
// handling.
|
||||
//
|
||||
// cublas_func: cuBLAS function pointer.
|
||||
// cublas_name: cuBLAS function name.
|
||||
// stream: Stream to enqueue the BLAS operation onto.
|
||||
// pointer_mode_host: Indicate if the pointer to a scalar value is from host
|
||||
// (true) or device (false).
|
||||
// err_on_failure: Whether to print an error if the cublas function fails.
|
||||
// args: Arguments of cuBLAS function.
|
||||
template <typename FuncT, typename... Args>
|
||||
bool DoBlasInternalImpl(FuncT cublas_func, Stream *stream,
|
||||
bool pointer_mode_host, bool err_on_failure,
|
||||
bool use_tensor_op_math, Args... args);
|
||||
|
||||
// Convenience functions that call DoBlasInternalImpl with different values
|
||||
// for err_on_failure.
|
||||
template <typename FuncT, typename... Args>
|
||||
bool DoBlasInternal(FuncT cublas_func, Stream *stream, bool pointer_mode_host,
|
||||
Args... args) {
|
||||
return DoBlasInternalImpl(cublas_func, stream, pointer_mode_host,
|
||||
/*err_on_failure=*/true, /*use_tensor_ops=*/false,
|
||||
args...);
|
||||
}
|
||||
template <typename FuncT, typename... Args>
|
||||
bool DoBlasInternalFailureOK(FuncT cublas_func, Stream *stream,
|
||||
bool pointer_mode_host, Args... args) {
|
||||
// Tensor ops are hard-coded off in this path, but can still be enabled with
|
||||
// a specific algorithm choice as in DoBlasGemmWithAlgorithmImpl().
|
||||
return DoBlasInternalImpl(cublas_func, stream, pointer_mode_host,
|
||||
/*err_on_failure=*/false,
|
||||
/*use_tensor_ops=*/false, args...);
|
||||
}
|
||||
|
||||
// A helper function to implement DoBlasGemmBatched interfaces for generic
|
||||
// types.
|
||||
template <typename T, typename Scalar, typename FuncT>
|
||||
port::Status DoBlasGemmBatchedInternal(
|
||||
FuncT cublas_func, Stream *stream, blas::Transpose transa,
|
||||
blas::Transpose transb, uint64 m, uint64 n, uint64 k, Scalar alpha,
|
||||
const port::ArraySlice<DeviceMemory<T> *> &a_array, int lda,
|
||||
const port::ArraySlice<DeviceMemory<T> *> &b_array, int ldb, Scalar beta,
|
||||
const port::ArraySlice<DeviceMemory<T> *> &c_array, int ldc,
|
||||
int batch_count, ScratchAllocator *scratch_allocator);
|
||||
|
||||
// Helper function for implementing DoBlasGemmWithAlgorithm.
|
||||
template <typename InT, typename OutT, typename CompT>
|
||||
bool DoBlasGemmWithAlgorithmImpl(
|
||||
Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m,
|
||||
uint64 n, uint64 k, const HostOrDeviceScalar<CompT> &alpha,
|
||||
const DeviceMemory<InT> &a, int lda, const DeviceMemory<InT> &b, int ldb,
|
||||
const HostOrDeviceScalar<CompT> &beta, DeviceMemory<OutT> *c, int ldc,
|
||||
blas::ComputationType computation_type, blas::AlgorithmType algorithm,
|
||||
blas::ProfileResult *output_profile_result);
|
||||
|
||||
// Helper function for implementing DoBlasGemmWithProfiling.
|
||||
template <typename T, typename ParamType>
|
||||
bool DoBlasGemmWithProfilingImpl(
|
||||
Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m,
|
||||
uint64 n, uint64 k, const ParamType &alpha, const DeviceMemory<T> &a,
|
||||
int lda, const DeviceMemory<T> &b, int ldb, const ParamType &beta,
|
||||
DeviceMemory<T> *c, int ldc, blas::ProfileResult *output_profile_result);
|
||||
|
||||
// Helper function for implementing DoBlasGemvWithProfiling.
|
||||
template <typename T>
|
||||
bool DoBlasGemvWithProfilingImpl(Stream *stream, blas::Transpose trans,
|
||||
uint64 m, uint64 n, const T &alpha,
|
||||
const DeviceMemory<T> &a, int lda,
|
||||
const DeviceMemory<T> &x, int incx,
|
||||
const T &beta, DeviceMemory<T> *y, int incy,
|
||||
blas::ProfileResult *output_profile_result);
|
||||
|
||||
// mutex that guards the cuBLAS handle for this device.
|
||||
mutex mu_;
|
||||
|
||||
// CUDAExecutor which instantiated this CUDABlas.
|
||||
// Immutable post-initialization.
|
||||
CUDAExecutor *parent_;
|
||||
|
||||
// cuBLAS library handle on the device.
|
||||
cublasHandle_t blas_ GUARDED_BY(mu_);
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CUDABlas);
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_BLAS_H_
|
||||
@@ -0,0 +1,97 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DIAGNOSTICS_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DIAGNOSTICS_H_
|
||||
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include <tuple>
|
||||
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
// e.g. DriverVersion{346, 3, 4}
|
||||
using DriverVersion = std::tuple<int, int, int>;
|
||||
|
||||
// Converts a parsed driver version to string form.
|
||||
string DriverVersionToString(DriverVersion version);
|
||||
|
||||
// Converts a parsed driver version or status value to natural string form.
|
||||
string DriverVersionStatusToString(port::StatusOr<DriverVersion> version);
|
||||
|
||||
// Converts a string of a form like "331.79" to a DriverVersion{331, 79}.
|
||||
port::StatusOr<DriverVersion> StringToDriverVersion(const string &value);
|
||||
|
||||
class Diagnostician {
|
||||
public:
|
||||
// Logs diagnostic information when CUDA appears to be misconfigured (e.g. is
|
||||
// not initializing).
|
||||
//
|
||||
// Note: if we're running on a machine that has no GPUs, we don't want to
|
||||
// produce very much log spew beyond saying, "looks like there's no CUDA
|
||||
// kernel
|
||||
// module running".
|
||||
//
|
||||
// Note: we use non-Google-File:: API here because we may be called before
|
||||
// InitGoogle has completed.
|
||||
static void LogDiagnosticInformation();
|
||||
|
||||
// Given the driver version file contents, finds the kernel module version and
|
||||
// returns it as a string.
|
||||
//
|
||||
// This is solely used for more informative log messages when the user is
|
||||
// running on a machine that happens to have a libcuda/kernel driver mismatch.
|
||||
static port::StatusOr<DriverVersion> FindKernelModuleVersion(
|
||||
const string &driver_version_file_contents);
|
||||
|
||||
// Extracts the kernel driver version from the current host.
|
||||
static port::StatusOr<DriverVersion> FindKernelDriverVersion();
|
||||
|
||||
// Iterates through loaded DSOs with DlIteratePhdrCallback to find the
|
||||
// driver-interfacing DSO version number. Returns it as a string.
|
||||
static port::StatusOr<DriverVersion> FindDsoVersion();
|
||||
|
||||
// Logs information about the kernel driver version and userspace driver
|
||||
// library version.
|
||||
static void LogDriverVersionInformation();
|
||||
|
||||
private:
|
||||
|
||||
// Given the DSO version number and the driver version file contents, extracts
|
||||
// the driver version and compares, warning the user in the case of
|
||||
// incompatibility.
|
||||
//
|
||||
// This is solely used for more informative log messages when the user is
|
||||
// running on a machine that happens to have a libcuda/kernel driver mismatch.
|
||||
static void WarnOnDsoKernelMismatch(
|
||||
port::StatusOr<DriverVersion> dso_version,
|
||||
port::StatusOr<DriverVersion> kernel_version);
|
||||
|
||||
// Logs information about the dev nodes present on this machine: their
|
||||
// existence, permissions, accessibility from this uid/gid.
|
||||
static void LogDevNodeDiagnosticInformation();
|
||||
|
||||
static string GetDevNodePath(int dev_node_ordinal);
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(Diagnostician);
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DIAGNOSTICS_H_
|
||||
@@ -0,0 +1,778 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// The CUDA-specific DNN library support, implementing the general DnnSupport
|
||||
// interface.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DNN_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DNN_H_
|
||||
|
||||
#include "tensorflow/stream_executor/cuda/cuda_activation.h"
|
||||
#include "tensorflow/stream_executor/dnn.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/platform/thread_annotations.h"
|
||||
#include "tensorflow/stream_executor/plugin_registry.h"
|
||||
#include "tensorflow/stream_executor/temporary_device_memory.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
class CUDAExecutor;
|
||||
class CudnnRnnDescriptor;
|
||||
class CudnnRnnSequenceTensorDescriptor;
|
||||
class CudnnRnnStateTensorDescriptor;
|
||||
|
||||
// Opaque and unique identifier for the cuDNN plugin.
|
||||
extern const PluginId kCuDnnPlugin;
|
||||
|
||||
// cudnn-library based DNN support. For details on overridden interface
|
||||
// functions, see dnn.h.
|
||||
class CudnnSupport : public dnn::DnnSupport {
|
||||
public:
|
||||
explicit CudnnSupport(CUDAExecutor* parent);
|
||||
|
||||
port::Status Init() override;
|
||||
port::StatusOr<perftools::gputools::dnn::VersionInfo> GetVersion() override;
|
||||
|
||||
port::StatusOr<std::unique_ptr<dnn::RnnDescriptor>> createRnnDescriptor(
|
||||
int num_layers, int hidden_size, int input_size, int batch_size,
|
||||
dnn::RnnInputMode input_mode, dnn::RnnDirectionMode direction_mode,
|
||||
dnn::RnnMode rnn_mode, dnn::DataType data_type,
|
||||
const dnn::AlgorithmConfig& algorithm_config, float dropout, uint64 seed,
|
||||
ScratchAllocator* state_allocator) override;
|
||||
|
||||
port::StatusOr<std::unique_ptr<dnn::RnnSequenceTensorDescriptor>>
|
||||
createRnnSequenceTensorDescriptor(int seq_length, int batch_size,
|
||||
int data_size,
|
||||
dnn::DataType data_type) override;
|
||||
|
||||
port::StatusOr<std::unique_ptr<dnn::RnnStateTensorDescriptor>>
|
||||
createRnnStateTensorDescriptor(int num_layer, int batch_size, int data_size,
|
||||
dnn::DataType data_type) override;
|
||||
|
||||
bool DoRnnForward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
|
||||
const dnn::RnnSequenceTensorDescriptor& input_desc,
|
||||
const DeviceMemory<Eigen::half>& input_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_h_desc,
|
||||
const DeviceMemory<Eigen::half>& input_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_c_desc,
|
||||
const DeviceMemory<Eigen::half>& input_c_data,
|
||||
const DeviceMemory<Eigen::half>& params,
|
||||
const dnn::RnnSequenceTensorDescriptor& output_desc,
|
||||
DeviceMemory<Eigen::half>* output_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_h_desc,
|
||||
DeviceMemory<Eigen::half>* output_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_c_desc,
|
||||
DeviceMemory<Eigen::half>* output_c_data, bool is_training,
|
||||
ScratchAllocator* reserve_space_allocator,
|
||||
ScratchAllocator* workspace_allocator,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoRnnForward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
|
||||
const dnn::RnnSequenceTensorDescriptor& input_desc,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_h_desc,
|
||||
const DeviceMemory<float>& input_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_c_desc,
|
||||
const DeviceMemory<float>& input_c_data,
|
||||
const DeviceMemory<float>& params,
|
||||
const dnn::RnnSequenceTensorDescriptor& output_desc,
|
||||
DeviceMemory<float>* output_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_h_desc,
|
||||
DeviceMemory<float>* output_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_c_desc,
|
||||
DeviceMemory<float>* output_c_data, bool is_training,
|
||||
ScratchAllocator* reserve_space_allocator,
|
||||
ScratchAllocator* workspace_allocator,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoRnnForward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
|
||||
const dnn::RnnSequenceTensorDescriptor& input_desc,
|
||||
const DeviceMemory<double>& input_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_h_desc,
|
||||
const DeviceMemory<double>& input_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_c_desc,
|
||||
const DeviceMemory<double>& input_c_data,
|
||||
const DeviceMemory<double>& params,
|
||||
const dnn::RnnSequenceTensorDescriptor& output_desc,
|
||||
DeviceMemory<double>* output_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_h_desc,
|
||||
DeviceMemory<double>* output_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_c_desc,
|
||||
DeviceMemory<double>* output_c_data, bool is_training,
|
||||
ScratchAllocator* reserve_space_allocator,
|
||||
ScratchAllocator* workspace_allocator,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoRnnBackward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
|
||||
const dnn::RnnSequenceTensorDescriptor& input_desc,
|
||||
const DeviceMemory<Eigen::half>& input_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_h_desc,
|
||||
const DeviceMemory<Eigen::half>& input_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_c_desc,
|
||||
const DeviceMemory<Eigen::half>& input_c_data,
|
||||
const DeviceMemory<Eigen::half>& params,
|
||||
const dnn::RnnSequenceTensorDescriptor& output_desc,
|
||||
const DeviceMemory<Eigen::half>& output_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_h_desc,
|
||||
const DeviceMemory<Eigen::half>& output_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_c_desc,
|
||||
const DeviceMemory<Eigen::half>& output_c_data,
|
||||
const DeviceMemory<Eigen::half>& output_backprop_data,
|
||||
const DeviceMemory<Eigen::half>& output_h_backprop_data,
|
||||
const DeviceMemory<Eigen::half>& output_c_backprop_data,
|
||||
DeviceMemory<Eigen::half>* input_backprop_data,
|
||||
DeviceMemory<Eigen::half>* input_h_backprop_data,
|
||||
DeviceMemory<Eigen::half>* input_c_backprop_data,
|
||||
DeviceMemory<Eigen::half>* params_backprop_data,
|
||||
DeviceMemory<uint8>* reserve_space_data,
|
||||
ScratchAllocator* workspace_allocator,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoRnnBackward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
|
||||
const dnn::RnnSequenceTensorDescriptor& input_desc,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_h_desc,
|
||||
const DeviceMemory<float>& input_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_c_desc,
|
||||
const DeviceMemory<float>& input_c_data,
|
||||
const DeviceMemory<float>& params,
|
||||
const dnn::RnnSequenceTensorDescriptor& output_desc,
|
||||
const DeviceMemory<float>& output_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_h_desc,
|
||||
const DeviceMemory<float>& output_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_c_desc,
|
||||
const DeviceMemory<float>& output_c_data,
|
||||
const DeviceMemory<float>& output_backprop_data,
|
||||
const DeviceMemory<float>& output_h_backprop_data,
|
||||
const DeviceMemory<float>& output_c_backprop_data,
|
||||
DeviceMemory<float>* input_backprop_data,
|
||||
DeviceMemory<float>* input_h_backprop_data,
|
||||
DeviceMemory<float>* input_c_backprop_data,
|
||||
DeviceMemory<float>* params_backprop_data,
|
||||
DeviceMemory<uint8>* reserve_space_data,
|
||||
ScratchAllocator* workspace_allocator,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoRnnBackward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
|
||||
const dnn::RnnSequenceTensorDescriptor& input_desc,
|
||||
const DeviceMemory<double>& input_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_h_desc,
|
||||
const DeviceMemory<double>& input_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& input_c_desc,
|
||||
const DeviceMemory<double>& input_c_data,
|
||||
const DeviceMemory<double>& params,
|
||||
const dnn::RnnSequenceTensorDescriptor& output_desc,
|
||||
const DeviceMemory<double>& output_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_h_desc,
|
||||
const DeviceMemory<double>& output_h_data,
|
||||
const dnn::RnnStateTensorDescriptor& output_c_desc,
|
||||
const DeviceMemory<double>& output_c_data,
|
||||
const DeviceMemory<double>& output_backprop_data,
|
||||
const DeviceMemory<double>& output_h_backprop_data,
|
||||
const DeviceMemory<double>& output_c_backprop_data,
|
||||
DeviceMemory<double>* input_backprop_data,
|
||||
DeviceMemory<double>* input_h_backprop_data,
|
||||
DeviceMemory<double>* input_c_backprop_data,
|
||||
DeviceMemory<double>* params_backprop_data,
|
||||
DeviceMemory<uint8>* reserve_space_data,
|
||||
ScratchAllocator* workspace_allocator,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool GetConvolveAlgorithms(
|
||||
bool with_winograd_nonfused, int cc_major, int cc_minor,
|
||||
std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
|
||||
|
||||
bool GetRnnAlgorithms(
|
||||
std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
|
||||
|
||||
bool GetConvolveBackwardDataAlgorithms(
|
||||
bool with_winograd_nonfused, int cc_major, int cc_minor,
|
||||
std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
|
||||
|
||||
bool GetConvolveBackwardFilterAlgorithms(
|
||||
bool with_winograd_nonfused, int cc_major, int cc_minor,
|
||||
std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
|
||||
|
||||
bool DoBatchNormalizationForward(
|
||||
Stream* stream, const DeviceMemory<float>& x,
|
||||
const DeviceMemory<float>& scale, const DeviceMemory<float>& offset,
|
||||
const DeviceMemory<float>& estimated_mean,
|
||||
const DeviceMemory<float>& estimated_variance,
|
||||
const dnn::BatchDescriptor& x_desc,
|
||||
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
|
||||
DeviceMemory<float>* y, DeviceMemory<float>* batch_mean,
|
||||
DeviceMemory<float>* batch_var, DeviceMemory<float>* saved_mean,
|
||||
DeviceMemory<float>* saved_inv_var, bool is_training,
|
||||
std::function<const DeviceMemory<float>&()> var_to_inv_var,
|
||||
std::function<void()> inv_var_to_var) override;
|
||||
|
||||
bool DoBatchNormalizationForward(
|
||||
Stream* stream, const DeviceMemory<Eigen::half>& x,
|
||||
const DeviceMemory<float>& scale, const DeviceMemory<float>& offset,
|
||||
const DeviceMemory<float>& estimated_mean,
|
||||
const DeviceMemory<float>& estimated_variance,
|
||||
const dnn::BatchDescriptor& x_desc,
|
||||
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
|
||||
DeviceMemory<Eigen::half>* y, DeviceMemory<float>* batch_mean,
|
||||
DeviceMemory<float>* batch_var, DeviceMemory<float>* saved_mean,
|
||||
DeviceMemory<float>* saved_inv_var, bool is_training,
|
||||
std::function<const DeviceMemory<float>&()> var_to_inv_var,
|
||||
std::function<void()> inv_var_to_var) override;
|
||||
|
||||
bool DoBatchNormalizationBackward(
|
||||
Stream* stream, const DeviceMemory<float>& y_backprop,
|
||||
const DeviceMemory<float>& x, const DeviceMemory<float>& scale,
|
||||
const DeviceMemory<float>& mean, const DeviceMemory<float>& inv_var,
|
||||
const dnn::BatchDescriptor& x_desc,
|
||||
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
|
||||
DeviceMemory<float>* x_backprop, DeviceMemory<float>* scale_backprop,
|
||||
DeviceMemory<float>* offset_backprop) override;
|
||||
|
||||
bool DoBatchNormalizationBackward(
|
||||
Stream* stream, const DeviceMemory<Eigen::half>& y_backprop,
|
||||
const DeviceMemory<Eigen::half>& x, const DeviceMemory<float>& scale,
|
||||
const DeviceMemory<float>& mean, const DeviceMemory<float>& inv_var,
|
||||
const dnn::BatchDescriptor& x_desc,
|
||||
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
|
||||
DeviceMemory<Eigen::half>* x_backprop,
|
||||
DeviceMemory<float>* scale_backprop,
|
||||
DeviceMemory<float>* offset_backprop) override;
|
||||
|
||||
bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<float>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<float>* output_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
|
||||
const DeviceMemory<double>& input_data,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<double>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<double>* output_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
|
||||
const DeviceMemory<Eigen::half>& input_data,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<Eigen::half>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<Eigen::half>* output_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoFusedConvolve(
|
||||
Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
|
||||
const DeviceMemory<double>& conv_input_data, double conv_input_scale,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<double>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const DeviceMemory<double>& side_input_data, double side_input_scale,
|
||||
const dnn::BatchDescriptor& bias_descriptor,
|
||||
const DeviceMemory<double>& biases, dnn::ActivationMode activation_mode,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoFusedConvolve(
|
||||
Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
|
||||
const DeviceMemory<float>& conv_input_data, float conv_input_scale,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<float>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const DeviceMemory<float>& side_input_data, float side_input_scale,
|
||||
const dnn::BatchDescriptor& bias_descriptor,
|
||||
const DeviceMemory<float>& biases, dnn::ActivationMode activation_mode,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoFusedConvolve(Stream* stream,
|
||||
const dnn::BatchDescriptor& conv_input_descriptor,
|
||||
const DeviceMemory<Eigen::half>& conv_input_data,
|
||||
float conv_input_scale,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<Eigen::half>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const DeviceMemory<Eigen::half>& side_input_data,
|
||||
float side_input_scale,
|
||||
const dnn::BatchDescriptor& bias_descriptor,
|
||||
const DeviceMemory<Eigen::half>& biases,
|
||||
dnn::ActivationMode activation_mode,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<Eigen::half>* output_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoFusedConvolve(
|
||||
Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
|
||||
const DeviceMemory<int8>& conv_input_data, float conv_input_scale,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<int8>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const DeviceMemory<int8>& side_input_data, float side_input_scale,
|
||||
const dnn::BatchDescriptor& bias_descriptor,
|
||||
const DeviceMemory<float>& biases, dnn::ActivationMode activation_mode,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<int8>* output_data, ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolveQuantized(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<int8>& filter_coefficients,
|
||||
const DeviceMemory<float>& coefficient_scales,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<float>* output_data) override {
|
||||
LOG(ERROR) << "DoConvolveQuantized not supported by cuDNN";
|
||||
return false;
|
||||
}
|
||||
|
||||
bool DoConvolveQuantized(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<int16>& filter_coefficients,
|
||||
const DeviceMemory<float>& coefficient_scales,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<float>* output_data) override {
|
||||
LOG(ERROR) << "DoConvolveQuantized not supported by cuDNN";
|
||||
return false;
|
||||
}
|
||||
|
||||
bool DoSeparableConvolve(
|
||||
Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::FilterDescriptor& filter_descriptor, int depth_multiplier,
|
||||
const DeviceMemory<float>& first_weights,
|
||||
const DeviceMemory<float>& second_weights,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<float>* output_data) override {
|
||||
LOG(ERROR) << "separable convolution not supported by CUDNN";
|
||||
return false;
|
||||
}
|
||||
|
||||
bool DoConvolveBackwardData(
|
||||
Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<double>& filter_data,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<double> backward_output_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& input_descriptor,
|
||||
DeviceMemory<double>* backward_input_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolveBackwardData(
|
||||
Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<float>& filter_data,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<float> backward_output_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& input_descriptor,
|
||||
DeviceMemory<float>* backward_input_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolveBackwardData(
|
||||
Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<Eigen::half>& filter_data,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<Eigen::half> backward_output_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& input_descriptor,
|
||||
DeviceMemory<Eigen::half>* backward_input_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolveBackwardFilter(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<double>& input_data,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<double> backward_output_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
DeviceMemory<double>* backward_filter_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolveBackwardFilter(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<float> backward_output_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
DeviceMemory<float>* backward_filter_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolveBackwardFilter(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<Eigen::half>& input_data,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<Eigen::half> backward_output_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
DeviceMemory<Eigen::half>* backward_filter_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result) override;
|
||||
|
||||
bool DoConvolveBackwardBias(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<double>& input_data,
|
||||
const dnn::BatchDescriptor& bias_descriptor,
|
||||
DeviceMemory<double>* backward_bias_data) override;
|
||||
|
||||
bool DoConvolveBackwardBias(Stream* stream,
|
||||
const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::BatchDescriptor& bias_descriptor,
|
||||
DeviceMemory<float>* backward_bias_data) override;
|
||||
|
||||
bool DoConvolveBackwardBias(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<Eigen::half>& input_data,
|
||||
const dnn::BatchDescriptor& bias_descriptor,
|
||||
DeviceMemory<Eigen::half>* backward_bias_data) override;
|
||||
|
||||
bool DoMatMul(Stream* stream, const DeviceMemory<float>& input_data,
|
||||
const DeviceMemory<float>& weights,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
DeviceMemory<float>* output_data) override;
|
||||
|
||||
bool DoMatMulQuantized(Stream* stream, const DeviceMemory<float>& input_data,
|
||||
const DeviceMemory<int8>& quantized_weights,
|
||||
const DeviceMemory<float>& weight_scales,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
DeviceMemory<float>* output_data) override {
|
||||
LOG(ERROR) << "DNN MatMulQuantized not supported by CUDNN";
|
||||
return false;
|
||||
}
|
||||
|
||||
bool DoMatMulQuantized(Stream* stream, const DeviceMemory<float>& input_data,
|
||||
const DeviceMemory<int16>& quantized_weights,
|
||||
const DeviceMemory<float>& weight_scales,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
DeviceMemory<float>* output_data) override {
|
||||
LOG(ERROR) << "DNN MatMulQuantized not supported by CUDNN";
|
||||
return false;
|
||||
}
|
||||
|
||||
bool DoBiasAdd(Stream* stream, const DeviceMemory<float>& input_data,
|
||||
const DeviceMemory<float>& biases,
|
||||
const dnn::BatchDescriptor& dimensions,
|
||||
DeviceMemory<float>* output_data) override;
|
||||
|
||||
bool DoActivate(Stream* stream, dnn::ActivationMode activation_mode,
|
||||
const dnn::BatchDescriptor& dimensions,
|
||||
const DeviceMemory<float>& input_data,
|
||||
DeviceMemory<float>* output_data, uint64 options) override;
|
||||
|
||||
bool DoPoolForward(Stream* stream,
|
||||
const dnn::PoolingDescriptor& pooling_dimensions,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const DeviceMemory<double>& input_data,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
DeviceMemory<double>* output_data,
|
||||
ScratchAllocator* workspace_allocator) override;
|
||||
|
||||
bool DoPoolForward(Stream* stream,
|
||||
const dnn::PoolingDescriptor& pooling_dimensions,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
DeviceMemory<float>* output_data,
|
||||
ScratchAllocator* workspace_allocator) override;
|
||||
|
||||
bool DoPoolForward(Stream* stream,
|
||||
const dnn::PoolingDescriptor& pooling_dimensions,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const DeviceMemory<Eigen::half>& input_data,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
DeviceMemory<Eigen::half>* output_data,
|
||||
ScratchAllocator* workspace_allocator) override;
|
||||
|
||||
bool DoPoolBackward(Stream* stream,
|
||||
const dnn::PoolingDescriptor& pooling_dimensions,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const DeviceMemory<double>& input_data,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
const DeviceMemory<double>& output_data,
|
||||
const DeviceMemory<double>& input_diff_data,
|
||||
DeviceMemory<double>* output_diff_data,
|
||||
ScratchAllocator* workspace_allocator) override;
|
||||
|
||||
bool DoPoolBackward(Stream* stream,
|
||||
const dnn::PoolingDescriptor& pooling_dimensions,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const DeviceMemory<float>& input_data,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
const DeviceMemory<float>& output_data,
|
||||
const DeviceMemory<float>& input_diff_data,
|
||||
DeviceMemory<float>* output_diff_data,
|
||||
ScratchAllocator* workspace_allocator) override;
|
||||
|
||||
bool DoPoolBackward(Stream* stream,
|
||||
const dnn::PoolingDescriptor& pooling_dimensions,
|
||||
const dnn::BatchDescriptor& input_dimensions,
|
||||
const DeviceMemory<Eigen::half>& input_data,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
const DeviceMemory<Eigen::half>& output_data,
|
||||
const DeviceMemory<Eigen::half>& input_diff_data,
|
||||
DeviceMemory<Eigen::half>* output_diff_data,
|
||||
ScratchAllocator* workspace_allocator) override;
|
||||
|
||||
bool DoNormalize(Stream* stream,
|
||||
const dnn::NormalizeDescriptor& normalize_descriptor,
|
||||
const DeviceMemory<float>& input_data,
|
||||
DeviceMemory<float>* output_data) override;
|
||||
|
||||
bool DoNormalizeWithDimensions(
|
||||
Stream* stream, const dnn::NormalizeDescriptor& normalize_descriptor,
|
||||
const dnn::BatchDescriptor& dimensions,
|
||||
const DeviceMemory<float>& input_data,
|
||||
DeviceMemory<float>* output_data) override;
|
||||
|
||||
bool DoNormalizeBackwardWithDimensions(
|
||||
Stream* stream, const dnn::NormalizeDescriptor& normalize_descriptor,
|
||||
const dnn::BatchDescriptor& dimensions,
|
||||
const DeviceMemory<float>& raw_data,
|
||||
const DeviceMemory<float>& normalized_data,
|
||||
const DeviceMemory<float>& normalized_variable_gradient,
|
||||
DeviceMemory<float>* raw_variable_gradient,
|
||||
ScratchAllocator* workspace_allocator) override;
|
||||
|
||||
bool DoDepthConcatenate(
|
||||
Stream* stream, port::ArraySlice<dnn::BatchDescriptor> input_dimensions,
|
||||
port::ArraySlice<const DeviceMemory<float>*> input_data,
|
||||
DeviceMemory<float>* output_data) override;
|
||||
|
||||
bool DoElementwiseOperate(
|
||||
Stream* stream, dnn::ElementwiseOperation operation,
|
||||
port::ArraySlice<dnn::BatchDescriptor> input_dimensions,
|
||||
port::ArraySlice<const DeviceMemory<float>*> input_data,
|
||||
const dnn::BatchDescriptor& output_dimensions,
|
||||
DeviceMemory<float>* output_data) override;
|
||||
|
||||
bool DoXYPad(Stream* stream, const dnn::BatchDescriptor &dimensions,
|
||||
const DeviceMemory<float> &input_data,
|
||||
int64 left_pad, int64 right_pad, int64 top_pad,
|
||||
int64 bottom_pad, DeviceMemory<float> *output_data) override;
|
||||
|
||||
bool DoXYSlice(Stream* stream, const dnn::BatchDescriptor &dimensions,
|
||||
const DeviceMemory<float> &input_data,
|
||||
int64 left_trim, int64 right_trim, int64 top_trim,
|
||||
int64 bottom_trim, DeviceMemory<float> *output_data) override;
|
||||
|
||||
bool DoMemcpyD2HQuantized(Stream* stream,
|
||||
const DeviceMemory<float>& device_unquantized_src,
|
||||
dnn::QuantizedActivationMode mode, void* host_dst,
|
||||
int64 size) override;
|
||||
|
||||
bool DoMemcpyH2DQuantized(
|
||||
Stream* stream, const void* host_src, int64 size,
|
||||
dnn::QuantizedActivationMode mode,
|
||||
DeviceMemory<float>* device_unquantized_dst) override;
|
||||
|
||||
// Derives an output batch descriptor from an input batch and convolution
|
||||
// descriptors.
|
||||
bool DeriveOutputBatchDescriptor(
|
||||
const dnn::BatchDescriptor& batch_descriptor,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
dnn::BatchDescriptor* output_batch_descriptor);
|
||||
|
||||
bool DoTransformTensor(Stream* stream, const dnn::BatchDescriptor& input_desc,
|
||||
dnn::DataType input_type,
|
||||
const DeviceMemoryBase& input_data,
|
||||
const dnn::BatchDescriptor& output_desc,
|
||||
dnn::DataType output_type, float scale,
|
||||
DeviceMemoryBase* output_data) override;
|
||||
|
||||
private:
|
||||
CUDAExecutor* parent_; // Parent executor object. Not owned.
|
||||
|
||||
// Provides access to the cuDNN handle.
|
||||
std::unique_ptr<class CudnnAccess> cudnn_;
|
||||
|
||||
template <class T, class U>
|
||||
port::Status DoBatchNormalizationForwardImpl(
|
||||
Stream* stream, dnn::DataType input_data_type,
|
||||
dnn::DataType scale_data_type, const DeviceMemory<T>& x,
|
||||
const DeviceMemory<U>& scale, const DeviceMemory<U>& offset,
|
||||
const DeviceMemory<U>& estimated_mean,
|
||||
const DeviceMemory<U>& estimated_variance,
|
||||
const dnn::BatchDescriptor& x_desc,
|
||||
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
|
||||
DeviceMemory<T>* y, DeviceMemory<U>* batch_mean,
|
||||
DeviceMemory<U>* batch_var, DeviceMemory<U>* saved_mean,
|
||||
DeviceMemory<U>* saved_inv_var, bool is_training,
|
||||
std::function<const DeviceMemory<U>&()> var_to_inv_var,
|
||||
std::function<void()> inv_var_to_var);
|
||||
|
||||
template <class T, class U>
|
||||
port::Status DoBatchNormalizationBackwardImpl(
|
||||
Stream* stream, int cudnn_input_type, int cudnn_scale_type,
|
||||
const DeviceMemory<T>& y_backprop, const DeviceMemory<T>& x,
|
||||
const DeviceMemory<U>& scale, const DeviceMemory<U>& mean,
|
||||
const DeviceMemory<U>& inv_var, const dnn::BatchDescriptor& x_desc,
|
||||
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
|
||||
DeviceMemory<T>* x_backprop, DeviceMemory<U>* scale_backprop,
|
||||
DeviceMemory<U>* offset_backprop);
|
||||
|
||||
template <class T>
|
||||
port::Status DoConvolveImpl(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<T>& input_data,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<T>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<T>* output_data, ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result);
|
||||
|
||||
template <typename AccumulatorType, typename ElementType, typename BiasType,
|
||||
typename ScaleType>
|
||||
port::Status DoFusedConvolveImpl(
|
||||
Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
|
||||
const DeviceMemory<ElementType>& conv_input_data,
|
||||
ScaleType conv_input_scale,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<ElementType>& filter_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const DeviceMemory<ElementType>& side_input_data,
|
||||
ScaleType side_input_scale, const dnn::BatchDescriptor& bias_descriptor,
|
||||
const DeviceMemory<BiasType>& biases, dnn::ActivationMode activation_mode,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<ElementType>* output_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result);
|
||||
|
||||
template <class T>
|
||||
port::Status DoConvolveBackwardDataImpl(
|
||||
Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
|
||||
const DeviceMemory<T>& filter_data,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<T> backward_output_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::BatchDescriptor& input_descriptor,
|
||||
DeviceMemory<T>* backward_input_data, ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result);
|
||||
|
||||
template <class T>
|
||||
port::Status DoConvolveBackwardFilterImpl(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<T>& input_data,
|
||||
const dnn::BatchDescriptor& output_descriptor,
|
||||
DeviceMemory<T> backward_output_data,
|
||||
const dnn::ConvolutionDescriptor& convolution_descriptor,
|
||||
const dnn::FilterDescriptor& filter_descriptor,
|
||||
DeviceMemory<T>* backward_filter_data,
|
||||
ScratchAllocator* scratch_allocator,
|
||||
const dnn::AlgorithmConfig& algorithm_config,
|
||||
dnn::ProfileResult* output_profile_result);
|
||||
|
||||
template <class T>
|
||||
port::Status DoConvolveBackwardBiasImpl(
|
||||
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
|
||||
const DeviceMemory<T>& input_data,
|
||||
const dnn::BatchDescriptor& bias_descriptor,
|
||||
DeviceMemory<T>* backward_bias_data);
|
||||
|
||||
template <class T>
|
||||
port::Status DoRnnForwardImpl(
|
||||
Stream* stream, const CudnnRnnDescriptor& rnn_desc,
|
||||
const CudnnRnnSequenceTensorDescriptor& input_desc,
|
||||
const DeviceMemory<T>& input_data,
|
||||
const CudnnRnnStateTensorDescriptor& input_h_desc,
|
||||
const DeviceMemory<T>& input_h_data,
|
||||
const CudnnRnnStateTensorDescriptor& input_c_desc,
|
||||
const DeviceMemory<T>& input_c_data, const DeviceMemory<T>& params,
|
||||
const CudnnRnnSequenceTensorDescriptor& output_desc,
|
||||
DeviceMemory<T>* output_data,
|
||||
const CudnnRnnStateTensorDescriptor& output_h_desc,
|
||||
DeviceMemory<T>* output_h_data,
|
||||
const CudnnRnnStateTensorDescriptor& output_c_desc,
|
||||
DeviceMemory<T>* output_c_data, bool is_training,
|
||||
ScratchAllocator* reserve_space_allocator,
|
||||
ScratchAllocator* workspace_allocator,
|
||||
dnn::ProfileResult* output_profile_result);
|
||||
|
||||
template <class T>
|
||||
port::Status DoRnnBackwardImpl(
|
||||
Stream* stream, const CudnnRnnDescriptor& rnn_desc,
|
||||
const CudnnRnnSequenceTensorDescriptor& input_desc,
|
||||
const DeviceMemory<T>& input_data,
|
||||
const CudnnRnnStateTensorDescriptor& input_h_desc,
|
||||
const DeviceMemory<T>& input_h_data,
|
||||
const CudnnRnnStateTensorDescriptor& input_c_desc,
|
||||
const DeviceMemory<T>& input_c_data, const DeviceMemory<T>& params,
|
||||
const CudnnRnnSequenceTensorDescriptor& output_desc,
|
||||
const DeviceMemory<T>& output_data,
|
||||
const CudnnRnnStateTensorDescriptor& output_h_desc,
|
||||
const DeviceMemory<T>& output_h_data,
|
||||
const CudnnRnnStateTensorDescriptor& output_c_desc,
|
||||
const DeviceMemory<T>& output_c_data,
|
||||
const DeviceMemory<T>& output_backprop_data,
|
||||
const DeviceMemory<T>& output_h_backprop_data,
|
||||
const DeviceMemory<T>& output_c_backprop_data,
|
||||
DeviceMemory<T>* input_backprop_data,
|
||||
DeviceMemory<T>* input_h_backprop_data,
|
||||
DeviceMemory<T>* input_c_backprop_data,
|
||||
DeviceMemory<T>* params_backprop_data,
|
||||
DeviceMemory<uint8>* reserve_space_data,
|
||||
ScratchAllocator* workspace_allocator,
|
||||
dnn::ProfileResult* output_profile_result);
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CudnnSupport);
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DNN_H_
|
||||
@@ -0,0 +1,520 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// CUDA userspace driver library wrapper functionality.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_H_
|
||||
|
||||
#include <stddef.h>
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
|
||||
#include "tensorflow/stream_executor/device_options.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "cuda/include/cuda.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
// Identifies the memory space where an allocation resides. See
|
||||
// CUDADriver::GetPointerMemorySpace().
|
||||
enum class MemorySpace { kHost, kDevice };
|
||||
|
||||
// Returns a casual string, such as "host" for the provided memory space.
|
||||
string MemorySpaceString(MemorySpace memory_space);
|
||||
|
||||
class CudaContext;
|
||||
|
||||
// CUDADriver contains wrappers for calls to the userspace library driver. It's
|
||||
// useful to isolate these calls and put basic wrappers around them to separate
|
||||
// userspace library driver behaviors from the rest of the program.
|
||||
//
|
||||
// At the moment it's simply used as a namespace.
|
||||
//
|
||||
// The calls log any specific errors internally and return whether the operation
|
||||
// was successful to the caller.
|
||||
//
|
||||
// The order of parameters is generally kept symmetric with the underlying CUDA
|
||||
// driver API.
|
||||
//
|
||||
// Links on functions are to specific documentation under
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/
|
||||
//
|
||||
// Thread safety: these functions should not be used from signal handlers.
|
||||
class CUDADriver {
|
||||
public:
|
||||
// Wraps a call to cuInit with logging to help indicate what has gone wrong in
|
||||
// the case of failure. Safe to call multiple times; will be fast on all calls
|
||||
// after the first.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__INITIALIZE.html#group__CUDA__INITIALIZE_1g0a2f1517e1bd8502c7194c3a8c134bc3
|
||||
static port::Status Init();
|
||||
|
||||
// Returns the device associated with the given context.
|
||||
// device is an outparam owned by the caller, must not be null.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g4e84b109eba36cdaaade167f34ae881e
|
||||
static port::StatusOr<CUdevice> DeviceFromContext(CudaContext* context);
|
||||
|
||||
// Creates a new CUDA stream associated with the given context via
|
||||
// cuStreamCreate.
|
||||
// stream is an outparam owned by the caller, must not be null.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1ga581f0c5833e21ded8b5a56594e243f4
|
||||
static bool CreateStream(CudaContext* context, CUstream *stream);
|
||||
|
||||
// Destroys a CUDA stream associated with the given context.
|
||||
// stream is owned by the caller, must not be null, and *stream is set to null
|
||||
// if the stream is successfully destroyed.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g244c8833de4596bcd31a06cdf21ee758
|
||||
static void DestroyStream(CudaContext* context, CUstream *stream);
|
||||
|
||||
// CUDA events can explicitly disable event TSC retrieval for some presumed
|
||||
// performance improvement if timing is unnecessary.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g450687e75f3ff992fe01662a43d9d3db
|
||||
enum class EventFlags { kDefault, kDisableTiming };
|
||||
|
||||
// Creates a new event associated with the given context.
|
||||
// result is an outparam owned by the caller and must not be null.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g450687e75f3ff992fe01662a43d9d3db
|
||||
static port::Status CreateEvent(CudaContext* context, CUevent *result,
|
||||
EventFlags flags);
|
||||
|
||||
// Destroys *event and turns it into a nullptr. event may not be null, but
|
||||
// *event may be, via cuEventDestroy
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g593ec73a8ec5a5fc031311d3e4dca1ef
|
||||
static port::Status DestroyEvent(CudaContext* context, CUevent *event);
|
||||
|
||||
// Allocates a GPU memory space of size bytes associated with the given
|
||||
// context via cuMemAlloc.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gb82d2a09844a58dd9e744dc31e8aa467
|
||||
static void *DeviceAllocate(CudaContext* context, uint64 bytes);
|
||||
|
||||
// Deallocates a GPU memory space of size bytes associated with the given
|
||||
// context via cuMemFree.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g89b3f154e17cc89b6eea277dbdf5c93a
|
||||
static void DeviceDeallocate(CudaContext* context, void *location);
|
||||
|
||||
// Allocates a unified memory space of size bytes associated with the given
|
||||
// context via cuMemAllocManaged.
|
||||
// https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gb347ded34dc326af404aa02af5388a32
|
||||
static void* UnifiedMemoryAllocate(CudaContext* context, uint64 bytes);
|
||||
|
||||
// Deallocates a unified memory space of size bytes associated with the given
|
||||
// context via cuMemFree.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g89b3f154e17cc89b6eea277dbdf5c93a
|
||||
static void UnifiedMemoryDeallocate(CudaContext* context, void* location);
|
||||
|
||||
// Allocates page-locked and CUDA-registered memory on the host via
|
||||
// cuMemAllocHost.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gdd8311286d2c2691605362c689bc64e0
|
||||
static void *HostAllocate(CudaContext* context, uint64 bytes);
|
||||
|
||||
// Deallocates a location created by HostAllocate, via cuMemFreeHost.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g62e0fdbe181dab6b1c90fa1a51c7b92c
|
||||
static void HostDeallocate(CudaContext* context, void *location);
|
||||
|
||||
// Registers a memory region at location of size bytes via cuMemHostRegister.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gf0a9fe11544326dabd743b7aa6b54223
|
||||
static bool HostRegister(CudaContext* context, void *location, uint64 bytes);
|
||||
|
||||
// Unregisters a memory region that was previously registered at location via
|
||||
// cuMemHostUnregister.
|
||||
//
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g63f450c8125359be87b7623b1c0b2a14
|
||||
//
|
||||
// TODO(leary) verify an error will be returned if the location wasn't
|
||||
// previously registered.
|
||||
static bool HostUnregister(CudaContext* context, void *location);
|
||||
|
||||
// Given a device ordinal, returns a device handle into the device outparam,
|
||||
// which must not be null.
|
||||
//
|
||||
// N.B. these device handles do not have a corresponding destroy function in
|
||||
// the CUDA driver API.
|
||||
static port::Status GetDevice(int device_ordinal, CUdevice *device);
|
||||
|
||||
// Given a device handle, returns the name reported by the driver for the
|
||||
// device.
|
||||
static bool GetDeviceName(CUdevice device, string *name_out);
|
||||
|
||||
// Given a device to create a context for, returns a context handle into the
|
||||
// context outparam, which must not be null.
|
||||
//
|
||||
// N.B. CUDA contexts are weird. They are implicitly associated with the
|
||||
// calling thread. Current documentation on contexts and their influence on
|
||||
// userspace processes is given here:
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g65dc0012348bc84810e2103a40d8e2cf
|
||||
static port::Status CreateContext(CUdevice device,
|
||||
const DeviceOptions& device_options,
|
||||
CudaContext** context);
|
||||
|
||||
// Destroys the provided context via cuCtxDestroy.
|
||||
// Don't do this while clients could still be using the context, per the docs
|
||||
// bad things will happen.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g27a365aebb0eb548166309f58a1e8b8e
|
||||
static void DestroyContext(CudaContext* context);
|
||||
|
||||
// Queries the runtime for the specified attribute of the specified function.
|
||||
// cuFuncGetAttribute (the underlying CUDA driver API routine) only operates
|
||||
// in terms of integer-sized values, so there's no potential for overrun (as
|
||||
// of CUDA 5.5).
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1g5e92a1b0d8d1b82cb00dcfb2de15961b
|
||||
static bool FuncGetAttribute(CUfunction_attribute attribute,
|
||||
CUfunction function, int *attribute_value);
|
||||
|
||||
// Sets the preferred cache configuration for the specified function.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1g40f8c11e81def95dc0072a375f965681
|
||||
static bool FuncSetCacheConfig(CUfunction function,
|
||||
CUfunc_cache cache_config);
|
||||
|
||||
// Gets the preferred shared memory bank configuration for the specified
|
||||
// CONTEXT (not function!), either default or four- or eight-byte bank size.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g17153a1b8b8c756f7ab8505686a4ad74
|
||||
static port::StatusOr<CUsharedconfig> ContextGetSharedMemConfig(
|
||||
CudaContext* context);
|
||||
|
||||
// Sets the preferred shared memory bank configuration for the specified
|
||||
// CONTEXT (not function!), either default or four- or eight-byte bank size.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g2574235fa643f8f251bf7bc28fac3692
|
||||
static port::Status ContextSetSharedMemConfig(
|
||||
CudaContext* context, CUsharedconfig shared_mem_config);
|
||||
|
||||
// Launches a CUDA kernel via cuLaunchKernel.
|
||||
// TODO(leary) describe the structure of kernel_params and extra in a readable
|
||||
// way.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1gb8f3dc3031b40da29d5f9a7139e52e15
|
||||
static bool LaunchKernel(CudaContext* context, CUfunction function,
|
||||
unsigned int grid_dim_x, unsigned int grid_dim_y,
|
||||
unsigned int grid_dim_z, unsigned int block_dim_x,
|
||||
unsigned int block_dim_y, unsigned int block_dim_z,
|
||||
unsigned int shared_mem_bytes, CUstream stream,
|
||||
void **kernel_params, void **extra);
|
||||
|
||||
// Loads ptx_contents with the CUDA driver's PTX JIT and stores the resulting
|
||||
// handle in "module". Any error logs that are produced are logged internally.
|
||||
static bool LoadPtx(CudaContext* context, const char *ptx_contents,
|
||||
CUmodule *module);
|
||||
|
||||
// Loads cubin_bytes with the CUDA driver's blob loading interface and stores
|
||||
// the resulting handle in "module".
|
||||
static port::Status LoadCubin(CudaContext* context, const char *cubin_bytes,
|
||||
CUmodule *module);
|
||||
|
||||
// Retrieves a named kernel from a loaded module, and places the resulting
|
||||
// handle into function (outparam) on success. Neither kernel_name nor
|
||||
// function may be null. No ownership is taken of kernel_name.
|
||||
static bool GetModuleFunction(CudaContext* context, CUmodule module,
|
||||
const char *kernel_name, CUfunction *function);
|
||||
|
||||
// Retrieves a named global/constant symbol from a loaded module, and returns
|
||||
// a device pointer and size of the symbol on success. symbol_name may not be
|
||||
// null. At least one of dptr or bytes should not be null. No ownership is
|
||||
// taken of symbol_name.
|
||||
static bool GetModuleSymbol(CudaContext* context, CUmodule module,
|
||||
const char *symbol_name, CUdeviceptr *dptr,
|
||||
size_t *bytes);
|
||||
|
||||
// Unloads module from the current context via cuModuleUnload.
|
||||
// TODO(leary) the documentation doesn't say what kind of disasters happen
|
||||
// if you try to unload a module while its CUfunctions are in use.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MODULE.html#group__CUDA__MODULE_1g8ea3d716524369de3763104ced4ea57b
|
||||
static void UnloadModule(CudaContext* context, CUmodule module);
|
||||
|
||||
// Performs a synchronous memset of the device memory segment via cuMemsetD8.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g6e582bf866e9e2fb014297bfaf354d7b
|
||||
static bool SynchronousMemsetUint8(CudaContext* context, CUdeviceptr location,
|
||||
uint8 value, size_t size);
|
||||
|
||||
// Performs a synchronous memset of the device memory segment via cuMemsetD32.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g983e8d8759acd1b64326317481fbf132
|
||||
static bool SynchronousMemsetUint32(CudaContext* context,
|
||||
CUdeviceptr location, uint32 value,
|
||||
size_t uint32_count);
|
||||
|
||||
// Performs an asynchronous memset of the device memory segment via
|
||||
// cuMemsetD8Async.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gaef08a7ccd61112f94e82f2b30d43627
|
||||
static bool AsynchronousMemsetUint8(CudaContext* context, CUdeviceptr location,
|
||||
uint8 value, size_t uint32_count,
|
||||
CUstream stream);
|
||||
|
||||
// Performs an asynchronous memset of the device memory segment via
|
||||
// cuMemsetD32Async.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g58229da5d30f1c0cdf667b320ec2c0f5
|
||||
static bool AsynchronousMemsetUint32(CudaContext* context,
|
||||
CUdeviceptr location, uint32 value,
|
||||
size_t uint32_count, CUstream stream);
|
||||
|
||||
// -- Synchronous memcopies.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g4d32266788c440b0220b1a9ba5795169
|
||||
|
||||
static port::Status SynchronousMemcpyD2H(CudaContext* context, void* host_dst,
|
||||
CUdeviceptr gpu_src, uint64 size);
|
||||
static port::Status SynchronousMemcpyH2D(CudaContext* context,
|
||||
CUdeviceptr gpu_dst,
|
||||
const void* host_src, uint64 size);
|
||||
static port::Status SynchronousMemcpyD2D(CudaContext* context,
|
||||
CUdeviceptr gpu_dst,
|
||||
CUdeviceptr gpu_src, uint64 size);
|
||||
|
||||
// -- Asynchronous memcopies.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g56f30236c7c5247f8e061b59d3268362
|
||||
|
||||
static bool AsynchronousMemcpyD2H(CudaContext* context, void *host_dst,
|
||||
CUdeviceptr gpu_src, uint64 size,
|
||||
CUstream stream);
|
||||
static bool AsynchronousMemcpyH2D(CudaContext* context, CUdeviceptr gpu_dst,
|
||||
const void *host_src, uint64 size,
|
||||
CUstream stream);
|
||||
static bool AsynchronousMemcpyD2D(CudaContext* context, CUdeviceptr gpu_dst,
|
||||
CUdeviceptr gpu_src, uint64 size,
|
||||
CUstream stream);
|
||||
|
||||
// The CUDA stream callback type signature.
|
||||
// The data passed to AddStreamCallback is subsequently passed to this
|
||||
// callback when it fires.
|
||||
//
|
||||
// Some notable things:
|
||||
// * Callbacks must not make any CUDA API calls.
|
||||
// * Callbacks from independent streams execute in an undefined order and may
|
||||
// be serialized.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g613d97a277d7640f4cb1c03bd51c2483
|
||||
typedef void (*StreamCallback)(CUstream stream, CUresult status, void *data);
|
||||
|
||||
// Enqueues a callback operation into stream.
|
||||
// See StreamCallback above and the NVIDIA documentation for additional
|
||||
// details.
|
||||
static bool AddStreamCallback(CudaContext* context, CUstream stream,
|
||||
StreamCallback callback, void *data);
|
||||
|
||||
// Causes stream to wait for event to trigger before proceeding via
|
||||
// cuStreamWaitEvent.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#axzz334nAXAhM
|
||||
static bool WaitStreamOnEvent(CudaContext* context, CUstream stream,
|
||||
CUevent event);
|
||||
|
||||
// Blocks the calling thread until the operations enqueued onto stream have
|
||||
// been completed, via cuStreamSynchronize.
|
||||
//
|
||||
// TODO(leary) if a pathological thread enqueues operations onto the stream
|
||||
// while another thread blocks like this, can you wind up waiting an unbounded
|
||||
// amount of time?
|
||||
//
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g15e49dd91ec15991eb7c0a741beb7dad
|
||||
static port::Status SynchronizeStream(CudaContext* context, CUstream stream);
|
||||
|
||||
// Blocks the calling thread until the operations associated with the context
|
||||
// have been completed, via cuCtxSynchronize.
|
||||
//
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g7a54725f28d34b8c6299f0c6ca579616
|
||||
static bool SynchronizeContext(CudaContext* context);
|
||||
|
||||
// Returns true if all stream tasks have completed at time of the call. Note
|
||||
// the potential for races around this call (if another thread adds work to
|
||||
// the stream immediately after this returns).
|
||||
static bool IsStreamIdle(CudaContext* context, CUstream stream);
|
||||
|
||||
// Returns whether code in the from context can access memory in the to
|
||||
// context via cuDeviceCanAccessPeer.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__PEER__ACCESS.html#group__CUDA__PEER__ACCESS_1g496bdaae1f632ebfb695b99d2c40f19e
|
||||
static bool CanEnablePeerAccess(CudaContext* from, CudaContext* to);
|
||||
|
||||
// Enables peer access per CanEnablePeerAccess, via cuCtxEnablePeerAccess.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__PEER__ACCESS.html#group__CUDA__PEER__ACCESS_1g0889ec6728e61c05ed359551d67b3f5a
|
||||
static port::Status EnablePeerAccess(CudaContext* from, CudaContext* to);
|
||||
|
||||
// Returns the elapsed milliseconds between start and stop via
|
||||
// cuEventElapsedTime.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1gdfb1178807353bbcaa9e245da497cf97
|
||||
static bool GetEventElapsedTime(CudaContext* context,
|
||||
float *elapsed_milliseconds, CUevent start,
|
||||
CUevent stop);
|
||||
|
||||
// Records that an event occurred when execution reaches the current point in
|
||||
// thestream via cuEventRecord.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g95424d3be52c4eb95d83861b70fb89d1
|
||||
static port::Status RecordEvent(CudaContext* context, CUevent event,
|
||||
CUstream stream);
|
||||
|
||||
// Polls (without blocking) to determine the status of an event - pending or
|
||||
// complete (or an error status).
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g6f0704d755066b0ee705749ae911deef
|
||||
static port::StatusOr<CUresult> QueryEvent(CudaContext* context,
|
||||
CUevent event);
|
||||
|
||||
// -- Pointer-specific calls.
|
||||
|
||||
// Returns the context in which pointer was allocated or registered.
|
||||
static port::StatusOr<CudaContext*> GetPointerContext(CUdeviceptr pointer);
|
||||
|
||||
// Returns the device associated with the context from GetPointerContext().
|
||||
static port::StatusOr<CUdevice> GetPointerDevice(CUdeviceptr pointer);
|
||||
|
||||
// Returns the memory space addressed by pointer.
|
||||
static port::StatusOr<MemorySpace> GetPointerMemorySpace(CUdeviceptr pointer);
|
||||
|
||||
// Returns the base address and size of the device pointer dptr.
|
||||
static port::Status GetPointerAddressRange(CUdeviceptr dptr,
|
||||
CUdeviceptr *base, size_t *size);
|
||||
|
||||
// -- Device-specific calls.
|
||||
|
||||
// Returns the compute capability for the device; i.e (3, 5).
|
||||
// This is currently done via the deprecated device API.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE__DEPRECATED.html#group__CUDA__DEVICE__DEPRECATED_1ge2091bbac7e1fb18c2821612115607ea
|
||||
static port::Status GetComputeCapability(int *cc_major, int *cc_minor,
|
||||
CUdevice device);
|
||||
|
||||
// Returns the number of multiprocessors on the device (note that the device
|
||||
// may be multi-GPU-per-board).
|
||||
static port::StatusOr<int> GetMultiprocessorCount(CUdevice device);
|
||||
|
||||
// Returns the limit on number of threads that can be resident in a single
|
||||
// multiprocessor.
|
||||
static port::StatusOr<int64> GetMaxThreadsPerMultiprocessor(CUdevice device);
|
||||
|
||||
// Returns the limit on number of threads which may be resident for a single
|
||||
// block (cooperative thread array).
|
||||
static port::StatusOr<int64> GetMaxThreadsPerBlock(CUdevice device);
|
||||
|
||||
// Returns the amount of shared memory available on a single GPU core (i.e.
|
||||
// SM on NVIDIA devices).
|
||||
static port::StatusOr<int64> GetMaxSharedMemoryPerCore(CUdevice device);
|
||||
|
||||
// Returns the amount of shared memory available for a single block
|
||||
// (cooperative thread array).
|
||||
static port::StatusOr<int64> GetMaxSharedMemoryPerBlock(CUdevice device);
|
||||
|
||||
// Returns the maximum supported number of registers per block.
|
||||
static port::StatusOr<int64> GetMaxRegistersPerBlock(CUdevice device);
|
||||
|
||||
// Returns the number of threads per warp.
|
||||
static port::StatusOr<int64> GetThreadsPerWarp(CUdevice device);
|
||||
|
||||
// Queries the grid limits for device with cuDeviceGetAttribute calls.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
|
||||
static bool GetGridLimits(int *x, int *y, int *z, CUdevice device);
|
||||
|
||||
// Returns a grab-bag of device properties in a caller-owned device_properties
|
||||
// structure for device_ordinal via cuDeviceGetProperties.
|
||||
//
|
||||
// This call is deprecated in the NVIDIA driver API; its replacement is
|
||||
// GetDeviceAttribute
|
||||
//
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE__DEPRECATED.html#group__CUDA__DEVICE__DEPRECATED_1g65a5b4e25186bd257df80b98c98cffe6
|
||||
static bool GetDeviceProperties(CUdevprop *device_properties,
|
||||
int device_ordinal);
|
||||
|
||||
// Gets a specific integer-valued property about the given device.
|
||||
//
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
|
||||
static port::StatusOr<int> GetDeviceAttribute(CUdevice_attribute attribute,
|
||||
CUdevice device);
|
||||
|
||||
// Returns whether ECC is enabled for the given CUdevice via
|
||||
// cuDeviceGetattribute with CU_DEVICE_ATTRIBUTE_ECC_ENABLED.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
|
||||
static bool IsEccEnabled(CUdevice device, bool *result);
|
||||
|
||||
// Returns the total amount of memory available for allocation by the CUDA
|
||||
// context, in bytes, via cuDeviceTotalMem.
|
||||
static bool GetDeviceTotalMemory(CUdevice device, uint64 *result);
|
||||
|
||||
// Returns the free amount of memory and total amount of memory, as reported
|
||||
// by cuMemGetInfo.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g808f555540d0143a331cc42aa98835c0
|
||||
static bool GetDeviceMemoryInfo(CudaContext* context, int64* free,
|
||||
int64* total);
|
||||
|
||||
// Returns a PCI bus id string for the device.
|
||||
// [domain]:[bus]:[device].[function]
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g85295e7d9745ab8f0aa80dd1e172acfc
|
||||
static string GetPCIBusID(CUdevice device);
|
||||
|
||||
// -- Context- and device-independent calls.
|
||||
|
||||
// Returns the number of visible CUDA device via cuDeviceGetCount.
|
||||
// This should correspond to the set of device ordinals available.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g52b5ce05cb8c5fb6831b2c0ff2887c74
|
||||
static int GetDeviceCount();
|
||||
|
||||
// Returns the driver version number via cuDriverGetVersion.
|
||||
// This is, surprisingly, NOT the actual driver version (e.g. 331.79) but,
|
||||
// instead, the CUDA toolkit release number that this driver is compatible
|
||||
// with; e.g. 6000 (for a CUDA 6.0 compatible driver) or 6050 (for a CUDA 6.5
|
||||
// compatible driver).
|
||||
//
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__VERSION.html#group__CUDA__VERSION_1g8b7a10395392e049006e61bcdc8ebe71
|
||||
static bool GetDriverVersion(int *driver_version);
|
||||
|
||||
// -- Other calls
|
||||
|
||||
// Returns the maximum number of blocks (per multiprocessor) occupied by the
|
||||
// specified kernel/CUfunction when launched with the specified parameters.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__OCCUPANCY.html#group__CUDA__OCCUPANCY_1gcc6e1094d05cba2cee17fe33ddd04a98
|
||||
static port::StatusOr<int> GetMaxOccupiedBlocksPerCore(
|
||||
CudaContext* context, CUfunction kernel, int threads_per_block,
|
||||
size_t dynamic_shared_memory_bytes);
|
||||
|
||||
// Returns the current context set in CUDA. This is done by calling the cuda
|
||||
// driver (e.g., this value is not our cached view of the current context).
|
||||
static CUcontext CurrentContextOrDie();
|
||||
|
||||
// Seam for injecting an error at CUDA initialization time for testing
|
||||
// purposes.
|
||||
static bool driver_inject_init_error_;
|
||||
};
|
||||
|
||||
// Ensures a context is activated within a scope.
|
||||
class ScopedActivateContext {
|
||||
public:
|
||||
// Activates the context via cuCtxSetCurrent, if it is not the currently
|
||||
// active context (a la cuCtxGetCurrent). Note the alternative push/pop
|
||||
// mechanism is said by NVIDIA to be relatively slow and deprecated.
|
||||
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1gbe562ee6258b4fcc272ca6478ca2a2f7
|
||||
explicit ScopedActivateContext(CudaContext* context);
|
||||
|
||||
// Checks that the context has remained activated for the duration of the
|
||||
// scope.
|
||||
~ScopedActivateContext();
|
||||
|
||||
private:
|
||||
CudaContext* to_restore_ = nullptr;
|
||||
};
|
||||
|
||||
// CudaContext wraps a cuda CUcontext handle, and includes a unique id. The
|
||||
// unique id is positive, and ids are not repeated within the process.
|
||||
class CudaContext {
|
||||
public:
|
||||
CudaContext(CUcontext context, int64 id) : context_(context), id_(id) { }
|
||||
|
||||
CUcontext context() const { return context_; }
|
||||
int64 id() const { return id_; }
|
||||
|
||||
// Disallow copying and moving.
|
||||
CudaContext(CudaContext&&) = delete;
|
||||
CudaContext(const CudaContext&) = delete;
|
||||
CudaContext& operator=(CudaContext&&) = delete;
|
||||
CudaContext& operator=(const CudaContext&) = delete;
|
||||
|
||||
private:
|
||||
CUcontext const context_;
|
||||
const int64 id_;
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_H_
|
||||
@@ -0,0 +1,62 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_EVENT_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_EVENT_H_
|
||||
|
||||
#include "tensorflow/stream_executor/cuda/cuda_driver.h"
|
||||
#include "tensorflow/stream_executor/cuda/cuda_stream.h"
|
||||
#include "tensorflow/stream_executor/event.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
// CUDAEvent wraps a CUevent in the platform-independent EventInterface
|
||||
// interface.
|
||||
class CUDAEvent : public internal::EventInterface {
|
||||
public:
|
||||
explicit CUDAEvent(CUDAExecutor* parent);
|
||||
|
||||
~CUDAEvent() override;
|
||||
|
||||
// Populates the CUDA-platform-specific elements of this object.
|
||||
port::Status Init();
|
||||
|
||||
// Deallocates any platform-specific elements of this object. This is broken
|
||||
// out (not part of the destructor) to allow for error reporting.
|
||||
port::Status Destroy();
|
||||
|
||||
// Inserts the event at the current position into the specified stream.
|
||||
port::Status Record(CUDAStream* stream);
|
||||
|
||||
// Polls the CUDA platform for the event's current status.
|
||||
Event::Status PollForStatus();
|
||||
|
||||
// The underlying CUDA event element.
|
||||
const CUevent& cuda_event();
|
||||
|
||||
private:
|
||||
// The Executor used to which this object and CUevent are bound.
|
||||
CUDAExecutor* parent_;
|
||||
|
||||
// The underlying CUDA event element.
|
||||
CUevent cuda_event_;
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_EVENT_H_
|
||||
@@ -0,0 +1,137 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// CUDA-specific support for FFT functionality -- this wraps the cuFFT library
|
||||
// capabilities, and is only included into CUDA implementation code -- it will
|
||||
// not introduce cuda headers into other code.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_FFT_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_FFT_H_
|
||||
|
||||
#include "cuda/include/cufft.h"
|
||||
#include "tensorflow/stream_executor/fft.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/plugin_registry.h"
|
||||
#include "tensorflow/stream_executor/scratch_allocator.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class Stream;
|
||||
|
||||
namespace cuda {
|
||||
|
||||
class CUDAExecutor;
|
||||
|
||||
// Opaque and unique indentifier for the cuFFT plugin.
|
||||
extern const PluginId kCuFftPlugin;
|
||||
|
||||
// CUDAFftPlan uses deferred initialization. Only a single call of
|
||||
// Initialize() is allowed to properly create cufft plan and set member
|
||||
// variable is_initialized_ to true. Newly added interface that uses member
|
||||
// variables should first check is_initialized_ to make sure that the values of
|
||||
// member variables are valid.
|
||||
class CUDAFftPlan : public fft::Plan {
|
||||
public:
|
||||
CUDAFftPlan()
|
||||
: parent_(nullptr),
|
||||
plan_(-1),
|
||||
fft_type_(fft::Type::kInvalid),
|
||||
scratch_(nullptr),
|
||||
scratch_size_bytes_(0),
|
||||
is_initialized_(false) {}
|
||||
~CUDAFftPlan() override;
|
||||
|
||||
// Get FFT direction in cuFFT based on FFT type.
|
||||
int GetFftDirection() const;
|
||||
cufftHandle GetPlan() const {
|
||||
if (IsInitialized()) {
|
||||
return plan_;
|
||||
} else {
|
||||
LOG(FATAL) << "Try to get cufftHandle value before initialization.";
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize function for batched plan
|
||||
port::Status Initialize(CUDAExecutor *parent, Stream *stream, int rank,
|
||||
uint64 *elem_count, uint64 *input_embed,
|
||||
uint64 input_stride, uint64 input_distance,
|
||||
uint64 *output_embed, uint64 output_stride,
|
||||
uint64 output_distance, fft::Type type,
|
||||
int batch_count, ScratchAllocator *scratch_allocator);
|
||||
|
||||
// Initialize function for 1d,2d, and 3d plan
|
||||
port::Status Initialize(CUDAExecutor *parent, Stream *stream, int rank,
|
||||
uint64 *elem_count, fft::Type type,
|
||||
ScratchAllocator *scratch_allocator);
|
||||
|
||||
port::Status UpdateScratchAllocator(Stream *stream,
|
||||
ScratchAllocator *scratch_allocator);
|
||||
|
||||
protected:
|
||||
bool IsInitialized() const { return is_initialized_; }
|
||||
|
||||
private:
|
||||
CUDAExecutor *parent_;
|
||||
cufftHandle plan_;
|
||||
fft::Type fft_type_;
|
||||
DeviceMemory<uint8> scratch_;
|
||||
size_t scratch_size_bytes_;
|
||||
bool is_initialized_;
|
||||
};
|
||||
|
||||
// FFT support for CUDA platform via cuFFT library.
|
||||
//
|
||||
// This satisfies the platform-agnostic FftSupport interface.
|
||||
//
|
||||
// Note that the cuFFT handle that this encapsulates is implicitly tied to the
|
||||
// context (and, as a result, the device) that the parent CUDAExecutor is tied
|
||||
// to. This simply happens as an artifact of creating the cuFFT handle when a
|
||||
// CUDA context is active.
|
||||
//
|
||||
// Thread-safe. The CUDA context associated with all operations is the CUDA
|
||||
// context of parent_, so all context is explicit.
|
||||
class CUDAFft : public fft::FftSupport {
|
||||
public:
|
||||
explicit CUDAFft(CUDAExecutor *parent) : parent_(parent) {}
|
||||
~CUDAFft() override {}
|
||||
|
||||
TENSORFLOW_STREAM_EXECUTOR_GPU_FFT_SUPPORT_OVERRIDES
|
||||
|
||||
private:
|
||||
CUDAExecutor *parent_;
|
||||
|
||||
// Two helper functions that execute dynload::cufftExec?2?.
|
||||
|
||||
// This is for complex to complex FFT, when the direction is required.
|
||||
template <typename FuncT, typename InputT, typename OutputT>
|
||||
bool DoFftWithDirectionInternal(Stream *stream, fft::Plan *plan,
|
||||
FuncT cufft_exec,
|
||||
const DeviceMemory<InputT> &input,
|
||||
DeviceMemory<OutputT> *output);
|
||||
|
||||
// This is for complex to real or real to complex FFT, when the direction
|
||||
// is implied.
|
||||
template <typename FuncT, typename InputT, typename OutputT>
|
||||
bool DoFftInternal(Stream *stream, fft::Plan *plan, FuncT cufft_exec,
|
||||
const DeviceMemory<InputT> &input,
|
||||
DeviceMemory<OutputT> *output);
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CUDAFft);
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_FFT_H_
|
||||
@@ -0,0 +1,299 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// The CUDA implementation of the StreamExecutorInterface functionality.
|
||||
// CUDA inclusions are ideally confined to this implementation file.
|
||||
//
|
||||
// The notions from the StreamExecutor basically correspond to the CUDA streams
|
||||
// programming model provided by the libcuda.so driver APIs, so we don't have
|
||||
// to do much more than wrap the calls to the libraries appropriately.
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_GPU_EXECUTOR_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_GPU_EXECUTOR_H_
|
||||
|
||||
#include <set>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "tensorflow/stream_executor/cuda/cuda_kernel.h"
|
||||
#include "tensorflow/stream_executor/event.h"
|
||||
#include "tensorflow/stream_executor/lib/status.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/platform.h"
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/platform/thread_annotations.h"
|
||||
#include "tensorflow/stream_executor/stream_executor_internal.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
// CUDA-platform implementation of the platform-agnostic
|
||||
// StreamExecutorInferface.
|
||||
class CUDAExecutor : public internal::StreamExecutorInterface {
|
||||
public:
|
||||
// sub_platform indicates the subplatform used in this executor; it must
|
||||
// be a CUDA type.
|
||||
explicit CUDAExecutor(const PluginConfig &plugin_config)
|
||||
: device_(0),
|
||||
context_(nullptr),
|
||||
device_ordinal_(0),
|
||||
cc_major_(0),
|
||||
cc_minor_(0),
|
||||
plugin_config_(plugin_config) {}
|
||||
|
||||
// See the corresponding StreamExecutor methods for method comments on the
|
||||
// following overrides.
|
||||
|
||||
~CUDAExecutor() override;
|
||||
|
||||
port::Status Init(int device_ordinal, DeviceOptions device_options) override;
|
||||
|
||||
bool GetKernel(const MultiKernelLoaderSpec &spec,
|
||||
KernelBase *kernel) override;
|
||||
void UnloadKernel(const KernelBase *kernel) override;
|
||||
bool LoadModule(const MultiModuleLoaderSpec &spec,
|
||||
ModuleHandle *module_handle) override;
|
||||
bool UnloadModule(ModuleHandle module_handle) override;
|
||||
|
||||
bool Launch(Stream *stream, const ThreadDim &thread_dims,
|
||||
const BlockDim &block_dims, const KernelBase &k,
|
||||
const KernelArgsArrayBase &args) override;
|
||||
|
||||
void *Allocate(uint64 size) override;
|
||||
|
||||
void *AllocateSubBuffer(DeviceMemoryBase *mem, uint64 offset_bytes,
|
||||
uint64 size_bytes) override;
|
||||
|
||||
void Deallocate(DeviceMemoryBase *mem) override;
|
||||
|
||||
void *UnifiedMemoryAllocate(uint64 size) override {
|
||||
return CUDADriver::UnifiedMemoryAllocate(context_, size);
|
||||
}
|
||||
|
||||
void UnifiedMemoryDeallocate(void *location) override {
|
||||
return CUDADriver::UnifiedMemoryDeallocate(context_, location);
|
||||
}
|
||||
|
||||
// CUDA allocation/registration functions are necessary because the driver
|
||||
// internally sets up buffers for DMA operations (and page locks them).
|
||||
// There's no external interface for us to otherwise control these DMA
|
||||
// settings.
|
||||
void *HostMemoryAllocate(uint64 size) override {
|
||||
return CUDADriver::HostAllocate(context_, size);
|
||||
}
|
||||
|
||||
void HostMemoryDeallocate(void *location) override {
|
||||
return CUDADriver::HostDeallocate(context_, location);
|
||||
}
|
||||
|
||||
bool HostMemoryRegister(void *location, uint64 size) override;
|
||||
|
||||
bool HostMemoryUnregister(void *location) override;
|
||||
|
||||
bool SynchronizeAllActivity() override;
|
||||
|
||||
bool SynchronousMemZero(DeviceMemoryBase *location, uint64 size) override;
|
||||
|
||||
bool SynchronousMemSet(DeviceMemoryBase *location, int value,
|
||||
uint64 size) override;
|
||||
|
||||
port::Status SynchronousMemcpy(DeviceMemoryBase *gpu_dst,
|
||||
const void *host_src, uint64 size) override;
|
||||
|
||||
port::Status SynchronousMemcpy(void *host_dst,
|
||||
const DeviceMemoryBase &gpu_src,
|
||||
uint64 size) override;
|
||||
|
||||
port::Status SynchronousMemcpyDeviceToDevice(DeviceMemoryBase *gpu_dst,
|
||||
const DeviceMemoryBase &gpu_src,
|
||||
uint64 size) override;
|
||||
|
||||
bool MemZero(Stream *stream, DeviceMemoryBase *location,
|
||||
uint64 size) override;
|
||||
bool Memset(Stream *stream, DeviceMemoryBase *location, uint8 pattern,
|
||||
uint64 size) override;
|
||||
bool Memset32(Stream *stream, DeviceMemoryBase *location, uint32 pattern,
|
||||
uint64 size) override;
|
||||
|
||||
bool Memcpy(Stream *stream, void *host_dst, const DeviceMemoryBase &gpu_src,
|
||||
uint64 size) override;
|
||||
|
||||
bool Memcpy(Stream *stream, DeviceMemoryBase *gpu_dst, const void *host_src,
|
||||
uint64 size) override;
|
||||
|
||||
bool MemcpyDeviceToDevice(Stream *stream, DeviceMemoryBase *gpu_dst,
|
||||
const DeviceMemoryBase &gpu_src,
|
||||
uint64 size) override;
|
||||
|
||||
bool HostCallback(Stream *stream, std::function<void()> callback) override;
|
||||
|
||||
bool AllocateStream(Stream *stream) override;
|
||||
|
||||
void DeallocateStream(Stream *stream) override;
|
||||
|
||||
bool CreateStreamDependency(Stream *dependent, Stream *other) override;
|
||||
|
||||
bool AllocateTimer(Timer *timer) override;
|
||||
|
||||
void DeallocateTimer(Timer *timer) override;
|
||||
|
||||
bool StartTimer(Stream *stream, Timer *timer) override;
|
||||
|
||||
bool StopTimer(Stream *stream, Timer *timer) override;
|
||||
|
||||
port::Status AllocateEvent(Event *event) override;
|
||||
|
||||
port::Status DeallocateEvent(Event *event) override;
|
||||
|
||||
port::Status RecordEvent(Stream *stream, Event *event) override;
|
||||
|
||||
port::Status WaitForEvent(Stream *stream, Event *event) override;
|
||||
|
||||
Event::Status PollForEventStatus(Event *event) override;
|
||||
|
||||
port::Status BlockHostUntilDone(Stream *stream) override;
|
||||
|
||||
int PlatformDeviceCount() override { return CUDADriver::GetDeviceCount(); }
|
||||
|
||||
port::Status EnablePeerAccessTo(StreamExecutorInterface *other) override;
|
||||
|
||||
bool CanEnablePeerAccessTo(StreamExecutorInterface *other) override;
|
||||
|
||||
SharedMemoryConfig GetDeviceSharedMemoryConfig() override;
|
||||
|
||||
port::Status SetDeviceSharedMemoryConfig(SharedMemoryConfig config) override;
|
||||
|
||||
bool DeviceMemoryUsage(int64 *free, int64 *total) const override;
|
||||
|
||||
// Search for the symbol and returns a device pointer and size.
|
||||
// Returns false if symbol does not exist.
|
||||
bool GetSymbol(const string &symbol_name, ModuleHandle module_handle,
|
||||
void **mem, size_t *bytes) override;
|
||||
|
||||
DeviceDescription *PopulateDeviceDescription() const override;
|
||||
|
||||
// Populates the block_dim_limit by querying the device driver API. If an
|
||||
// error occurs at any point while asking the driver for block dim limits, it
|
||||
// will be only partially populated as a result, and an error will be logged.
|
||||
bool FillBlockDimLimit(BlockDim *block_dim_limit) const;
|
||||
|
||||
bool SupportsBlas() const override;
|
||||
|
||||
blas::BlasSupport *CreateBlas() override;
|
||||
|
||||
bool SupportsFft() const override;
|
||||
|
||||
fft::FftSupport *CreateFft() override;
|
||||
|
||||
bool SupportsRng() const override;
|
||||
|
||||
rng::RngSupport *CreateRng() override;
|
||||
|
||||
bool SupportsDnn() const override;
|
||||
|
||||
dnn::DnnSupport *CreateDnn() override;
|
||||
|
||||
std::unique_ptr<internal::EventInterface> CreateEventImplementation()
|
||||
override;
|
||||
|
||||
std::unique_ptr<internal::KernelInterface> CreateKernelImplementation()
|
||||
override;
|
||||
|
||||
std::unique_ptr<internal::StreamInterface> GetStreamImplementation() override;
|
||||
|
||||
std::unique_ptr<internal::TimerInterface> GetTimerImplementation() override;
|
||||
|
||||
void *GpuContextHack() override;
|
||||
|
||||
CudaContext* cuda_context();
|
||||
|
||||
private:
|
||||
// Attempts to find a more specific version of the file indicated by
|
||||
// filename by looking for compute-capability-specific suffixed versions; i.e.
|
||||
// looking for "foo.ptx" will check to see if "foo.ptx.cc30.ptx" is present if
|
||||
// we're on a compute capability 3.0 machine.
|
||||
bool FindOnDiskForComputeCapability(port::StringPiece filename,
|
||||
port::StringPiece canonical_suffix,
|
||||
string *found_filename) const;
|
||||
|
||||
// Host callback landing routine invoked by CUDA.
|
||||
// data: User-provided callback provided to HostCallback() above, captured
|
||||
// as a std::function<void()>. Allocated/initialized inside
|
||||
// HostCallback() and owned and deleted by this call.
|
||||
static void InternalHostCallback(CUstream stream, CUresult status,
|
||||
void *data);
|
||||
|
||||
// Collects metadata for the specified kernel.
|
||||
bool GetKernelMetadata(CUDAKernel *cuda_kernel,
|
||||
KernelMetadata *kernel_metadata);
|
||||
|
||||
// Prints to VLOG(2) information about the kernel's occupancy and how it might
|
||||
// be improved.
|
||||
void VlogOccupancyInfo(const KernelBase &kernel, const ThreadDim &thread_dims,
|
||||
const BlockDim &block_dims);
|
||||
|
||||
bool LoadModuleFromCuBin(const char *cubin, CUmodule *module)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
|
||||
|
||||
// Loads the PTX text `ptx` as a CUDA module. `ptx` must be null terminated.
|
||||
bool LoadModuleFromPtx(const char *ptx, CUmodule *module)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
|
||||
|
||||
bool UnloadGpuBinary(const void *gpu_binary)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
|
||||
|
||||
// Guards the in-memory-module mapping.
|
||||
mutex in_memory_modules_mu_;
|
||||
|
||||
// Kernel -> loaded GPU binary. Many kernels may load the same binary.
|
||||
std::unordered_map<const KernelBase *, const void *> kernel_to_gpu_binary_
|
||||
GUARDED_BY(in_memory_modules_mu_);
|
||||
// GPU binary (PTX or CUBIN) -> {CUDA module, reference count}.
|
||||
std::unordered_map<const void *, std::pair<CUmodule, uint64>>
|
||||
gpu_binary_to_module_ GUARDED_BY(in_memory_modules_mu_);
|
||||
|
||||
// Guards the launched kernel set.
|
||||
mutex launched_kernels_mu_;
|
||||
|
||||
// Keeps track of the set of launched kernels. Currently used to suppress the
|
||||
// occupancy check on subsequent launches.
|
||||
std::set<CUfunction> launched_kernels_ GUARDED_BY(launched_kernels_mu_);
|
||||
|
||||
// Handle for the CUDA device being operated on. Immutable
|
||||
// post-initialization.
|
||||
CUdevice device_;
|
||||
|
||||
// Handle for session with the library/driver. Immutable post-initialization.
|
||||
CudaContext* context_;
|
||||
|
||||
// The device ordinal value that this executor was initialized with; recorded
|
||||
// for use in getting device metadata. Immutable post-initialization.
|
||||
int device_ordinal_;
|
||||
|
||||
// The major verion of the compute capability for device_.
|
||||
int cc_major_;
|
||||
|
||||
// The minor verion of the compute capability for device_.
|
||||
int cc_minor_;
|
||||
|
||||
// The plugin configuration associated with this instance.
|
||||
PluginConfig plugin_config_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CUDAExecutor);
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_GPU_EXECUTOR_H_
|
||||
@@ -0,0 +1,105 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Common helper functions used for dealing with CUDA API datatypes.
|
||||
//
|
||||
// These are typically placed here for use by multiple source components (for
|
||||
// example, BLAS and executor components).
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_HELPERS_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_HELPERS_H_
|
||||
|
||||
#include <stddef.h>
|
||||
#include <complex>
|
||||
|
||||
#include "cuda/include/cuComplex.h"
|
||||
#include "cuda/include/cuda.h"
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
template <typename ElemT>
|
||||
class DeviceMemory;
|
||||
|
||||
namespace cuda {
|
||||
|
||||
// Converts a const DeviceMemory reference to its underlying typed pointer in
|
||||
// CUDA
|
||||
// device memory.
|
||||
template <typename T>
|
||||
const T *CUDAMemory(const DeviceMemory<T> &mem) {
|
||||
return static_cast<const T *>(mem.opaque());
|
||||
}
|
||||
|
||||
// Converts a (non-const) DeviceMemory pointer reference to its underlying typed
|
||||
// pointer in CUDA device memory.
|
||||
template <typename T>
|
||||
T *CUDAMemoryMutable(DeviceMemory<T> *mem) {
|
||||
return static_cast<T *>(mem->opaque());
|
||||
}
|
||||
|
||||
static_assert(sizeof(std::complex<float>) == sizeof(cuComplex),
|
||||
"std::complex<float> and cuComplex should have the same size");
|
||||
static_assert(offsetof(cuComplex, x) == 0,
|
||||
"The real part of cuComplex should appear first.");
|
||||
static_assert(sizeof(std::complex<double>) == sizeof(cuDoubleComplex),
|
||||
"std::complex<double> and cuDoubleComplex should have the same "
|
||||
"size");
|
||||
static_assert(offsetof(cuDoubleComplex, x) == 0,
|
||||
"The real part of cuDoubleComplex should appear first.");
|
||||
|
||||
// Type traits to get CUDA complex types from std::complex<>.
|
||||
|
||||
template <typename T>
|
||||
struct CUDAComplexT {
|
||||
typedef T type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct CUDAComplexT<std::complex<float>> {
|
||||
typedef cuComplex type;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct CUDAComplexT<std::complex<double>> {
|
||||
typedef cuDoubleComplex type;
|
||||
};
|
||||
|
||||
// Converts pointers of std::complex<> to pointers of
|
||||
// cuComplex/cuDoubleComplex. No type conversion for non-complex types.
|
||||
|
||||
template <typename T>
|
||||
inline const typename CUDAComplexT<T>::type *CUDAComplex(const T *p) {
|
||||
return reinterpret_cast<const typename CUDAComplexT<T>::type *>(p);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline typename CUDAComplexT<T>::type *CUDAComplex(T *p) {
|
||||
return reinterpret_cast<typename CUDAComplexT<T>::type *>(p);
|
||||
}
|
||||
|
||||
// Converts values of std::complex<float/double> to values of
|
||||
// cuComplex/cuDoubleComplex.
|
||||
inline cuComplex CUDAComplexValue(std::complex<float> val) {
|
||||
return {val.real(), val.imag()};
|
||||
}
|
||||
|
||||
inline cuDoubleComplex CUDAComplexValue(std::complex<double> val) {
|
||||
return {val.real(), val.imag()};
|
||||
}
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_HELPERS_H_
|
||||
@@ -0,0 +1,128 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// The CUDA implementation of the StreamExecutorInterface functionality.
|
||||
// CUDA inclusions are ideally confined to this implementation file.
|
||||
//
|
||||
// The notions from the StreamExecutor basically correspond to the CUDA streams
|
||||
// programming model provided by the libcuda.so driver APIs, so we don't have
|
||||
// to do much more than wrap the calls to the libraries appropriately.
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_KERNEL_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_KERNEL_H_
|
||||
|
||||
#include "tensorflow/stream_executor/kernel_cache_config.h"
|
||||
#include "tensorflow/stream_executor/stream_executor_internal.h"
|
||||
#include "tensorflow/stream_executor/cuda/cuda_driver.h"
|
||||
#include "tensorflow/stream_executor/lib/casts.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/platform/logging.h"
|
||||
#include "cuda/include/cuda.h"
|
||||
|
||||
#ifdef PLATFORMS_GPUS_CUDA_DYNAMIC_LIBCUDA_DYNAMIC_LIBCUDA_H_
|
||||
#error \
|
||||
"No driver calls in this file, wrap driver functionality in cuda_driver.cc."
|
||||
#endif
|
||||
|
||||
#ifdef __CUDA_RUNTIME_H__
|
||||
#error \
|
||||
"CUDA runtime being included into CUDA GPU executor; should be driver only."
|
||||
#endif
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
// Wraps a CUfunction to implement the platform-independent KernelInterface.
|
||||
class CUDAKernel : public internal::KernelInterface {
|
||||
public:
|
||||
CUDAKernel() : cuda_function_(nullptr), arity_(0),
|
||||
preferred_cache_config_(KernelCacheConfig::kNoPreference) {}
|
||||
|
||||
// Note that the function is unloaded when the module is unloaded, and the
|
||||
// module that the function is contained in is owned by the CUDAExecutor.
|
||||
~CUDAKernel() override {}
|
||||
|
||||
// As arity cannot be reflected upon using the CUDA API, the arity is
|
||||
// explicitly set during the CUDAExecutor::GetKernel initialization process.
|
||||
void set_arity(unsigned arity) { arity_ = arity; }
|
||||
unsigned Arity() const override { return arity_; }
|
||||
|
||||
// Returns the CUfunction value for passing to the CUDA API.
|
||||
CUfunction AsCUDAFunctionValue() const {
|
||||
DCHECK(cuda_function_ != nullptr);
|
||||
return const_cast<CUfunction>(cuda_function_);
|
||||
}
|
||||
|
||||
// Returns the slot that the CUfunction is stored within for this object,
|
||||
// for the CUDA API which wants to load into a CUfunction*.
|
||||
CUfunction *cuda_function_ptr() { return &cuda_function_; }
|
||||
|
||||
// CUDA supports setting the preferred cache configuration of a CUfunction
|
||||
// (more-or-less equivalent to a CUDAKernel). We support this via the below
|
||||
// functions; users can set a preference, and that is applied when the kernel
|
||||
// is [lazy-]loaded (in CUDAExecutor::Launch). The alternative would be to
|
||||
// load the kernel & set the preference when the user calls the setter below;
|
||||
// either approach is valid.
|
||||
// Sets the current kernel cache configuration preference.
|
||||
void SetPreferredCacheConfig(KernelCacheConfig config) override {
|
||||
preferred_cache_config_ = config;
|
||||
}
|
||||
|
||||
// Returns the current kernel cache configuration preference.
|
||||
KernelCacheConfig GetPreferredCacheConfig() const override {
|
||||
return preferred_cache_config_;
|
||||
}
|
||||
|
||||
// Returns the current kernel cache configuration preference as a
|
||||
// CUfunc_cache.
|
||||
CUfunc_cache GetCUDACacheConfig() const {
|
||||
switch (preferred_cache_config_) {
|
||||
case KernelCacheConfig::kNoPreference:
|
||||
return CU_FUNC_CACHE_PREFER_NONE;
|
||||
case KernelCacheConfig::kPreferShared:
|
||||
return CU_FUNC_CACHE_PREFER_SHARED;
|
||||
case KernelCacheConfig::kPreferL1:
|
||||
return CU_FUNC_CACHE_PREFER_L1;
|
||||
case KernelCacheConfig::kPreferEqual:
|
||||
return CU_FUNC_CACHE_PREFER_EQUAL;
|
||||
default:
|
||||
LOG(FATAL) << "Unknown KernelCacheConfig"
|
||||
<< static_cast<int32>(preferred_cache_config_);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
CUfunction cuda_function_; // Wrapped CUDA kernel handle.
|
||||
unsigned arity_; // Number of formal parameters the kernel takes.
|
||||
|
||||
// Preferred (but not required) cache configuration for this kernel.
|
||||
KernelCacheConfig preferred_cache_config_;
|
||||
};
|
||||
|
||||
// Given a platform-independent kernel datatype, returns the (const) internal
|
||||
// CUDA platform implementation pointer.
|
||||
inline const CUDAKernel *AsCUDAKernel(const KernelBase *kernel) {
|
||||
return static_cast<const CUDAKernel *>(kernel->implementation());
|
||||
}
|
||||
|
||||
// Given a platform-independent kernel datatype, returns the (non-const)
|
||||
// internal CUDA platform implementation pointer.
|
||||
inline CUDAKernel *AsCUDAKernel(KernelBase *kernel) {
|
||||
return static_cast<CUDAKernel *>(kernel->implementation());
|
||||
}
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_KERNEL_H_
|
||||
@@ -0,0 +1,108 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_H_
|
||||
|
||||
#include <memory>
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/stream_executor/executor_cache.h"
|
||||
#include "tensorflow/stream_executor/lib/statusor.h"
|
||||
#include "tensorflow/stream_executor/multi_platform_manager.h"
|
||||
#include "tensorflow/stream_executor/platform.h"
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/platform/thread_annotations.h"
|
||||
#include "tensorflow/stream_executor/stream_executor_internal.h"
|
||||
#include "tensorflow/stream_executor/stream_executor_pimpl.h"
|
||||
#include "tensorflow/stream_executor/trace_listener.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
// Opaque and unique identifier for the CUDA platform plugin.
|
||||
// This is needed so that plugins can refer to/identify this platform without
|
||||
// instantiating a CudaPlatform object.
|
||||
extern const Platform::Id kCudaPlatformId;
|
||||
|
||||
// Cuda-specific platform plugin, registered as a singleton value via module
|
||||
// initializer.
|
||||
class CudaPlatform : public Platform {
|
||||
public:
|
||||
CudaPlatform();
|
||||
~CudaPlatform() override;
|
||||
|
||||
// CudaPlatform-specific functionality
|
||||
// Returns the number of distinct buses / NUMA nodes on the machine.
|
||||
int BusCount();
|
||||
|
||||
// Returns the bus/NUMA node for the specified device ordinal.
|
||||
int DeviceToBus(int device_ordinal);
|
||||
|
||||
// Returns the lowest-ordinal-number StreamExecutor on the specified bus.
|
||||
port::StatusOr<StreamExecutor*> FirstExecutorForBus(int bus_ordinal);
|
||||
|
||||
// Platform interface implementation:
|
||||
// Returns the same value as kCudaPlatform above.
|
||||
Platform::Id id() const override;
|
||||
|
||||
// Returns -1 as a sentinel on internal failure (and logs the error).
|
||||
int VisibleDeviceCount() const override;
|
||||
|
||||
const string& Name() const override;
|
||||
|
||||
port::StatusOr<StreamExecutor*> ExecutorForDevice(int ordinal) override;
|
||||
|
||||
port::StatusOr<StreamExecutor*> ExecutorForDeviceWithPluginConfig(
|
||||
int ordinal, const PluginConfig& config) override;
|
||||
|
||||
port::StatusOr<StreamExecutor*> GetExecutor(
|
||||
const StreamExecutorConfig& config) override;
|
||||
|
||||
port::StatusOr<std::unique_ptr<StreamExecutor>> GetUncachedExecutor(
|
||||
const StreamExecutorConfig& config) override;
|
||||
|
||||
void RegisterTraceListener(std::unique_ptr<TraceListener> listener) override;
|
||||
|
||||
void UnregisterTraceListener(TraceListener* listener) override;
|
||||
|
||||
private:
|
||||
// Determines the number of NUMA nodes and the assignment of executor to each.
|
||||
void InspectNumaNodes();
|
||||
|
||||
// This platform's name.
|
||||
string name_;
|
||||
|
||||
// Cache of created executors.
|
||||
ExecutorCache executor_cache_;
|
||||
|
||||
// The smallest NUMA node value for any device managed by this machine
|
||||
// manager. Used, along with limit_numa_node_, to convert NUMA nodes into bus
|
||||
// ordinals. The NUMA node space occupied by GPUs is assumed to be dense./
|
||||
int min_numa_node_;
|
||||
|
||||
// Larger than the NUMA node value for any device managed by this machine
|
||||
// manager.
|
||||
int limit_numa_node_;
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CudaPlatform);
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_H_
|
||||
@@ -0,0 +1,34 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_ID_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_ID_H_
|
||||
|
||||
#include "tensorflow/stream_executor/platform.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
// Opaque and unique identifier for the cuda platform.
|
||||
// This is needed so that plugins can refer to/identify this platform without
|
||||
// instantiating a CudaPlatform object.
|
||||
// This is broken out here to avoid a circular dependency between CudaPlatform
|
||||
// and CudaExecutor.
|
||||
extern const Platform::Id kCudaPlatformId;
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_ID_H_
|
||||
@@ -0,0 +1,102 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_RNG_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_RNG_H_
|
||||
|
||||
#include "tensorflow/stream_executor/platform/mutex.h"
|
||||
#include "tensorflow/stream_executor/platform/port.h"
|
||||
#include "tensorflow/stream_executor/platform/thread_annotations.h"
|
||||
#include "tensorflow/stream_executor/plugin_registry.h"
|
||||
#include "tensorflow/stream_executor/rng.h"
|
||||
|
||||
typedef struct curandGenerator_st *curandGenerator_t;
|
||||
|
||||
namespace stream_executor {
|
||||
|
||||
class Stream;
|
||||
template <typename ElemT>
|
||||
class DeviceMemory;
|
||||
|
||||
namespace cuda {
|
||||
|
||||
// Opaque and unique identifier for the cuRAND plugin.
|
||||
extern const PluginId kCuRandPlugin;
|
||||
|
||||
class CUDAExecutor;
|
||||
|
||||
// CUDA-platform implementation of the random number generation support
|
||||
// interface.
|
||||
//
|
||||
// Thread-safe post-initialization.
|
||||
class CUDARng : public rng::RngSupport {
|
||||
public:
|
||||
explicit CUDARng(CUDAExecutor *parent);
|
||||
|
||||
// Retrieves a curand library generator handle. This is necessary for
|
||||
// enqueuing random number generation work onto the device.
|
||||
// TODO(leary) provide a way for users to select the RNG algorithm.
|
||||
bool Init();
|
||||
|
||||
// Releases a curand library generator handle, if one was acquired.
|
||||
~CUDARng() override;
|
||||
|
||||
// See rng::RngSupport for details on the following overrides.
|
||||
bool DoPopulateRandUniform(Stream *stream, DeviceMemory<float> *v) override;
|
||||
bool DoPopulateRandUniform(Stream *stream, DeviceMemory<double> *v) override;
|
||||
bool DoPopulateRandUniform(Stream *stream,
|
||||
DeviceMemory<std::complex<float>> *v) override;
|
||||
bool DoPopulateRandUniform(Stream *stream,
|
||||
DeviceMemory<std::complex<double>> *v) override;
|
||||
bool DoPopulateRandGaussian(Stream *stream, float mean, float stddev,
|
||||
DeviceMemory<float> *v) override;
|
||||
bool DoPopulateRandGaussian(Stream *stream, double mean, double stddev,
|
||||
DeviceMemory<double> *v) override;
|
||||
|
||||
bool SetSeed(Stream *stream, const uint8 *seed, uint64 seed_bytes) override;
|
||||
|
||||
private:
|
||||
// Actually performs the work of generating random numbers - the public
|
||||
// methods are thin wrappers to this interface.
|
||||
template <typename T>
|
||||
bool DoPopulateRandUniformInternal(Stream *stream, DeviceMemory<T> *v);
|
||||
template <typename ElemT, typename FuncT>
|
||||
bool DoPopulateRandGaussianInternal(Stream *stream, ElemT mean, ElemT stddev,
|
||||
DeviceMemory<ElemT> *v, FuncT func);
|
||||
|
||||
// Sets the stream for the internal curand generator.
|
||||
//
|
||||
// This is a stateful operation, as the handle can only have one stream set at
|
||||
// a given time, so it is usually performed right before enqueuing work to do
|
||||
// with random number generation.
|
||||
bool SetStream(Stream *stream) EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
||||
|
||||
// mutex that guards the cuRAND handle for this device.
|
||||
mutex mu_;
|
||||
|
||||
// CUDAExecutor which instantiated this CUDARng.
|
||||
// Immutable post-initialization.
|
||||
CUDAExecutor *parent_;
|
||||
|
||||
// cuRANDalibrary handle on the device.
|
||||
curandGenerator_t rng_ GUARDED_BY(mu_);
|
||||
|
||||
SE_DISALLOW_COPY_AND_ASSIGN(CUDARng);
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_RNG_H_
|
||||
@@ -0,0 +1,93 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Defines the CUDAStream type - the CUDA-specific implementation of the generic
|
||||
// StreamExecutor Stream interface.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_STREAM_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_STREAM_H_
|
||||
|
||||
#include "tensorflow/stream_executor/cuda/cuda_driver.h"
|
||||
#include "tensorflow/stream_executor/platform/thread_annotations.h"
|
||||
#include "tensorflow/stream_executor/stream_executor_internal.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
class CUDAExecutor;
|
||||
|
||||
// Wraps a CUstream in order to satisfy the platform-independent
|
||||
// StreamInterface.
|
||||
//
|
||||
// Thread-safe post-initialization.
|
||||
class CUDAStream : public internal::StreamInterface {
|
||||
public:
|
||||
explicit CUDAStream(CUDAExecutor *parent)
|
||||
: parent_(parent), cuda_stream_(nullptr), completed_event_(nullptr) {}
|
||||
|
||||
// Note: teardown is handled by a parent's call to DeallocateStream.
|
||||
~CUDAStream() override {}
|
||||
|
||||
void *GpuStreamHack() override { return cuda_stream_; }
|
||||
void **GpuStreamMemberHack() override {
|
||||
return reinterpret_cast<void **>(&cuda_stream_);
|
||||
}
|
||||
|
||||
// Explicitly initialize the CUDA resources associated with this stream, used
|
||||
// by StreamExecutor::AllocateStream().
|
||||
bool Init();
|
||||
|
||||
// Explicitly destroy the CUDA resources associated with this stream, used by
|
||||
// StreamExecutor::DeallocateStream().
|
||||
void Destroy();
|
||||
|
||||
// Returns true if no work is pending or executing on the stream.
|
||||
bool IsIdle() const;
|
||||
|
||||
// Retrieves an event which indicates that all work enqueued into the stream
|
||||
// has completed. Ownership of the event is not transferred to the caller, the
|
||||
// event is owned by this stream.
|
||||
CUevent* completed_event() { return &completed_event_; }
|
||||
|
||||
// Returns the CUstream value for passing to the CUDA API.
|
||||
//
|
||||
// Precond: this CUDAStream has been allocated (otherwise passing a nullptr
|
||||
// into the NVIDIA library causes difficult-to-understand faults).
|
||||
CUstream cuda_stream() const {
|
||||
DCHECK(cuda_stream_ != nullptr);
|
||||
return const_cast<CUstream>(cuda_stream_);
|
||||
}
|
||||
|
||||
CUDAExecutor *parent() const { return parent_; }
|
||||
|
||||
private:
|
||||
CUDAExecutor *parent_; // Executor that spawned this stream.
|
||||
CUstream cuda_stream_; // Wrapped CUDA stream handle.
|
||||
|
||||
// Event that indicates this stream has completed.
|
||||
CUevent completed_event_ = nullptr;
|
||||
};
|
||||
|
||||
// Helper functions to simplify extremely common flows.
|
||||
// Converts a Stream to the underlying CUDAStream implementation.
|
||||
CUDAStream *AsCUDAStream(Stream *stream);
|
||||
|
||||
// Extracts a CUstream from a CUDAStream-backed Stream object.
|
||||
CUstream AsCUDAStreamValue(Stream *stream);
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_STREAM_H_
|
||||
@@ -0,0 +1,90 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Defines the CUDATimer type - the CUDA-specific implementation of the generic
|
||||
// StreamExecutor Timer interface.
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_TIMER_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_TIMER_H_
|
||||
|
||||
#include "tensorflow/stream_executor/stream_executor_internal.h"
|
||||
#include "tensorflow/stream_executor/cuda/cuda_driver.h"
|
||||
#include "tensorflow/stream_executor/cuda/cuda_gpu_executor.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
class CUDAExecutor;
|
||||
class CUDAStream;
|
||||
|
||||
// Wraps a pair of CUevents in order to satisfy the platform-independent
|
||||
// TimerInferface -- both a start and a stop event are present which may be
|
||||
// recorded in a stream.
|
||||
class CUDATimer : public internal::TimerInterface {
|
||||
public:
|
||||
explicit CUDATimer(CUDAExecutor *parent)
|
||||
: parent_(parent), start_event_(nullptr), stop_event_(nullptr) {}
|
||||
|
||||
// Note: teardown needs to be explicitly handled in this API by a call to
|
||||
// StreamExecutor::DeallocateTimer(), which invokes Destroy().
|
||||
// TODO(csigg): Change to RAII.
|
||||
~CUDATimer() override {}
|
||||
|
||||
// Allocates the platform-specific pieces of the timer, called as part of
|
||||
// StreamExecutor::AllocateTimer().
|
||||
bool Init();
|
||||
|
||||
// Deallocates the platform-specific pieces of the timer, called as part of
|
||||
// StreamExecutor::DeallocateTimer().
|
||||
void Destroy();
|
||||
|
||||
// Records the "timer start" event at the current point in the stream.
|
||||
bool Start(CUDAStream *stream);
|
||||
|
||||
// Records the "timer stop" event at the current point in the stream.
|
||||
bool Stop(CUDAStream *stream);
|
||||
|
||||
// Returns the elapsed time, in milliseconds, between the start and stop
|
||||
// events.
|
||||
float GetElapsedMilliseconds() const;
|
||||
|
||||
// See Timer::Microseconds().
|
||||
// TODO(leary) make this into an error code interface...
|
||||
uint64 Microseconds() const override {
|
||||
return GetElapsedMilliseconds() * 1e3;
|
||||
}
|
||||
|
||||
// See Timer::Nanoseconds().
|
||||
uint64 Nanoseconds() const override { return GetElapsedMilliseconds() * 1e6; }
|
||||
|
||||
private:
|
||||
CUDAExecutor *parent_;
|
||||
CUevent start_event_; // Event recorded to indicate the "start" timestamp
|
||||
// executing in a stream.
|
||||
CUevent stop_event_; // Event recorded to indicate the "stop" timestamp
|
||||
// executing in a stream.
|
||||
};
|
||||
|
||||
struct TimerDeleter {
|
||||
void operator()(CUDATimer *t) {
|
||||
t->Destroy();
|
||||
delete t;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_TIMER_H_
|
||||
@@ -0,0 +1,50 @@
|
||||
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDNN_VERSION_H_
|
||||
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDNN_VERSION_H_
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "tensorflow/core/lib/strings/strcat.h"
|
||||
|
||||
namespace stream_executor {
|
||||
namespace cuda {
|
||||
|
||||
struct CudnnVersion {
|
||||
CudnnVersion() = default;
|
||||
|
||||
CudnnVersion(int major, int minor, int patch)
|
||||
: major_version(major), minor_version(minor), patch_level(patch) {}
|
||||
|
||||
tensorflow::string ToString() const {
|
||||
return tensorflow::strings::StrCat(major_version, ".", minor_version, ".",
|
||||
patch_level);
|
||||
}
|
||||
|
||||
int major_version;
|
||||
int minor_version;
|
||||
int patch_level;
|
||||
};
|
||||
|
||||
// Returns true if the given source CuDNN version is compatible with the given
|
||||
// loaded version.
|
||||
bool IsSourceCompatibleWithCudnnLibrary(CudnnVersion source_version,
|
||||
CudnnVersion loaded_version);
|
||||
|
||||
} // namespace cuda
|
||||
} // namespace stream_executor
|
||||
|
||||
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDNN_VERSION_H_
|
||||
Reference in new issue
Block a user