This commit is contained in:
Kevin Darby committed 2019-01-02 17:04:10 -05:00
1 parent eb1b42a5e8
commit 458eb2aef1
1283 files changed
+323373

No files matched your search

File diff suppressed because it is too large. Load diff
@@ -0,0 +1,61 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This file contains APIs that assume a StreamExecutor is backed by CUDA.
// It reaches into the CUDA implementation to activate an underlying CUDA
// context.
//
// Having this file separate from cuda_gpu_executor.h means that dependent
// code does not also have to depend on cuda.h.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_ACTIVATION_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_ACTIVATION_H_
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
class StreamExecutor;
namespace cuda {
class CUDAExecutor;
class ScopedActivateContext;
// Activates a CUDA context within an enclosing scope.
class ScopedActivateExecutorContext {
public:
// Form that takes a CUDA executor implementation.
explicit ScopedActivateExecutorContext(CUDAExecutor* cuda_exec);
// Form that takes a pImpl executor and extracts a CUDA implementation --
// fatal failure if it is not CUDA inside.
explicit ScopedActivateExecutorContext(StreamExecutor* stream_exec);
ScopedActivateExecutorContext(ScopedActivateExecutorContext&& other);
~ScopedActivateExecutorContext();
private:
// The cuda.h-using datatype that we wrap.
ScopedActivateContext* driver_scoped_activate_context_;
SE_DISALLOW_COPY_AND_ASSIGN(ScopedActivateExecutorContext);
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_ACTIVATION_H_
@@ -0,0 +1,162 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// CUDA-specific support for BLAS functionality -- this wraps the cuBLAS library
// capabilities, and is only included into CUDA implementation code -- it will
// not introduce cuda headers into other code.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_BLAS_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_BLAS_H_
#include "tensorflow/stream_executor/blas.h"
#include "tensorflow/stream_executor/host_or_device_scalar.h"
#include "tensorflow/stream_executor/lib/stringpiece.h"
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/platform/thread_annotations.h"
#include "tensorflow/stream_executor/plugin_registry.h"
typedef struct cublasContext *cublasHandle_t;
namespace stream_executor {
class Stream;
namespace cuda {
// Opaque and unique identifier for the cuBLAS plugin.
extern const PluginId kCuBlasPlugin;
class CUDAExecutor;
// BLAS plugin for CUDA platform via cuBLAS library.
//
// This satisfies the platform-agnostic BlasSupport interface.
//
// Note that the cuBLAS handle that this encapsulates is implicitly tied to the
// context (and, as a result, the device) that the parent CUDAExecutor is tied
// to. This simply happens as an artifact of creating the cuBLAS handle when a
// CUDA context is active.
//
// Thread-safe post-initialization.
class CUDABlas : public blas::BlasSupport {
public:
explicit CUDABlas(CUDAExecutor *parent);
// Allocates a cuBLAS handle.
bool Init();
// Releases the cuBLAS handle, if present.
~CUDABlas() override;
TENSORFLOW_STREAM_EXECUTOR_GPU_BLAS_SUPPORT_OVERRIDES
private:
// Tells cuBLAS to enqueue the BLAS operation onto a particular Stream.
//
// cuBLAS is stateful, and only be associated with one stream (in order to
// enqueue dispatch) at a given time. As a result, this generally must be
// invoked before calling into cuBLAS.
bool SetStream(Stream *stream) EXCLUSIVE_LOCKS_REQUIRED(mu_);
// A helper function that calls the real cuBLAS function together with error
// handling.
//
// cublas_func: cuBLAS function pointer.
// cublas_name: cuBLAS function name.
// stream: Stream to enqueue the BLAS operation onto.
// pointer_mode_host: Indicate if the pointer to a scalar value is from host
// (true) or device (false).
// err_on_failure: Whether to print an error if the cublas function fails.
// args: Arguments of cuBLAS function.
template <typename FuncT, typename... Args>
bool DoBlasInternalImpl(FuncT cublas_func, Stream *stream,
bool pointer_mode_host, bool err_on_failure,
bool use_tensor_op_math, Args... args);
// Convenience functions that call DoBlasInternalImpl with different values
// for err_on_failure.
template <typename FuncT, typename... Args>
bool DoBlasInternal(FuncT cublas_func, Stream *stream, bool pointer_mode_host,
Args... args) {
return DoBlasInternalImpl(cublas_func, stream, pointer_mode_host,
/*err_on_failure=*/true, /*use_tensor_ops=*/false,
args...);
}
template <typename FuncT, typename... Args>
bool DoBlasInternalFailureOK(FuncT cublas_func, Stream *stream,
bool pointer_mode_host, Args... args) {
// Tensor ops are hard-coded off in this path, but can still be enabled with
// a specific algorithm choice as in DoBlasGemmWithAlgorithmImpl().
return DoBlasInternalImpl(cublas_func, stream, pointer_mode_host,
/*err_on_failure=*/false,
/*use_tensor_ops=*/false, args...);
}
// A helper function to implement DoBlasGemmBatched interfaces for generic
// types.
template <typename T, typename Scalar, typename FuncT>
port::Status DoBlasGemmBatchedInternal(
FuncT cublas_func, Stream *stream, blas::Transpose transa,
blas::Transpose transb, uint64 m, uint64 n, uint64 k, Scalar alpha,
const port::ArraySlice<DeviceMemory<T> *> &a_array, int lda,
const port::ArraySlice<DeviceMemory<T> *> &b_array, int ldb, Scalar beta,
const port::ArraySlice<DeviceMemory<T> *> &c_array, int ldc,
int batch_count, ScratchAllocator *scratch_allocator);
// Helper function for implementing DoBlasGemmWithAlgorithm.
template <typename InT, typename OutT, typename CompT>
bool DoBlasGemmWithAlgorithmImpl(
Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m,
uint64 n, uint64 k, const HostOrDeviceScalar<CompT> &alpha,
const DeviceMemory<InT> &a, int lda, const DeviceMemory<InT> &b, int ldb,
const HostOrDeviceScalar<CompT> &beta, DeviceMemory<OutT> *c, int ldc,
blas::ComputationType computation_type, blas::AlgorithmType algorithm,
blas::ProfileResult *output_profile_result);
// Helper function for implementing DoBlasGemmWithProfiling.
template <typename T, typename ParamType>
bool DoBlasGemmWithProfilingImpl(
Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m,
uint64 n, uint64 k, const ParamType &alpha, const DeviceMemory<T> &a,
int lda, const DeviceMemory<T> &b, int ldb, const ParamType &beta,
DeviceMemory<T> *c, int ldc, blas::ProfileResult *output_profile_result);
// Helper function for implementing DoBlasGemvWithProfiling.
template <typename T>
bool DoBlasGemvWithProfilingImpl(Stream *stream, blas::Transpose trans,
uint64 m, uint64 n, const T &alpha,
const DeviceMemory<T> &a, int lda,
const DeviceMemory<T> &x, int incx,
const T &beta, DeviceMemory<T> *y, int incy,
blas::ProfileResult *output_profile_result);
// mutex that guards the cuBLAS handle for this device.
mutex mu_;
// CUDAExecutor which instantiated this CUDABlas.
// Immutable post-initialization.
CUDAExecutor *parent_;
// cuBLAS library handle on the device.
cublasHandle_t blas_ GUARDED_BY(mu_);
SE_DISALLOW_COPY_AND_ASSIGN(CUDABlas);
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_BLAS_H_
@@ -0,0 +1,97 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DIAGNOSTICS_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DIAGNOSTICS_H_
#include "tensorflow/stream_executor/platform/port.h"
#include <tuple>
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
namespace cuda {
// e.g. DriverVersion{346, 3, 4}
using DriverVersion = std::tuple<int, int, int>;
// Converts a parsed driver version to string form.
string DriverVersionToString(DriverVersion version);
// Converts a parsed driver version or status value to natural string form.
string DriverVersionStatusToString(port::StatusOr<DriverVersion> version);
// Converts a string of a form like "331.79" to a DriverVersion{331, 79}.
port::StatusOr<DriverVersion> StringToDriverVersion(const string &value);
class Diagnostician {
public:
// Logs diagnostic information when CUDA appears to be misconfigured (e.g. is
// not initializing).
//
// Note: if we're running on a machine that has no GPUs, we don't want to
// produce very much log spew beyond saying, "looks like there's no CUDA
// kernel
// module running".
//
// Note: we use non-Google-File:: API here because we may be called before
// InitGoogle has completed.
static void LogDiagnosticInformation();
// Given the driver version file contents, finds the kernel module version and
// returns it as a string.
//
// This is solely used for more informative log messages when the user is
// running on a machine that happens to have a libcuda/kernel driver mismatch.
static port::StatusOr<DriverVersion> FindKernelModuleVersion(
const string &driver_version_file_contents);
// Extracts the kernel driver version from the current host.
static port::StatusOr<DriverVersion> FindKernelDriverVersion();
// Iterates through loaded DSOs with DlIteratePhdrCallback to find the
// driver-interfacing DSO version number. Returns it as a string.
static port::StatusOr<DriverVersion> FindDsoVersion();
// Logs information about the kernel driver version and userspace driver
// library version.
static void LogDriverVersionInformation();
private:
// Given the DSO version number and the driver version file contents, extracts
// the driver version and compares, warning the user in the case of
// incompatibility.
//
// This is solely used for more informative log messages when the user is
// running on a machine that happens to have a libcuda/kernel driver mismatch.
static void WarnOnDsoKernelMismatch(
port::StatusOr<DriverVersion> dso_version,
port::StatusOr<DriverVersion> kernel_version);
// Logs information about the dev nodes present on this machine: their
// existence, permissions, accessibility from this uid/gid.
static void LogDevNodeDiagnosticInformation();
static string GetDevNodePath(int dev_node_ordinal);
SE_DISALLOW_COPY_AND_ASSIGN(Diagnostician);
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DIAGNOSTICS_H_
@@ -0,0 +1,778 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// The CUDA-specific DNN library support, implementing the general DnnSupport
// interface.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DNN_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DNN_H_
#include "tensorflow/stream_executor/cuda/cuda_activation.h"
#include "tensorflow/stream_executor/dnn.h"
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/platform/thread_annotations.h"
#include "tensorflow/stream_executor/plugin_registry.h"
#include "tensorflow/stream_executor/temporary_device_memory.h"
namespace stream_executor {
namespace cuda {
class CUDAExecutor;
class CudnnRnnDescriptor;
class CudnnRnnSequenceTensorDescriptor;
class CudnnRnnStateTensorDescriptor;
// Opaque and unique identifier for the cuDNN plugin.
extern const PluginId kCuDnnPlugin;
// cudnn-library based DNN support. For details on overridden interface
// functions, see dnn.h.
class CudnnSupport : public dnn::DnnSupport {
public:
explicit CudnnSupport(CUDAExecutor* parent);
port::Status Init() override;
port::StatusOr<perftools::gputools::dnn::VersionInfo> GetVersion() override;
port::StatusOr<std::unique_ptr<dnn::RnnDescriptor>> createRnnDescriptor(
int num_layers, int hidden_size, int input_size, int batch_size,
dnn::RnnInputMode input_mode, dnn::RnnDirectionMode direction_mode,
dnn::RnnMode rnn_mode, dnn::DataType data_type,
const dnn::AlgorithmConfig& algorithm_config, float dropout, uint64 seed,
ScratchAllocator* state_allocator) override;
port::StatusOr<std::unique_ptr<dnn::RnnSequenceTensorDescriptor>>
createRnnSequenceTensorDescriptor(int seq_length, int batch_size,
int data_size,
dnn::DataType data_type) override;
port::StatusOr<std::unique_ptr<dnn::RnnStateTensorDescriptor>>
createRnnStateTensorDescriptor(int num_layer, int batch_size, int data_size,
dnn::DataType data_type) override;
bool DoRnnForward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
const dnn::RnnSequenceTensorDescriptor& input_desc,
const DeviceMemory<Eigen::half>& input_data,
const dnn::RnnStateTensorDescriptor& input_h_desc,
const DeviceMemory<Eigen::half>& input_h_data,
const dnn::RnnStateTensorDescriptor& input_c_desc,
const DeviceMemory<Eigen::half>& input_c_data,
const DeviceMemory<Eigen::half>& params,
const dnn::RnnSequenceTensorDescriptor& output_desc,
DeviceMemory<Eigen::half>* output_data,
const dnn::RnnStateTensorDescriptor& output_h_desc,
DeviceMemory<Eigen::half>* output_h_data,
const dnn::RnnStateTensorDescriptor& output_c_desc,
DeviceMemory<Eigen::half>* output_c_data, bool is_training,
ScratchAllocator* reserve_space_allocator,
ScratchAllocator* workspace_allocator,
dnn::ProfileResult* output_profile_result) override;
bool DoRnnForward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
const dnn::RnnSequenceTensorDescriptor& input_desc,
const DeviceMemory<float>& input_data,
const dnn::RnnStateTensorDescriptor& input_h_desc,
const DeviceMemory<float>& input_h_data,
const dnn::RnnStateTensorDescriptor& input_c_desc,
const DeviceMemory<float>& input_c_data,
const DeviceMemory<float>& params,
const dnn::RnnSequenceTensorDescriptor& output_desc,
DeviceMemory<float>* output_data,
const dnn::RnnStateTensorDescriptor& output_h_desc,
DeviceMemory<float>* output_h_data,
const dnn::RnnStateTensorDescriptor& output_c_desc,
DeviceMemory<float>* output_c_data, bool is_training,
ScratchAllocator* reserve_space_allocator,
ScratchAllocator* workspace_allocator,
dnn::ProfileResult* output_profile_result) override;
bool DoRnnForward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
const dnn::RnnSequenceTensorDescriptor& input_desc,
const DeviceMemory<double>& input_data,
const dnn::RnnStateTensorDescriptor& input_h_desc,
const DeviceMemory<double>& input_h_data,
const dnn::RnnStateTensorDescriptor& input_c_desc,
const DeviceMemory<double>& input_c_data,
const DeviceMemory<double>& params,
const dnn::RnnSequenceTensorDescriptor& output_desc,
DeviceMemory<double>* output_data,
const dnn::RnnStateTensorDescriptor& output_h_desc,
DeviceMemory<double>* output_h_data,
const dnn::RnnStateTensorDescriptor& output_c_desc,
DeviceMemory<double>* output_c_data, bool is_training,
ScratchAllocator* reserve_space_allocator,
ScratchAllocator* workspace_allocator,
dnn::ProfileResult* output_profile_result) override;
bool DoRnnBackward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
const dnn::RnnSequenceTensorDescriptor& input_desc,
const DeviceMemory<Eigen::half>& input_data,
const dnn::RnnStateTensorDescriptor& input_h_desc,
const DeviceMemory<Eigen::half>& input_h_data,
const dnn::RnnStateTensorDescriptor& input_c_desc,
const DeviceMemory<Eigen::half>& input_c_data,
const DeviceMemory<Eigen::half>& params,
const dnn::RnnSequenceTensorDescriptor& output_desc,
const DeviceMemory<Eigen::half>& output_data,
const dnn::RnnStateTensorDescriptor& output_h_desc,
const DeviceMemory<Eigen::half>& output_h_data,
const dnn::RnnStateTensorDescriptor& output_c_desc,
const DeviceMemory<Eigen::half>& output_c_data,
const DeviceMemory<Eigen::half>& output_backprop_data,
const DeviceMemory<Eigen::half>& output_h_backprop_data,
const DeviceMemory<Eigen::half>& output_c_backprop_data,
DeviceMemory<Eigen::half>* input_backprop_data,
DeviceMemory<Eigen::half>* input_h_backprop_data,
DeviceMemory<Eigen::half>* input_c_backprop_data,
DeviceMemory<Eigen::half>* params_backprop_data,
DeviceMemory<uint8>* reserve_space_data,
ScratchAllocator* workspace_allocator,
dnn::ProfileResult* output_profile_result) override;
bool DoRnnBackward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
const dnn::RnnSequenceTensorDescriptor& input_desc,
const DeviceMemory<float>& input_data,
const dnn::RnnStateTensorDescriptor& input_h_desc,
const DeviceMemory<float>& input_h_data,
const dnn::RnnStateTensorDescriptor& input_c_desc,
const DeviceMemory<float>& input_c_data,
const DeviceMemory<float>& params,
const dnn::RnnSequenceTensorDescriptor& output_desc,
const DeviceMemory<float>& output_data,
const dnn::RnnStateTensorDescriptor& output_h_desc,
const DeviceMemory<float>& output_h_data,
const dnn::RnnStateTensorDescriptor& output_c_desc,
const DeviceMemory<float>& output_c_data,
const DeviceMemory<float>& output_backprop_data,
const DeviceMemory<float>& output_h_backprop_data,
const DeviceMemory<float>& output_c_backprop_data,
DeviceMemory<float>* input_backprop_data,
DeviceMemory<float>* input_h_backprop_data,
DeviceMemory<float>* input_c_backprop_data,
DeviceMemory<float>* params_backprop_data,
DeviceMemory<uint8>* reserve_space_data,
ScratchAllocator* workspace_allocator,
dnn::ProfileResult* output_profile_result) override;
bool DoRnnBackward(Stream* stream, const dnn::RnnDescriptor& rnn_desc,
const dnn::RnnSequenceTensorDescriptor& input_desc,
const DeviceMemory<double>& input_data,
const dnn::RnnStateTensorDescriptor& input_h_desc,
const DeviceMemory<double>& input_h_data,
const dnn::RnnStateTensorDescriptor& input_c_desc,
const DeviceMemory<double>& input_c_data,
const DeviceMemory<double>& params,
const dnn::RnnSequenceTensorDescriptor& output_desc,
const DeviceMemory<double>& output_data,
const dnn::RnnStateTensorDescriptor& output_h_desc,
const DeviceMemory<double>& output_h_data,
const dnn::RnnStateTensorDescriptor& output_c_desc,
const DeviceMemory<double>& output_c_data,
const DeviceMemory<double>& output_backprop_data,
const DeviceMemory<double>& output_h_backprop_data,
const DeviceMemory<double>& output_c_backprop_data,
DeviceMemory<double>* input_backprop_data,
DeviceMemory<double>* input_h_backprop_data,
DeviceMemory<double>* input_c_backprop_data,
DeviceMemory<double>* params_backprop_data,
DeviceMemory<uint8>* reserve_space_data,
ScratchAllocator* workspace_allocator,
dnn::ProfileResult* output_profile_result) override;
bool GetConvolveAlgorithms(
bool with_winograd_nonfused, int cc_major, int cc_minor,
std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
bool GetRnnAlgorithms(
std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
bool GetConvolveBackwardDataAlgorithms(
bool with_winograd_nonfused, int cc_major, int cc_minor,
std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
bool GetConvolveBackwardFilterAlgorithms(
bool with_winograd_nonfused, int cc_major, int cc_minor,
std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
bool DoBatchNormalizationForward(
Stream* stream, const DeviceMemory<float>& x,
const DeviceMemory<float>& scale, const DeviceMemory<float>& offset,
const DeviceMemory<float>& estimated_mean,
const DeviceMemory<float>& estimated_variance,
const dnn::BatchDescriptor& x_desc,
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
DeviceMemory<float>* y, DeviceMemory<float>* batch_mean,
DeviceMemory<float>* batch_var, DeviceMemory<float>* saved_mean,
DeviceMemory<float>* saved_inv_var, bool is_training,
std::function<const DeviceMemory<float>&()> var_to_inv_var,
std::function<void()> inv_var_to_var) override;
bool DoBatchNormalizationForward(
Stream* stream, const DeviceMemory<Eigen::half>& x,
const DeviceMemory<float>& scale, const DeviceMemory<float>& offset,
const DeviceMemory<float>& estimated_mean,
const DeviceMemory<float>& estimated_variance,
const dnn::BatchDescriptor& x_desc,
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
DeviceMemory<Eigen::half>* y, DeviceMemory<float>* batch_mean,
DeviceMemory<float>* batch_var, DeviceMemory<float>* saved_mean,
DeviceMemory<float>* saved_inv_var, bool is_training,
std::function<const DeviceMemory<float>&()> var_to_inv_var,
std::function<void()> inv_var_to_var) override;
bool DoBatchNormalizationBackward(
Stream* stream, const DeviceMemory<float>& y_backprop,
const DeviceMemory<float>& x, const DeviceMemory<float>& scale,
const DeviceMemory<float>& mean, const DeviceMemory<float>& inv_var,
const dnn::BatchDescriptor& x_desc,
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
DeviceMemory<float>* x_backprop, DeviceMemory<float>* scale_backprop,
DeviceMemory<float>* offset_backprop) override;
bool DoBatchNormalizationBackward(
Stream* stream, const DeviceMemory<Eigen::half>& y_backprop,
const DeviceMemory<Eigen::half>& x, const DeviceMemory<float>& scale,
const DeviceMemory<float>& mean, const DeviceMemory<float>& inv_var,
const dnn::BatchDescriptor& x_desc,
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
DeviceMemory<Eigen::half>* x_backprop,
DeviceMemory<float>* scale_backprop,
DeviceMemory<float>* offset_backprop) override;
bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
const DeviceMemory<float>& input_data,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<float>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<float>* output_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
const DeviceMemory<double>& input_data,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<double>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<double>* output_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
const DeviceMemory<Eigen::half>& input_data,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<Eigen::half>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<Eigen::half>* output_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoFusedConvolve(
Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
const DeviceMemory<double>& conv_input_data, double conv_input_scale,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<double>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const DeviceMemory<double>& side_input_data, double side_input_scale,
const dnn::BatchDescriptor& bias_descriptor,
const DeviceMemory<double>& biases, dnn::ActivationMode activation_mode,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoFusedConvolve(
Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
const DeviceMemory<float>& conv_input_data, float conv_input_scale,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<float>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const DeviceMemory<float>& side_input_data, float side_input_scale,
const dnn::BatchDescriptor& bias_descriptor,
const DeviceMemory<float>& biases, dnn::ActivationMode activation_mode,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoFusedConvolve(Stream* stream,
const dnn::BatchDescriptor& conv_input_descriptor,
const DeviceMemory<Eigen::half>& conv_input_data,
float conv_input_scale,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<Eigen::half>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const DeviceMemory<Eigen::half>& side_input_data,
float side_input_scale,
const dnn::BatchDescriptor& bias_descriptor,
const DeviceMemory<Eigen::half>& biases,
dnn::ActivationMode activation_mode,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<Eigen::half>* output_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoFusedConvolve(
Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
const DeviceMemory<int8>& conv_input_data, float conv_input_scale,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<int8>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const DeviceMemory<int8>& side_input_data, float side_input_scale,
const dnn::BatchDescriptor& bias_descriptor,
const DeviceMemory<float>& biases, dnn::ActivationMode activation_mode,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<int8>* output_data, ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolveQuantized(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<float>& input_data,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<int8>& filter_coefficients,
const DeviceMemory<float>& coefficient_scales,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<float>* output_data) override {
LOG(ERROR) << "DoConvolveQuantized not supported by cuDNN";
return false;
}
bool DoConvolveQuantized(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<float>& input_data,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<int16>& filter_coefficients,
const DeviceMemory<float>& coefficient_scales,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<float>* output_data) override {
LOG(ERROR) << "DoConvolveQuantized not supported by cuDNN";
return false;
}
bool DoSeparableConvolve(
Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
const DeviceMemory<float>& input_data,
const dnn::FilterDescriptor& filter_descriptor, int depth_multiplier,
const DeviceMemory<float>& first_weights,
const DeviceMemory<float>& second_weights,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<float>* output_data) override {
LOG(ERROR) << "separable convolution not supported by CUDNN";
return false;
}
bool DoConvolveBackwardData(
Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<double>& filter_data,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<double> backward_output_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& input_descriptor,
DeviceMemory<double>* backward_input_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolveBackwardData(
Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<float>& filter_data,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<float> backward_output_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& input_descriptor,
DeviceMemory<float>* backward_input_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolveBackwardData(
Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<Eigen::half>& filter_data,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<Eigen::half> backward_output_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& input_descriptor,
DeviceMemory<Eigen::half>* backward_input_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolveBackwardFilter(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<double>& input_data,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<double> backward_output_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::FilterDescriptor& filter_descriptor,
DeviceMemory<double>* backward_filter_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolveBackwardFilter(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<float>& input_data,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<float> backward_output_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::FilterDescriptor& filter_descriptor,
DeviceMemory<float>* backward_filter_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolveBackwardFilter(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<Eigen::half>& input_data,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<Eigen::half> backward_output_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::FilterDescriptor& filter_descriptor,
DeviceMemory<Eigen::half>* backward_filter_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result) override;
bool DoConvolveBackwardBias(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<double>& input_data,
const dnn::BatchDescriptor& bias_descriptor,
DeviceMemory<double>* backward_bias_data) override;
bool DoConvolveBackwardBias(Stream* stream,
const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<float>& input_data,
const dnn::BatchDescriptor& bias_descriptor,
DeviceMemory<float>* backward_bias_data) override;
bool DoConvolveBackwardBias(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<Eigen::half>& input_data,
const dnn::BatchDescriptor& bias_descriptor,
DeviceMemory<Eigen::half>* backward_bias_data) override;
bool DoMatMul(Stream* stream, const DeviceMemory<float>& input_data,
const DeviceMemory<float>& weights,
const dnn::BatchDescriptor& input_dimensions,
const dnn::BatchDescriptor& output_dimensions,
DeviceMemory<float>* output_data) override;
bool DoMatMulQuantized(Stream* stream, const DeviceMemory<float>& input_data,
const DeviceMemory<int8>& quantized_weights,
const DeviceMemory<float>& weight_scales,
const dnn::BatchDescriptor& input_dimensions,
const dnn::BatchDescriptor& output_dimensions,
DeviceMemory<float>* output_data) override {
LOG(ERROR) << "DNN MatMulQuantized not supported by CUDNN";
return false;
}
bool DoMatMulQuantized(Stream* stream, const DeviceMemory<float>& input_data,
const DeviceMemory<int16>& quantized_weights,
const DeviceMemory<float>& weight_scales,
const dnn::BatchDescriptor& input_dimensions,
const dnn::BatchDescriptor& output_dimensions,
DeviceMemory<float>* output_data) override {
LOG(ERROR) << "DNN MatMulQuantized not supported by CUDNN";
return false;
}
bool DoBiasAdd(Stream* stream, const DeviceMemory<float>& input_data,
const DeviceMemory<float>& biases,
const dnn::BatchDescriptor& dimensions,
DeviceMemory<float>* output_data) override;
bool DoActivate(Stream* stream, dnn::ActivationMode activation_mode,
const dnn::BatchDescriptor& dimensions,
const DeviceMemory<float>& input_data,
DeviceMemory<float>* output_data, uint64 options) override;
bool DoPoolForward(Stream* stream,
const dnn::PoolingDescriptor& pooling_dimensions,
const dnn::BatchDescriptor& input_dimensions,
const DeviceMemory<double>& input_data,
const dnn::BatchDescriptor& output_dimensions,
DeviceMemory<double>* output_data,
ScratchAllocator* workspace_allocator) override;
bool DoPoolForward(Stream* stream,
const dnn::PoolingDescriptor& pooling_dimensions,
const dnn::BatchDescriptor& input_dimensions,
const DeviceMemory<float>& input_data,
const dnn::BatchDescriptor& output_dimensions,
DeviceMemory<float>* output_data,
ScratchAllocator* workspace_allocator) override;
bool DoPoolForward(Stream* stream,
const dnn::PoolingDescriptor& pooling_dimensions,
const dnn::BatchDescriptor& input_dimensions,
const DeviceMemory<Eigen::half>& input_data,
const dnn::BatchDescriptor& output_dimensions,
DeviceMemory<Eigen::half>* output_data,
ScratchAllocator* workspace_allocator) override;
bool DoPoolBackward(Stream* stream,
const dnn::PoolingDescriptor& pooling_dimensions,
const dnn::BatchDescriptor& input_dimensions,
const DeviceMemory<double>& input_data,
const dnn::BatchDescriptor& output_dimensions,
const DeviceMemory<double>& output_data,
const DeviceMemory<double>& input_diff_data,
DeviceMemory<double>* output_diff_data,
ScratchAllocator* workspace_allocator) override;
bool DoPoolBackward(Stream* stream,
const dnn::PoolingDescriptor& pooling_dimensions,
const dnn::BatchDescriptor& input_dimensions,
const DeviceMemory<float>& input_data,
const dnn::BatchDescriptor& output_dimensions,
const DeviceMemory<float>& output_data,
const DeviceMemory<float>& input_diff_data,
DeviceMemory<float>* output_diff_data,
ScratchAllocator* workspace_allocator) override;
bool DoPoolBackward(Stream* stream,
const dnn::PoolingDescriptor& pooling_dimensions,
const dnn::BatchDescriptor& input_dimensions,
const DeviceMemory<Eigen::half>& input_data,
const dnn::BatchDescriptor& output_dimensions,
const DeviceMemory<Eigen::half>& output_data,
const DeviceMemory<Eigen::half>& input_diff_data,
DeviceMemory<Eigen::half>* output_diff_data,
ScratchAllocator* workspace_allocator) override;
bool DoNormalize(Stream* stream,
const dnn::NormalizeDescriptor& normalize_descriptor,
const DeviceMemory<float>& input_data,
DeviceMemory<float>* output_data) override;
bool DoNormalizeWithDimensions(
Stream* stream, const dnn::NormalizeDescriptor& normalize_descriptor,
const dnn::BatchDescriptor& dimensions,
const DeviceMemory<float>& input_data,
DeviceMemory<float>* output_data) override;
bool DoNormalizeBackwardWithDimensions(
Stream* stream, const dnn::NormalizeDescriptor& normalize_descriptor,
const dnn::BatchDescriptor& dimensions,
const DeviceMemory<float>& raw_data,
const DeviceMemory<float>& normalized_data,
const DeviceMemory<float>& normalized_variable_gradient,
DeviceMemory<float>* raw_variable_gradient,
ScratchAllocator* workspace_allocator) override;
bool DoDepthConcatenate(
Stream* stream, port::ArraySlice<dnn::BatchDescriptor> input_dimensions,
port::ArraySlice<const DeviceMemory<float>*> input_data,
DeviceMemory<float>* output_data) override;
bool DoElementwiseOperate(
Stream* stream, dnn::ElementwiseOperation operation,
port::ArraySlice<dnn::BatchDescriptor> input_dimensions,
port::ArraySlice<const DeviceMemory<float>*> input_data,
const dnn::BatchDescriptor& output_dimensions,
DeviceMemory<float>* output_data) override;
bool DoXYPad(Stream* stream, const dnn::BatchDescriptor &dimensions,
const DeviceMemory<float> &input_data,
int64 left_pad, int64 right_pad, int64 top_pad,
int64 bottom_pad, DeviceMemory<float> *output_data) override;
bool DoXYSlice(Stream* stream, const dnn::BatchDescriptor &dimensions,
const DeviceMemory<float> &input_data,
int64 left_trim, int64 right_trim, int64 top_trim,
int64 bottom_trim, DeviceMemory<float> *output_data) override;
bool DoMemcpyD2HQuantized(Stream* stream,
const DeviceMemory<float>& device_unquantized_src,
dnn::QuantizedActivationMode mode, void* host_dst,
int64 size) override;
bool DoMemcpyH2DQuantized(
Stream* stream, const void* host_src, int64 size,
dnn::QuantizedActivationMode mode,
DeviceMemory<float>* device_unquantized_dst) override;
// Derives an output batch descriptor from an input batch and convolution
// descriptors.
bool DeriveOutputBatchDescriptor(
const dnn::BatchDescriptor& batch_descriptor,
const dnn::FilterDescriptor& filter_descriptor,
const dnn::ConvolutionDescriptor& convolution_descriptor,
dnn::BatchDescriptor* output_batch_descriptor);
bool DoTransformTensor(Stream* stream, const dnn::BatchDescriptor& input_desc,
dnn::DataType input_type,
const DeviceMemoryBase& input_data,
const dnn::BatchDescriptor& output_desc,
dnn::DataType output_type, float scale,
DeviceMemoryBase* output_data) override;
private:
CUDAExecutor* parent_; // Parent executor object. Not owned.
// Provides access to the cuDNN handle.
std::unique_ptr<class CudnnAccess> cudnn_;
template <class T, class U>
port::Status DoBatchNormalizationForwardImpl(
Stream* stream, dnn::DataType input_data_type,
dnn::DataType scale_data_type, const DeviceMemory<T>& x,
const DeviceMemory<U>& scale, const DeviceMemory<U>& offset,
const DeviceMemory<U>& estimated_mean,
const DeviceMemory<U>& estimated_variance,
const dnn::BatchDescriptor& x_desc,
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
DeviceMemory<T>* y, DeviceMemory<U>* batch_mean,
DeviceMemory<U>* batch_var, DeviceMemory<U>* saved_mean,
DeviceMemory<U>* saved_inv_var, bool is_training,
std::function<const DeviceMemory<U>&()> var_to_inv_var,
std::function<void()> inv_var_to_var);
template <class T, class U>
port::Status DoBatchNormalizationBackwardImpl(
Stream* stream, int cudnn_input_type, int cudnn_scale_type,
const DeviceMemory<T>& y_backprop, const DeviceMemory<T>& x,
const DeviceMemory<U>& scale, const DeviceMemory<U>& mean,
const DeviceMemory<U>& inv_var, const dnn::BatchDescriptor& x_desc,
const dnn::BatchDescriptor& scale_offset_desc, const double epsilon,
DeviceMemory<T>* x_backprop, DeviceMemory<U>* scale_backprop,
DeviceMemory<U>* offset_backprop);
template <class T>
port::Status DoConvolveImpl(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<T>& input_data,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<T>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<T>* output_data, ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result);
template <typename AccumulatorType, typename ElementType, typename BiasType,
typename ScaleType>
port::Status DoFusedConvolveImpl(
Stream* stream, const dnn::BatchDescriptor& conv_input_descriptor,
const DeviceMemory<ElementType>& conv_input_data,
ScaleType conv_input_scale,
const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<ElementType>& filter_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const DeviceMemory<ElementType>& side_input_data,
ScaleType side_input_scale, const dnn::BatchDescriptor& bias_descriptor,
const DeviceMemory<BiasType>& biases, dnn::ActivationMode activation_mode,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<ElementType>* output_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result);
template <class T>
port::Status DoConvolveBackwardDataImpl(
Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
const DeviceMemory<T>& filter_data,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<T> backward_output_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::BatchDescriptor& input_descriptor,
DeviceMemory<T>* backward_input_data, ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result);
template <class T>
port::Status DoConvolveBackwardFilterImpl(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<T>& input_data,
const dnn::BatchDescriptor& output_descriptor,
DeviceMemory<T> backward_output_data,
const dnn::ConvolutionDescriptor& convolution_descriptor,
const dnn::FilterDescriptor& filter_descriptor,
DeviceMemory<T>* backward_filter_data,
ScratchAllocator* scratch_allocator,
const dnn::AlgorithmConfig& algorithm_config,
dnn::ProfileResult* output_profile_result);
template <class T>
port::Status DoConvolveBackwardBiasImpl(
Stream* stream, const dnn::BatchDescriptor& input_descriptor,
const DeviceMemory<T>& input_data,
const dnn::BatchDescriptor& bias_descriptor,
DeviceMemory<T>* backward_bias_data);
template <class T>
port::Status DoRnnForwardImpl(
Stream* stream, const CudnnRnnDescriptor& rnn_desc,
const CudnnRnnSequenceTensorDescriptor& input_desc,
const DeviceMemory<T>& input_data,
const CudnnRnnStateTensorDescriptor& input_h_desc,
const DeviceMemory<T>& input_h_data,
const CudnnRnnStateTensorDescriptor& input_c_desc,
const DeviceMemory<T>& input_c_data, const DeviceMemory<T>& params,
const CudnnRnnSequenceTensorDescriptor& output_desc,
DeviceMemory<T>* output_data,
const CudnnRnnStateTensorDescriptor& output_h_desc,
DeviceMemory<T>* output_h_data,
const CudnnRnnStateTensorDescriptor& output_c_desc,
DeviceMemory<T>* output_c_data, bool is_training,
ScratchAllocator* reserve_space_allocator,
ScratchAllocator* workspace_allocator,
dnn::ProfileResult* output_profile_result);
template <class T>
port::Status DoRnnBackwardImpl(
Stream* stream, const CudnnRnnDescriptor& rnn_desc,
const CudnnRnnSequenceTensorDescriptor& input_desc,
const DeviceMemory<T>& input_data,
const CudnnRnnStateTensorDescriptor& input_h_desc,
const DeviceMemory<T>& input_h_data,
const CudnnRnnStateTensorDescriptor& input_c_desc,
const DeviceMemory<T>& input_c_data, const DeviceMemory<T>& params,
const CudnnRnnSequenceTensorDescriptor& output_desc,
const DeviceMemory<T>& output_data,
const CudnnRnnStateTensorDescriptor& output_h_desc,
const DeviceMemory<T>& output_h_data,
const CudnnRnnStateTensorDescriptor& output_c_desc,
const DeviceMemory<T>& output_c_data,
const DeviceMemory<T>& output_backprop_data,
const DeviceMemory<T>& output_h_backprop_data,
const DeviceMemory<T>& output_c_backprop_data,
DeviceMemory<T>* input_backprop_data,
DeviceMemory<T>* input_h_backprop_data,
DeviceMemory<T>* input_c_backprop_data,
DeviceMemory<T>* params_backprop_data,
DeviceMemory<uint8>* reserve_space_data,
ScratchAllocator* workspace_allocator,
dnn::ProfileResult* output_profile_result);
SE_DISALLOW_COPY_AND_ASSIGN(CudnnSupport);
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DNN_H_
@@ -0,0 +1,520 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// CUDA userspace driver library wrapper functionality.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_H_
#include <stddef.h>
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/device_options.h"
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "cuda/include/cuda.h"
namespace stream_executor {
namespace cuda {
// Identifies the memory space where an allocation resides. See
// CUDADriver::GetPointerMemorySpace().
enum class MemorySpace { kHost, kDevice };
// Returns a casual string, such as "host" for the provided memory space.
string MemorySpaceString(MemorySpace memory_space);
class CudaContext;
// CUDADriver contains wrappers for calls to the userspace library driver. It's
// useful to isolate these calls and put basic wrappers around them to separate
// userspace library driver behaviors from the rest of the program.
//
// At the moment it's simply used as a namespace.
//
// The calls log any specific errors internally and return whether the operation
// was successful to the caller.
//
// The order of parameters is generally kept symmetric with the underlying CUDA
// driver API.
//
// Links on functions are to specific documentation under
// http://docs.nvidia.com/cuda/cuda-driver-api/
//
// Thread safety: these functions should not be used from signal handlers.
class CUDADriver {
public:
// Wraps a call to cuInit with logging to help indicate what has gone wrong in
// the case of failure. Safe to call multiple times; will be fast on all calls
// after the first.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__INITIALIZE.html#group__CUDA__INITIALIZE_1g0a2f1517e1bd8502c7194c3a8c134bc3
static port::Status Init();
// Returns the device associated with the given context.
// device is an outparam owned by the caller, must not be null.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g4e84b109eba36cdaaade167f34ae881e
static port::StatusOr<CUdevice> DeviceFromContext(CudaContext* context);
// Creates a new CUDA stream associated with the given context via
// cuStreamCreate.
// stream is an outparam owned by the caller, must not be null.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1ga581f0c5833e21ded8b5a56594e243f4
static bool CreateStream(CudaContext* context, CUstream *stream);
// Destroys a CUDA stream associated with the given context.
// stream is owned by the caller, must not be null, and *stream is set to null
// if the stream is successfully destroyed.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g244c8833de4596bcd31a06cdf21ee758
static void DestroyStream(CudaContext* context, CUstream *stream);
// CUDA events can explicitly disable event TSC retrieval for some presumed
// performance improvement if timing is unnecessary.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g450687e75f3ff992fe01662a43d9d3db
enum class EventFlags { kDefault, kDisableTiming };
// Creates a new event associated with the given context.
// result is an outparam owned by the caller and must not be null.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g450687e75f3ff992fe01662a43d9d3db
static port::Status CreateEvent(CudaContext* context, CUevent *result,
EventFlags flags);
// Destroys *event and turns it into a nullptr. event may not be null, but
// *event may be, via cuEventDestroy
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g593ec73a8ec5a5fc031311d3e4dca1ef
static port::Status DestroyEvent(CudaContext* context, CUevent *event);
// Allocates a GPU memory space of size bytes associated with the given
// context via cuMemAlloc.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gb82d2a09844a58dd9e744dc31e8aa467
static void *DeviceAllocate(CudaContext* context, uint64 bytes);
// Deallocates a GPU memory space of size bytes associated with the given
// context via cuMemFree.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g89b3f154e17cc89b6eea277dbdf5c93a
static void DeviceDeallocate(CudaContext* context, void *location);
// Allocates a unified memory space of size bytes associated with the given
// context via cuMemAllocManaged.
// https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gb347ded34dc326af404aa02af5388a32
static void* UnifiedMemoryAllocate(CudaContext* context, uint64 bytes);
// Deallocates a unified memory space of size bytes associated with the given
// context via cuMemFree.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g89b3f154e17cc89b6eea277dbdf5c93a
static void UnifiedMemoryDeallocate(CudaContext* context, void* location);
// Allocates page-locked and CUDA-registered memory on the host via
// cuMemAllocHost.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gdd8311286d2c2691605362c689bc64e0
static void *HostAllocate(CudaContext* context, uint64 bytes);
// Deallocates a location created by HostAllocate, via cuMemFreeHost.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g62e0fdbe181dab6b1c90fa1a51c7b92c
static void HostDeallocate(CudaContext* context, void *location);
// Registers a memory region at location of size bytes via cuMemHostRegister.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gf0a9fe11544326dabd743b7aa6b54223
static bool HostRegister(CudaContext* context, void *location, uint64 bytes);
// Unregisters a memory region that was previously registered at location via
// cuMemHostUnregister.
//
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g63f450c8125359be87b7623b1c0b2a14
//
// TODO(leary) verify an error will be returned if the location wasn't
// previously registered.
static bool HostUnregister(CudaContext* context, void *location);
// Given a device ordinal, returns a device handle into the device outparam,
// which must not be null.
//
// N.B. these device handles do not have a corresponding destroy function in
// the CUDA driver API.
static port::Status GetDevice(int device_ordinal, CUdevice *device);
// Given a device handle, returns the name reported by the driver for the
// device.
static bool GetDeviceName(CUdevice device, string *name_out);
// Given a device to create a context for, returns a context handle into the
// context outparam, which must not be null.
//
// N.B. CUDA contexts are weird. They are implicitly associated with the
// calling thread. Current documentation on contexts and their influence on
// userspace processes is given here:
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g65dc0012348bc84810e2103a40d8e2cf
static port::Status CreateContext(CUdevice device,
const DeviceOptions& device_options,
CudaContext** context);
// Destroys the provided context via cuCtxDestroy.
// Don't do this while clients could still be using the context, per the docs
// bad things will happen.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g27a365aebb0eb548166309f58a1e8b8e
static void DestroyContext(CudaContext* context);
// Queries the runtime for the specified attribute of the specified function.
// cuFuncGetAttribute (the underlying CUDA driver API routine) only operates
// in terms of integer-sized values, so there's no potential for overrun (as
// of CUDA 5.5).
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1g5e92a1b0d8d1b82cb00dcfb2de15961b
static bool FuncGetAttribute(CUfunction_attribute attribute,
CUfunction function, int *attribute_value);
// Sets the preferred cache configuration for the specified function.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1g40f8c11e81def95dc0072a375f965681
static bool FuncSetCacheConfig(CUfunction function,
CUfunc_cache cache_config);
// Gets the preferred shared memory bank configuration for the specified
// CONTEXT (not function!), either default or four- or eight-byte bank size.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g17153a1b8b8c756f7ab8505686a4ad74
static port::StatusOr<CUsharedconfig> ContextGetSharedMemConfig(
CudaContext* context);
// Sets the preferred shared memory bank configuration for the specified
// CONTEXT (not function!), either default or four- or eight-byte bank size.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g2574235fa643f8f251bf7bc28fac3692
static port::Status ContextSetSharedMemConfig(
CudaContext* context, CUsharedconfig shared_mem_config);
// Launches a CUDA kernel via cuLaunchKernel.
// TODO(leary) describe the structure of kernel_params and extra in a readable
// way.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EXEC.html#group__CUDA__EXEC_1gb8f3dc3031b40da29d5f9a7139e52e15
static bool LaunchKernel(CudaContext* context, CUfunction function,
unsigned int grid_dim_x, unsigned int grid_dim_y,
unsigned int grid_dim_z, unsigned int block_dim_x,
unsigned int block_dim_y, unsigned int block_dim_z,
unsigned int shared_mem_bytes, CUstream stream,
void **kernel_params, void **extra);
// Loads ptx_contents with the CUDA driver's PTX JIT and stores the resulting
// handle in "module". Any error logs that are produced are logged internally.
static bool LoadPtx(CudaContext* context, const char *ptx_contents,
CUmodule *module);
// Loads cubin_bytes with the CUDA driver's blob loading interface and stores
// the resulting handle in "module".
static port::Status LoadCubin(CudaContext* context, const char *cubin_bytes,
CUmodule *module);
// Retrieves a named kernel from a loaded module, and places the resulting
// handle into function (outparam) on success. Neither kernel_name nor
// function may be null. No ownership is taken of kernel_name.
static bool GetModuleFunction(CudaContext* context, CUmodule module,
const char *kernel_name, CUfunction *function);
// Retrieves a named global/constant symbol from a loaded module, and returns
// a device pointer and size of the symbol on success. symbol_name may not be
// null. At least one of dptr or bytes should not be null. No ownership is
// taken of symbol_name.
static bool GetModuleSymbol(CudaContext* context, CUmodule module,
const char *symbol_name, CUdeviceptr *dptr,
size_t *bytes);
// Unloads module from the current context via cuModuleUnload.
// TODO(leary) the documentation doesn't say what kind of disasters happen
// if you try to unload a module while its CUfunctions are in use.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MODULE.html#group__CUDA__MODULE_1g8ea3d716524369de3763104ced4ea57b
static void UnloadModule(CudaContext* context, CUmodule module);
// Performs a synchronous memset of the device memory segment via cuMemsetD8.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g6e582bf866e9e2fb014297bfaf354d7b
static bool SynchronousMemsetUint8(CudaContext* context, CUdeviceptr location,
uint8 value, size_t size);
// Performs a synchronous memset of the device memory segment via cuMemsetD32.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g983e8d8759acd1b64326317481fbf132
static bool SynchronousMemsetUint32(CudaContext* context,
CUdeviceptr location, uint32 value,
size_t uint32_count);
// Performs an asynchronous memset of the device memory segment via
// cuMemsetD8Async.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1gaef08a7ccd61112f94e82f2b30d43627
static bool AsynchronousMemsetUint8(CudaContext* context, CUdeviceptr location,
uint8 value, size_t uint32_count,
CUstream stream);
// Performs an asynchronous memset of the device memory segment via
// cuMemsetD32Async.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g58229da5d30f1c0cdf667b320ec2c0f5
static bool AsynchronousMemsetUint32(CudaContext* context,
CUdeviceptr location, uint32 value,
size_t uint32_count, CUstream stream);
// -- Synchronous memcopies.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g4d32266788c440b0220b1a9ba5795169
static port::Status SynchronousMemcpyD2H(CudaContext* context, void* host_dst,
CUdeviceptr gpu_src, uint64 size);
static port::Status SynchronousMemcpyH2D(CudaContext* context,
CUdeviceptr gpu_dst,
const void* host_src, uint64 size);
static port::Status SynchronousMemcpyD2D(CudaContext* context,
CUdeviceptr gpu_dst,
CUdeviceptr gpu_src, uint64 size);
// -- Asynchronous memcopies.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g56f30236c7c5247f8e061b59d3268362
static bool AsynchronousMemcpyD2H(CudaContext* context, void *host_dst,
CUdeviceptr gpu_src, uint64 size,
CUstream stream);
static bool AsynchronousMemcpyH2D(CudaContext* context, CUdeviceptr gpu_dst,
const void *host_src, uint64 size,
CUstream stream);
static bool AsynchronousMemcpyD2D(CudaContext* context, CUdeviceptr gpu_dst,
CUdeviceptr gpu_src, uint64 size,
CUstream stream);
// The CUDA stream callback type signature.
// The data passed to AddStreamCallback is subsequently passed to this
// callback when it fires.
//
// Some notable things:
// * Callbacks must not make any CUDA API calls.
// * Callbacks from independent streams execute in an undefined order and may
// be serialized.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g613d97a277d7640f4cb1c03bd51c2483
typedef void (*StreamCallback)(CUstream stream, CUresult status, void *data);
// Enqueues a callback operation into stream.
// See StreamCallback above and the NVIDIA documentation for additional
// details.
static bool AddStreamCallback(CudaContext* context, CUstream stream,
StreamCallback callback, void *data);
// Causes stream to wait for event to trigger before proceeding via
// cuStreamWaitEvent.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#axzz334nAXAhM
static bool WaitStreamOnEvent(CudaContext* context, CUstream stream,
CUevent event);
// Blocks the calling thread until the operations enqueued onto stream have
// been completed, via cuStreamSynchronize.
//
// TODO(leary) if a pathological thread enqueues operations onto the stream
// while another thread blocks like this, can you wind up waiting an unbounded
// amount of time?
//
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__STREAM.html#group__CUDA__STREAM_1g15e49dd91ec15991eb7c0a741beb7dad
static port::Status SynchronizeStream(CudaContext* context, CUstream stream);
// Blocks the calling thread until the operations associated with the context
// have been completed, via cuCtxSynchronize.
//
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1g7a54725f28d34b8c6299f0c6ca579616
static bool SynchronizeContext(CudaContext* context);
// Returns true if all stream tasks have completed at time of the call. Note
// the potential for races around this call (if another thread adds work to
// the stream immediately after this returns).
static bool IsStreamIdle(CudaContext* context, CUstream stream);
// Returns whether code in the from context can access memory in the to
// context via cuDeviceCanAccessPeer.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__PEER__ACCESS.html#group__CUDA__PEER__ACCESS_1g496bdaae1f632ebfb695b99d2c40f19e
static bool CanEnablePeerAccess(CudaContext* from, CudaContext* to);
// Enables peer access per CanEnablePeerAccess, via cuCtxEnablePeerAccess.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__PEER__ACCESS.html#group__CUDA__PEER__ACCESS_1g0889ec6728e61c05ed359551d67b3f5a
static port::Status EnablePeerAccess(CudaContext* from, CudaContext* to);
// Returns the elapsed milliseconds between start and stop via
// cuEventElapsedTime.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1gdfb1178807353bbcaa9e245da497cf97
static bool GetEventElapsedTime(CudaContext* context,
float *elapsed_milliseconds, CUevent start,
CUevent stop);
// Records that an event occurred when execution reaches the current point in
// thestream via cuEventRecord.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g95424d3be52c4eb95d83861b70fb89d1
static port::Status RecordEvent(CudaContext* context, CUevent event,
CUstream stream);
// Polls (without blocking) to determine the status of an event - pending or
// complete (or an error status).
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__EVENT.html#group__CUDA__EVENT_1g6f0704d755066b0ee705749ae911deef
static port::StatusOr<CUresult> QueryEvent(CudaContext* context,
CUevent event);
// -- Pointer-specific calls.
// Returns the context in which pointer was allocated or registered.
static port::StatusOr<CudaContext*> GetPointerContext(CUdeviceptr pointer);
// Returns the device associated with the context from GetPointerContext().
static port::StatusOr<CUdevice> GetPointerDevice(CUdeviceptr pointer);
// Returns the memory space addressed by pointer.
static port::StatusOr<MemorySpace> GetPointerMemorySpace(CUdeviceptr pointer);
// Returns the base address and size of the device pointer dptr.
static port::Status GetPointerAddressRange(CUdeviceptr dptr,
CUdeviceptr *base, size_t *size);
// -- Device-specific calls.
// Returns the compute capability for the device; i.e (3, 5).
// This is currently done via the deprecated device API.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE__DEPRECATED.html#group__CUDA__DEVICE__DEPRECATED_1ge2091bbac7e1fb18c2821612115607ea
static port::Status GetComputeCapability(int *cc_major, int *cc_minor,
CUdevice device);
// Returns the number of multiprocessors on the device (note that the device
// may be multi-GPU-per-board).
static port::StatusOr<int> GetMultiprocessorCount(CUdevice device);
// Returns the limit on number of threads that can be resident in a single
// multiprocessor.
static port::StatusOr<int64> GetMaxThreadsPerMultiprocessor(CUdevice device);
// Returns the limit on number of threads which may be resident for a single
// block (cooperative thread array).
static port::StatusOr<int64> GetMaxThreadsPerBlock(CUdevice device);
// Returns the amount of shared memory available on a single GPU core (i.e.
// SM on NVIDIA devices).
static port::StatusOr<int64> GetMaxSharedMemoryPerCore(CUdevice device);
// Returns the amount of shared memory available for a single block
// (cooperative thread array).
static port::StatusOr<int64> GetMaxSharedMemoryPerBlock(CUdevice device);
// Returns the maximum supported number of registers per block.
static port::StatusOr<int64> GetMaxRegistersPerBlock(CUdevice device);
// Returns the number of threads per warp.
static port::StatusOr<int64> GetThreadsPerWarp(CUdevice device);
// Queries the grid limits for device with cuDeviceGetAttribute calls.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
static bool GetGridLimits(int *x, int *y, int *z, CUdevice device);
// Returns a grab-bag of device properties in a caller-owned device_properties
// structure for device_ordinal via cuDeviceGetProperties.
//
// This call is deprecated in the NVIDIA driver API; its replacement is
// GetDeviceAttribute
//
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE__DEPRECATED.html#group__CUDA__DEVICE__DEPRECATED_1g65a5b4e25186bd257df80b98c98cffe6
static bool GetDeviceProperties(CUdevprop *device_properties,
int device_ordinal);
// Gets a specific integer-valued property about the given device.
//
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
static port::StatusOr<int> GetDeviceAttribute(CUdevice_attribute attribute,
CUdevice device);
// Returns whether ECC is enabled for the given CUdevice via
// cuDeviceGetattribute with CU_DEVICE_ATTRIBUTE_ECC_ENABLED.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
static bool IsEccEnabled(CUdevice device, bool *result);
// Returns the total amount of memory available for allocation by the CUDA
// context, in bytes, via cuDeviceTotalMem.
static bool GetDeviceTotalMemory(CUdevice device, uint64 *result);
// Returns the free amount of memory and total amount of memory, as reported
// by cuMemGetInfo.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g808f555540d0143a331cc42aa98835c0
static bool GetDeviceMemoryInfo(CudaContext* context, int64* free,
int64* total);
// Returns a PCI bus id string for the device.
// [domain]:[bus]:[device].[function]
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g85295e7d9745ab8f0aa80dd1e172acfc
static string GetPCIBusID(CUdevice device);
// -- Context- and device-independent calls.
// Returns the number of visible CUDA device via cuDeviceGetCount.
// This should correspond to the set of device ordinals available.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g52b5ce05cb8c5fb6831b2c0ff2887c74
static int GetDeviceCount();
// Returns the driver version number via cuDriverGetVersion.
// This is, surprisingly, NOT the actual driver version (e.g. 331.79) but,
// instead, the CUDA toolkit release number that this driver is compatible
// with; e.g. 6000 (for a CUDA 6.0 compatible driver) or 6050 (for a CUDA 6.5
// compatible driver).
//
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__VERSION.html#group__CUDA__VERSION_1g8b7a10395392e049006e61bcdc8ebe71
static bool GetDriverVersion(int *driver_version);
// -- Other calls
// Returns the maximum number of blocks (per multiprocessor) occupied by the
// specified kernel/CUfunction when launched with the specified parameters.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__OCCUPANCY.html#group__CUDA__OCCUPANCY_1gcc6e1094d05cba2cee17fe33ddd04a98
static port::StatusOr<int> GetMaxOccupiedBlocksPerCore(
CudaContext* context, CUfunction kernel, int threads_per_block,
size_t dynamic_shared_memory_bytes);
// Returns the current context set in CUDA. This is done by calling the cuda
// driver (e.g., this value is not our cached view of the current context).
static CUcontext CurrentContextOrDie();
// Seam for injecting an error at CUDA initialization time for testing
// purposes.
static bool driver_inject_init_error_;
};
// Ensures a context is activated within a scope.
class ScopedActivateContext {
public:
// Activates the context via cuCtxSetCurrent, if it is not the currently
// active context (a la cuCtxGetCurrent). Note the alternative push/pop
// mechanism is said by NVIDIA to be relatively slow and deprecated.
// http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__CTX.html#group__CUDA__CTX_1gbe562ee6258b4fcc272ca6478ca2a2f7
explicit ScopedActivateContext(CudaContext* context);
// Checks that the context has remained activated for the duration of the
// scope.
~ScopedActivateContext();
private:
CudaContext* to_restore_ = nullptr;
};
// CudaContext wraps a cuda CUcontext handle, and includes a unique id. The
// unique id is positive, and ids are not repeated within the process.
class CudaContext {
public:
CudaContext(CUcontext context, int64 id) : context_(context), id_(id) { }
CUcontext context() const { return context_; }
int64 id() const { return id_; }
// Disallow copying and moving.
CudaContext(CudaContext&&) = delete;
CudaContext(const CudaContext&) = delete;
CudaContext& operator=(CudaContext&&) = delete;
CudaContext& operator=(const CudaContext&) = delete;
private:
CUcontext const context_;
const int64 id_;
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_H_
@@ -0,0 +1,62 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_EVENT_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_EVENT_H_
#include "tensorflow/stream_executor/cuda/cuda_driver.h"
#include "tensorflow/stream_executor/cuda/cuda_stream.h"
#include "tensorflow/stream_executor/event.h"
#include "tensorflow/stream_executor/lib/status.h"
namespace stream_executor {
namespace cuda {
// CUDAEvent wraps a CUevent in the platform-independent EventInterface
// interface.
class CUDAEvent : public internal::EventInterface {
public:
explicit CUDAEvent(CUDAExecutor* parent);
~CUDAEvent() override;
// Populates the CUDA-platform-specific elements of this object.
port::Status Init();
// Deallocates any platform-specific elements of this object. This is broken
// out (not part of the destructor) to allow for error reporting.
port::Status Destroy();
// Inserts the event at the current position into the specified stream.
port::Status Record(CUDAStream* stream);
// Polls the CUDA platform for the event's current status.
Event::Status PollForStatus();
// The underlying CUDA event element.
const CUevent& cuda_event();
private:
// The Executor used to which this object and CUevent are bound.
CUDAExecutor* parent_;
// The underlying CUDA event element.
CUevent cuda_event_;
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_EVENT_H_
@@ -0,0 +1,137 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// CUDA-specific support for FFT functionality -- this wraps the cuFFT library
// capabilities, and is only included into CUDA implementation code -- it will
// not introduce cuda headers into other code.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_FFT_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_FFT_H_
#include "cuda/include/cufft.h"
#include "tensorflow/stream_executor/fft.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/plugin_registry.h"
#include "tensorflow/stream_executor/scratch_allocator.h"
namespace stream_executor {
class Stream;
namespace cuda {
class CUDAExecutor;
// Opaque and unique indentifier for the cuFFT plugin.
extern const PluginId kCuFftPlugin;
// CUDAFftPlan uses deferred initialization. Only a single call of
// Initialize() is allowed to properly create cufft plan and set member
// variable is_initialized_ to true. Newly added interface that uses member
// variables should first check is_initialized_ to make sure that the values of
// member variables are valid.
class CUDAFftPlan : public fft::Plan {
public:
CUDAFftPlan()
: parent_(nullptr),
plan_(-1),
fft_type_(fft::Type::kInvalid),
scratch_(nullptr),
scratch_size_bytes_(0),
is_initialized_(false) {}
~CUDAFftPlan() override;
// Get FFT direction in cuFFT based on FFT type.
int GetFftDirection() const;
cufftHandle GetPlan() const {
if (IsInitialized()) {
return plan_;
} else {
LOG(FATAL) << "Try to get cufftHandle value before initialization.";
}
}
// Initialize function for batched plan
port::Status Initialize(CUDAExecutor *parent, Stream *stream, int rank,
uint64 *elem_count, uint64 *input_embed,
uint64 input_stride, uint64 input_distance,
uint64 *output_embed, uint64 output_stride,
uint64 output_distance, fft::Type type,
int batch_count, ScratchAllocator *scratch_allocator);
// Initialize function for 1d,2d, and 3d plan
port::Status Initialize(CUDAExecutor *parent, Stream *stream, int rank,
uint64 *elem_count, fft::Type type,
ScratchAllocator *scratch_allocator);
port::Status UpdateScratchAllocator(Stream *stream,
ScratchAllocator *scratch_allocator);
protected:
bool IsInitialized() const { return is_initialized_; }
private:
CUDAExecutor *parent_;
cufftHandle plan_;
fft::Type fft_type_;
DeviceMemory<uint8> scratch_;
size_t scratch_size_bytes_;
bool is_initialized_;
};
// FFT support for CUDA platform via cuFFT library.
//
// This satisfies the platform-agnostic FftSupport interface.
//
// Note that the cuFFT handle that this encapsulates is implicitly tied to the
// context (and, as a result, the device) that the parent CUDAExecutor is tied
// to. This simply happens as an artifact of creating the cuFFT handle when a
// CUDA context is active.
//
// Thread-safe. The CUDA context associated with all operations is the CUDA
// context of parent_, so all context is explicit.
class CUDAFft : public fft::FftSupport {
public:
explicit CUDAFft(CUDAExecutor *parent) : parent_(parent) {}
~CUDAFft() override {}
TENSORFLOW_STREAM_EXECUTOR_GPU_FFT_SUPPORT_OVERRIDES
private:
CUDAExecutor *parent_;
// Two helper functions that execute dynload::cufftExec?2?.
// This is for complex to complex FFT, when the direction is required.
template <typename FuncT, typename InputT, typename OutputT>
bool DoFftWithDirectionInternal(Stream *stream, fft::Plan *plan,
FuncT cufft_exec,
const DeviceMemory<InputT> &input,
DeviceMemory<OutputT> *output);
// This is for complex to real or real to complex FFT, when the direction
// is implied.
template <typename FuncT, typename InputT, typename OutputT>
bool DoFftInternal(Stream *stream, fft::Plan *plan, FuncT cufft_exec,
const DeviceMemory<InputT> &input,
DeviceMemory<OutputT> *output);
SE_DISALLOW_COPY_AND_ASSIGN(CUDAFft);
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_FFT_H_
@@ -0,0 +1,299 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// The CUDA implementation of the StreamExecutorInterface functionality.
// CUDA inclusions are ideally confined to this implementation file.
//
// The notions from the StreamExecutor basically correspond to the CUDA streams
// programming model provided by the libcuda.so driver APIs, so we don't have
// to do much more than wrap the calls to the libraries appropriately.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_GPU_EXECUTOR_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_GPU_EXECUTOR_H_
#include <set>
#include <unordered_map>
#include "tensorflow/stream_executor/cuda/cuda_kernel.h"
#include "tensorflow/stream_executor/event.h"
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/platform.h"
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/platform/thread_annotations.h"
#include "tensorflow/stream_executor/stream_executor_internal.h"
namespace stream_executor {
namespace cuda {
// CUDA-platform implementation of the platform-agnostic
// StreamExecutorInferface.
class CUDAExecutor : public internal::StreamExecutorInterface {
public:
// sub_platform indicates the subplatform used in this executor; it must
// be a CUDA type.
explicit CUDAExecutor(const PluginConfig &plugin_config)
: device_(0),
context_(nullptr),
device_ordinal_(0),
cc_major_(0),
cc_minor_(0),
plugin_config_(plugin_config) {}
// See the corresponding StreamExecutor methods for method comments on the
// following overrides.
~CUDAExecutor() override;
port::Status Init(int device_ordinal, DeviceOptions device_options) override;
bool GetKernel(const MultiKernelLoaderSpec &spec,
KernelBase *kernel) override;
void UnloadKernel(const KernelBase *kernel) override;
bool LoadModule(const MultiModuleLoaderSpec &spec,
ModuleHandle *module_handle) override;
bool UnloadModule(ModuleHandle module_handle) override;
bool Launch(Stream *stream, const ThreadDim &thread_dims,
const BlockDim &block_dims, const KernelBase &k,
const KernelArgsArrayBase &args) override;
void *Allocate(uint64 size) override;
void *AllocateSubBuffer(DeviceMemoryBase *mem, uint64 offset_bytes,
uint64 size_bytes) override;
void Deallocate(DeviceMemoryBase *mem) override;
void *UnifiedMemoryAllocate(uint64 size) override {
return CUDADriver::UnifiedMemoryAllocate(context_, size);
}
void UnifiedMemoryDeallocate(void *location) override {
return CUDADriver::UnifiedMemoryDeallocate(context_, location);
}
// CUDA allocation/registration functions are necessary because the driver
// internally sets up buffers for DMA operations (and page locks them).
// There's no external interface for us to otherwise control these DMA
// settings.
void *HostMemoryAllocate(uint64 size) override {
return CUDADriver::HostAllocate(context_, size);
}
void HostMemoryDeallocate(void *location) override {
return CUDADriver::HostDeallocate(context_, location);
}
bool HostMemoryRegister(void *location, uint64 size) override;
bool HostMemoryUnregister(void *location) override;
bool SynchronizeAllActivity() override;
bool SynchronousMemZero(DeviceMemoryBase *location, uint64 size) override;
bool SynchronousMemSet(DeviceMemoryBase *location, int value,
uint64 size) override;
port::Status SynchronousMemcpy(DeviceMemoryBase *gpu_dst,
const void *host_src, uint64 size) override;
port::Status SynchronousMemcpy(void *host_dst,
const DeviceMemoryBase &gpu_src,
uint64 size) override;
port::Status SynchronousMemcpyDeviceToDevice(DeviceMemoryBase *gpu_dst,
const DeviceMemoryBase &gpu_src,
uint64 size) override;
bool MemZero(Stream *stream, DeviceMemoryBase *location,
uint64 size) override;
bool Memset(Stream *stream, DeviceMemoryBase *location, uint8 pattern,
uint64 size) override;
bool Memset32(Stream *stream, DeviceMemoryBase *location, uint32 pattern,
uint64 size) override;
bool Memcpy(Stream *stream, void *host_dst, const DeviceMemoryBase &gpu_src,
uint64 size) override;
bool Memcpy(Stream *stream, DeviceMemoryBase *gpu_dst, const void *host_src,
uint64 size) override;
bool MemcpyDeviceToDevice(Stream *stream, DeviceMemoryBase *gpu_dst,
const DeviceMemoryBase &gpu_src,
uint64 size) override;
bool HostCallback(Stream *stream, std::function<void()> callback) override;
bool AllocateStream(Stream *stream) override;
void DeallocateStream(Stream *stream) override;
bool CreateStreamDependency(Stream *dependent, Stream *other) override;
bool AllocateTimer(Timer *timer) override;
void DeallocateTimer(Timer *timer) override;
bool StartTimer(Stream *stream, Timer *timer) override;
bool StopTimer(Stream *stream, Timer *timer) override;
port::Status AllocateEvent(Event *event) override;
port::Status DeallocateEvent(Event *event) override;
port::Status RecordEvent(Stream *stream, Event *event) override;
port::Status WaitForEvent(Stream *stream, Event *event) override;
Event::Status PollForEventStatus(Event *event) override;
port::Status BlockHostUntilDone(Stream *stream) override;
int PlatformDeviceCount() override { return CUDADriver::GetDeviceCount(); }
port::Status EnablePeerAccessTo(StreamExecutorInterface *other) override;
bool CanEnablePeerAccessTo(StreamExecutorInterface *other) override;
SharedMemoryConfig GetDeviceSharedMemoryConfig() override;
port::Status SetDeviceSharedMemoryConfig(SharedMemoryConfig config) override;
bool DeviceMemoryUsage(int64 *free, int64 *total) const override;
// Search for the symbol and returns a device pointer and size.
// Returns false if symbol does not exist.
bool GetSymbol(const string &symbol_name, ModuleHandle module_handle,
void **mem, size_t *bytes) override;
DeviceDescription *PopulateDeviceDescription() const override;
// Populates the block_dim_limit by querying the device driver API. If an
// error occurs at any point while asking the driver for block dim limits, it
// will be only partially populated as a result, and an error will be logged.
bool FillBlockDimLimit(BlockDim *block_dim_limit) const;
bool SupportsBlas() const override;
blas::BlasSupport *CreateBlas() override;
bool SupportsFft() const override;
fft::FftSupport *CreateFft() override;
bool SupportsRng() const override;
rng::RngSupport *CreateRng() override;
bool SupportsDnn() const override;
dnn::DnnSupport *CreateDnn() override;
std::unique_ptr<internal::EventInterface> CreateEventImplementation()
override;
std::unique_ptr<internal::KernelInterface> CreateKernelImplementation()
override;
std::unique_ptr<internal::StreamInterface> GetStreamImplementation() override;
std::unique_ptr<internal::TimerInterface> GetTimerImplementation() override;
void *GpuContextHack() override;
CudaContext* cuda_context();
private:
// Attempts to find a more specific version of the file indicated by
// filename by looking for compute-capability-specific suffixed versions; i.e.
// looking for "foo.ptx" will check to see if "foo.ptx.cc30.ptx" is present if
// we're on a compute capability 3.0 machine.
bool FindOnDiskForComputeCapability(port::StringPiece filename,
port::StringPiece canonical_suffix,
string *found_filename) const;
// Host callback landing routine invoked by CUDA.
// data: User-provided callback provided to HostCallback() above, captured
// as a std::function<void()>. Allocated/initialized inside
// HostCallback() and owned and deleted by this call.
static void InternalHostCallback(CUstream stream, CUresult status,
void *data);
// Collects metadata for the specified kernel.
bool GetKernelMetadata(CUDAKernel *cuda_kernel,
KernelMetadata *kernel_metadata);
// Prints to VLOG(2) information about the kernel's occupancy and how it might
// be improved.
void VlogOccupancyInfo(const KernelBase &kernel, const ThreadDim &thread_dims,
const BlockDim &block_dims);
bool LoadModuleFromCuBin(const char *cubin, CUmodule *module)
EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
// Loads the PTX text `ptx` as a CUDA module. `ptx` must be null terminated.
bool LoadModuleFromPtx(const char *ptx, CUmodule *module)
EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
bool UnloadGpuBinary(const void *gpu_binary)
EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
// Guards the in-memory-module mapping.
mutex in_memory_modules_mu_;
// Kernel -> loaded GPU binary. Many kernels may load the same binary.
std::unordered_map<const KernelBase *, const void *> kernel_to_gpu_binary_
GUARDED_BY(in_memory_modules_mu_);
// GPU binary (PTX or CUBIN) -> {CUDA module, reference count}.
std::unordered_map<const void *, std::pair<CUmodule, uint64>>
gpu_binary_to_module_ GUARDED_BY(in_memory_modules_mu_);
// Guards the launched kernel set.
mutex launched_kernels_mu_;
// Keeps track of the set of launched kernels. Currently used to suppress the
// occupancy check on subsequent launches.
std::set<CUfunction> launched_kernels_ GUARDED_BY(launched_kernels_mu_);
// Handle for the CUDA device being operated on. Immutable
// post-initialization.
CUdevice device_;
// Handle for session with the library/driver. Immutable post-initialization.
CudaContext* context_;
// The device ordinal value that this executor was initialized with; recorded
// for use in getting device metadata. Immutable post-initialization.
int device_ordinal_;
// The major verion of the compute capability for device_.
int cc_major_;
// The minor verion of the compute capability for device_.
int cc_minor_;
// The plugin configuration associated with this instance.
PluginConfig plugin_config_;
SE_DISALLOW_COPY_AND_ASSIGN(CUDAExecutor);
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_GPU_EXECUTOR_H_
@@ -0,0 +1,105 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Common helper functions used for dealing with CUDA API datatypes.
//
// These are typically placed here for use by multiple source components (for
// example, BLAS and executor components).
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_HELPERS_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_HELPERS_H_
#include <stddef.h>
#include <complex>
#include "cuda/include/cuComplex.h"
#include "cuda/include/cuda.h"
namespace stream_executor {
template <typename ElemT>
class DeviceMemory;
namespace cuda {
// Converts a const DeviceMemory reference to its underlying typed pointer in
// CUDA
// device memory.
template <typename T>
const T *CUDAMemory(const DeviceMemory<T> &mem) {
return static_cast<const T *>(mem.opaque());
}
// Converts a (non-const) DeviceMemory pointer reference to its underlying typed
// pointer in CUDA device memory.
template <typename T>
T *CUDAMemoryMutable(DeviceMemory<T> *mem) {
return static_cast<T *>(mem->opaque());
}
static_assert(sizeof(std::complex<float>) == sizeof(cuComplex),
"std::complex<float> and cuComplex should have the same size");
static_assert(offsetof(cuComplex, x) == 0,
"The real part of cuComplex should appear first.");
static_assert(sizeof(std::complex<double>) == sizeof(cuDoubleComplex),
"std::complex<double> and cuDoubleComplex should have the same "
"size");
static_assert(offsetof(cuDoubleComplex, x) == 0,
"The real part of cuDoubleComplex should appear first.");
// Type traits to get CUDA complex types from std::complex<>.
template <typename T>
struct CUDAComplexT {
typedef T type;
};
template <>
struct CUDAComplexT<std::complex<float>> {
typedef cuComplex type;
};
template <>
struct CUDAComplexT<std::complex<double>> {
typedef cuDoubleComplex type;
};
// Converts pointers of std::complex<> to pointers of
// cuComplex/cuDoubleComplex. No type conversion for non-complex types.
template <typename T>
inline const typename CUDAComplexT<T>::type *CUDAComplex(const T *p) {
return reinterpret_cast<const typename CUDAComplexT<T>::type *>(p);
}
template <typename T>
inline typename CUDAComplexT<T>::type *CUDAComplex(T *p) {
return reinterpret_cast<typename CUDAComplexT<T>::type *>(p);
}
// Converts values of std::complex<float/double> to values of
// cuComplex/cuDoubleComplex.
inline cuComplex CUDAComplexValue(std::complex<float> val) {
return {val.real(), val.imag()};
}
inline cuDoubleComplex CUDAComplexValue(std::complex<double> val) {
return {val.real(), val.imag()};
}
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_HELPERS_H_
@@ -0,0 +1,128 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// The CUDA implementation of the StreamExecutorInterface functionality.
// CUDA inclusions are ideally confined to this implementation file.
//
// The notions from the StreamExecutor basically correspond to the CUDA streams
// programming model provided by the libcuda.so driver APIs, so we don't have
// to do much more than wrap the calls to the libraries appropriately.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_KERNEL_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_KERNEL_H_
#include "tensorflow/stream_executor/kernel_cache_config.h"
#include "tensorflow/stream_executor/stream_executor_internal.h"
#include "tensorflow/stream_executor/cuda/cuda_driver.h"
#include "tensorflow/stream_executor/lib/casts.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/platform/logging.h"
#include "cuda/include/cuda.h"
#ifdef PLATFORMS_GPUS_CUDA_DYNAMIC_LIBCUDA_DYNAMIC_LIBCUDA_H_
#error \
"No driver calls in this file, wrap driver functionality in cuda_driver.cc."
#endif
#ifdef __CUDA_RUNTIME_H__
#error \
"CUDA runtime being included into CUDA GPU executor; should be driver only."
#endif
namespace stream_executor {
namespace cuda {
// Wraps a CUfunction to implement the platform-independent KernelInterface.
class CUDAKernel : public internal::KernelInterface {
public:
CUDAKernel() : cuda_function_(nullptr), arity_(0),
preferred_cache_config_(KernelCacheConfig::kNoPreference) {}
// Note that the function is unloaded when the module is unloaded, and the
// module that the function is contained in is owned by the CUDAExecutor.
~CUDAKernel() override {}
// As arity cannot be reflected upon using the CUDA API, the arity is
// explicitly set during the CUDAExecutor::GetKernel initialization process.
void set_arity(unsigned arity) { arity_ = arity; }
unsigned Arity() const override { return arity_; }
// Returns the CUfunction value for passing to the CUDA API.
CUfunction AsCUDAFunctionValue() const {
DCHECK(cuda_function_ != nullptr);
return const_cast<CUfunction>(cuda_function_);
}
// Returns the slot that the CUfunction is stored within for this object,
// for the CUDA API which wants to load into a CUfunction*.
CUfunction *cuda_function_ptr() { return &cuda_function_; }
// CUDA supports setting the preferred cache configuration of a CUfunction
// (more-or-less equivalent to a CUDAKernel). We support this via the below
// functions; users can set a preference, and that is applied when the kernel
// is [lazy-]loaded (in CUDAExecutor::Launch). The alternative would be to
// load the kernel & set the preference when the user calls the setter below;
// either approach is valid.
// Sets the current kernel cache configuration preference.
void SetPreferredCacheConfig(KernelCacheConfig config) override {
preferred_cache_config_ = config;
}
// Returns the current kernel cache configuration preference.
KernelCacheConfig GetPreferredCacheConfig() const override {
return preferred_cache_config_;
}
// Returns the current kernel cache configuration preference as a
// CUfunc_cache.
CUfunc_cache GetCUDACacheConfig() const {
switch (preferred_cache_config_) {
case KernelCacheConfig::kNoPreference:
return CU_FUNC_CACHE_PREFER_NONE;
case KernelCacheConfig::kPreferShared:
return CU_FUNC_CACHE_PREFER_SHARED;
case KernelCacheConfig::kPreferL1:
return CU_FUNC_CACHE_PREFER_L1;
case KernelCacheConfig::kPreferEqual:
return CU_FUNC_CACHE_PREFER_EQUAL;
default:
LOG(FATAL) << "Unknown KernelCacheConfig"
<< static_cast<int32>(preferred_cache_config_);
}
}
private:
CUfunction cuda_function_; // Wrapped CUDA kernel handle.
unsigned arity_; // Number of formal parameters the kernel takes.
// Preferred (but not required) cache configuration for this kernel.
KernelCacheConfig preferred_cache_config_;
};
// Given a platform-independent kernel datatype, returns the (const) internal
// CUDA platform implementation pointer.
inline const CUDAKernel *AsCUDAKernel(const KernelBase *kernel) {
return static_cast<const CUDAKernel *>(kernel->implementation());
}
// Given a platform-independent kernel datatype, returns the (non-const)
// internal CUDA platform implementation pointer.
inline CUDAKernel *AsCUDAKernel(KernelBase *kernel) {
return static_cast<CUDAKernel *>(kernel->implementation());
}
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_KERNEL_H_
@@ -0,0 +1,108 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_H_
#include <memory>
#include "tensorflow/stream_executor/platform/port.h"
#include <vector>
#include "tensorflow/stream_executor/executor_cache.h"
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/multi_platform_manager.h"
#include "tensorflow/stream_executor/platform.h"
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/platform/thread_annotations.h"
#include "tensorflow/stream_executor/stream_executor_internal.h"
#include "tensorflow/stream_executor/stream_executor_pimpl.h"
#include "tensorflow/stream_executor/trace_listener.h"
namespace stream_executor {
namespace cuda {
// Opaque and unique identifier for the CUDA platform plugin.
// This is needed so that plugins can refer to/identify this platform without
// instantiating a CudaPlatform object.
extern const Platform::Id kCudaPlatformId;
// Cuda-specific platform plugin, registered as a singleton value via module
// initializer.
class CudaPlatform : public Platform {
public:
CudaPlatform();
~CudaPlatform() override;
// CudaPlatform-specific functionality
// Returns the number of distinct buses / NUMA nodes on the machine.
int BusCount();
// Returns the bus/NUMA node for the specified device ordinal.
int DeviceToBus(int device_ordinal);
// Returns the lowest-ordinal-number StreamExecutor on the specified bus.
port::StatusOr<StreamExecutor*> FirstExecutorForBus(int bus_ordinal);
// Platform interface implementation:
// Returns the same value as kCudaPlatform above.
Platform::Id id() const override;
// Returns -1 as a sentinel on internal failure (and logs the error).
int VisibleDeviceCount() const override;
const string& Name() const override;
port::StatusOr<StreamExecutor*> ExecutorForDevice(int ordinal) override;
port::StatusOr<StreamExecutor*> ExecutorForDeviceWithPluginConfig(
int ordinal, const PluginConfig& config) override;
port::StatusOr<StreamExecutor*> GetExecutor(
const StreamExecutorConfig& config) override;
port::StatusOr<std::unique_ptr<StreamExecutor>> GetUncachedExecutor(
const StreamExecutorConfig& config) override;
void RegisterTraceListener(std::unique_ptr<TraceListener> listener) override;
void UnregisterTraceListener(TraceListener* listener) override;
private:
// Determines the number of NUMA nodes and the assignment of executor to each.
void InspectNumaNodes();
// This platform's name.
string name_;
// Cache of created executors.
ExecutorCache executor_cache_;
// The smallest NUMA node value for any device managed by this machine
// manager. Used, along with limit_numa_node_, to convert NUMA nodes into bus
// ordinals. The NUMA node space occupied by GPUs is assumed to be dense./
int min_numa_node_;
// Larger than the NUMA node value for any device managed by this machine
// manager.
int limit_numa_node_;
SE_DISALLOW_COPY_AND_ASSIGN(CudaPlatform);
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_H_
@@ -0,0 +1,34 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_ID_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_ID_H_
#include "tensorflow/stream_executor/platform.h"
namespace stream_executor {
namespace cuda {
// Opaque and unique identifier for the cuda platform.
// This is needed so that plugins can refer to/identify this platform without
// instantiating a CudaPlatform object.
// This is broken out here to avoid a circular dependency between CudaPlatform
// and CudaExecutor.
extern const Platform::Id kCudaPlatformId;
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_PLATFORM_ID_H_
@@ -0,0 +1,102 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_RNG_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_RNG_H_
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/platform/thread_annotations.h"
#include "tensorflow/stream_executor/plugin_registry.h"
#include "tensorflow/stream_executor/rng.h"
typedef struct curandGenerator_st *curandGenerator_t;
namespace stream_executor {
class Stream;
template <typename ElemT>
class DeviceMemory;
namespace cuda {
// Opaque and unique identifier for the cuRAND plugin.
extern const PluginId kCuRandPlugin;
class CUDAExecutor;
// CUDA-platform implementation of the random number generation support
// interface.
//
// Thread-safe post-initialization.
class CUDARng : public rng::RngSupport {
public:
explicit CUDARng(CUDAExecutor *parent);
// Retrieves a curand library generator handle. This is necessary for
// enqueuing random number generation work onto the device.
// TODO(leary) provide a way for users to select the RNG algorithm.
bool Init();
// Releases a curand library generator handle, if one was acquired.
~CUDARng() override;
// See rng::RngSupport for details on the following overrides.
bool DoPopulateRandUniform(Stream *stream, DeviceMemory<float> *v) override;
bool DoPopulateRandUniform(Stream *stream, DeviceMemory<double> *v) override;
bool DoPopulateRandUniform(Stream *stream,
DeviceMemory<std::complex<float>> *v) override;
bool DoPopulateRandUniform(Stream *stream,
DeviceMemory<std::complex<double>> *v) override;
bool DoPopulateRandGaussian(Stream *stream, float mean, float stddev,
DeviceMemory<float> *v) override;
bool DoPopulateRandGaussian(Stream *stream, double mean, double stddev,
DeviceMemory<double> *v) override;
bool SetSeed(Stream *stream, const uint8 *seed, uint64 seed_bytes) override;
private:
// Actually performs the work of generating random numbers - the public
// methods are thin wrappers to this interface.
template <typename T>
bool DoPopulateRandUniformInternal(Stream *stream, DeviceMemory<T> *v);
template <typename ElemT, typename FuncT>
bool DoPopulateRandGaussianInternal(Stream *stream, ElemT mean, ElemT stddev,
DeviceMemory<ElemT> *v, FuncT func);
// Sets the stream for the internal curand generator.
//
// This is a stateful operation, as the handle can only have one stream set at
// a given time, so it is usually performed right before enqueuing work to do
// with random number generation.
bool SetStream(Stream *stream) EXCLUSIVE_LOCKS_REQUIRED(mu_);
// mutex that guards the cuRAND handle for this device.
mutex mu_;
// CUDAExecutor which instantiated this CUDARng.
// Immutable post-initialization.
CUDAExecutor *parent_;
// cuRANDalibrary handle on the device.
curandGenerator_t rng_ GUARDED_BY(mu_);
SE_DISALLOW_COPY_AND_ASSIGN(CUDARng);
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_RNG_H_
@@ -0,0 +1,93 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Defines the CUDAStream type - the CUDA-specific implementation of the generic
// StreamExecutor Stream interface.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_STREAM_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_STREAM_H_
#include "tensorflow/stream_executor/cuda/cuda_driver.h"
#include "tensorflow/stream_executor/platform/thread_annotations.h"
#include "tensorflow/stream_executor/stream_executor_internal.h"
namespace stream_executor {
namespace cuda {
class CUDAExecutor;
// Wraps a CUstream in order to satisfy the platform-independent
// StreamInterface.
//
// Thread-safe post-initialization.
class CUDAStream : public internal::StreamInterface {
public:
explicit CUDAStream(CUDAExecutor *parent)
: parent_(parent), cuda_stream_(nullptr), completed_event_(nullptr) {}
// Note: teardown is handled by a parent's call to DeallocateStream.
~CUDAStream() override {}
void *GpuStreamHack() override { return cuda_stream_; }
void **GpuStreamMemberHack() override {
return reinterpret_cast<void **>(&cuda_stream_);
}
// Explicitly initialize the CUDA resources associated with this stream, used
// by StreamExecutor::AllocateStream().
bool Init();
// Explicitly destroy the CUDA resources associated with this stream, used by
// StreamExecutor::DeallocateStream().
void Destroy();
// Returns true if no work is pending or executing on the stream.
bool IsIdle() const;
// Retrieves an event which indicates that all work enqueued into the stream
// has completed. Ownership of the event is not transferred to the caller, the
// event is owned by this stream.
CUevent* completed_event() { return &completed_event_; }
// Returns the CUstream value for passing to the CUDA API.
//
// Precond: this CUDAStream has been allocated (otherwise passing a nullptr
// into the NVIDIA library causes difficult-to-understand faults).
CUstream cuda_stream() const {
DCHECK(cuda_stream_ != nullptr);
return const_cast<CUstream>(cuda_stream_);
}
CUDAExecutor *parent() const { return parent_; }
private:
CUDAExecutor *parent_; // Executor that spawned this stream.
CUstream cuda_stream_; // Wrapped CUDA stream handle.
// Event that indicates this stream has completed.
CUevent completed_event_ = nullptr;
};
// Helper functions to simplify extremely common flows.
// Converts a Stream to the underlying CUDAStream implementation.
CUDAStream *AsCUDAStream(Stream *stream);
// Extracts a CUstream from a CUDAStream-backed Stream object.
CUstream AsCUDAStreamValue(Stream *stream);
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_STREAM_H_
@@ -0,0 +1,90 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Defines the CUDATimer type - the CUDA-specific implementation of the generic
// StreamExecutor Timer interface.
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_TIMER_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_TIMER_H_
#include "tensorflow/stream_executor/stream_executor_internal.h"
#include "tensorflow/stream_executor/cuda/cuda_driver.h"
#include "tensorflow/stream_executor/cuda/cuda_gpu_executor.h"
namespace stream_executor {
namespace cuda {
class CUDAExecutor;
class CUDAStream;
// Wraps a pair of CUevents in order to satisfy the platform-independent
// TimerInferface -- both a start and a stop event are present which may be
// recorded in a stream.
class CUDATimer : public internal::TimerInterface {
public:
explicit CUDATimer(CUDAExecutor *parent)
: parent_(parent), start_event_(nullptr), stop_event_(nullptr) {}
// Note: teardown needs to be explicitly handled in this API by a call to
// StreamExecutor::DeallocateTimer(), which invokes Destroy().
// TODO(csigg): Change to RAII.
~CUDATimer() override {}
// Allocates the platform-specific pieces of the timer, called as part of
// StreamExecutor::AllocateTimer().
bool Init();
// Deallocates the platform-specific pieces of the timer, called as part of
// StreamExecutor::DeallocateTimer().
void Destroy();
// Records the "timer start" event at the current point in the stream.
bool Start(CUDAStream *stream);
// Records the "timer stop" event at the current point in the stream.
bool Stop(CUDAStream *stream);
// Returns the elapsed time, in milliseconds, between the start and stop
// events.
float GetElapsedMilliseconds() const;
// See Timer::Microseconds().
// TODO(leary) make this into an error code interface...
uint64 Microseconds() const override {
return GetElapsedMilliseconds() * 1e3;
}
// See Timer::Nanoseconds().
uint64 Nanoseconds() const override { return GetElapsedMilliseconds() * 1e6; }
private:
CUDAExecutor *parent_;
CUevent start_event_; // Event recorded to indicate the "start" timestamp
// executing in a stream.
CUevent stop_event_; // Event recorded to indicate the "stop" timestamp
// executing in a stream.
};
struct TimerDeleter {
void operator()(CUDATimer *t) {
t->Destroy();
delete t;
}
};
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_TIMER_H_
@@ -0,0 +1,50 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDNN_VERSION_H_
#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDNN_VERSION_H_
#include <string>
#include "tensorflow/core/lib/strings/strcat.h"
namespace stream_executor {
namespace cuda {
struct CudnnVersion {
CudnnVersion() = default;
CudnnVersion(int major, int minor, int patch)
: major_version(major), minor_version(minor), patch_level(patch) {}
tensorflow::string ToString() const {
return tensorflow::strings::StrCat(major_version, ".", minor_version, ".",
patch_level);
}
int major_version;
int minor_version;
int patch_level;
};
// Returns true if the given source CuDNN version is compatible with the given
// loaded version.
bool IsSourceCompatibleWithCudnnLibrary(CudnnVersion source_version,
CudnnVersion loaded_version);
} // namespace cuda
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDNN_VERSION_H_
@@ -0,0 +1,390 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Describes the underlying platform for a StreamExecutor; e.g. OpenCL or CUDA
// device and platform properties. Also contains convenience functions for
// checking/calculating launch dimensionality based on device properties.
#ifndef TENSORFLOW_STREAM_EXECUTOR_DEVICE_DESCRIPTION_H_
#define TENSORFLOW_STREAM_EXECUTOR_DEVICE_DESCRIPTION_H_
#include <map>
#include <memory>
#include "absl/base/macros.h"
#include "tensorflow/stream_executor/launch_dim.h"
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
namespace internal {
class DeviceDescriptionBuilder;
} // namespace internal
// Data that describes the execution target of the StreamExecutor, in terms of
// important logical parameters. These include dimensionality limits and
// physical parameters of interest, such as number of cores present on the
// device.
//
// Thread-safe: immutable post-initialization.
class DeviceDescription {
public:
// Returns the platform being run on; this value is primarily intended for
// printing, and comes out something like "OpenCL 1.2" or "Compute Capability
// 3.5".
const string &platform_version() const { return platform_version_; }
// Returns the driver version interfacing with the underlying platform. Vendor
// dependent format.
const string &driver_version() const { return driver_version_; }
// Return the runtime version, if one is provided by the underlying platform.
// Vendor dependent format / usefulness.
const string &runtime_version() const { return runtime_version_; }
// Returns the name that the device reports. Vendor dependent.
const string &name() const { return name_; }
// Returns the PCI bus identifier for this device, of the form
// [domain]:[bus]:[device].[function]
const string &pci_bus_id() const { return pci_bus_id_; }
// Returns the NUMA node associated with this device, for use in
// determining socket locality. If the NUMA node could not be determined, -1
// is returned.
int numa_node() const { return numa_node_; }
// Number of cores (traditional notion of core; i.e. an SM on an NVIDIA device
// or an AMD Compute Unit.
int core_count() const { return core_count_; }
// Returns the limit on the thread dimensionality values in each of the
// respective dimensions. These limits affect what constitutes a legitimate
// kernel launch request.
const ThreadDim &thread_dim_limit() const { return thread_dim_limit_; }
// Returns the limit on the block dimensionality values in each of the
// respective dimensions. These limits may affect what constitutes a
// legitimate kernel launch request.
const BlockDim &block_dim_limit() const { return block_dim_limit_; }
// Returns the limit on the number of simultaneously resident blocks
// on a multiprocessor.
uint64 blocks_per_core_limit() const { return blocks_per_core_limit_; }
// Returns the limit on the total number of threads that can be launched in a
// single block; i.e. the limit on x * y * z dimensions of a ThreadDim.
// This limit affects what constitutes a legitimate kernel launch request.
const uint64 &threads_per_block_limit() const {
return threads_per_block_limit_;
}
// Returns the limit on the total number of threads that can be simultaneously
// launched on a given multiprocessor.
const uint64 &threads_per_core_limit() const {
return threads_per_core_limit_;
}
// Returns the number of threads per warp/wavefront.
const uint64 &threads_per_warp() const { return threads_per_warp_; }
// Returns the limit on the total number of registers per core.
const uint64 &registers_per_core_limit() const {
return registers_per_core_limit_;
}
// Returns the limit on the total number of registers that can be
// simultaneously used by a block.
const uint64 &registers_per_block_limit() const {
return registers_per_block_limit_;
}
// Returns the limit on the total number of registers that can be
// allocated to a thread.
const uint64 &registers_per_thread_limit() const {
return registers_per_thread_limit_;
}
// Returns the granularity at which warps are allocated resources.
const uint64 &warp_alloc_granularity() const {
return warp_alloc_granularity_;
}
// Returns the granularity at which registers are allocated to warps.
const uint64 &register_alloc_granularity() const {
return register_alloc_granularity_;
}
// Returns the granularity at which shared memory is allocated to warps.
const uint64 &shared_memory_alloc_granularity() const {
return shared_memory_alloc_granularity_;
}
// Returns the number of address bits available to kernel code running on the
// platform. This affects things like the maximum allocation size and perhaps
// types used in kernel code such as size_t.
const uint64 &device_address_bits() const { return device_address_bits_; }
// Returns the device memory size in bytes.
uint64 device_memory_size() const { return device_memory_size_; }
// Returns the device's memory bandwidth in bytes/sec. (This is for
// reads/writes to/from the device's own memory, not for transfers between the
// host and device.)
uint64 memory_bandwidth() const { return memory_bandwidth_; }
// Returns the device's core clock rate in GHz.
float clock_rate_ghz() const { return clock_rate_ghz_; }
// Returns whether ECC is enabled.
bool ecc_enabled() const { return ecc_enabled_; }
// Returns the device vendor string, e.g., "NVIDIA Corporation", "Advanced
// Micro Devices, Inc.", or "GenuineIntel".
const string &device_vendor() const { return device_vendor_; }
// Returns the CUDA compute capability if we're running on the CUDA platform.
// If a CUDA compute capability is not available, the major version will be
// zero, and the return value will be false.
bool cuda_compute_capability(int *major, int *minor) const;
// Returns the maximum amount of shared memory present on a single core
// (i.e. Streaming Multiprocessor on NVIDIA GPUs; Compute Unit for OpenCL
// devices). Note that some devices, such as NVIDIA's have a configurable
// partitioning between shared memory and L1 cache.
uint64 shared_memory_per_core() const { return shared_memory_per_core_; }
// Returns the maximum amount of shared memory available for a single block.
uint64 shared_memory_per_block() const { return shared_memory_per_block_; }
// TODO(leary): resident blocks per core will be useful.
// Convenience typedef for the string-based DeviceDescription mapping.
typedef std::map<string, string> Map;
// Returns a mapping from readable names to readable values that describe the
// device. This is useful for things like printing.
std::unique_ptr<Map> ToMap() const;
// For string values that are not available via the underlying platform, this
// value will be provided.
static const char *kUndefinedString;
private:
friend class internal::DeviceDescriptionBuilder;
DeviceDescription();
// For description of the following members, see the corresponding accessor
// above.
//
// N.B. If another field is added, update ToMap() above.
string device_vendor_;
string platform_version_;
string driver_version_;
string runtime_version_;
string pci_bus_id_;
string name_;
ThreadDim thread_dim_limit_;
BlockDim block_dim_limit_;
uint64 blocks_per_core_limit_;
uint64 threads_per_core_limit_;
uint64 threads_per_block_limit_;
uint64 threads_per_warp_;
uint64 registers_per_core_limit_;
uint64 registers_per_block_limit_;
uint64 registers_per_thread_limit_;
uint64 warp_alloc_granularity_;
uint64 register_alloc_granularity_;
uint64 shared_memory_alloc_granularity_;
uint64 device_address_bits_;
uint64 device_memory_size_;
uint64 memory_bandwidth_;
// Shared memory limits on a given device.
uint64 shared_memory_per_core_;
uint64 shared_memory_per_block_;
float clock_rate_ghz_;
// CUDA "CC" major value, -1 if not available.
int cuda_compute_capability_major_;
int cuda_compute_capability_minor_;
int numa_node_;
int core_count_;
bool ecc_enabled_;
SE_DISALLOW_COPY_AND_ASSIGN(DeviceDescription);
};
namespace internal {
// Helper class the builds a device description, given that it has a large
// number of fields that would be easily confused in constructor form.
class DeviceDescriptionBuilder {
public:
DeviceDescriptionBuilder();
// For descriptions of the following fields, see comments on the corresponding
// DeviceDescription::* accessors above.
void set_device_vendor(const string &value) {
device_description_->device_vendor_ = value;
}
void set_platform_version(const string &value) {
device_description_->platform_version_ = value;
}
void set_driver_version(const string &value) {
device_description_->driver_version_ = value;
}
void set_runtime_version(const string &value) {
device_description_->runtime_version_ = value;
}
void set_pci_bus_id(const string &value) {
device_description_->pci_bus_id_ = value;
}
void set_name(const string &value) { device_description_->name_ = value; }
void set_thread_dim_limit(const ThreadDim &value) {
device_description_->thread_dim_limit_ = value;
}
void set_block_dim_limit(const BlockDim &value) {
device_description_->block_dim_limit_ = value;
}
void set_blocks_per_core_limit(uint64 value) {
device_description_->blocks_per_core_limit_ = value;
}
void set_threads_per_core_limit(uint64 value) {
device_description_->threads_per_core_limit_ = value;
}
void set_threads_per_block_limit(uint64 value) {
device_description_->threads_per_block_limit_ = value;
}
void set_threads_per_warp(uint64 value) {
device_description_->threads_per_warp_ = value;
}
void set_registers_per_core_limit(uint64 value) {
device_description_->registers_per_core_limit_ = value;
}
void set_registers_per_block_limit(uint64 value) {
device_description_->registers_per_block_limit_ = value;
}
void set_registers_per_thread_limit(uint64 value) {
device_description_->registers_per_thread_limit_ = value;
}
void set_warp_alloc_granularity(uint64 value) {
device_description_->warp_alloc_granularity_ = value;
}
void set_register_alloc_granularity(uint64 value) {
device_description_->register_alloc_granularity_ = value;
}
void set_shared_memory_alloc_granularity(uint64 value) {
device_description_->shared_memory_alloc_granularity_ = value;
}
void set_device_address_bits(uint64 value) {
device_description_->device_address_bits_ = value;
}
void set_device_memory_size(uint64 value) {
device_description_->device_memory_size_ = value;
}
void set_memory_bandwidth(uint64 value) {
device_description_->memory_bandwidth_ = value;
}
void set_shared_memory_per_core(int64 value) {
device_description_->shared_memory_per_core_ = value;
}
void set_shared_memory_per_block(int64 value) {
device_description_->shared_memory_per_block_ = value;
}
void set_clock_rate_ghz(float value) {
device_description_->clock_rate_ghz_ = value;
}
void set_cuda_compute_capability(int major, int minor) {
device_description_->cuda_compute_capability_major_ = major;
device_description_->cuda_compute_capability_minor_ = minor;
}
void set_numa_node(int value) { device_description_->numa_node_ = value; }
void set_core_count(int value) { device_description_->core_count_ = value; }
void set_ecc_enabled(bool value) {
device_description_->ecc_enabled_ = value;
}
// Returns a built DeviceDescription with ownership transferred to the
// caller. There are currently no restrictions on which fields must be set in
// order to build the descriptor.
//
// Once the description is built, this builder object should be discarded.
std::unique_ptr<DeviceDescription> Build() {
return std::move(device_description_);
}
private:
std::unique_ptr<DeviceDescription> device_description_;
SE_DISALLOW_COPY_AND_ASSIGN(DeviceDescriptionBuilder);
};
} // namespace internal
// Returns whether the given thread_dim is acceptable given the limits described
// in device_description. For detailed reasons for failing the predicate, enable
// VLOG(2) for this module.
bool ThreadDimOk(const DeviceDescription &device_description,
const ThreadDim &thread_dim);
// Equivalent to ceil(double(element_count) / threads_per_block).
ABSL_DEPRECATED("Use MathUtil::CeilOfRatio directly instead.")
uint64 DivideCeil(uint64 x, uint64 y);
// Calculate the number of threads/blocks required to process element_count
// elements. Note that you can still end up with more threads than
// element_count due to rounding, so kernels often start with an "is this
// thread id in the element_count range?" test.
void CalculateDimensionality(const DeviceDescription &device_description,
uint64 element_count, uint64 *threads_per_block,
uint64 *block_count);
// Compute and return maximum blocks per core (occupancy) based on the
// device description, some kernel characteristics and the number of threads per
// block. If unable to compute occupancy, zero is returned.
uint64 CalculateOccupancy(const DeviceDescription &device_description,
uint64 registers_per_thread,
uint64 shared_memory_per_block,
const ThreadDim &thread_dims);
// Compute and return the maximum number of registers per thread which
// achieves the target occupancy. If the target is not possible then
// zero is returned.
uint64 CalculateRegisterLimitForTargetOccupancy(
const DeviceDescription &device_description, uint64 shared_memory_per_block,
const ThreadDim &thread_dims, uint64 target_blocks_per_core);
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_DEVICE_DESCRIPTION_H_
@@ -0,0 +1,295 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Suite of types that represent device memory allocations. These are
// allocated by the StreamExecutor interface, which produces values appropriate
// for the underlying platform (whether it be CUDA or OpenCL).
//
// The untyped base class (like a device void*) is DeviceMemoryBase, which can
// be specialized for a given allocation type (like a device T*) using
// DeviceMemory<T>.
#ifndef TENSORFLOW_STREAM_EXECUTOR_DEVICE_MEMORY_H_
#define TENSORFLOW_STREAM_EXECUTOR_DEVICE_MEMORY_H_
#include <stddef.h>
#include "tensorflow/stream_executor/lib/casts.h"
#include "tensorflow/stream_executor/platform/port.h"
namespace perftools {
namespace gputools {
// Temporarily pull stream_executor into perftools::gputools while we migrate
// code to the new namespace. TODO(b/77980417): Remove this once we've
// completed the migration.
using namespace stream_executor; // NOLINT[build/namespaces]
} // namespace gputools
} // namespace perftools
namespace stream_executor {
class StreamExecutor;
// void*-analogous device memory allocation. For the typed variation, see
// DeviceMemory<T>.
//
// This is effectively a two-tuple of a pointer and size; however, note that the
// pointer may not be to the virtual address itself -- in OpenCL the pointer is
// to a cl_mem handle that describes the device allocation. Therefore,
// DeviceMemoryBase::opaque does not necessarily produce a pointer that can be
// referenced directly, so use it with caution.
//
// Thread-compatible.
class DeviceMemoryBase {
public:
// Default constructor instantiates a null-pointed, zero-sized device memory
// region. An opaque pointer may be provided -- see header for details on the
// opacity of that pointer.
explicit DeviceMemoryBase(void *opaque = nullptr, uint64 size = 0,
bool is_sub_buffer = false)
: opaque_(opaque), size_(size), is_sub_buffer_(is_sub_buffer) {}
// Returns whether the backing memory is the null pointer.
// A `== nullptr` convenience method is also provided.
bool is_null() const { return opaque_ == nullptr; }
bool operator==(std::nullptr_t other) const { return is_null(); }
bool operator!=(std::nullptr_t other) const { return !is_null(); }
// Provides a partial order between device memory values.
//
// This operator is provided so that this object can be used as a key in an
// ordered map.
bool operator<(const DeviceMemoryBase &other) const {
return opaque() < other.opaque();
}
// Returns the size, in bytes, for the backing memory.
uint64 size() const { return size_; }
// Warning: note that the pointer returned is not necessarily directly to
// device virtual address space, but is platform-dependent.
void *opaque() { return opaque_; }
const void *opaque() const { return opaque_; }
// Returns true if this is an offset into another primary allocation.
bool is_sub_buffer() const { return is_sub_buffer_; }
// Returns whether the two DeviceMemoryBase segments are identical (both in
// their opaque pointer and size).
bool IsSameAs(const DeviceMemoryBase &other) const {
return opaque() == other.opaque() && size() == other.size();
}
protected:
friend class StreamExecutor;
// Resets the internal values of the opaque pointer and number of bytes in the
// memory region, just as in the constructor.
void Reset(void *opaque, uint64 bytes) {
opaque_ = opaque;
size_ = bytes;
}
private:
void *opaque_; // Platform-dependent value representing allocated memory.
uint64 size_; // Size in bytes of this allocation.
bool is_sub_buffer_; // Is this a primary allocation or a sub-buffer?
};
// Typed wrapper around "void *"-like DeviceMemoryBase.
//
// For example, DeviceMemory<int> is a simple wrapper around DeviceMemoryBase
// that represents one or more integers in Device memory.
//
// Thread-compatible.
template <typename ElemT>
class DeviceMemory final : public DeviceMemoryBase {
public:
// Default constructor instantiates a null-pointed, zero-sized memory region.
DeviceMemory() : DeviceMemoryBase(nullptr, 0) {}
DeviceMemory(std::nullptr_t) : DeviceMemory() {}
// Typed device memory regions may be constructed from untyped device memory
// regions, this effectively amounts to a cast from a void*.
explicit DeviceMemory(const DeviceMemoryBase &other)
: DeviceMemoryBase(const_cast<DeviceMemoryBase &>(other).opaque(),
other.size(), other.is_sub_buffer()) {}
// Returns the number of elements of type ElemT that constitute this
// allocation.
uint64 ElementCount() const { return size() / sizeof(ElemT); }
// Returns whether this is a single-element allocation.
bool IsScalar() const { return ElementCount() == 1; }
// Create a typed area of DeviceMemory with a given opaque pointer and the
// quantity of bytes in the allocation. This function is broken out to
// distinguish bytes from an element count.
static DeviceMemory<ElemT> MakeFromByteSize(void *opaque, uint64 bytes) {
return DeviceMemory<ElemT>(opaque, bytes);
}
// Resets the DeviceMemory data, in MakeFromByteSize fashion.
// This simply clobbers the prior values.
void ResetFromByteSize(void *opaque, uint64 bytes) {
// TODO(leary) when NVCC is eliminated we can add this check (and the
// logging include it requires).
// CHECK_EQ(0, bytes % sizeof(ElemT));
DeviceMemoryBase::Reset(opaque, bytes);
}
// ------------------------------------------------------------
protected:
// This constructor is solely used from derived classes; it is made protected
// because it accepts a byte-size instead of an element count, which could
// potentially be misused given the ElementCount() nature of this interface.
//
// In order to specify the desire to use byte size instead of element count
// explicitly, use MakeFromByteSize.
DeviceMemory(void *opaque, uint64 size) : DeviceMemoryBase(opaque, size) {}
};
// A class to encapsulate the type and size of a dynamic shared memory
// buffer. Because the buffer exists solely on the device and is not copyable
// to the host, memory objects of this type do not maintain buffer pointers
// on the host.
template <typename ElemT>
class SharedDeviceMemory final : public DeviceMemoryBase {
public:
explicit SharedDeviceMemory(uint64 elem_count)
: DeviceMemoryBase(nullptr, elem_count * kElemSize) {}
static constexpr size_t kElemSize = sizeof(ElemT);
// Returns the number of elements of type ElemT that constitute this
// allocation.
uint64 ElementCount() const { return size() / kElemSize; }
// Returns whether this is a single-element allocation.
bool IsScalar() const { return ElementCount() == 1; }
};
// Similar to the typed DeviceMemory, but is the unique owner of its
// memory, if any. ScopedDeviceMemory is thread-compatible. It is also
// movable and uncopyable to represent unique ownership.
template <typename ElemT>
class ScopedDeviceMemory {
public:
// Default construction initializes the internal state to nullptr. This
// mirrors the std::unique_ptr<> functionality, where default construction
// produces a nullptr unique_ptr, which can be assigned later.
ScopedDeviceMemory();
// Parameters:
// parent: Executor used to deallocate memory when this instance goes
// out of scope.
// value: Already-allocated device memory value for this scoped mechanism to
// deallocate. This memory must have been allocated by parent.
ScopedDeviceMemory(StreamExecutor *parent, DeviceMemoryBase value);
// Constructor overload that places a literal array into device memory
ScopedDeviceMemory(StreamExecutor *parent,
std::initializer_list<ElemT> values);
// Moves ownership of the memory from other to the constructed
// object.
//
// Postcondition: other == nullptr.
ScopedDeviceMemory(ScopedDeviceMemory &&other) noexcept:
ScopedDeviceMemory(other.parent_, other.Release()) {}
// Releases the memory that was provided in the constructor, through the
// "parent" StreamExecutor.
~ScopedDeviceMemory();
// Moves ownership of the memory from other to this object.
//
// Postcondition: other == nullptr.
ScopedDeviceMemory& operator=(ScopedDeviceMemory &&other) {
Reset(other.Release());
parent_ = other.parent_;
return *this;
}
// Returns the memory that backs this scoped allocation converted to
// DeviceMemory<T> apparent type. This is useful for cases where the
// DeviceMemory must be passed by const-ref, as the ScopedDeviceMemory doesn't
// allow copying, for scoped-object-lifetime reasons.
const DeviceMemory<ElemT> &cref() const { return wrapped_; }
// Returns a pointer to the DeviceMemory<T> apparent type for use in mutable
// operations. The value returned should not be used outside the scope of this
// ScopedDeviceMemory object's lifetime.
DeviceMemory<ElemT> *ptr() { return &wrapped_; }
const DeviceMemory<ElemT> *ptr() const { return &wrapped_; }
// Smart-pointer-like operators for the wrapped DeviceMemory.
// This reference must not be used outside the lifetime of this
// ScopedDeviceMemory.
const DeviceMemory<ElemT> &operator*() const { return cref(); }
DeviceMemory<ElemT> *operator->() { return ptr(); }
const DeviceMemory<ElemT> *operator->() const { return ptr(); }
bool operator==(std::nullptr_t other) const { return wrapped_.is_null(); }
bool operator!=(std::nullptr_t other) const { return !wrapped_.is_null(); }
// Analogous to std::unique_ptr::reset, frees the existing memory held in
// this scoped memory container and replaces it with updated. Ownership
// of updated is transferred to this object.
void Reset(DeviceMemory<ElemT> updated);
void Reset(std::nullptr_t);
// Analogous to std::unique_ptr::release, releases ownership of the held
// memory and transfers it to the caller.
//
// Postcondition: *this == nullptr
DeviceMemory<ElemT> Release() {
auto tmp = wrapped_;
wrapped_.ResetFromByteSize(nullptr, 0);
return tmp;
}
private:
DeviceMemory<ElemT> wrapped_; // Value we wrap with scoped-release.
StreamExecutor *parent_; // See constructor.
SE_DISALLOW_COPY_AND_ASSIGN(ScopedDeviceMemory);
};
// Host-side representation of packed-and-aligned vector datatypes on the device
// side. Since these can appear in device kernel signatures, we support
// launching them with these datatypes in launch signatures.
struct Float2 {
float x, y;
};
struct Float4 {
Float2 xz, yw;
};
struct Double2 {
double x, y;
};
static_assert(sizeof(Float2) == 2 * sizeof(float), "Float2 must be packed");
static_assert(sizeof(Float4) == 4 * sizeof(float), "Float4 must be packed");
static_assert(sizeof(Double2) == 2 * sizeof(double), "Double2 must be packed");
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_DEVICE_MEMORY_H_
@@ -0,0 +1,88 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Contains device-level options that can be specified at a platform level.
// Example usage:
// auto device_options = DeviceOptions::Default();
#ifndef TENSORFLOW_STREAM_EXECUTOR_DEVICE_OPTIONS_H_
#define TENSORFLOW_STREAM_EXECUTOR_DEVICE_OPTIONS_H_
#include <map>
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/platform/logging.h"
namespace stream_executor {
// Indicates a set of options for a device's usage, which generally must be
// provided at StreamExecutor device-initialization time.
//
// These are intended to be useful-but-not-mandatorily-supported options for
// using devices on the underlying platform. Presently, if the option requested
// is not available on the target platform, a warning will be emitted.
struct DeviceOptions {
public:
// When it is observed that more memory has to be allocated for thread stacks,
// this flag prevents it from ever being deallocated. Potentially saves
// thrashing the thread stack memory allocation, but at the potential cost of
// some memory space.
static const unsigned kDoNotReclaimStackAllocation = 0x1;
// The following options refer to synchronization options when
// using SynchronizeStream or SynchronizeContext.
// Synchronize with spinlocks.
static const unsigned kScheduleSpin = 0x02;
// Synchronize with spinlocks that also call CPU yield instructions.
static const unsigned kScheduleYield = 0x04;
// Synchronize with a "synchronization primitive" (e.g. mutex).
static const unsigned kScheduleBlockingSync = 0x08;
static const unsigned kMask = 0xf; // Mask of all available flags.
// Constructs an or-d together set of device options.
explicit DeviceOptions(unsigned flags) : flags_(flags) {
CHECK((flags & kMask) == flags);
}
// Factory for the default set of device options.
static DeviceOptions Default() { return DeviceOptions(0); }
unsigned flags() const { return flags_; }
bool operator==(const DeviceOptions& other) const {
return flags_ == other.flags_;
}
bool operator!=(const DeviceOptions& other) const {
return !(*this == other);
}
string ToString() {
return flags_ == 0 ? "none" : "kDoNotReclaimStackAllocation";
}
// Platform-specific device options. Expressed as key-value pairs to avoid
// DeviceOptions subclass proliferation.
std::map<string, string> non_portable_tags;
private:
unsigned flags_;
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_DEVICE_OPTIONS_H_
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,118 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Common DSO loading functionality: exposes callables that dlopen DSOs
// in either the runfiles directories
#ifndef TENSORFLOW_STREAM_EXECUTOR_DSO_LOADER_H_
#define TENSORFLOW_STREAM_EXECUTOR_DSO_LOADER_H_
#include "tensorflow/stream_executor/platform/port.h"
#include <vector>
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/lib/stringpiece.h"
#include "tensorflow/stream_executor/platform.h"
#include "tensorflow/stream_executor/platform/mutex.h"
namespace stream_executor {
namespace internal {
// Permits StreamExecutor code to dynamically load a pre-determined set of
// relevant DSOs via dlopen.
//
// Thread-safe.
class DsoLoader {
public:
// The following methods either load the DSO of interest and return a dlopen
// handle or error status in the canonical namespace.
static port::Status GetCublasDsoHandle(void** dso_handle);
static port::Status GetCudnnDsoHandle(void** dso_handle);
static port::Status GetCufftDsoHandle(void** dso_handle);
static port::Status GetCurandDsoHandle(void** dso_handle);
static port::Status GetLibcudaDsoHandle(void** dso_handle);
static port::Status GetLibcuptiDsoHandle(void** dso_handle);
// Registers a new binary-relative path to use as a dlopen search path.
static void RegisterRpath(port::StringPiece path);
private:
// Registered rpaths (singleton vector) and a mutex that guards it.
static std::vector<string>* GetRpaths();
// Descriptive boolean wrapper to indicate whether symbols are made available
// to resolve in later-loaded libraries.
enum class LoadKind { kLocal, kGlobal };
// Loads a DSO from the given "path" (which can technically be any dlopen-able
// name). If the load kind is global, the symbols in the loaded DSO are
// visible to subsequent DSO loading operations.
static port::Status GetDsoHandle(port::StringPiece path, void** dso_handle,
LoadKind load_kind = LoadKind::kLocal);
// Returns the binary directory (or binary path) associated with the currently
// executing program. If strip_executable_name is true, the executable file is
// stripped off of the path.
static string GetBinaryDirectory(bool strip_executable_name);
// Invokes realpath on the original path; updates candidate and returns true
// if it succeeds (i.e. a file exists at the path); otherwise, returns false.
static bool TrySymbolicDereference(string* candidate);
// Attempts to find a path to the DSO of interest, otherwise returns the
// bare library name:
// Arguments:
// library_name: the filename in tree; e.g. libOpenCL.so.1.0.0
// runfiles_relpath: where to look for the library relative to the runfiles
// root; e.g. third_party/gpus/cuda/lib64
static string FindDsoPath(port::StringPiece library_name,
port::StringPiece runfiles_relpath);
// Return platform dependent paths for DSOs
static string GetCudaLibraryDirPath();
static string GetCudaDriverLibraryPath();
static string GetCudaCuptiLibraryPath();
SE_DISALLOW_COPY_AND_ASSIGN(DsoLoader);
};
// Wrapper around the DsoLoader that prevents us from dlopen'ing any of the DSOs
// more than once.
class CachedDsoLoader {
public:
// Cached versions of the corresponding DsoLoader methods above.
static port::StatusOr<void*> GetCublasDsoHandle();
static port::StatusOr<void*> GetCudnnDsoHandle();
static port::StatusOr<void*> GetCufftDsoHandle();
static port::StatusOr<void*> GetCurandDsoHandle();
static port::StatusOr<void*> GetLibcudaDsoHandle();
static port::StatusOr<void*> GetLibcuptiDsoHandle();
private:
// Fetches a DSO handle via "load_dso" and returns the StatusOr form of the
// result.
static port::StatusOr<void*> FetchHandleResult(
std::function<port::Status(void**)> load_dso);
SE_DISALLOW_COPY_AND_ASSIGN(CachedDsoLoader);
};
} // namespace internal
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_DSO_LOADER_H_
@@ -0,0 +1,83 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_EVENT_H_
#define TENSORFLOW_STREAM_EXECUTOR_EVENT_H_
#include <memory>
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
namespace internal {
class EventInterface;
}
class Stream;
class StreamExecutor;
// The Event class, when supported by a platform, enables low-overhead status
// reporting for a Stream. An Event is inserted at a location in a stream via
// the Stream::ThenRecordEvent() API. From then on, the Event's status can be
// monitored via the nonblocking Event::PollForStatus() call.
class Event {
public:
// Potential states for an Event. If PollForStatus() returns anything aside
// from kPending or kComplete, an error has occurred; kUnknown is a bad state.
// Not all implementations are able to return all enumeration values. Refer to
// the platform-specific implementation for details.
enum class Status {
kUnknown,
kError,
kPending,
kComplete,
};
explicit Event(StreamExecutor* stream_exec); // NOLINT
// Releases any resources held by the Event object.
~Event();
// Performs any platform-specific or potentially error-generating
// initialization.
bool Init();
// Returns the current Status for the event.
Status PollForStatus();
// Returns a pointer to the underlying platform-specific implementation.
internal::EventInterface* implementation() { return implementation_.get(); }
Event(Event&&) = default;
Event& operator=(Event&&) = default;
private:
friend class Stream;
// Pointer to the StreamExecutor interface used to create this object.
// Not owned.
StreamExecutor* stream_exec_;
// Pointer to the platform-specific EventInterface implementation underlying
// the object. Owned.
std::unique_ptr<internal::EventInterface> implementation_;
SE_DISALLOW_COPY_AND_ASSIGN(Event);
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_EVENT_H_
@@ -0,0 +1,80 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_EXECUTOR_CACHE_H_
#define TENSORFLOW_STREAM_EXECUTOR_EXECUTOR_CACHE_H_
#include <functional>
#include <map>
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/stream_executor_pimpl.h"
namespace stream_executor {
// Utility class to allow Platform objects to manage cached StreamExecutors.
// Thread-safe.
class ExecutorCache {
public:
ExecutorCache() {}
// Looks up 'config' in the cache. Returns a pointer to the existing executor,
// if already present, or creates it using 'factory', if it does not.
// Factories may be executed concurrently for different device ordinals.
typedef port::StatusOr<std::unique_ptr<StreamExecutor>> ExecutorFactory();
port::StatusOr<StreamExecutor*> GetOrCreate(
const StreamExecutorConfig& config,
const std::function<ExecutorFactory>& factory);
// Returns a pointer to the described executor (if one with a matching config
// has been created), or a NOT_FOUND status.
port::StatusOr<StreamExecutor*> Get(const StreamExecutorConfig& config);
// Destroys all Executors and clears the cache.
// Performs no synchronization with the executors - undefined behavior may
// occur if any executors are active!
void DestroyAllExecutors();
private:
// Each Entry contains zero or more cached executors for a device ordinal.
struct Entry {
~Entry();
// Mutex that locks the contents of each entry. The 'mutex_' of the
// ExecutorCache class protects both the 'cache_' and the existence of each
// Entry, but not the Entry's contents. 'configurations_mutex' protects the
// contents of the entry after 'mutex_' has been dropped.
mutex configurations_mutex;
// Vector of cached {config, executor} pairs.
std::vector<
std::pair<StreamExecutorConfig, std::unique_ptr<StreamExecutor>>>
configurations GUARDED_BY(configurations_mutex);
};
// Maps ordinal number to a list of cached executors for that ordinal.
// We key off of ordinal (instead of just looking up all fields in the
// StreamExecutorConfig) for a slight improvement in lookup time.
mutex mutex_;
std::map<int, Entry> cache_ GUARDED_BY(mutex_);
SE_DISALLOW_COPY_AND_ASSIGN(ExecutorCache);
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_EXECUTOR_CACHE_H_
@@ -0,0 +1,269 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Exposes the family of FFT routines as pre-canned high performance calls for
// use in conjunction with the StreamExecutor abstraction.
//
// Note that this interface is optionally supported by platforms; see
// StreamExecutor::SupportsFft() for details.
//
// This abstraction makes it simple to entrain FFT operations on GPU data into
// a Stream -- users typically will not use this API directly, but will use the
// Stream builder methods to entrain these operations "under the hood". For
// example:
//
// DeviceMemory<std::complex<float>> x =
// stream_exec->AllocateArray<std::complex<float>>(1024);
// DeviceMemory<std::complex<float>> y =
// stream_exec->AllocateArray<std::complex<float>>(1024);
// // ... populate x and y ...
// Stream stream{stream_exec};
// std::unique_ptr<Plan> plan =
// stream_exec.AsFft()->Create1dPlan(&stream, 1024, Type::kC2CForward);
// stream
// .Init()
// .ThenFft(plan.get(), x, &y);
// SE_CHECK_OK(stream.BlockHostUntilDone());
//
// By using stream operations in this manner the user can easily intermix custom
// kernel launches (via StreamExecutor::ThenLaunch()) with these pre-canned FFT
// routines.
#ifndef TENSORFLOW_STREAM_EXECUTOR_FFT_H_
#define TENSORFLOW_STREAM_EXECUTOR_FFT_H_
#include <complex>
#include <memory>
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
class Stream;
template <typename ElemT>
class DeviceMemory;
class ScratchAllocator;
namespace fft {
// Specifies FFT input and output types, and the direction.
// R, D, C, and Z stand for SP real, DP real, SP complex, and DP complex.
enum class Type {
kInvalid,
kC2CForward,
kC2CInverse,
kC2R,
kR2C,
kZ2ZForward,
kZ2ZInverse,
kZ2D,
kD2Z
};
// FFT plan class. Each FFT implementation should define a plan class that is
// derived from this class. It does not provide any interface but serves
// as a common type that is used to execute the plan.
class Plan {
public:
virtual ~Plan() {}
};
// FFT support interface -- this can be derived from a GPU executor when the
// underlying platform has an FFT library implementation available. See
// StreamExecutor::AsFft().
//
// This support interface is not generally thread-safe; it is only thread-safe
// for the CUDA platform (cuFFT) usage; host side FFT support is known
// thread-compatible, but not thread-safe.
class FftSupport {
public:
virtual ~FftSupport() {}
// Creates a 1d FFT plan.
virtual std::unique_ptr<Plan> Create1dPlan(Stream *stream, uint64 num_x,
Type type, bool in_place_fft) = 0;
// Creates a 2d FFT plan.
virtual std::unique_ptr<Plan> Create2dPlan(Stream *stream, uint64 num_x,
uint64 num_y, Type type,
bool in_place_fft) = 0;
// Creates a 3d FFT plan.
virtual std::unique_ptr<Plan> Create3dPlan(Stream *stream, uint64 num_x,
uint64 num_y, uint64 num_z,
Type type, bool in_place_fft) = 0;
// Creates a 1d FFT plan with scratch allocator.
virtual std::unique_ptr<Plan> Create1dPlanWithScratchAllocator(
Stream *stream, uint64 num_x, Type type, bool in_place_fft,
ScratchAllocator *scratch_allocator) = 0;
// Creates a 2d FFT plan with scratch allocator.
virtual std::unique_ptr<Plan> Create2dPlanWithScratchAllocator(
Stream *stream, uint64 num_x, uint64 num_y, Type type, bool in_place_fft,
ScratchAllocator *scratch_allocator) = 0;
// Creates a 3d FFT plan with scratch allocator.
virtual std::unique_ptr<Plan> Create3dPlanWithScratchAllocator(
Stream *stream, uint64 num_x, uint64 num_y, uint64 num_z, Type type,
bool in_place_fft, ScratchAllocator *scratch_allocator) = 0;
// Creates a batched FFT plan.
//
// stream: The GPU stream in which the FFT runs.
// rank: Dimensionality of the transform (1, 2, or 3).
// elem_count: Array of size rank, describing the size of each dimension.
// input_embed, output_embed:
// Pointer of size rank that indicates the storage dimensions
// of the input/output data in memory. If set to null_ptr all
// other advanced data layout parameters are ignored.
// input_stride: Indicates the distance (number of elements; same below)
// between two successive input elements.
// input_distance: Indicates the distance between the first element of two
// consecutive signals in a batch of the input data.
// output_stride: Indicates the distance between two successive output
// elements.
// output_distance: Indicates the distance between the first element of two
// consecutive signals in a batch of the output data.
virtual std::unique_ptr<Plan> CreateBatchedPlan(
Stream *stream, int rank, uint64 *elem_count, uint64 *input_embed,
uint64 input_stride, uint64 input_distance, uint64 *output_embed,
uint64 output_stride, uint64 output_distance, Type type,
bool in_place_fft, int batch_count) = 0;
// Creates a batched FFT plan with scratch allocator.
//
// stream: The GPU stream in which the FFT runs.
// rank: Dimensionality of the transform (1, 2, or 3).
// elem_count: Array of size rank, describing the size of each dimension.
// input_embed, output_embed:
// Pointer of size rank that indicates the storage dimensions
// of the input/output data in memory. If set to null_ptr all
// other advanced data layout parameters are ignored.
// input_stride: Indicates the distance (number of elements; same below)
// between two successive input elements.
// input_distance: Indicates the distance between the first element of two
// consecutive signals in a batch of the input data.
// output_stride: Indicates the distance between two successive output
// elements.
// output_distance: Indicates the distance between the first element of two
// consecutive signals in a batch of the output data.
virtual std::unique_ptr<Plan> CreateBatchedPlanWithScratchAllocator(
Stream *stream, int rank, uint64 *elem_count, uint64 *input_embed,
uint64 input_stride, uint64 input_distance, uint64 *output_embed,
uint64 output_stride, uint64 output_distance, Type type,
bool in_place_fft, int batch_count,
ScratchAllocator *scratch_allocator) = 0;
// Updates the plan's work area with space allocated by a new scratch
// allocator. This facilitates plan reuse with scratch allocators.
//
// This requires that the plan was originally created using a scratch
// allocator, as otherwise scratch space will have been allocated internally
// by cuFFT.
virtual void UpdatePlanWithScratchAllocator(
Stream *stream, Plan *plan, ScratchAllocator *scratch_allocator) = 0;
// Computes complex-to-complex FFT in the transform direction as specified
// by direction parameter.
virtual bool DoFft(Stream *stream, Plan *plan,
const DeviceMemory<std::complex<float>> &input,
DeviceMemory<std::complex<float>> *output) = 0;
virtual bool DoFft(Stream *stream, Plan *plan,
const DeviceMemory<std::complex<double>> &input,
DeviceMemory<std::complex<double>> *output) = 0;
// Computes real-to-complex FFT in forward direction.
virtual bool DoFft(Stream *stream, Plan *plan,
const DeviceMemory<float> &input,
DeviceMemory<std::complex<float>> *output) = 0;
virtual bool DoFft(Stream *stream, Plan *plan,
const DeviceMemory<double> &input,
DeviceMemory<std::complex<double>> *output) = 0;
// Computes complex-to-real FFT in inverse direction.
virtual bool DoFft(Stream *stream, Plan *plan,
const DeviceMemory<std::complex<float>> &input,
DeviceMemory<float> *output) = 0;
virtual bool DoFft(Stream *stream, Plan *plan,
const DeviceMemory<std::complex<double>> &input,
DeviceMemory<double> *output) = 0;
protected:
FftSupport() {}
private:
SE_DISALLOW_COPY_AND_ASSIGN(FftSupport);
};
// Macro used to quickly declare overrides for abstract virtuals in the
// fft::FftSupport base class. Assumes that it's emitted somewhere inside the
// ::stream_executor namespace.
#define TENSORFLOW_STREAM_EXECUTOR_GPU_FFT_SUPPORT_OVERRIDES \
std::unique_ptr<fft::Plan> Create1dPlan(Stream *stream, uint64 num_x, \
fft::Type type, bool in_place_fft) \
override; \
std::unique_ptr<fft::Plan> Create2dPlan(Stream *stream, uint64 num_x, \
uint64 num_y, fft::Type type, \
bool in_place_fft) override; \
std::unique_ptr<fft::Plan> Create3dPlan( \
Stream *stream, uint64 num_x, uint64 num_y, uint64 num_z, \
fft::Type type, bool in_place_fft) override; \
std::unique_ptr<fft::Plan> Create1dPlanWithScratchAllocator( \
Stream *stream, uint64 num_x, fft::Type type, bool in_place_fft, \
ScratchAllocator *scratch_allocator) override; \
std::unique_ptr<fft::Plan> Create2dPlanWithScratchAllocator( \
Stream *stream, uint64 num_x, uint64 num_y, fft::Type type, \
bool in_place_fft, ScratchAllocator *scratch_allocator) override; \
std::unique_ptr<fft::Plan> Create3dPlanWithScratchAllocator( \
Stream *stream, uint64 num_x, uint64 num_y, uint64 num_z, \
fft::Type type, bool in_place_fft, ScratchAllocator *scratch_allocator) \
override; \
std::unique_ptr<fft::Plan> CreateBatchedPlan( \
Stream *stream, int rank, uint64 *elem_count, uint64 *input_embed, \
uint64 input_stride, uint64 input_distance, uint64 *output_embed, \
uint64 output_stride, uint64 output_distance, fft::Type type, \
bool in_place_fft, int batch_count) override; \
std::unique_ptr<fft::Plan> CreateBatchedPlanWithScratchAllocator( \
Stream *stream, int rank, uint64 *elem_count, uint64 *input_embed, \
uint64 input_stride, uint64 input_distance, uint64 *output_embed, \
uint64 output_stride, uint64 output_distance, fft::Type type, \
bool in_place_fft, int batch_count, ScratchAllocator *scratch_allocator) \
override; \
void UpdatePlanWithScratchAllocator(Stream *stream, fft::Plan *plan, \
ScratchAllocator *scratch_allocator) \
override; \
bool DoFft(Stream *stream, fft::Plan *plan, \
const DeviceMemory<std::complex<float>> &input, \
DeviceMemory<std::complex<float>> *output) override; \
bool DoFft(Stream *stream, fft::Plan *plan, \
const DeviceMemory<std::complex<double>> &input, \
DeviceMemory<std::complex<double>> *output) override; \
bool DoFft(Stream *stream, fft::Plan *plan, \
const DeviceMemory<float> &input, \
DeviceMemory<std::complex<float>> *output) override; \
bool DoFft(Stream *stream, fft::Plan *plan, \
const DeviceMemory<double> &input, \
DeviceMemory<std::complex<double>> *output) override; \
bool DoFft(Stream *stream, fft::Plan *plan, \
const DeviceMemory<std::complex<float>> &input, \
DeviceMemory<float> *output) override; \
bool DoFft(Stream *stream, fft::Plan *plan, \
const DeviceMemory<std::complex<double>> &input, \
DeviceMemory<double> *output) override;
} // namespace fft
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_FFT_H_
@@ -0,0 +1,23 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_GPU_LAUNCH_DIM_H_
#define TENSORFLOW_STREAM_EXECUTOR_GPU_LAUNCH_DIM_H_
// TODO(rspringer): Temporary redirection until all users - including gcudacc -
// are using the new file.
#include "tensorflow/stream_executor/launch_dim.h"
#endif // TENSORFLOW_STREAM_EXECUTOR_GPU_LAUNCH_DIM_H_
@@ -0,0 +1,214 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Declares the HostExecutor class, which is a CPU-only implementation of
// the StreamExecutor interface. For now, this is used for testing and to
// examine the performance of host-based StreamExecutor code.
#ifndef TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_GPU_EXECUTOR_H_
#define TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_GPU_EXECUTOR_H_
#include "tensorflow/stream_executor/blas.h"
#include "tensorflow/stream_executor/host/host_stream.h"
#include "tensorflow/stream_executor/host/host_timer.h"
#include "tensorflow/stream_executor/lib/error.h"
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/rng.h"
#include "tensorflow/stream_executor/stream_executor.h"
#include "tensorflow/stream_executor/stream_executor_internal.h"
namespace stream_executor {
namespace host {
// An implementation of StreamExecutor that does no communication or interaction
// with a device, but DOES perform memory operations backed by the host.
// Plugin routines (RNG, BLAS) are also supported and functional.
// Kernel invocations will fail, but host callbacks may be enqueued on this
// executor and its associated stream, and should follow standard ordering
// semantics.
//
// This is useful for evaluating the performance of host-based or fallback
// routines executed under the context of a GPU executor.
// See stream_executor.h for description of the below operations.
class HostExecutor : public internal::StreamExecutorInterface {
public:
explicit HostExecutor(const PluginConfig &plugin_config);
~HostExecutor() override;
port::Status Init(int device_ordinal, DeviceOptions device_options) override {
return port::Status::OK();
}
bool GetKernel(const MultiKernelLoaderSpec &spec,
KernelBase *kernel) override {
return false;
}
bool Launch(Stream *stream, const ThreadDim &thread_dims,
const BlockDim &block_dims, const KernelBase &kernel,
const KernelArgsArrayBase &args) override {
return false;
}
void *Allocate(uint64 size) override;
void *AllocateSubBuffer(DeviceMemoryBase *mem, uint64 offset_bytes,
uint64 size_bytes) override;
void Deallocate(DeviceMemoryBase *mem) override;
void *HostMemoryAllocate(uint64 size) override { return new char[size]; }
void HostMemoryDeallocate(void *mem) override {
delete[] static_cast<char *>(mem);
}
bool HostMemoryRegister(void *mem, uint64 size) override { return true; }
bool HostMemoryUnregister(void *mem) override { return true; }
bool Memcpy(Stream *stream, void *host_dst, const DeviceMemoryBase &gpu_src,
uint64 size) override;
bool Memcpy(Stream *stream, DeviceMemoryBase *gpu_dst, const void *host_src,
uint64 size) override;
bool MemcpyDeviceToDevice(Stream *stream, DeviceMemoryBase *gpu_dst,
const DeviceMemoryBase &host_src,
uint64 size) override;
bool MemZero(Stream *stream, DeviceMemoryBase *location,
uint64 size) override;
bool Memset(Stream *stream, DeviceMemoryBase *location, uint8 pattern,
uint64 size) override;
bool Memset32(Stream *stream, DeviceMemoryBase *location, uint32 pattern,
uint64 size) override;
// No "synchronize all activity" implemented for this platform at the moment.
bool SynchronizeAllActivity() override { return true; }
bool SynchronousMemZero(DeviceMemoryBase *location, uint64 size) override;
bool SynchronousMemSet(DeviceMemoryBase *location, int value,
uint64 size) override;
port::Status SynchronousMemcpy(DeviceMemoryBase *gpu_dst,
const void *host_src, uint64 size) override;
port::Status SynchronousMemcpy(void *host_dst,
const DeviceMemoryBase &gpu_src,
uint64 size) override;
port::Status SynchronousMemcpyDeviceToDevice(DeviceMemoryBase *gpu_dst,
const DeviceMemoryBase &gpu_src,
uint64 size) override;
bool HostCallback(Stream *stream, std::function<void()> callback) override;
port::Status AllocateEvent(Event *event) override {
return port::Status(port::error::UNIMPLEMENTED, "");
}
port::Status DeallocateEvent(Event *event) override {
return port::Status(port::error::UNIMPLEMENTED, "");
}
port::Status RecordEvent(Stream *stream, Event *event) override {
return port::Status(port::error::UNIMPLEMENTED, "");
}
port::Status WaitForEvent(Stream *stream, Event *event) override {
return port::Status(port::error::UNIMPLEMENTED, "");
}
Event::Status PollForEventStatus(Event *event) override {
return Event::Status::kError;
}
bool AllocateStream(Stream *stream) override;
void DeallocateStream(Stream *stream) override;
bool CreateStreamDependency(Stream *dependent, Stream *other) override;
// No special initialization is necessary for host timers.
bool AllocateTimer(Timer *timer) override { return true; }
void DeallocateTimer(Timer *timer) override {}
bool StartTimer(Stream *stream, Timer *timer) override;
bool StopTimer(Stream *stream, Timer *timer) override;
port::Status BlockHostUntilDone(Stream *stream) override;
int PlatformDeviceCount() override { return 1; }
bool DeviceMemoryUsage(int64 *free, int64 *total) const override {
return false;
}
DeviceDescription *PopulateDeviceDescription() const override;
port::Status EnablePeerAccessTo(StreamExecutorInterface *other) override {
return port::Status::OK();
}
bool CanEnablePeerAccessTo(StreamExecutorInterface *other) override {
return true;
}
SharedMemoryConfig GetDeviceSharedMemoryConfig() override {
LOG(INFO) << "Shared memory configuration is unsupported for host "
<< "executors.";
return SharedMemoryConfig::kDefault;
}
port::Status SetDeviceSharedMemoryConfig(SharedMemoryConfig config) override {
string error_msg{
"Shared memory configuration is unsupported for host "
"executors."};
LOG(INFO) << error_msg;
return port::Status(port::error::UNIMPLEMENTED, error_msg);
}
bool SupportsBlas() const override;
blas::BlasSupport *CreateBlas() override;
bool SupportsDnn() const override { return false; }
dnn::DnnSupport *CreateDnn() override { return nullptr; }
bool SupportsFft() const override;
fft::FftSupport *CreateFft() override;
bool SupportsRng() const override;
rng::RngSupport *CreateRng() override;
std::unique_ptr<internal::EventInterface> CreateEventImplementation()
override {
LOG(WARNING) << "Events not currently supported by HostExecutor.";
return nullptr;
}
std::unique_ptr<internal::KernelInterface> CreateKernelImplementation()
override {
return nullptr;
}
std::unique_ptr<internal::StreamInterface> GetStreamImplementation()
override {
return std::unique_ptr<internal::StreamInterface>(new HostStream());
}
std::unique_ptr<internal::TimerInterface> GetTimerImplementation() override {
return std::unique_ptr<internal::TimerInterface>(new HostTimer());
}
void *GpuContextHack() override { return nullptr; }
private:
const PluginConfig plugin_config_;
};
} // namespace host
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_GPU_EXECUTOR_H_
@@ -0,0 +1,83 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Declares the "host" platform, which is a CPU-only implementation of the
// StreamExecutor. The host platform only supports memory operations and plugin
// routines, and is primarily used for testing.
#ifndef TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_PLATFORM_H_
#define TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_PLATFORM_H_
#include <memory>
#include <string>
#include <vector>
#include "tensorflow/stream_executor/executor_cache.h"
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/multi_platform_manager.h"
#include "tensorflow/stream_executor/platform.h"
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/platform/thread_annotations.h"
#include "tensorflow/stream_executor/stream_executor_pimpl.h"
#include "tensorflow/stream_executor/trace_listener.h"
namespace stream_executor {
namespace host {
// Host (CPU) platform plugin, registered as a singleton value via module
// initializer.
class HostPlatform : public Platform {
public:
HostPlatform();
~HostPlatform() override;
Platform::Id id() const override;
// Device count is less clear-cut for CPUs than accelerators. This call
// currently returns the number of thread units in the host, as reported by
// base::NumCPUs().
int VisibleDeviceCount() const override;
const string& Name() const override;
port::StatusOr<StreamExecutor*> ExecutorForDevice(int ordinal) override;
port::StatusOr<StreamExecutor*> ExecutorForDeviceWithPluginConfig(
int ordinal, const PluginConfig& config) override;
port::StatusOr<StreamExecutor*> GetExecutor(
const StreamExecutorConfig& config) override;
port::StatusOr<std::unique_ptr<StreamExecutor>> GetUncachedExecutor(
const StreamExecutorConfig& config) override;
void RegisterTraceListener(std::unique_ptr<TraceListener> listener) override;
void UnregisterTraceListener(TraceListener* listener) override;
private:
// This platform's name.
string name_;
// Cache of created StreamExecutors.
ExecutorCache executor_cache_;
SE_DISALLOW_COPY_AND_ASSIGN(HostPlatform);
};
} // namespace host
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_PLATFORM_H_
@@ -0,0 +1,34 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_PLATFORM_ID_H_
#define TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_PLATFORM_ID_H_
#include "tensorflow/stream_executor/platform.h"
namespace stream_executor {
namespace host {
// Opaque and unique identifier for the host platform.
// This is needed so that plugins can refer to/identify this platform without
// instantiating a HostPlatform object.
// This is broken out here to avoid a circular dependency between HostPlatform
// and HostStreamExecutor.
extern const Platform::Id kHostPlatformId;
} // namespace host
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_PLATFORM_ID_H_
@@ -0,0 +1,56 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Class declaration for Stream type that enqueues tasks onto a host/CPU-based
// execution context (as opposed to a GPU device), HostExecutor.
#ifndef TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_STREAM_H_
#define TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_STREAM_H_
#include <functional>
#include <memory>
#include "tensorflow/stream_executor/lib/threadpool.h"
#include "tensorflow/stream_executor/stream_executor_internal.h"
namespace stream_executor {
namespace host {
class HostStream : public internal::StreamInterface {
public:
HostStream();
~HostStream() override;
bool EnqueueTask(std::function<void()> task);
void *GpuStreamHack() override { return nullptr; }
void **GpuStreamMemberHack() override { return nullptr; }
void BlockUntilDone();
private:
// Use only one thread and own task queue to preserve FIFO ordering
// for the operations enqueued by any given stream.
static const int kExecutorThreads = 1;
std::unique_ptr<port::ThreadPool> host_executor_;
mutex mu_;
int pending_tasks_ GUARDED_BY(mu_) = 0;
condition_variable completion_condition_;
};
} // namespace host
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_STREAM_H_
@@ -0,0 +1,61 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_TIMER_H_
#define TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_TIMER_H_
#include <chrono>
#include "tensorflow/stream_executor/stream_executor_internal.h"
namespace stream_executor {
namespace host {
class HostTimer : public internal::TimerInterface {
public:
HostTimer() {}
~HostTimer() override {}
// Begins the timer at the present point in the stream.
bool Start(Stream *stream);
// Stops the timer at the present point in the stream.
bool Stop(Stream *stream);
// Returns the most recent value recorded for a start/stopcycle, in
// microseconds.
uint64 Microseconds() const override;
// Returns the most recent value recorded for a start/stopcycle, in
// nanoseconds.
uint64 Nanoseconds() const override;
private:
using clock = std::chrono::high_resolution_clock;
clock::time_point start_time_;
clock::duration duration_;
// Actually starts (rather than enqueues starting) the timer.
void StartNow();
// Actually stops (rather than enqueues stopping) the timer.
void StopNow();
};
} // namespace host
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_HOST_HOST_TIMER_H_
@@ -0,0 +1,46 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_HOST_BUFFER_H_
#define TENSORFLOW_STREAM_EXECUTOR_HOST_BUFFER_H_
#include "tensorflow/stream_executor/dnn.h"
namespace stream_executor {
// A HostBuffer is a block of memory in host memory containing the data for a
// dnn::BatchDescriptor using a device-dependent memory layout.
// Derived classes provide methods to construct a HostBuffer for a specific
// device, and to copy data in and out of the buffer.
class HostBuffer {
public:
const dnn::BatchDescriptor& descriptor() const { return descriptor_; }
// Returns a string describing the HostBuffer.
virtual string AsString() const = 0;
protected:
// Construct a HostBuffer from the supplied dnn::BatchDescriptor.
explicit HostBuffer(const dnn::BatchDescriptor& descriptor)
: descriptor_(descriptor) {}
virtual ~HostBuffer() {}
private:
const dnn::BatchDescriptor descriptor_;
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_HOST_BUFFER_H_
@@ -0,0 +1,56 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_HOST_OR_DEVICE_SCALAR_H_
#define TENSORFLOW_STREAM_EXECUTOR_HOST_OR_DEVICE_SCALAR_H_
#include "tensorflow/stream_executor/device_memory.h"
#include "tensorflow/stream_executor/platform/logging.h"
namespace stream_executor {
// Allows to represent a value that is either a host scalar or a scalar stored
// on the GPU device.
template <typename ElemT>
class HostOrDeviceScalar {
public:
// Not marked as explicit because when using this constructor, we usually want
// to set this to a compile-time constant.
HostOrDeviceScalar(ElemT value) : value_(value), is_pointer_(false) {}
explicit HostOrDeviceScalar(const DeviceMemory<ElemT>& pointer)
: pointer_(pointer), is_pointer_(true) {
CHECK_EQ(1, pointer.ElementCount());
}
bool is_pointer() const { return is_pointer_; }
const DeviceMemory<ElemT>& pointer() const {
CHECK(is_pointer());
return pointer_;
}
const ElemT& value() const {
CHECK(!is_pointer());
return value_;
}
private:
union {
ElemT value_;
DeviceMemory<ElemT> pointer_;
};
bool is_pointer_;
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_HOST_OR_DEVICE_SCALAR_H_
@@ -0,0 +1,715 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Suite of datatypes to represent data-parallel kernel objects (code entities).
// Kernel is the untyped variant, whereas TypedKernel takes a type signature
// to do some template-based helper generation and give compile-time type
// checking for kernel launch parameters.
//
// Users typically don't see KernelBase, they see typed kernels, analogous to a
// typed function pointer. TypedKernels express their argument types via
// template parameters like so:
//
// TypedKernel<DeviceMemory<int>*, int>
//
// Which expresses a data parallel kernel signature for:
//
// void(int*, int);
//
// And for a const memory region:
//
// TypedKernel<const DeviceMemory<int>&, int>
//
// Corresponds to a data parallel kernel signature for:
//
// void(const int*, int)
//
// Note that kernels always have a void return type, so results typically must
// be memcpy'ied from device memory to the host.
//
// Also note that a scalar integer residing in device memory and an array of
// integers residing in device memory have the same signature: DeviceMemory<T>.
// However, in the future, checks may be added for additional safety that arrays
// of minimum sizes are passed when those minimum sizes are contractually
// expected by the kernel.
//
// For user-defined types whose definitions are appropriately shared between the
// host code doing the launching and the kernel code being launched, the user
// defined types are similarly permitted to be expressed as residing in device
// memory:
//
// TypedKernel<DeviceMemory<MyUserDefinedStructure>>
//
// And, when the alignment and padding are agreed upon, POD types will also be
// able to be passed by value; for example, it is a common idiom to specify a
// bunch of options simultaneously with a structure:
//
// TypedKernel<MyOptionsStructurePassedByValue, DeviceMemory<float>>
//
// Which corresponds to a data parallel kernel signature like:
//
// void(MyOptionsStructurePassedByValue value, float *result);
//
// Users typically won't need to type out the TypedKernel signature in full, it
// will be typedef'd by automatically generated code; for example, see
// stream_executor::executor_sample::VecReduceAddKernel.
#ifndef TENSORFLOW_STREAM_EXECUTOR_KERNEL_H_
#define TENSORFLOW_STREAM_EXECUTOR_KERNEL_H_
#include <array>
#include <memory>
#include <tuple>
#include <type_traits>
#include <vector>
#include "tensorflow/stream_executor/device_memory.h"
#include "tensorflow/stream_executor/kernel_cache_config.h"
#include "tensorflow/stream_executor/lib/array_slice.h"
#include "tensorflow/stream_executor/lib/inlined_vector.h"
#include "tensorflow/stream_executor/lib/stringpiece.h"
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
class DeviceMemoryBase;
template <typename ElemT>
class DeviceMemory;
class StreamExecutor;
namespace internal {
class KernelInterface;
} // namespace internal
// KernelMetadata holds runtime-queryable attributes of a loaded kernel, such as
// registers allocated, shared memory used, etc.
// Not all platforms support reporting of all information, so each accessor
// returns false if the associated field is not populated in the underlying
// platform.
class KernelMetadata {
public:
KernelMetadata()
: has_registers_per_thread_(false), has_shared_memory_bytes_(false) {}
// Returns the number of registers used per thread executing this kernel.
bool registers_per_thread(int *registers_per_thread) const;
// Sets the number of registers used per thread executing this kernel.
void set_registers_per_thread(int registers_per_thread);
// Returns the amount of [static] shared memory used per block executing this
// kernel. Note that dynamic shared memory allocations are not (and can not)
// be reported here (since they're not specified until kernel launch time).
bool shared_memory_bytes(int *shared_memory_bytes) const;
// Sets the amount of [static] shared memory used per block executing this
// kernel.
void set_shared_memory_bytes(int shared_memory_bytes);
private:
// Holds the value returned by registers_per_thread above.
bool has_registers_per_thread_;
int registers_per_thread_;
// Holds the value returned by shared_memory_bytes above.
bool has_shared_memory_bytes_;
int64 shared_memory_bytes_;
};
// A data-parallel kernel (code entity) for launching via the StreamExecutor,
// analogous to a void* device function pointer. See TypedKernel for the typed
// variant.
//
// Thread-compatible.
class KernelBase {
public:
KernelBase(KernelBase &&from);
// Constructs an "empty" (not-yet-loaded) kernel instance.
//
// parent is the StreamExecutor that will be responsible for loading the
// implementation of this kernel. It must not be null.
explicit KernelBase(StreamExecutor *parent);
// Test-only constructor that can take a mock KernelInterface implementation.
KernelBase(StreamExecutor *parent, internal::KernelInterface *implementation);
// Releases resources associated with the kernel instance (i.e.
// platform-specific implementation).
~KernelBase();
// Returns the number of parameters that this kernel accepts. (Arity refers to
// nullary, unary, ...).
unsigned Arity() const;
// Returns the StreamExecutor that represents the platform this kernel
// executes upon.
StreamExecutor *parent() const { return parent_; }
// Returns a const pointer to the (opaque) platform-dependent implementation.
const internal::KernelInterface *implementation() const {
return implementation_.get();
}
// Returns a non-const pointer to the (opaque) platform-dependent
// implementation.
internal::KernelInterface *implementation() { return implementation_.get(); }
void set_metadata(const KernelMetadata &metadata) { metadata_ = metadata; }
const KernelMetadata &metadata() const { return metadata_; }
// Sets the preferred cache configuration for a kernel. This is just a
// suggestion to the runtime, and may not be honored during execution.
void SetPreferredCacheConfig(KernelCacheConfig config);
// Gets the preferred cache configuration for a kernel.
KernelCacheConfig GetPreferredCacheConfig() const;
void set_name(port::StringPiece name);
const string &name() const { return name_; }
const string &demangled_name() const { return demangled_name_; }
private:
// The StreamExecutor that loads this kernel object.
StreamExecutor *parent_;
// Implementation delegated to for platform-specific functionality.
std::unique_ptr<internal::KernelInterface> implementation_;
string name_;
string demangled_name_;
KernelMetadata metadata_;
SE_DISALLOW_COPY_AND_ASSIGN(KernelBase);
};
// Whether T is a DeviceMemory-family pointer.
template <typename T>
struct IsDeviceMemoryPointer {
static constexpr bool value = false;
};
template <typename U>
struct IsDeviceMemoryPointer<DeviceMemory<U> *> {
static constexpr bool value = true;
};
template <>
struct IsDeviceMemoryPointer<DeviceMemoryBase *> {
static constexpr bool value = true;
};
// Whether T is a DeviceMemory-family value-like thing (which includes a
// reference). This trait is useful because we pack values in the same manner as
// references.
template <typename T>
struct IsDeviceMemoryValueLike {
static constexpr bool value = false;
};
template <typename U>
struct IsDeviceMemoryValueLike<DeviceMemory<U> &> {
static constexpr bool value = true;
};
// We need to treat SharedDeviceMemory types differently than other DeviceMemory
// types (since they maintain no allocations), hence these specializations.
template <typename U>
struct IsDeviceMemoryValueLike<SharedDeviceMemory<U> &> {
static constexpr bool value = false;
};
template <>
struct IsDeviceMemoryValueLike<DeviceMemoryBase &> {
static constexpr bool value = true;
};
template <typename U>
struct IsDeviceMemoryValueLike<DeviceMemory<U>> {
static constexpr bool value = true;
};
template <typename U>
struct IsDeviceMemoryValueLike<SharedDeviceMemory<U>> {
static constexpr bool value = false;
};
template <>
struct IsDeviceMemoryValueLike<DeviceMemoryBase> {
static constexpr bool value = true;
};
template <typename U>
struct IsSharedDeviceMemory {
static constexpr bool value = false;
};
template <typename U>
struct IsSharedDeviceMemory<SharedDeviceMemory<U> &> {
static constexpr bool value = true;
};
template <typename U>
struct IsSharedDeviceMemory<SharedDeviceMemory<U>> {
static constexpr bool value = true;
};
// Basic data about a kernel argument.
struct KernelArg {
bool is_shared;
const void *address;
size_t size;
};
// An iterator for traversing all the arguments of a KernelArgsArray.
class KernelArgIterator {
public:
KernelArgIterator(int number_of_argument_addresses,
int number_of_shared_memory_arguments,
const void *const *arg_addresses_data,
const size_t *arg_sizes_data,
const size_t *shmem_bytes_data,
const size_t *shmem_indices_data)
: arg_index_(0),
number_of_arguments_(number_of_argument_addresses +
number_of_shared_memory_arguments),
arg_address_iter_(arg_addresses_data),
arg_size_iter_(arg_sizes_data),
shmem_bytes_iter_(shmem_bytes_data),
shmem_indices_iter_(shmem_indices_data),
shmem_indices_end_(shmem_indices_data +
number_of_shared_memory_arguments) {}
// Returns true if another argument is present in the iterator.
bool has_next() { return arg_index_ < number_of_arguments_; }
// Returns the next argument in the iterator.
//
// Returns a default-constructed KernelArg if there is no next argument.
KernelArg next() {
KernelArg result = {};
if (!has_next()) {
return result;
} else if ((shmem_indices_iter_ != shmem_indices_end_) &&
(arg_index_ == *shmem_indices_iter_)) {
result.is_shared = true;
result.address = nullptr;
result.size = *shmem_bytes_iter_;
++shmem_indices_iter_;
++shmem_bytes_iter_;
} else {
result.is_shared = false;
result.address = *arg_address_iter_;
result.size = *arg_size_iter_;
++arg_address_iter_;
++arg_size_iter_;
}
++arg_index_;
return result;
}
private:
size_t arg_index_;
size_t number_of_arguments_;
const void *const *arg_address_iter_;
const size_t *arg_size_iter_;
const size_t *shmem_bytes_iter_;
const size_t *shmem_indices_iter_;
const size_t *const shmem_indices_end_;
};
// Base class for KernelArgsArray.
//
// Supports all the getter methods that do not depend on the compile-time number
// of arguments template parameter.
//
// This class exists as a way to pass kernel arguments to
// StreamExecutorInterface::Launch. That Launch method is virtual, so it can't
// be templated to accept any KernelArgsArray type, therefore a reference to
// this base type is passed instead.
//
// Performance is not a concern here because each of these methods will be
// called at most once per kernel launch. Past performance concerns with
// KernelArgsArray have been in reference to the argument packing routines which
// are called once per kernel argument. Those packing routines are now handled
// by the templated KernelArgsArray subclass of this class where they can take
// advantage of compile-time knowledge of the number of arguments in order to be
// very efficient.
class KernelArgsArrayBase {
public:
virtual ~KernelArgsArrayBase() = default;
// Gets the number of arguments added so far, including shared memory
// arguments.
virtual size_t number_of_arguments() const = 0;
// Gets the total number of shared memory bytes added so far.
virtual uint64 number_of_shared_bytes() const = 0;
// Gets the list of argument addresses.
virtual port::ArraySlice<const void *> argument_addresses() const = 0;
// Gets an iterator to the arguments in the array.
virtual KernelArgIterator arg_iterator() const = 0;
};
// A list of arguments for a kernel call.
//
// The template parameter kNumArgs is the maximum number of arguments which can
// be stored in the list.
//
// Contains a list of addresses for non-shared-memory arguments and a list of
// sizes for shared-memory arguments. Since the shared-memory arguments may be
// interspersed with the non-shared-memory arguments, it also stores a list of
// the indices at which the shared-memory arguments appeared.
//
// For example, if the argument address list contains {a, b, c, d, e}, the
// shared-memory arguments list contains the sizes of {A, B, C}, and the
// shared-memory indices list contains {0, 3, 5}, then the original list of
// arguments was {A, a, b, B, c, C, d, e}.
//
// This way of storing the arguments makes CUDA kernel calls efficient because
// they only require the argument address list and the total number of shared
// bytes, but it also makes it possible for OpenCL kernel calls because they
// depend on the location of each shared-memory argument and its size.
//
// Note that the code for adding arguments has been identified as a performance
// hotspot in some real-world applications so this structure has been optimized
// for the performance of argument adding.
template <size_t kNumArgs>
class KernelArgsArray : public KernelArgsArrayBase {
public:
explicit KernelArgsArray()
: total_shared_memory_bytes_(0),
number_of_argument_addresses_(0),
number_of_shared_memory_arguments_(0) {}
// Adds an argument to the list.
//
// Note that the address of the argument is stored, so the input must not go
// out of scope before the instance of this class that calls this method does.
template <typename T>
void add_argument(const T &arg) {
argument_addresses_[number_of_argument_addresses_] =
static_cast<const void *>(&arg);
argument_sizes_[number_of_argument_addresses_] = sizeof(arg);
++number_of_argument_addresses_;
}
// Adds a device memory argument to the list.
void add_device_memory_argument(const DeviceMemoryBase &arg) {
const void **copy_ptr =
&device_memory_opaque_pointers_[number_of_argument_addresses_];
*copy_ptr = arg.opaque();
argument_addresses_[number_of_argument_addresses_] = copy_ptr;
argument_sizes_[number_of_argument_addresses_] = sizeof(void *);
++number_of_argument_addresses_;
}
// Adds a shared memory argument to the list.
//
// The only significant information about a shared argument is its size, so
// that is the only parameter in this function.
void add_shared_bytes(size_t number_of_bytes) {
shared_memory_indices_[number_of_shared_memory_arguments_] =
number_of_argument_addresses_ + number_of_shared_memory_arguments_;
shared_memory_bytes_[number_of_shared_memory_arguments_] = number_of_bytes;
++number_of_shared_memory_arguments_;
total_shared_memory_bytes_ += number_of_bytes;
}
// Gets the number of arguments added so far, including shared memory
// arguments.
size_t number_of_arguments() const override {
return number_of_argument_addresses_ + number_of_shared_memory_arguments_;
}
// Gets the total number of shared memory bytes added so far.
uint64 number_of_shared_bytes() const override {
return total_shared_memory_bytes_;
}
// Gets the list of argument addresses.
port::ArraySlice<const void *> argument_addresses() const override {
return port::ArraySlice<const void *>(argument_addresses_.data(),
number_of_argument_addresses_);
}
// Gets an iterator to the arguments in the array.
KernelArgIterator arg_iterator() const override {
return KernelArgIterator(
number_of_argument_addresses_, number_of_shared_memory_arguments_,
argument_addresses_.data(), argument_sizes_.data(),
shared_memory_bytes_.data(), shared_memory_indices_.data());
}
private:
// A place to store copies of opaque pointers from device memory arguments.
std::array<const void *, kNumArgs> device_memory_opaque_pointers_;
// Addresses for non-shared-memory arguments.
std::array<const void *, kNumArgs> argument_addresses_;
// Sizes for non-shared-memory arguments.
std::array<size_t, kNumArgs> argument_sizes_;
// Size in bytes for each shared memory argument.
std::array<size_t, kNumArgs> shared_memory_bytes_;
// Indices in the arguments array for shared memory arguments.
std::array<size_t, kNumArgs> shared_memory_indices_;
// Total of all shared memory sizes.
size_t total_shared_memory_bytes_;
// Number of significant entries in argument_addresses_ and argument_sizes_.
size_t number_of_argument_addresses_;
// Number of significant entries in shared_memory_bytes_ and
// shared_memory_indices_.
size_t number_of_shared_memory_arguments_;
};
// Typed variant of KernelBase, like a typed device function pointer. See the
// file comment for details and example usage.
//
// This class contains template metaprogramming magic to type check the
// parameters passed to a kernel launch are acceptable, and subsequently pack
// them into a form which can be used by the StreamExecutorInterface
// implementation. (i.e. CUDA and OpenCL both bind void*s with associated
// sizes as kernel arguments.)
//
// Thread-compatible.
template <typename... Params>
class TypedKernel : public KernelBase {
public:
static constexpr size_t kNumberOfParameters = sizeof...(Params);
// Delegates to KernelBase::KernelBase(), see that constructor.
explicit TypedKernel(StreamExecutor *parent) : KernelBase(parent) {}
// Test-only constructor that can take a mock KernelInterface implementation.
// Takes ownership of implementation, it should not be null.
TypedKernel(StreamExecutor *parent, internal::KernelInterface *implementation)
: KernelBase(parent, implementation) {}
private:
// Stream needs access to the specific parameter-packing functionality that
// the TypedKernel provides for its corresponding type signature (and no other
// type signatures).
friend class Stream;
// This is the main entry point into the magic. Packs the parameters (which
// must type check against the class template) into the args and sizes
// arrays.
//
// Const refs are taken as parameters on all of the handlers to avoid
// implicit type promotion of integers.
//
// WARNING: as a performance optimization this method may store pointers to
// some of the input parameters in the kernel args structure, so any params
// passed into this method must live at least as long as the kernel args
// structure.
void PackParams(KernelArgsArray<kNumberOfParameters> *args,
Params &... params) const {
PackOneParam(args, params...);
}
template <typename T, typename... RestOfParams>
void PackOneParam(KernelArgsArray<kNumberOfParameters> *args, const T &arg,
const RestOfParams &... rest) const {
PackOneParam(args, arg);
PackOneParam(args, rest...);
}
// Packs one (non-DeviceMemoryBase) parameter into the arg and sizes array.
// The enable_if<> is for excluding DeviceMemoryBase args, which have a
// separate implementation below.
template <typename T>
void PackOneParam(
KernelArgsArray<kNumberOfParameters> *args, const T &arg,
typename std::enable_if<!IsDeviceMemoryValueLike<T>::value &&
!IsDeviceMemoryPointer<T>::value &&
!IsSharedDeviceMemory<T>::value>::type * =
nullptr) const {
static_assert(!std::is_pointer<T>::value,
"cannot pass raw pointer to the device");
static_assert(!std::is_convertible<T, DeviceMemoryBase>::value,
"cannot pass device memory as a normal value");
args->add_argument(arg);
}
// DeviceMemoryBase family reference override.
template <typename T>
void PackOneParam(
KernelArgsArray<kNumberOfParameters> *args, const T &arg,
typename std::enable_if<IsDeviceMemoryValueLike<T>::value>::type * =
nullptr) const {
args->add_device_memory_argument(arg);
}
// DeviceMemoryBase family pointer override.
template <typename T>
void PackOneParam(
KernelArgsArray<kNumberOfParameters> *args, T arg,
typename std::enable_if<IsDeviceMemoryPointer<T>::value>::type * =
nullptr) const {
DeviceMemoryBase *ptr = static_cast<DeviceMemoryBase *>(arg);
args->add_device_memory_argument(*ptr);
}
// Dynamic shared device memory has a size, but no associated allocation on
// the host; internally, the device will allocate storage.
template <typename T>
void PackOneParam(
KernelArgsArray<kNumberOfParameters> *args, T arg,
typename std::enable_if<IsSharedDeviceMemory<T>::value>::type * =
nullptr) const {
args->add_shared_bytes(arg.size());
}
// Base case for variadic template expansion - nothing to do!
void PackOneParam(KernelArgsArray<kNumberOfParameters> *args) const {}
SE_DISALLOW_COPY_AND_ASSIGN(TypedKernel);
};
// Template metaprogramming helper type that helps us produce better error
// messages at compile time when the are mismatches between the parameter
// type list and the argument type list.
template <typename ParamTuple, typename ArgTuple>
struct KernelInvocationChecker {
// Whether the parameter tuple and argument tuple match in length.
static constexpr bool kLengthMatches =
std::tuple_size<ParamTuple>::value == std::tuple_size<ArgTuple>::value;
// The (matching) length of the parameters and arguments type lists.
static constexpr int kTupleLength =
static_cast<int>(std::tuple_size<ArgTuple>::value);
// Helper trait to say whether the parameter wants a DeviceMemory-reference
// compatible type. This is for inexact type matches, so that it doesn't have
// to be precisely a const DeviceMemory<T>&, but can also be a value that
// represents the same.
template <typename ParamType, typename ArgType>
struct IsCompatibleDeviceMemoryRef {
static constexpr bool value = false;
};
// See type trait definition above.
template <typename U>
struct IsCompatibleDeviceMemoryRef<const DeviceMemory<U> &, DeviceMemory<U>> {
static constexpr bool value = true;
};
// See type trait definition above.
template <typename U>
struct IsCompatibleDeviceMemoryRef<const SharedDeviceMemory<U> &,
SharedDeviceMemory<U>> {
static constexpr bool value = true;
};
// Returns whether ParamT and ArgT are compatible for data parallel kernel
// parameter packing without any assert functionality.
template <typename ParamT, typename ArgT>
static constexpr bool CompatibleNoAssert() {
return std::is_same<typename std::remove_const<ParamT>::type,
ArgT>::value ||
IsCompatibleDeviceMemoryRef<ParamT, ArgT>::value;
}
// Checks whether ParamT and ArgT are compatible for data parallel kernel
// parameter packing. kArgumentNumber is unused, it just for error display.
//
// NOTE: if you encounter an error here, you can see the mismatch by looking
// at the end of the last error message, which will be of the form:
//
// ...::Compatible<const stream_executor::DeviceMemory<OneThing> &,
// stream_executor::DeviceMemory<AnotherThing>, true,
// 0>'
// requested here
//
// This means that the 0th argument you passed to the kernel invocation should
// have been DeviceMemory<OneThing> but was observed to be
// DeviceMemory<AnotherThing>.
template <typename ParamT, typename ArgT, bool kShouldStaticAssert,
int kArgumentNumber>
static constexpr bool Compatible() {
static_assert(
kShouldStaticAssert ? CompatibleNoAssert<ParamT, ArgT>() : true,
"parameter type (LHS) is not compatible with argument type (RHS)");
return CompatibleNoAssert<ParamT, ArgT>();
}
// Checks the parameter/argument match at kArgumentNumber for an out of bounds
// argument number.
//
// This is the base case: we've run out of argument to check, so we're all
// good.
template <int kArgumentNumber, bool kShouldStaticAssert>
static constexpr bool CheckParam(
typename std::enable_if<(kArgumentNumber < 0)>::type *dummy = nullptr) {
return true;
}
// Checks the parameter/argument match at kArgumentNumber.
// kShouldStaticAssert determines whether to assert out on a mismatch, or just
// yield the constexpr boolean value.
template <int kArgumentNumber, bool kShouldStaticAssert>
static constexpr bool CheckParam(
typename std::enable_if<kArgumentNumber >= 0>::type *dummy = nullptr) {
typedef typename std::tuple_element<kArgumentNumber, ParamTuple>::type
ParamT;
typedef typename std::tuple_element<kArgumentNumber, ArgTuple>::type ArgT;
return Compatible<ParamT, ArgT, kShouldStaticAssert, kArgumentNumber>() &&
CheckParam<kArgumentNumber - 1, kShouldStaticAssert>();
}
// Checks the parameters/arguments for match, but doesn't static assert out.
// This is useful for testing/inspecting whether a set of parameters match in
// things like tests.
static constexpr bool CheckAllNoStaticAssert() {
return kLengthMatches && CheckParam<kTupleLength - 1, false>();
}
// Checks the parameters and static asserts out with a helpful error message
// (and useful template parameters in the instantiation stack) if there is an
// error.
static constexpr bool CheckAllStaticAssert() {
static_assert(kLengthMatches,
"argument length mismatched against typed kernel parameters");
return kLengthMatches && CheckParam<kTupleLength - 1, true>();
}
};
// This is a convenience type for checking whether a typed kernel matches
// against a type list.
template <typename KernelT, typename... Params>
struct KernelParamsOk {
static constexpr bool kResult = false;
};
// See above.
template <typename... Params, typename... Args>
struct KernelParamsOk<TypedKernel<Params...>, Args...> {
static constexpr bool kResult = KernelInvocationChecker<
std::tuple<Params...>, std::tuple<Args...>>::CheckAllNoStaticAssert();
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_KERNEL_H_
@@ -0,0 +1,42 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This file contains declarations relating to kernel cache configuration
// parameters recognized by the StreamExecutor.
#ifndef TENSORFLOW_STREAM_EXECUTOR_KERNEL_CACHE_CONFIG_H_
#define TENSORFLOW_STREAM_EXECUTOR_KERNEL_CACHE_CONFIG_H_
namespace stream_executor {
// This enum represents potential configurations of L1/shared memory when
// running a particular kernel. These values represent user preference, and
// the runtime is not required to respect these choices.
enum class KernelCacheConfig {
// Indicates no preference for device L1/shared memory configuration.
kNoPreference,
// Indicates a preference for more shared memory than L1 cache.
kPreferShared,
// Indicates a preference for more L1 cache than shared memory.
kPreferL1,
// Indicates a preference for equal amounts of L1 cache and shared memory.
kPreferEqual,
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_KERNEL_CACHE_CONFIG_H_
@@ -0,0 +1,378 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Kernel-loader specs are structures that describe how to load a data-parallel
// kernel on a given platform for subsequent launching. Headers that instantiate
// these data structures will typically be auto-generated. However, users can
// also instantiate them by hand.
//
// A kernel with the same exact functionality and type signature may be
// implemented on several different platforms. Typical usage is to create a
// singleton that describes how to load a kernel on the various supported
// platforms:
//
// static const MultiKernelLoaderSpec &SaxpySpec() {
// static auto *mkls =
// (new MultiKernelLoaderSpec{4 /* = arity */})
// ->AddCudaPtxOnDisk(ptx_file_path, ptx_kernelname)
// ->AddOpenCLTextOnDisk(opencl_text_file_path, ocl_kernelname);
// };
//
// return *mkls;
// }
//
// This lazily instantiates an object that describes how to load CUDA PTX
// present on disk that implements saxpy for the for the CUDA platform, or
// OpenCL text present on disk that implements saxpy for an OpenCL-based
// platform. The CudaPtxOnDisk and OpenCLTextOnDisk objects are subtypes of
// KernelLoaderSpec -- KernelLoaderSpec describes how to load a kernel for
// subsequent launching on a single platform.
//
// For the loader functionality that accepts these KernelLoaderSpecs in order
// to grab the kernel appropriately, see StreamExecutor::GetKernel().
#ifndef TENSORFLOW_STREAM_EXECUTOR_KERNEL_SPEC_H_
#define TENSORFLOW_STREAM_EXECUTOR_KERNEL_SPEC_H_
#include <stddef.h>
#include <map>
#include <memory>
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/lib/stringpiece.h"
#include "tensorflow/stream_executor/platform/logging.h"
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
// Describes how to load a kernel on a target platform.
//
// This is an abstract base class, subclassed for specific platforms.
// The filename_or_text field represents the program location (i.e. PTX or
// OpenCL loadable translation unit path) and is simply stored; whether it is a
// filename or text is exposed via more specifically named accessors in
// subclasses.
//
// These kernel loader specifications are typically auto-generated into header
// files at build time, but can also be specified manually.
class KernelLoaderSpec {
public:
virtual ~KernelLoaderSpec() {}
// Returns the kernel name to load out of the program.
const string &kernelname() const { return kernelname_; }
protected:
explicit KernelLoaderSpec(port::StringPiece kernelname);
private:
// The kernel name that should be loaded out of the program description given
// above.
string kernelname_;
SE_DISALLOW_COPY_AND_ASSIGN(KernelLoaderSpec);
};
// An abstract kernel loader spec that has an associated file path, where
// there's a canonical suffix for the filename; e.g. see CudaPtxOnDisk whose
// canonical filename suffix is ".ptx".
class OnDiskKernelLoaderSpec : public KernelLoaderSpec {
public:
~OnDiskKernelLoaderSpec() override {}
// Returns the path to the on-disk loadable kernel file.
const string &filename() const { return filename_; }
// Returns the canonical suffix for this on-disk kernel loader spec format;
// e.g. PTX files on disk have a canonical suffix of ".ptx".
virtual const char *CanonicalSuffix() const = 0;
protected:
OnDiskKernelLoaderSpec(port::StringPiece filename,
port::StringPiece kernelname);
string filename_;
private:
SE_DISALLOW_COPY_AND_ASSIGN(OnDiskKernelLoaderSpec);
};
// Kernel loader specification for PTX text that resides on disk.
class CudaPtxOnDisk : public OnDiskKernelLoaderSpec {
public:
CudaPtxOnDisk(port::StringPiece filename, port::StringPiece kernelname);
~CudaPtxOnDisk() override {}
const char *CanonicalSuffix() const override { return ".ptx"; }
private:
SE_DISALLOW_COPY_AND_ASSIGN(CudaPtxOnDisk);
};
// Kernel loader specification for CUBIN binary that resides on disk.
class CudaCubinOnDisk : public OnDiskKernelLoaderSpec {
public:
CudaCubinOnDisk(port::StringPiece filename, port::StringPiece kernelname);
~CudaCubinOnDisk() override {}
const string &filename() const { return filename_; }
const char *CanonicalSuffix() const override { return ".cubin"; }
private:
string filename_;
SE_DISALLOW_COPY_AND_ASSIGN(CudaCubinOnDisk);
};
// Kernel loader specification for PTX text that resides in memory.
class CudaPtxInMemory : public KernelLoaderSpec {
public:
// Components: compute capability major number, compute capability minor
// number, and PTX source.
typedef std::tuple<int, int, port::StringPiece> PtxSpec;
// Single-PTX constructor. Adds the provided PTX version with an unknown
// compute capability. Since the CC is unknown, the PTX is assumed to be very
// generally usable - in other words, PTX specified in this manner is VERY
// likely to be used as the default! Note that the PTX can be compressed,
// which is indicated by the argument ptx_compressed.
//
// Warning: the string backing the provided port::StringPiece ptx must outlive this
// instance.
CudaPtxInMemory(port::StringPiece ptx, port::StringPiece kernelname,
bool ptx_compressed = false);
// Multiple-PTX-version constructor. Adds each item in spec_list to this
// object. Note that the PTX can be compressed, which is indicated by the
// argument ptx_compressed.
CudaPtxInMemory(const std::initializer_list<PtxSpec> &spec_list,
port::StringPiece kernel_name, bool ptx_compressed = false);
~CudaPtxInMemory() override {}
// Add the PTX implementation described by ptx_spec to this object. On
// collision (i.e., if a version with the same compute_capability already
// exists), the existing implementation will be overwritten.
void AddSpec(PtxSpec ptx_spec);
// Returns pointer to the ptx of available implementation with the
// lowest-valued compute capability. For example, if PTX written to CC2.0,
// 3.0, and 3.5 are all available, the version for CC2.0 will be set. Returns
// nullptr on failed lookup (if any version is not available).
// When the ptx is compressed, returns the decompressed ptx.
const char *default_text() const;
// Similar to default_text().
// When the ptx is compressed, returns the decompressed ptx.
const char *original_default_text() const;
// Returns pointer to the ptx for the requested compute capability.
// Returns nullptr on failed lookup (if the requested version is not
// available).
// When the ptx is compressed, returns the decompressed ptx.
const char *text(int compute_capability_major,
int compute_capability_minor) const;
// Similar to text().
// When the ptx is compressed, returns the original compressed ptx.
const char *original_text(int compute_capability_major,
int compute_capability_minor) const;
// Decompresses the PTX string using bzip2.
static string DecompressPtx(const char *ptx);
private:
// PTX translation unit text contents in memory. The key is of as a tuple
// "<cc_major>,<cc_minor>", i.e., "2,0", "3,0", "3,5". Because CC's
// represented in this way have a clear sorting order, map::begin() will give
// the lowest-numbered version available, i.e. the default.
std::map<std::tuple<int, int>, const char *,
bool (*)(const std::tuple<int, int> &, const std::tuple<int, int> &)>
ptx_by_compute_capability_;
// Stores all decompressed ptx strings, with original ptx string as keys.
// It is marked as mutable for lazy decompression.
mutable std::map<const char *, string> decompressed_ptx_;
mutable mutex mu_;
// Defines the minimum compute capability possible. Used when PTX has no
// compute capability specified (in the single-PTX constructor).
static const std::tuple<int, int> kMinimumCapability;
SE_DISALLOW_COPY_AND_ASSIGN(CudaPtxInMemory);
};
// Kernel loader specification for OpenCL text that resides on disk.
class OpenCLTextOnDisk : public OnDiskKernelLoaderSpec {
public:
OpenCLTextOnDisk(port::StringPiece filename, port::StringPiece kernelname);
~OpenCLTextOnDisk() override {}
const char *CanonicalSuffix() const override { return ".ocl"; }
private:
SE_DISALLOW_COPY_AND_ASSIGN(OpenCLTextOnDisk);
};
// Kernel loader specification for OpenCL binary that resides on disk.
class OpenCLBinaryOnDisk : public OnDiskKernelLoaderSpec {
public:
OpenCLBinaryOnDisk(port::StringPiece filename, port::StringPiece kernelname);
~OpenCLBinaryOnDisk() override {}
const char *CanonicalSuffix() const override { return ".aocx"; }
private:
SE_DISALLOW_COPY_AND_ASSIGN(OpenCLBinaryOnDisk);
};
// Kernel loader specification for OpenCL text that resides in memory.
class OpenCLTextInMemory : public KernelLoaderSpec {
public:
OpenCLTextInMemory(port::StringPiece text, port::StringPiece kernelname);
~OpenCLTextInMemory() override {}
// Returns the OpenCL text contents.
const string &text() const { return text_; }
private:
// OpenCL translation unit text contents in memory.
string text_;
SE_DISALLOW_COPY_AND_ASSIGN(OpenCLTextInMemory);
};
// Kernel loader specification for a CUBIN blob that resides in memory.
class CudaCubinInMemory : public KernelLoaderSpec {
public:
CudaCubinInMemory(const char *bytes, port::StringPiece kernelname);
~CudaCubinInMemory() override {}
const char *bytes() const { return bytes_; }
private:
const char *bytes_;
SE_DISALLOW_COPY_AND_ASSIGN(CudaCubinInMemory);
};
// Describes how to load a kernel on any subset of a number of target platforms.
class MultiKernelLoaderSpec {
public:
explicit MultiKernelLoaderSpec(size_t arity);
// Returns the number of arguments that this kernel accepts.
size_t arity() const { return arity_; }
// Convenience getters for testing whether these platform variants have
// kernel loader specifications available.
bool has_cuda_ptx_on_disk() const { return cuda_ptx_on_disk_ != nullptr; }
bool has_cuda_cubin_on_disk() const { return cuda_cubin_on_disk_ != nullptr; }
bool has_cuda_cubin_in_memory() const {
return cuda_cubin_in_memory_ != nullptr;
}
bool has_cuda_ptx_in_memory() const { return cuda_ptx_in_memory_ != nullptr; }
bool has_ocl_text_on_disk() const { return ocl_text_on_disk_ != nullptr; }
bool has_ocl_binary_on_disk() const { return ocl_binary_on_disk_ != nullptr; }
bool has_ocl_text_in_memory() const { return ocl_text_in_memory_ != nullptr; }
// Accessors for platform variant kernel load specifications.
// Precondition: corresponding has_* is true.
const CudaPtxOnDisk &cuda_ptx_on_disk() const {
CHECK(has_cuda_ptx_on_disk());
return *cuda_ptx_on_disk_;
}
const CudaCubinOnDisk &cuda_cubin_on_disk() const {
CHECK(has_cuda_cubin_on_disk());
return *cuda_cubin_on_disk_;
}
const CudaCubinInMemory &cuda_cubin_in_memory() const {
CHECK(has_cuda_cubin_in_memory());
return *cuda_cubin_in_memory_;
}
const CudaPtxInMemory &cuda_ptx_in_memory() const {
CHECK(has_cuda_ptx_in_memory());
return *cuda_ptx_in_memory_;
}
const OpenCLTextOnDisk &ocl_text_on_disk() const {
CHECK(has_ocl_text_on_disk());
return *ocl_text_on_disk_;
}
const OpenCLBinaryOnDisk &ocl_binary_on_disk() const {
CHECK(has_ocl_binary_on_disk());
return *ocl_binary_on_disk_;
}
const OpenCLTextInMemory &ocl_text_in_memory() const {
CHECK(has_ocl_text_in_memory());
return *ocl_text_in_memory_;
}
// Builder-pattern-like methods for use in initializing a
// MultiKernelLoaderSpec. Each of these should be used at most once for a
// single MultiKernelLoaderSpec object. See file comment for example usage.
//
// Note that the kernelname parameter must be consistent with the kernel in
// the PTX or OpenCL being loaded. Also be aware that in CUDA C++ the kernel
// name may be mangled by the compiler if it is not declared in an
// extern "C" scope.
MultiKernelLoaderSpec *AddOpenCLTextOnDisk(port::StringPiece filename,
port::StringPiece kernelname);
MultiKernelLoaderSpec *AddOpenCLBinaryOnDisk(port::StringPiece filename,
port::StringPiece kernelname);
MultiKernelLoaderSpec *AddOpenCLTextInMemory(port::StringPiece ocl_text,
port::StringPiece kernelname);
MultiKernelLoaderSpec *AddCudaPtxOnDisk(port::StringPiece filename,
port::StringPiece kernelname);
MultiKernelLoaderSpec *AddCudaCubinOnDisk(port::StringPiece filename,
port::StringPiece kernelname);
MultiKernelLoaderSpec *AddCudaCubinInMemory(const char *cubin_bytes,
port::StringPiece kernelname);
MultiKernelLoaderSpec *AddCudaPtxInMemory(port::StringPiece ptx,
port::StringPiece kernelname);
MultiKernelLoaderSpec *AddCudaCompressedPtxInMemory(
port::StringPiece ptx, port::StringPiece kernelname);
MultiKernelLoaderSpec *AddCudaPtxInMemory(
std::initializer_list<CudaPtxInMemory::PtxSpec> spec_list,
port::StringPiece kernelname);
MultiKernelLoaderSpec *AddCudaCompressedPtxInMemory(
std::initializer_list<CudaPtxInMemory::PtxSpec> spec_list,
port::StringPiece kernelname);
private:
std::unique_ptr<CudaPtxOnDisk>
cuda_ptx_on_disk_; // PTX text that resides in a file.
std::unique_ptr<CudaCubinOnDisk>
cuda_cubin_on_disk_; // Binary CUDA program in a file.
std::unique_ptr<CudaCubinInMemory>
cuda_cubin_in_memory_; // Binary CUDA program in memory.
std::unique_ptr<CudaPtxInMemory>
cuda_ptx_in_memory_; // PTX text that resides in memory.
std::unique_ptr<OpenCLTextOnDisk>
ocl_text_on_disk_; // OpenCL text that resides on disk.
std::unique_ptr<OpenCLBinaryOnDisk>
ocl_binary_on_disk_; // OpenCL binary that resides on disk.
std::unique_ptr<OpenCLTextInMemory>
ocl_text_in_memory_; // OpenCL text that resides in memory.
// Number of parameters that the kernel takes. (This is nicer to have in a
// constexpr than having to determine it from the types via template
// metaprogramming).
size_t arity_;
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_KERNEL_SPEC_H_
@@ -0,0 +1,78 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Types to express dimensionality of a kernel launch. Blocks and threads
// are (up to) 3-dimensional.
//
// A thread is conceptually like a SIMD lane. Some number, typically 32
// (though that fact should not be relied on) SIMD lanes are tied together with
// a single PC in a unit called a warp. There is a maximum number of threads
// that can execute in a shared-context entity called a block. Presently, that
// number is 1024 -- again, something that should not be relied on from this
// comment, but checked via stream_executor::DeviceDescription.
//
// For additional information, see
// http://docs.nvidia.com/cuda/kepler-tuning-guide/#device-utilization-and-occupancy
//
// Because of that modest thread-per-block limit, a kernel can be launched with
// multiple blocks. Each block is indivisibly scheduled onto a single core.
// Blocks can also be used in a multi-dimensional configuration, and the block
// count has much less modest limits -- typically they're similar to the maximum
// amount of addressable memory.
#ifndef TENSORFLOW_STREAM_EXECUTOR_LAUNCH_DIM_H_
#define TENSORFLOW_STREAM_EXECUTOR_LAUNCH_DIM_H_
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/lib/strcat.h"
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
// Basic type that represents a 3-dimensional index space.
struct Dim3D {
uint64 x, y, z;
Dim3D(uint64 x, uint64 y, uint64 z) : x(x), y(y), z(z) {}
};
// Thread dimensionality for use in a kernel launch. See file comment for
// details.
struct ThreadDim : public Dim3D {
explicit ThreadDim(uint64 x = 1, uint64 y = 1, uint64 z = 1)
: Dim3D(x, y, z) {}
// Returns a string representation of the thread dimensionality.
string ToString() const {
return port::StrCat("ThreadDim{", x, ", ", y, ", ", z, "}");
}
};
// Block dimensionality for use in a kernel launch. See file comment for
// details.
struct BlockDim : public Dim3D {
explicit BlockDim(uint64 x = 1, uint64 y = 1, uint64 z = 1)
: Dim3D(x, y, z) {}
// Returns a string representation of the block dimensionality.
string ToString() const {
return port::StrCat("BlockDim{", x, ", ", y, ", ", z, "}");
}
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LAUNCH_DIM_H_
@@ -0,0 +1,43 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_ARRAY_SLICE_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_ARRAY_SLICE_H_
#include "absl/types/span.h"
namespace stream_executor {
namespace port {
template <typename T>
using ArraySlice = absl::Span<const T>;
template <typename T>
using MutableArraySlice = absl::Span<T>;
} // namespace port
} // namespace stream_executor
namespace perftools {
namespace gputools {
// Temporarily pull stream_executor into perftools::gputools while we migrate
// code to the new namespace. TODO(b/77980417): Remove this once we've
// completed the migration.
using namespace stream_executor; // NOLINT[build/namespaces]
} // namespace gputools
} // namespace perftools
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_ARRAY_SLICE_H_
@@ -0,0 +1,100 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// IWYU pragma: private, include "third_party/tensorflow/stream_executor/stream_executor.h"
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_CASTS_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_CASTS_H_
#include <stdlib.h>
namespace stream_executor {
namespace port {
// port::bit_cast<Dest,Source> is a template function that implements the
// equivalent of "*reinterpret_cast<Dest*>(&source)". We need this in
// very low-level functions like the protobuf library and fast math
// support.
//
// float f = 3.14159265358979;
// int i = port::bit_cast<int32>(f);
// // i = 0x40490fdb
//
// The classical address-casting method is:
//
// // WRONG
// float f = 3.14159265358979; // WRONG
// int i = * reinterpret_cast<int*>(&f); // WRONG
//
// The address-casting method actually produces undefined behavior
// according to ISO C++ specification section 3.10 -15 -. Roughly, this
// section says: if an object in memory has one type, and a program
// accesses it with a different type, then the result is undefined
// behavior for most values of "different type".
//
// This is true for any cast syntax, either *(int*)&f or
// *reinterpret_cast<int*>(&f). And it is particularly true for
// conversions between integral lvalues and floating-point lvalues.
//
// The purpose of 3.10 -15- is to allow optimizing compilers to assume
// that expressions with different types refer to different memory. gcc
// 4.0.1 has an optimizer that takes advantage of this. So a
// non-conforming program quietly produces wildly incorrect output.
//
// The problem is not the use of reinterpret_cast. The problem is type
// punning: holding an object in memory of one type and reading its bits
// back using a different type.
//
// The C++ standard is more subtle and complex than this, but that
// is the basic idea.
//
// Anyways ...
//
// port::bit_cast<> calls memcpy() which is blessed by the standard,
// especially by the example in section 3.9 . Also, of course,
// port::bit_cast<> wraps up the nasty logic in one place.
//
// Fortunately memcpy() is very fast. In optimized mode, with a
// constant size, gcc 2.95.3, gcc 4.0.1, and msvc 7.1 produce inline
// code with the minimal amount of data movement. On a 32-bit system,
// memcpy(d,s,4) compiles to one load and one store, and memcpy(d,s,8)
// compiles to two loads and two stores.
//
// I tested this code with gcc 2.95.3, gcc 4.0.1, icc 8.1, and msvc 7.1.
//
// WARNING: if Dest or Source is a non-POD type, the result of the memcpy
// is likely to surprise you.
//
// Props to Bill Gibbons for the compile time assertion technique and
// Art Komninos and Igor Tandetnik for the msvc experiments.
//
// -- mec 2005-10-17
template <class Dest, class Source>
inline Dest bit_cast(const Source& source) {
// Compile time assertion: sizeof(Dest) == sizeof(Source)
// A compile error here means your Dest and Source have different sizes.
static_assert(sizeof(Dest) == sizeof(Source),
"src and dst types must have equal sizes");
Dest dest;
memcpy(&dest, &source, sizeof(dest));
return dest;
}
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_CASTS_H_
@@ -0,0 +1,29 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_DEMANGLE_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_DEMANGLE_H_
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
namespace port {
string Demangle(const char* mangled);
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_DEMANGLE_H_
@@ -0,0 +1,41 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_ENV_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_ENV_H_
#include "tensorflow/core/platform/env.h"
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/lib/stringpiece.h"
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
namespace port {
using tensorflow::Env;
using tensorflow::Thread;
inline Status FileExists(const string& filename) {
return Env::Default()->FileExists(filename);
}
inline Status FileExists(const port::StringPiece& filename) {
return Env::Default()->FileExists(string(filename));
}
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_ENV_H_
@@ -0,0 +1,31 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// IWYU pragma: private, include "third_party/tensorflow/stream_executor/stream_executor.h"
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_ERROR_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_ERROR_H_
#include "tensorflow/core/lib/core/error_codes.pb.h" // IWYU pragma: export
namespace stream_executor {
namespace port {
namespace error = tensorflow::error;
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_ERROR_H_
@@ -0,0 +1,71 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_HUMAN_READABLE_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_HUMAN_READABLE_H_
#include <assert.h>
#include <limits>
#include "tensorflow/stream_executor/lib/stringprintf.h"
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
namespace port {
class HumanReadableNumBytes {
public:
static string ToString(int64 num_bytes) {
if (num_bytes == std::numeric_limits<int64>::min()) {
// Special case for number with not representable nagation.
return "-8E";
}
const char* neg_str = GetNegStr(&num_bytes);
// Special case for bytes.
if (num_bytes < 1024LL) {
// No fractions for bytes.
return port::Printf("%s%lldB", neg_str, num_bytes);
}
static const char units[] = "KMGTPE"; // int64 only goes up to E.
const char* unit = units;
while (num_bytes >= (1024LL) * (1024LL)) {
num_bytes /= (1024LL);
++unit;
assert(unit < units + sizeof(units));
}
return port::Printf(((*unit == 'K') ? "%s%.1f%c" : "%s%.2f%c"), neg_str,
num_bytes / 1024.0, *unit);
}
private:
template <typename T>
static const char* GetNegStr(T* value) {
if (*value < 0) {
*value = -(*value);
return "-";
} else {
return "";
}
}
};
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_HUMAN_READABLE_H_
@@ -0,0 +1,70 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_INITIALIZE_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_INITIALIZE_H_
#include "tensorflow/stream_executor/platform/port.h"
#if defined(PLATFORM_GOOGLE)
#include "tensorflow/stream_executor/platform/google/initialize.h"
#else
#undef REGISTER_MODULE_INITIALIZER
#undef DECLARE_MODULE_INITIALIZER
#undef REGISTER_MODULE_INITIALIZER_SEQUENCE
namespace stream_executor {
namespace port {
class Initializer {
public:
typedef void (*InitializerFunc)();
explicit Initializer(InitializerFunc func) { func(); }
struct Dependency {
Dependency(const char *n, Initializer *i) : name(n), initializer(i) {}
const char *const name;
Initializer *const initializer;
};
struct DependencyRegisterer {
DependencyRegisterer(const char *type, const char *name,
Initializer *initializer,
const Dependency &dependency);
};
};
} // namespace port
} // namespace stream_executor
#define REGISTER_INITIALIZER(type, name, body) \
static void google_init_##type##_##name() { body; } \
::stream_executor::port::Initializer google_initializer_##type##_##name( \
google_init_##type##_##name)
#define REGISTER_MODULE_INITIALIZER(name, body) \
REGISTER_INITIALIZER(module, name, body)
#define DECLARE_INITIALIZER(type, name) \
extern ::stream_executor::port::Initializer google_initializer_##type##_##name
#define DECLARE_MODULE_INITIALIZER(name) DECLARE_INITIALIZER(module, name)
#define REGISTER_MODULE_INITIALIZER_SEQUENCE(name1, name2)
#endif // !defined(PLATFORM_GOOGLE)
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_INITIALIZE_H_
@@ -0,0 +1,29 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_INLINED_VECTOR_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_INLINED_VECTOR_H_
#include "absl/container/inlined_vector.h"
namespace stream_executor {
namespace port {
using absl::InlinedVector;
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_INLINED_VECTOR_H_
@@ -0,0 +1,101 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_MATHUTIL_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_MATHUTIL_H_
#include <algorithm>
#include <cmath>
#include <limits>
#include <type_traits>
#include <vector>
#include "tensorflow/stream_executor/platform/logging.h"
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
namespace port {
class MathUtil {
public:
template <typename IntegralType>
static IntegralType CeilOfRatio(IntegralType numerator,
IntegralType denominator) {
return CeilOrFloorOfRatio<IntegralType, true>(numerator, denominator);
}
template <typename IntegralType>
static IntegralType FloorOfRatio(IntegralType numerator,
IntegralType denominator) {
return CeilOrFloorOfRatio<IntegralType, false>(numerator, denominator);
}
template <typename IntegralType, bool ceil>
static IntegralType CeilOrFloorOfRatio(IntegralType numerator,
IntegralType denominator);
};
// ---- CeilOrFloorOfRatio ----
// This is a branching-free, cast-to-double-free implementation.
//
// Casting to double is in general incorrect because of loss of precision
// when casting an int64 into a double.
//
// There's a bunch of 'recipes' to compute a integer ceil (or floor) on the web,
// and most of them are incorrect.
template<typename IntegralType, bool ceil>
IntegralType MathUtil::CeilOrFloorOfRatio(IntegralType numerator,
IntegralType denominator) {
static_assert(std::is_integral<IntegralType>::value,
"CeilOfRatio_is_only_defined_for_integral_types");
assert(denominator != 0);
// Dividing the smallest signed integer by -1 is not supported: it would
// SIGFPE
assert(!std::is_signed<IntegralType>::value ||
numerator != std::numeric_limits<IntegralType>::min() ||
denominator != -1);
const IntegralType rounded_toward_zero = numerator / denominator;
const IntegralType intermediate_product = rounded_toward_zero * denominator;
if (ceil) { // Compile-time condition: not an actual branching
// When rounded_toward_zero is negative, then an adjustment is never needed:
// the real ratio is negative, and so rounded toward zero is the ceil.
// When rounded_toward_zero is non-negative, an adjustment is needed if the
// sign of the difference numerator - intermediate_product is the same as
// the sign of the denominator.
//
// Using a bool and then a static_cast to IntegralType is not strictly
// necessary, but it makes the code clear, and anyway the compiler should
// get rid of it.
const bool needs_adjustment = (rounded_toward_zero >= 0) &&
((denominator > 0 && numerator > intermediate_product) ||
(denominator < 0 && numerator < intermediate_product));
const IntegralType adjustment = static_cast<IntegralType>(needs_adjustment);
const IntegralType ceil_of_ratio = rounded_toward_zero + adjustment;
return ceil_of_ratio;
} else {
// Floor case: symmetrical to the previous one
const bool needs_adjustment = (rounded_toward_zero <= 0) &&
((denominator > 0 && numerator < intermediate_product) ||
(denominator < 0 && numerator > intermediate_product));
const IntegralType adjustment = static_cast<IntegralType>(needs_adjustment);
const IntegralType floor_of_ratio = rounded_toward_zero - adjustment;
return floor_of_ratio;
}
}
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_MATHUTIL_H_
@@ -0,0 +1,29 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_NOTIFICATION_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_NOTIFICATION_H_
#include "tensorflow/core/platform/notification.h"
namespace stream_executor {
namespace port {
using tensorflow::Notification;
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_NOTIFICATION_H_
@@ -0,0 +1,32 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_NUMBERS_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_NUMBERS_H_
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
namespace port {
// Convert strings to floating point values.
// Leading and trailing spaces are allowed.
// Values may be rounded on over- and underflow.
bool safe_strto32(const string& str, int32* value);
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_NUMBERS_H_
@@ -0,0 +1,60 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_PATH_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_PATH_H_
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/stream_executor/lib/stringpiece.h"
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
namespace port {
using tensorflow::io::Dirname;
namespace internal {
// TODO(rspringer): Move to cc/implementation file.
// Not part of the public API.
string JoinPathImpl(std::initializer_list<port::StringPiece> paths);
} // namespace internal
// Join multiple paths together.
// JoinPath unconditionally joins all paths together. For example:
//
// Arguments | JoinPath
// ---------------------------+---------------------
// '/foo', 'bar' | /foo/bar
// '/foo/', 'bar' | /foo/bar
// '/foo', '/bar' | /foo/bar
// '/foo', '/bar', '/baz' | /foo/bar/baz
//
// All paths will be treated as relative paths, regardless of whether or not
// they start with a leading '/'. That is, all paths will be concatenated
// together, with the appropriate path separator inserted in between.
// Arguments must be convertible to port::StringPiece.
//
// Usage:
// string path = file::JoinPath("/var/log", dirname, filename);
// string path = file::JoinPath(FLAGS_test_srcdir, filename);
template <typename... T>
inline string JoinPath(const T&... args) {
return internal::JoinPathImpl({args...});
}
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_PATH_H_
@@ -0,0 +1,30 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_PROCESS_STATE_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_PROCESS_STATE_H_
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
namespace port {
string Hostname();
bool GetCurrentDirectory(string* dir);
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_PROCESS_STATE_H_
@@ -0,0 +1,38 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_PTR_UTIL_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_PTR_UTIL_H_
#include <memory>
#include "tensorflow/core/util/ptr_util.h"
namespace stream_executor {
using tensorflow::MakeUnique;
using tensorflow::WrapUnique;
} // namespace stream_executor
namespace perftools {
namespace gputools {
// Temporarily pull stream_executor into perftools::gputools while we migrate
// code to the new namespace. TODO(jlebar): Remove this once we've completed
// the migration.
using namespace stream_executor; // NOLINT[build/namespaces]
} // namespace gputools
} // namespace perftools
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_PTR_UTIL_H_
@@ -0,0 +1,30 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STACKTRACE_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STACKTRACE_H_
#include "tensorflow/core/platform/stacktrace.h"
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
namespace port {
using tensorflow::CurrentStackTrace;
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STACKTRACE_H_
@@ -0,0 +1,46 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STATIC_THREADLOCAL_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STATIC_THREADLOCAL_H_
#ifdef _MSC_VER
#define __thread __declspec(thread)
#endif
// For POD types in TLS mode, s_obj_VAR is the thread-local variable.
#define SE_STATIC_THREAD_LOCAL_POD(_Type_, _var_) \
static __thread _Type_ s_obj_##_var_; \
namespace { \
class ThreadLocal_##_var_ { \
public: \
ThreadLocal_##_var_() {} \
void Init() {} \
inline _Type_ *pointer() const { \
return &s_obj_##_var_; \
} \
inline _Type_ *safe_pointer() const { \
return &s_obj_##_var_; \
} \
_Type_ &get() const { \
return s_obj_##_var_; \
} \
bool is_native_tls() const { return true; } \
private: \
SE_DISALLOW_COPY_AND_ASSIGN(ThreadLocal_##_var_); \
} _var_; \
}
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STATIC_THREADLOCAL_H_
@@ -0,0 +1,60 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// IWYU pragma: private, include "third_party/tensorflow/stream_executor/stream_executor.h"
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STATUS_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STATUS_H_
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/stream_executor/lib/error.h" // IWYU pragma: export
#include "tensorflow/stream_executor/lib/stringpiece.h"
#include "tensorflow/stream_executor/platform/logging.h"
namespace stream_executor {
namespace port {
using Status = tensorflow::Status;
#define SE_CHECK_OK(val) TF_CHECK_OK(val)
#define SE_ASSERT_OK(val) \
ASSERT_EQ(::stream_executor::port::Status::OK(), (val))
// Define some canonical error helpers.
inline Status UnimplementedError(StringPiece message) {
return Status(error::UNIMPLEMENTED, message);
}
inline Status InternalError(StringPiece message) {
return Status(error::INTERNAL, message);
}
inline Status FailedPreconditionError(StringPiece message) {
return Status(error::FAILED_PRECONDITION, message);
}
} // namespace port
} // namespace stream_executor
namespace perftools {
namespace gputools {
// Temporarily pull stream_executor into perftools::gputools while we migrate
// code to the new namespace. TODO(b/77980417): Remove this once we've
// completed the migration.
using namespace stream_executor; // NOLINT[build/namespaces]
} // namespace gputools
} // namespace perftools
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STATUS_H_
@@ -0,0 +1,53 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Helper macros for dealing with the port::Status datatype.
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STATUS_MACROS_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STATUS_MACROS_H_
// Early-returns the status if it is in error; otherwise, proceeds.
//
// The argument expression is guaranteed to be evaluated exactly once.
#define SE_RETURN_IF_ERROR(__status) \
do { \
auto status = __status; \
if (!status.ok()) { \
return status; \
} \
} while (false)
// Identifier concatenation helper macros.
#define SE_MACRO_CONCAT_INNER(__x, __y) __x##__y
#define SE_MACRO_CONCAT(__x, __y) SE_MACRO_CONCAT_INNER(__x, __y)
// Implementation of SE_ASSIGN_OR_RETURN that uses a unique temporary identifier
// for avoiding collision in the enclosing scope.
#define SE_ASSIGN_OR_RETURN_IMPL(__lhs, __rhs, __name) \
auto __name = (__rhs); \
if (!__name.ok()) { \
return __name.status(); \
} \
__lhs = std::move(__name.ValueOrDie());
// Early-returns the status if it is in error; otherwise, assigns the
// right-hand-side expression to the left-hand-side expression.
//
// The right-hand-side expression is guaranteed to be evaluated exactly once.
#define SE_ASSIGN_OR_RETURN(__lhs, __rhs) \
SE_ASSIGN_OR_RETURN_IMPL(__lhs, __rhs, \
SE_MACRO_CONCAT(__status_or_value, __COUNTER__))
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STATUS_MACROS_H_
@@ -0,0 +1,311 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// StatusOr<T> is the union of a Status object and a T object. StatusOr models
// the concept of an object that is either a value, or an error Status
// explaining why such a value is not present. To this end, StatusOr<T> does not
// allow its Status value to be Status::OK.
//
// The primary use-case for StatusOr<T> is as the return value of a
// function which may fail.
//
// Example client usage for a StatusOr<T>, where T is not a pointer:
//
// StatusOr<float> result = DoBigCalculationThatCouldFail();
// if (result.ok()) {
// float answer = result.ValueOrDie();
// printf("Big calculation yielded: %f", answer);
// } else {
// LOG(ERROR) << result.status();
// }
//
// Example client usage for a StatusOr<T*>:
//
// StatusOr<Foo*> result = FooFactory::MakeNewFoo(arg);
// if (result.ok()) {
// std::unique_ptr<Foo> foo(result.ValueOrDie());
// foo->DoSomethingCool();
// } else {
// LOG(ERROR) << result.status();
// }
//
// Example client usage for a StatusOr<std::unique_ptr<T>>:
//
// StatusOr<std::unique_ptr<Foo>> result = FooFactory::MakeNewFoo(arg);
// if (result.ok()) {
// std::unique_ptr<Foo> foo = std::move(result.ValueOrDie());
// foo->DoSomethingCool();
// } else {
// LOG(ERROR) << result.status();
// }
//
// Example factory implementation returning StatusOr<T*>:
//
// StatusOr<Foo*> FooFactory::MakeNewFoo(int arg) {
// if (arg <= 0) {
// return tensorflow::InvalidArgument("Arg must be positive");
// } else {
// return new Foo(arg);
// }
// }
//
// Note that the assignment operators require that destroying the currently
// stored value cannot invalidate the argument; in other words, the argument
// cannot be an alias for the current value, or anything owned by the current
// value.
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_H_
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/lib/statusor_internals.h"
namespace stream_executor {
namespace port {
#if defined(__clang__)
// Only clang supports warn_unused_result as a type annotation.
template <typename T>
class TF_MUST_USE_RESULT StatusOr;
#endif
template <typename T>
class StatusOr : private internal_statusor::StatusOrData<T>,
private internal_statusor::TraitsBase<
std::is_copy_constructible<T>::value,
std::is_move_constructible<T>::value> {
template <typename U>
friend class StatusOr;
typedef internal_statusor::StatusOrData<T> Base;
public:
typedef T element_type;
// Constructs a new StatusOr with Status::UNKNOWN status. This is marked
// 'explicit' to try to catch cases like 'return {};', where people think
// StatusOr<std::vector<int>> will be initialized with an empty vector,
// instead of a Status::UNKNOWN status.
explicit StatusOr();
// StatusOr<T> will be copy constructible/assignable if T is copy
// constructible.
StatusOr(const StatusOr&) = default;
StatusOr& operator=(const StatusOr&) = default;
// StatusOr<T> will be move constructible/assignable if T is move
// constructible.
StatusOr(StatusOr&&) = default;
StatusOr& operator=(StatusOr&&) = default;
// Conversion copy/move constructor, T must be convertible from U.
template <typename U, typename std::enable_if<
std::is_convertible<U, T>::value>::type* = nullptr>
StatusOr(const StatusOr<U>& other);
template <typename U, typename std::enable_if<
std::is_convertible<U, T>::value>::type* = nullptr>
StatusOr(StatusOr<U>&& other);
// Conversion copy/move assignment operator, T must be convertible from U.
template <typename U, typename std::enable_if<
std::is_convertible<U, T>::value>::type* = nullptr>
StatusOr& operator=(const StatusOr<U>& other);
template <typename U, typename std::enable_if<
std::is_convertible<U, T>::value>::type* = nullptr>
StatusOr& operator=(StatusOr<U>&& other);
// Constructs a new StatusOr with the given value. After calling this
// constructor, calls to ValueOrDie() will succeed, and calls to status() will
// return OK.
//
// NOTE: Not explicit - we want to use StatusOr<T> as a return type
// so it is convenient and sensible to be able to do 'return T()'
// when the return type is StatusOr<T>.
//
// REQUIRES: T is copy constructible.
StatusOr(const T& value);
// Constructs a new StatusOr with the given non-ok status. After calling
// this constructor, calls to ValueOrDie() will CHECK-fail.
//
// NOTE: Not explicit - we want to use StatusOr<T> as a return
// value, so it is convenient and sensible to be able to do 'return
// Status()' when the return type is StatusOr<T>.
//
// REQUIRES: !status.ok(). This requirement is DCHECKed.
// In optimized builds, passing Status::OK() here will have the effect
// of passing tensorflow::error::INTERNAL as a fallback.
StatusOr(const Status& status);
StatusOr& operator=(const Status& status);
// TODO(b/62186997): Add operator=(T) overloads.
// Similar to the `const T&` overload.
//
// REQUIRES: T is move constructible.
StatusOr(T&& value);
// RValue versions of the operations declared above.
StatusOr(Status&& status);
StatusOr& operator=(Status&& status);
// Returns this->status().ok()
bool ok() const { return this->status_.ok(); }
// Returns a reference to our status. If this contains a T, then
// returns Status::OK().
const Status& status() const &;
Status status() &&;
// Returns a reference to our current value, or CHECK-fails if !this->ok().
//
// Note: for value types that are cheap to copy, prefer simple code:
//
// T value = statusor.ValueOrDie();
//
// Otherwise, if the value type is expensive to copy, but can be left
// in the StatusOr, simply assign to a reference:
//
// T& value = statusor.ValueOrDie(); // or `const T&`
//
// Otherwise, if the value type supports an efficient move, it can be
// used as follows:
//
// T value = std::move(statusor).ValueOrDie();
//
// The std::move on statusor instead of on the whole expression enables
// warnings about possible uses of the statusor object after the move.
// C++ style guide waiver for ref-qualified overloads granted in cl/143176389
// See go/ref-qualifiers for more details on such overloads.
const T& ValueOrDie() const &;
T& ValueOrDie() &;
const T&& ValueOrDie() const &&;
T&& ValueOrDie() &&;
T ConsumeValueOrDie() { return std::move(ValueOrDie()); }
// Ignores any errors. This method does nothing except potentially suppress
// complaints from any tools that are checking that errors are not dropped on
// the floor.
void IgnoreError() const;
};
////////////////////////////////////////////////////////////////////////////////
// Implementation details for StatusOr<T>
template <typename T>
StatusOr<T>::StatusOr() : Base(Status(tensorflow::error::UNKNOWN, "")) {}
template <typename T>
StatusOr<T>::StatusOr(const T& value) : Base(value) {}
template <typename T>
StatusOr<T>::StatusOr(const Status& status) : Base(status) {}
template <typename T>
StatusOr<T>& StatusOr<T>::operator=(const Status& status) {
this->Assign(status);
return *this;
}
template <typename T>
StatusOr<T>::StatusOr(T&& value) : Base(std::move(value)) {}
template <typename T>
StatusOr<T>::StatusOr(Status&& status) : Base(std::move(status)) {}
template <typename T>
StatusOr<T>& StatusOr<T>::operator=(Status&& status) {
this->Assign(std::move(status));
return *this;
}
template <typename T>
template <typename U,
typename std::enable_if<std::is_convertible<U, T>::value>::type*>
inline StatusOr<T>::StatusOr(const StatusOr<U>& other)
: Base(static_cast<const typename StatusOr<U>::Base&>(other)) {}
template <typename T>
template <typename U,
typename std::enable_if<std::is_convertible<U, T>::value>::type*>
inline StatusOr<T>& StatusOr<T>::operator=(const StatusOr<U>& other) {
if (other.ok())
this->Assign(other.ValueOrDie());
else
this->Assign(other.status());
return *this;
}
template <typename T>
template <typename U,
typename std::enable_if<std::is_convertible<U, T>::value>::type*>
inline StatusOr<T>::StatusOr(StatusOr<U>&& other)
: Base(static_cast<typename StatusOr<U>::Base&&>(other)) {}
template <typename T>
template <typename U,
typename std::enable_if<std::is_convertible<U, T>::value>::type*>
inline StatusOr<T>& StatusOr<T>::operator=(StatusOr<U>&& other) {
if (other.ok()) {
this->Assign(std::move(other).ValueOrDie());
} else {
this->Assign(std::move(other).status());
}
return *this;
}
template <typename T>
const Status& StatusOr<T>::status() const & {
return this->status_;
}
template <typename T>
Status StatusOr<T>::status() && {
return ok() ? Status::OK() : std::move(this->status_);
}
template <typename T>
const T& StatusOr<T>::ValueOrDie() const & {
this->EnsureOk();
return this->data_;
}
template <typename T>
T& StatusOr<T>::ValueOrDie() & {
this->EnsureOk();
return this->data_;
}
template <typename T>
const T&& StatusOr<T>::ValueOrDie() const && {
this->EnsureOk();
return std::move(this->data_);
}
template <typename T>
T&& StatusOr<T>::ValueOrDie() && {
this->EnsureOk();
return std::move(this->data_);
}
template <typename T>
void StatusOr<T>::IgnoreError() const {
// no-op
}
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_H_
@@ -0,0 +1,247 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_INTERNALS_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_INTERNALS_H_
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/stream_executor/lib/status.h"
namespace stream_executor {
namespace port {
namespace internal_statusor {
class Helper {
public:
// Move type-agnostic error handling to the .cc.
static void HandleInvalidStatusCtorArg(Status*);
TF_ATTRIBUTE_NORETURN static void Crash(const Status& status);
};
// Construct an instance of T in `p` through placement new, passing Args... to
// the constructor.
// This abstraction is here mostly for the gcc performance fix.
template <typename T, typename... Args>
void PlacementNew(void* p, Args&&... args) {
#if defined(__GNUC__) && !defined(__clang__)
// Teach gcc that 'p' cannot be null, fixing code size issues.
if (p == nullptr) __builtin_unreachable();
#endif
new (p) T(std::forward<Args>(args)...);
}
// Helper base class to hold the data and all operations.
// We move all this to a base class to allow mixing with the appropriate
// TraitsBase specialization.
template <typename T>
class StatusOrData {
template <typename U>
friend class StatusOrData;
public:
StatusOrData() = delete;
StatusOrData(const StatusOrData& other) {
if (other.ok()) {
MakeValue(other.data_);
MakeStatus();
} else {
MakeStatus(other.status_);
}
}
StatusOrData(StatusOrData&& other) noexcept {
if (other.ok()) {
MakeValue(std::move(other.data_));
MakeStatus();
} else {
MakeStatus(std::move(other.status_));
}
}
template <typename U>
StatusOrData(const StatusOrData<U>& other) {
if (other.ok()) {
MakeValue(other.data_);
MakeStatus();
} else {
MakeStatus(other.status_);
}
}
template <typename U>
StatusOrData(StatusOrData<U>&& other) {
if (other.ok()) {
MakeValue(std::move(other.data_));
MakeStatus();
} else {
MakeStatus(std::move(other.status_));
}
}
explicit StatusOrData(const T& value) : data_(value) { MakeStatus(); }
explicit StatusOrData(T&& value) : data_(std::move(value)) { MakeStatus(); }
explicit StatusOrData(const Status& status) : status_(status) {
EnsureNotOk();
}
explicit StatusOrData(Status&& status) : status_(std::move(status)) {
EnsureNotOk();
}
StatusOrData& operator=(const StatusOrData& other) {
if (this == &other) return *this;
if (other.ok())
Assign(other.data_);
else
Assign(other.status_);
return *this;
}
StatusOrData& operator=(StatusOrData&& other) {
if (this == &other) return *this;
if (other.ok())
Assign(std::move(other.data_));
else
Assign(std::move(other.status_));
return *this;
}
~StatusOrData() {
if (ok()) {
status_.~Status();
data_.~T();
} else {
status_.~Status();
}
}
void Assign(const T& value) {
if (ok()) {
data_.~T();
MakeValue(value);
} else {
MakeValue(value);
status_ = Status::OK();
}
}
void Assign(T&& value) {
if (ok()) {
data_.~T();
MakeValue(std::move(value));
} else {
MakeValue(std::move(value));
status_ = Status::OK();
}
}
void Assign(const Status& status) {
Clear();
status_ = status;
EnsureNotOk();
}
void Assign(Status&& status) {
Clear();
status_ = std::move(status);
EnsureNotOk();
}
bool ok() const { return status_.ok(); }
protected:
// status_ will always be active after the constructor.
// We make it a union to be able to initialize exactly how we need without
// waste.
// Eg. in the copy constructor we use the default constructor of Status in
// the ok() path to avoid an extra Ref call.
union {
Status status_;
};
// data_ is active iff status_.ok()==true
struct Dummy {};
union {
// When T is const, we need some non-const object we can cast to void* for
// the placement new. dummy_ is that object.
Dummy dummy_;
T data_;
};
void Clear() {
if (ok()) data_.~T();
}
void EnsureOk() const {
if (!ok()) Helper::Crash(status_);
}
void EnsureNotOk() {
if (ok()) Helper::HandleInvalidStatusCtorArg(&status_);
}
// Construct the value (ie. data_) through placement new with the passed
// argument.
template <typename Arg>
void MakeValue(Arg&& arg) {
internal_statusor::PlacementNew<T>(&dummy_, std::forward<Arg>(arg));
}
// Construct the status (ie. status_) through placement new with the passed
// argument.
template <typename... Args>
void MakeStatus(Args&&... args) {
internal_statusor::PlacementNew<Status>(&status_,
std::forward<Args>(args)...);
}
};
// Helper base class to allow implicitly deleted constructors and assignment
// operations in StatusOr.
// TraitsBase will explicitly delete what it can't support and StatusOr will
// inherit that behavior implicitly.
template <bool Copy, bool Move>
struct TraitsBase {
TraitsBase() = default;
TraitsBase(const TraitsBase&) = default;
TraitsBase(TraitsBase&&) = default;
TraitsBase& operator=(const TraitsBase&) = default;
TraitsBase& operator=(TraitsBase&&) = default;
};
template <>
struct TraitsBase<false, true> {
TraitsBase() = default;
TraitsBase(const TraitsBase&) = delete;
TraitsBase(TraitsBase&&) = default;
TraitsBase& operator=(const TraitsBase&) = delete;
TraitsBase& operator=(TraitsBase&&) = default;
};
template <>
struct TraitsBase<false, false> {
TraitsBase() = default;
TraitsBase(const TraitsBase&) = delete;
TraitsBase(TraitsBase&&) = delete;
TraitsBase& operator=(const TraitsBase&) = delete;
TraitsBase& operator=(TraitsBase&&) = delete;
};
} // namespace internal_statusor
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_INTERNALS_H_
@@ -0,0 +1,42 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STR_UTIL_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STR_UTIL_H_
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/stream_executor/lib/stringpiece.h"
namespace stream_executor {
namespace port {
using tensorflow::str_util::Join;
using tensorflow::str_util::Split;
// Returns a copy of the input string 'str' with the given 'suffix'
// removed. If the suffix doesn't match, returns a copy of the original string.
inline string StripSuffixString(port::StringPiece str, port::StringPiece suffix) {
if (tensorflow::str_util::EndsWith(str, suffix)) {
str.remove_suffix(suffix.size());
}
return string(str);
}
using tensorflow::str_util::Lowercase;
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STR_UTIL_H_
@@ -0,0 +1,32 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// IWYU pragma: private, include "third_party/tensorflow/stream_executor/stream_executor.h"
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STRCAT_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STRCAT_H_
#include "absl/strings/str_cat.h"
namespace stream_executor {
namespace port {
using absl::StrAppend;
using absl::StrCat;
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STRCAT_H_
@@ -0,0 +1,29 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STRINGPIECE_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STRINGPIECE_H_
#include "absl/strings/string_view.h"
namespace stream_executor {
namespace port {
using StringPiece = absl::string_view;
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STRINGPIECE_H_
@@ -0,0 +1,30 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STRINGPRINTF_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_STRINGPRINTF_H_
#include "tensorflow/core/lib/strings/stringprintf.h"
namespace stream_executor {
namespace port {
using tensorflow::strings::Printf;
using tensorflow::strings::Appendf;
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STRINGPRINTF_H_
@@ -0,0 +1,29 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_THREAD_OPTIONS_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_THREAD_OPTIONS_H_
#include "tensorflow/core/platform/env.h"
namespace stream_executor {
namespace port {
using tensorflow::ThreadOptions;
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_THREAD_OPTIONS_H_
@@ -0,0 +1,32 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_THREADPOOL_H_
#define TENSORFLOW_STREAM_EXECUTOR_LIB_THREADPOOL_H_
#include "tensorflow/core/lib/core/threadpool.h"
#include "tensorflow/stream_executor/lib/env.h"
#include "tensorflow/stream_executor/lib/notification.h"
#include "tensorflow/stream_executor/lib/thread_options.h"
namespace stream_executor {
namespace port {
using tensorflow::thread::ThreadPool;
} // namespace port
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_THREADPOOL_H_
@@ -0,0 +1,66 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_MODULE_SPEC_H_
#define TENSORFLOW_STREAM_EXECUTOR_MODULE_SPEC_H_
#include "tensorflow/stream_executor/lib/array_slice.h"
#include "tensorflow/stream_executor/lib/stringpiece.h"
#include "tensorflow/stream_executor/platform/logging.h"
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
// Describes how to load a module on a target platform.
//
// The exact meaning of a "module" may differ from platform to platform but
// loosely speaking a module a collection of kernels and global variables. It
// corresponds to CUmodule when running on CUDA.
class MultiModuleLoaderSpec {
public:
bool has_cuda_cubin_in_memory() const { return has_cuda_cubin_in_memory_; }
port::ArraySlice<const uint8> cuda_cubin_in_memory() const {
CHECK(has_cuda_cubin_in_memory());
return {cuda_cubin_in_memory_.data(), cuda_cubin_in_memory_.size()};
}
bool has_cuda_ptx_in_memory() const { return has_cuda_ptx_in_memory_; }
const char* cuda_ptx_in_memory() const {
CHECK(has_cuda_ptx_in_memory());
return cuda_ptx_in_memory_;
}
void AddCudaCubinInMemory(port::ArraySlice<const uint8> cubin_bytes) {
CHECK(!cubin_bytes.empty());
has_cuda_cubin_in_memory_ = true;
cuda_cubin_in_memory_ = cubin_bytes;
}
void AddCudaPtxInMemory(const char* ptx) {
has_cuda_ptx_in_memory_ = true;
// The CUDA driver does not like getting an empty string as PTX.
cuda_ptx_in_memory_ = *ptx ? ptx : nullptr;
}
private:
port::ArraySlice<const uint8> cuda_cubin_in_memory_;
bool has_cuda_cubin_in_memory_ = false;
const char* cuda_ptx_in_memory_;
bool has_cuda_ptx_in_memory_ = false;
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_MODULE_SPEC_H_
@@ -0,0 +1,191 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This is a registration-oriented interface for multiple platforms. It will
// replace the MachineManager singleton interface, as MachineManager does not
// currently support simultaneous use of multiple platforms.
//
// Usage:
//
// In your BUILD rule, add a dependency on a platform plugin that you'd like
// to use, such as:
//
// //third_party/tensorflow/stream_executor/cuda:cuda_platform
// //third_party/tensorflow/stream_executor/opencl:opencl_platform
//
// This will register platform plugins that can be discovered via this
// interface. Sample API usage:
//
// port::StatusOr<Platform*> platform_status =
// se::MultiPlatformManager::PlatformWithName("OpenCL");
// if (!platform_status.ok()) { ... }
// Platform* platform = platform_status.ValueOrDie();
// LOG(INFO) << platform->VisibleDeviceCount() << " devices visible";
// if (platform->VisibleDeviceCount() <= 0) { return; }
//
// for (int i = 0; i < platform->VisibleDeviceCount(); ++i) {
// port::StatusOr<StreamExecutor*> executor_status =
// platform->ExecutorForDevice(i);
// if (!executor_status.ok()) {
// LOG(INFO) << "could not retrieve executor for device ordinal " << i
// << ": " << executor_status.status();
// continue;
// }
// LOG(INFO) << "found usable executor: " << executor_status.ValueOrDie();
// }
//
// A few things to note:
// - There is no standard formatting/practice for identifying the name of a
// platform. Ideally, a platform will list its registered name in its header
// or in other associated documentation.
// - Platform name lookup is case-insensitive. "OpenCL" or "opencl" (or even
// ("OpEnCl") would work correctly in the above example.
//
// And similarly, for standard interfaces (BLAS, RNG, etc.) you can add
// dependencies on support libraries, e.g.:
//
// //third_party/tensorflow/stream_executor/cuda:pluton_blas_plugin
// //third_party/tensorflow/stream_executor/cuda:cudnn_plugin
// //third_party/tensorflow/stream_executor/cuda:cublas_plugin
// //third_party/tensorflow/stream_executor/cuda:curand_plugin
#ifndef TENSORFLOW_STREAM_EXECUTOR_MULTI_PLATFORM_MANAGER_H_
#define TENSORFLOW_STREAM_EXECUTOR_MULTI_PLATFORM_MANAGER_H_
#include <functional>
#include <map>
#include <memory>
#include "tensorflow/stream_executor/lib/initialize.h"
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/platform.h"
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/platform/thread_annotations.h"
namespace stream_executor {
// Manages multiple platforms that may be present on the current machine.
class MultiPlatformManager {
public:
// Registers a platform object, returns an error status if the platform is
// already registered. The associated listener, if not null, will be used to
// trace events for ALL executors for that platform.
// Takes ownership of listener.
static port::Status RegisterPlatform(std::unique_ptr<Platform> platform)
LOCKS_EXCLUDED(platforms_mutex_);
// Retrieves the platform registered with the given platform name (e.g.
// "CUDA", "OpenCL", ...) or id (an opaque, comparable value provided by the
// Platform's Id() method).
//
// If the platform has not already been initialized, it will be initialized
// with a default set of parameters.
//
// If the requested platform is not registered, an error status is returned.
// Ownership of the platform is NOT transferred to the caller --
// the MultiPlatformManager owns the platforms in a singleton-like fashion.
static port::StatusOr<Platform*> PlatformWithName(const string& target)
LOCKS_EXCLUDED(platforms_mutex_);
static port::StatusOr<Platform*> PlatformWithId(const Platform::Id& id)
LOCKS_EXCLUDED(platforms_mutex_);
// Retrieves the platform registered with the given platform name (e.g.
// "CUDA", "OpenCL", ...) or id (an opaque, comparable value provided by the
// Platform's Id() method).
//
// The platform will be initialized with the given options. If the platform
// was already initialized, an error will be returned.
//
// If the requested platform is not registered, an error status is returned.
// Ownership of the platform is NOT transferred to the caller --
// the MultiPlatformManager owns the platforms in a singleton-like fashion.
static port::StatusOr<Platform*> InitializePlatformWithName(
const string& target, const std::map<string, string>& options)
LOCKS_EXCLUDED(platforms_mutex_);
static port::StatusOr<Platform*> InitializePlatformWithId(
const Platform::Id& id, const std::map<string, string>& options)
LOCKS_EXCLUDED(platforms_mutex_);
// Clears the set of registered platforms, primarily used for testing.
static void ClearPlatformRegistry() LOCKS_EXCLUDED(platforms_mutex_);
// Although the MultiPlatformManager "owns" its platforms, it holds them as
// undecorated pointers to prevent races during program exit (between this
// object's data and the underlying platforms (e.g., CUDA, OpenCL).
// Because certain platforms have unpredictable deinitialization
// times/sequences, it is not possible to strucure a safe deinitialization
// sequence. Thus, we intentionally "leak" allocated platforms to defer
// cleanup to the OS. This should be acceptable, as these are one-time
// allocations per program invocation.
// The MultiPlatformManager should be considered the owner
// of any platforms registered with it, and leak checking should be disabled
// during allocation of such Platforms, to avoid spurious reporting at program
// exit.
using PlatformMap = std::map<string, Platform*>;
// Provides access to the available set of platforms under a lock.
static port::Status WithPlatforms(
std::function<port::Status(PlatformMap*)> callback)
LOCKS_EXCLUDED(platforms_mutex_) {
mutex_lock lock(platforms_mutex_);
return callback(GetPlatformMap());
}
private:
using PlatformIdMap = std::map<Platform::Id, Platform*>;
static mutex platforms_mutex_;
// TODO(b/22689637): Clean up these two maps; make sure they coexist nicely.
// TODO(b/22689637): Move this (whatever the final/"official" map is) to
// plugin_regstry.h, along with the associated functionality.
// Platform-name-to-object mapping. These platforms are registered via module
// initializers, and linkage determines which platforms are available to a
// given target.
static PlatformMap* GetPlatformMap() {
static PlatformMap* instance = new PlatformMap;
return instance;
}
// Holds a Platform::Id-to-object mapping.
// Unlike platforms_ above, this map does not own its contents.
static PlatformIdMap* GetPlatformByIdMap() {
static PlatformIdMap* instance = new PlatformIdMap;
return instance;
}
// Looks up the platform object with the given name. Assumes the Platforms
// mutex is held.
static port::StatusOr<Platform*> LookupByNameLocked(const string& target)
EXCLUSIVE_LOCKS_REQUIRED(platforms_mutex_);
// Looks up the platform object with the given id. Assumes the Platforms
// mutex is held.
static port::StatusOr<Platform*> LookupByIdLocked(const Platform::Id& id)
EXCLUSIVE_LOCKS_REQUIRED(platforms_mutex_);
SE_DISALLOW_COPY_AND_ASSIGN(MultiPlatformManager);
};
} // namespace stream_executor
// multi_platform_manager.cc will define this instance. Includers of this header
// should use
// REGISTER_MODULE_INITIALIZER_SEQUENCE(my_platform, multi_platform_manager);
DECLARE_MODULE_INITIALIZER(multi_platform_manager);
#endif // TENSORFLOW_STREAM_EXECUTOR_MULTI_PLATFORM_MANAGER_H_
@@ -0,0 +1,209 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Defines types and declares functions for identifying and extracting
// information about the types of platforms and supporting libraries for which
// StreamExecutor implementations exist.
#ifndef TENSORFLOW_STREAM_EXECUTOR_PLATFORM_H_
#define TENSORFLOW_STREAM_EXECUTOR_PLATFORM_H_
#include <map>
#include "tensorflow/stream_executor/device_options.h"
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/lib/status_macros.h"
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/plugin.h"
#include "tensorflow/stream_executor/trace_listener.h"
namespace stream_executor {
class StreamExecutor;
// Describes the platform for a StreamExecutor instantiation to act upon.
//
// Implementors: if you add a value here be sure to update PlatformKindString
// and CheckPlatformKindIsValid.
enum class PlatformKind {
kInvalid,
kCuda,
kOpenCL,
kHost,
kMock,
kSize,
};
// Returns true if kind represents a valid platform capable of enqueuing items
// on a stream, but not necessarily on an accelerator device.
// Returns false for kMock and any invalid PlatformKind values.
bool PlatformIsRunnable(PlatformKind kind);
// Returns true if kind represents a valid platform capable of running kernels
// on an accelerator device. Returns false for kHost*, kMock and any invalid
// PlatformKind values.
bool PlatformIsRunnableOnDevice(PlatformKind kind);
// Returns a printable description of a PlatformKind.
string PlatformKindString(PlatformKind kind);
// Returns the PlatformKind corresponding to the input string; returns kInvalid
// in the case of no match.
PlatformKind PlatformKindFromString(string platform_string);
// Checks that kind takes on a valid value.
void CheckPlatformKindIsValid(PlatformKind kind);
// StreamExecutorConfig encapsulates the set of options for constructing a
// StreamExecutor for a given platform.
struct StreamExecutorConfig {
// Sets members to defaults: -1 for ordinal (must be changed), and default
// PluginConfig and DeviceOptions.
StreamExecutorConfig();
// Simple ordinal-setting constructor.
explicit StreamExecutorConfig(int ordinal);
// The ordinal of the device to be managed by the returned StreamExecutor.
int ordinal;
// The PluginConfig for the returned StreamExecutor.
PluginConfig plugin_config;
// The DeviceOptions for the returned StreamExecutor.
DeviceOptions device_options;
};
// Abstract base class for a platform registered with the MultiPlatformManager.
class Platform {
public:
virtual ~Platform();
// A platform ID is a unique identifier for each registered platform type -
// each platform is required to expose an ID to ensure unique registration and
// as a target against which plugins can register.
//
// The macro below is provided to help generate a [process-unique] identifier.
using Id = void*;
// Helper macro to define a plugin ID. To be used only inside plugin
// implementation files. Works by "reserving" an address/value (guaranteed to be
// unique) inside a process space.
#define PLATFORM_DEFINE_ID(ID_VAR_NAME) \
namespace { \
int plugin_id_value; \
} \
const ::stream_executor::Platform::Id ID_VAR_NAME = &plugin_id_value;
// Returns a key uniquely identifying this platform.
virtual Id id() const = 0;
// Name of this platform.
virtual const string& Name() const = 0;
// Returns the number of devices accessible on this platform.
//
// Note that, though these devices are visible, if there is only one userspace
// context allowed for the device at a time and another process is using this
// device, a call to ExecutorForDevice may return an error status.
virtual int VisibleDeviceCount() const = 0;
// Returns true iff the platform has been initialized.
virtual bool Initialized() const;
// Initializes the platform with a custom set of options. The platform must be
// initialized before obtaining StreamExecutor objects. The interpretation of
// the platform_options argument is implementation specific. This method may
// return an error if unrecognized options are provided. If using
// MultiPlatformManager, this method will be called automatically by
// InitializePlatformWithId/InitializePlatformWithName.
virtual port::Status Initialize(
const std::map<string, string>& platform_options);
// Returns a device with the given ordinal on this platform with a default
// plugin configuration or, if none can be found with the given ordinal or
// there is an error in opening a context to communicate with the device, an
// error status is returned.
//
// Ownership of the executor is NOT transferred to the caller --
// the Platform owns the executors in a singleton-like fashion.
virtual port::StatusOr<StreamExecutor*> ExecutorForDevice(int ordinal) = 0;
// Returns a device or error, as above, with the specified plugins.
//
// Ownership of the executor is NOT transferred to the caller.
virtual port::StatusOr<StreamExecutor*> ExecutorForDeviceWithPluginConfig(
int ordinal, const PluginConfig& plugin_config) = 0;
// Returns a device constructed with the options specified in "config".
// Ownership of the executor is NOT transferred to the caller.
virtual port::StatusOr<StreamExecutor*> GetExecutor(
const StreamExecutorConfig& config) = 0;
// Returns a device constructed with the options specified in "config" without
// looking in or storing to the Platform's executor cache.
// Ownership IS transferred to the caller.
virtual port::StatusOr<std::unique_ptr<StreamExecutor>> GetUncachedExecutor(
const StreamExecutorConfig& config) = 0;
// Warning: this is a dangerous API and should be used with caution.
//
// Forces the platform to delete executor instances, releasing their
// associated device contexts. There must be no held instances of the executor
// and there must be no outstanding activity on the devices for this platform.
//
// This is only useful on platforms which bind a device to a single process
// that has obtained the device context. May return UNIMPLEMENTED on platforms
// that have no reason to destroy device contexts.
//
// The platform must be reinitialized after this is called.
virtual port::Status ForceExecutorShutdown();
// Registers a TraceListener to listen to all StreamExecutors for this
// platform.
// Takes ownership of listener.
virtual void RegisterTraceListener(
std::unique_ptr<TraceListener> listener) = 0;
// Removes the specified TraceListener from all StreamExecutors.
virtual void UnregisterTraceListener(TraceListener* listener) = 0;
// Map of executor-to-executor coordinate and boolean, indicating if the first
// executor can access the second's memory.
using PeerAccessMap = std::map<std::pair<int, int>, bool>;
// Returns a matrix indicating which executors can access which other
// executors' memory.
virtual std::unique_ptr<PeerAccessMap> GetPeerAccessMap();
// Attempts to enable all peer-to-peer access links described by the result of
// GetPeerAccessMap(). Note that calling this routine will force the creation
// of a default-argument (see StreamExecutorConfig) StreamExecutor object for
// each device ordinal in the system, should any not yet exist.
virtual port::Status EnablePeerAccess();
protected:
// SE_DISALLOW_COPY_AND_ASSIGN declares a constructor, which suppresses the
// presence of the default constructor. This statement re-enables it, which
// simplifies subclassing.
Platform() = default;
private:
SE_DISALLOW_COPY_AND_ASSIGN(Platform);
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_PLATFORM_H_
@@ -0,0 +1,39 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_PLATFORM_DEFAULT_MUTEX_H_
#define TENSORFLOW_STREAM_EXECUTOR_PLATFORM_DEFAULT_MUTEX_H_
#include "tensorflow/stream_executor/platform/mutex.h"
namespace stream_executor {
#undef mutex_lock
#undef tf_shared_lock
using tensorflow::ConditionResult;
using tensorflow::WaitForMilliseconds;
using tensorflow::condition_variable;
using tensorflow::mutex;
using tensorflow::mutex_lock;
using tensorflow::tf_shared_lock;
#define mutex_lock(x) static_assert(0, "mutex_lock_decl_missing_var_name");
#define tf_shared_lock(x) \
static_assert(0, "tf_shared_lock_decl_missing_var_name");
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_PLATFORM_DEFAULT_MUTEX_H_
@@ -0,0 +1,36 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_PLATFORM_LOGGING_H_
#define TENSORFLOW_STREAM_EXECUTOR_PLATFORM_LOGGING_H_
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/stream_executor/platform/port.h"
#if !defined(PLATFORM_GOOGLE)
// A CHECK() macro that lets you assert the success of a function that
// returns -1 and sets errno in case of an error. E.g.
//
// CHECK_ERR(mkdir(path, 0700));
//
// or
//
// int fd = open(filename, flags); CHECK_ERR(fd) << ": open " << filename;
#define CHECK_ERR(invocation) CHECK((invocation) != -1) << #invocation
#endif
#endif // TENSORFLOW_STREAM_EXECUTOR_PLATFORM_LOGGING_H_
@@ -0,0 +1,28 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_PLATFORM_MUTEX_H_
#define TENSORFLOW_STREAM_EXECUTOR_PLATFORM_MUTEX_H_
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/types.h"
#if defined(PLATFORM_GOOGLE)
#include "tensorflow/stream_executor/platform/google/mutex.h"
#else
#include "tensorflow/stream_executor/platform/default/mutex.h"
#endif
#endif // TENSORFLOW_STREAM_EXECUTOR_PLATFORM_MUTEX_H_
@@ -0,0 +1,53 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// IWYU pragma: private, include "third_party/tensorflow/stream_executor/stream_executor.h"
#ifndef TENSORFLOW_STREAM_EXECUTOR_PLATFORM_PORT_H_
#define TENSORFLOW_STREAM_EXECUTOR_PLATFORM_PORT_H_
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/types.h"
namespace stream_executor {
using tensorflow::int8;
using tensorflow::int16;
using tensorflow::int32;
using tensorflow::int64;
using tensorflow::uint8;
using tensorflow::uint16;
using tensorflow::uint32;
using tensorflow::uint64;
#if !defined(PLATFORM_GOOGLE)
using std::string;
#endif
using tensorflow::LinkerInitialized;
using tensorflow::LINKER_INITIALIZED;
#define SE_FALLTHROUGH_INTENDED TF_FALLTHROUGH_INTENDED
} // namespace stream_executor
#define SE_DISALLOW_COPY_AND_ASSIGN TF_DISALLOW_COPY_AND_ASSIGN
#define SE_MUST_USE_RESULT TF_MUST_USE_RESULT
#define SE_PREDICT_TRUE TF_PREDICT_TRUE
#define SE_PREDICT_FALSE TF_PREDICT_FALSE
#endif // TENSORFLOW_STREAM_EXECUTOR_PLATFORM_PORT_H_
@@ -0,0 +1,21 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_PLATFORM_THREAD_ANNOTATIONS_H_
#define TENSORFLOW_STREAM_EXECUTOR_PLATFORM_THREAD_ANNOTATIONS_H_
#include "tensorflow/core/platform/thread_annotations.h"
#endif // TENSORFLOW_STREAM_EXECUTOR_PLATFORM_THREAD_ANNOTATIONS_H_
@@ -0,0 +1,87 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_PLUGIN_H_
#define TENSORFLOW_STREAM_EXECUTOR_PLUGIN_H_
namespace stream_executor {
// A plugin ID is a unique identifier for each registered plugin type.
typedef void* PluginId;
// Helper macro to define a plugin ID. To be used only inside plugin
// implementation files. Works by "reserving" an address/value (guaranteed to be
// unique) inside a process space.
#define PLUGIN_REGISTRY_DEFINE_PLUGIN_ID(ID_VAR_NAME) \
namespace { \
int plugin_id_value; \
} \
const PluginId ID_VAR_NAME = &plugin_id_value;
// kNullPlugin denotes an invalid plugin identifier.
extern const PluginId kNullPlugin;
// Enumeration to list the supported types of plugins / support libraries.
enum class PluginKind {
kInvalid,
kBlas,
kDnn,
kFft,
kRng,
};
// A PluginConfig describes the set of plugins to be used by a StreamExecutor
// instance. Each plugin is defined by an arbitrary identifier, usually best set
// to the address static member in the implementation (to avoid conflicts).
//
// A PluginConfig may be passed to the StreamExecutor constructor - the plugins
// described therein will be used to provide BLAS, DNN, FFT, and RNG
// functionality. Platform-appropriate defaults will be used for any un-set
// libraries. If a platform does not support a specified plugin (ex. cuBLAS on
// an OpenCL executor), then an error will be logged and no plugin operations
// will succeed.
//
// The StreamExecutor BUILD target does not link ANY plugin libraries - even
// common host fallbacks! Any plugins must be explicitly linked by dependent
// targets. See the cuda, opencl and host BUILD files for implemented plugin
// support (search for "plugin").
class PluginConfig {
public:
// Value specifying the platform's default option for that plugin.
static const PluginId kDefault;
// Initializes all members to the default options.
PluginConfig();
bool operator==(const PluginConfig& rhs) const;
// Sets the appropriate library kind to that passed in.
PluginConfig& SetBlas(PluginId blas);
PluginConfig& SetDnn(PluginId dnn);
PluginConfig& SetFft(PluginId fft);
PluginConfig& SetRng(PluginId rng);
PluginId blas() const { return blas_; }
PluginId dnn() const { return dnn_; }
PluginId fft() const { return fft_; }
PluginId rng() const { return rng_; }
private:
PluginId blas_, dnn_, fft_, rng_;
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_PLUGIN_H_
@@ -0,0 +1,166 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_PLUGIN_REGISTRY_H_
#define TENSORFLOW_STREAM_EXECUTOR_PLUGIN_REGISTRY_H_
#include <map>
#include "absl/base/macros.h"
#include "tensorflow/stream_executor/blas.h"
#include "tensorflow/stream_executor/dnn.h"
#include "tensorflow/stream_executor/fft.h"
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/platform.h"
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/plugin.h"
#include "tensorflow/stream_executor/rng.h"
namespace stream_executor {
namespace internal {
class StreamExecutorInterface;
}
// The PluginRegistry is a singleton that maintains the set of registered
// "support library" plugins. Currently, there are four kinds of plugins:
// BLAS, DNN, FFT, and RNG. Each interface is defined in the corresponding
// gpu_{kind}.h header.
//
// At runtime, a StreamExecutor object will query the singleton registry to
// retrieve the plugin kind that StreamExecutor was configured with (refer to
// the StreamExecutor and PluginConfig declarations).
//
// Plugin libraries are best registered using REGISTER_MODULE_INITIALIZER,
// but can be registered at any time. When registering a DSO-backed plugin, it
// is usually a good idea to load the DSO at registration time, to prevent
// late-loading from distorting performance/benchmarks as much as possible.
class PluginRegistry {
public:
typedef blas::BlasSupport* (*BlasFactory)(internal::StreamExecutorInterface*);
typedef dnn::DnnSupport* (*DnnFactory)(internal::StreamExecutorInterface*);
typedef fft::FftSupport* (*FftFactory)(internal::StreamExecutorInterface*);
typedef rng::RngSupport* (*RngFactory)(internal::StreamExecutorInterface*);
// Gets (and creates, if necessary) the singleton PluginRegistry instance.
static PluginRegistry* Instance();
// Registers the specified factory with the specified platform.
// Returns a non-successful status if the factory has already been registered
// with that platform (but execution should be otherwise unaffected).
template <typename FactoryT>
port::Status RegisterFactory(Platform::Id platform_id, PluginId plugin_id,
const string& name, FactoryT factory);
// Registers the specified factory as usable by _all_ platform types.
// Reports errors just as RegisterFactory.
template <typename FactoryT>
port::Status RegisterFactoryForAllPlatforms(PluginId plugin_id,
const string& name,
FactoryT factory);
// TODO(b/22689637): Setter for temporary mapping until all users are using
// MultiPlatformManager / PlatformId.
void MapPlatformKindToId(PlatformKind platform_kind,
Platform::Id platform_id);
// Potentially sets the plugin identified by plugin_id to be the default
// for the specified platform and plugin kind. If this routine is called
// multiple types for the same PluginKind, the PluginId given in the last call
// will be used.
bool SetDefaultFactory(Platform::Id platform_id, PluginKind plugin_kind,
PluginId plugin_id);
// Return true if the factory/id has been registered for the
// specified platform and plugin kind and false otherwise.
bool HasFactory(Platform::Id platform_id, PluginKind plugin_kind,
PluginId plugin) const;
// Retrieves the factory registered for the specified kind,
// or a port::Status on error.
template <typename FactoryT>
port::StatusOr<FactoryT> GetFactory(Platform::Id platform_id,
PluginId plugin_id);
// TODO(b/22689637): Deprecated/temporary. Will be deleted once all users are
// on MultiPlatformManager / PlatformId.
template <typename FactoryT>
ABSL_DEPRECATED("Use MultiPlatformManager / PlatformId instead.")
port::StatusOr<FactoryT> GetFactory(PlatformKind platform_kind,
PluginId plugin_id);
private:
// Containers for the sets of registered factories, by plugin kind.
struct PluginFactories {
std::map<PluginId, BlasFactory> blas;
std::map<PluginId, DnnFactory> dnn;
std::map<PluginId, FftFactory> fft;
std::map<PluginId, RngFactory> rng;
};
// Simple structure to hold the currently configured default plugins (for a
// particular Platform).
struct DefaultFactories {
DefaultFactories();
PluginId blas, dnn, fft, rng;
};
PluginRegistry();
// Actually performs the work of registration.
template <typename FactoryT>
port::Status RegisterFactoryInternal(PluginId plugin_id,
const string& plugin_name,
FactoryT factory,
std::map<PluginId, FactoryT>* factories);
// Actually performs the work of factory retrieval.
template <typename FactoryT>
port::StatusOr<FactoryT> GetFactoryInternal(
PluginId plugin_id, const std::map<PluginId, FactoryT>& factories,
const std::map<PluginId, FactoryT>& generic_factories) const;
// Returns true if the specified plugin has been registered with the specified
// platform factories. Unlike the other overload of this method, this does
// not implicitly examine the default factory lists.
bool HasFactory(const PluginFactories& factories, PluginKind plugin_kind,
PluginId plugin) const;
// The singleton itself.
static PluginRegistry* instance_;
// TODO(b/22689637): Temporary mapping until all users are using
// MultiPlatformManager / PlatformId.
std::map<PlatformKind, Platform::Id> platform_id_by_kind_;
// The set of registered factories, keyed by platform ID.
std::map<Platform::Id, PluginFactories> factories_;
// Plugins supported for all platform kinds.
PluginFactories generic_factories_;
// The sets of default factories, keyed by platform ID.
std::map<Platform::Id, DefaultFactories> default_factories_;
// Lookup table for plugin names.
std::map<PluginId, string> plugin_names_;
SE_DISALLOW_COPY_AND_ASSIGN(PluginRegistry);
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_PLUGIN_REGISTRY_H_
@@ -0,0 +1,93 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_RNG_H_
#define TENSORFLOW_STREAM_EXECUTOR_RNG_H_
#include <limits.h>
#include <complex>
#include "tensorflow/stream_executor/platform/logging.h"
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
class Stream;
template <typename ElemT>
class DeviceMemory;
namespace rng {
// Random-number-generation support interface -- this can be derived from a GPU
// executor when the underlying platform has an RNG library implementation
// available. See StreamExecutor::AsRng().
// When a seed is not specified, the backing RNG will be initialized with the
// default seed for that implementation.
//
// Thread-hostile: see StreamExecutor class comment for details on
// thread-hostility.
class RngSupport {
public:
static const int kMinSeedBytes = 16;
static const int kMaxSeedBytes = INT_MAX;
// Releases any random-number-generation resources associated with this
// support object in the underlying platform implementation.
virtual ~RngSupport() {}
// Populates a GPU memory allocation with random values appropriate for the
// DeviceMemory element type; i.e. populates DeviceMemory<float> with random
// float values.
virtual bool DoPopulateRandUniform(Stream *stream,
DeviceMemory<float> *v) = 0;
virtual bool DoPopulateRandUniform(Stream *stream,
DeviceMemory<double> *v) = 0;
virtual bool DoPopulateRandUniform(Stream *stream,
DeviceMemory<std::complex<float>> *v) = 0;
virtual bool DoPopulateRandUniform(Stream *stream,
DeviceMemory<std::complex<double>> *v) = 0;
// Populates a GPU memory allocation with random values sampled from a
// Gaussian distribution with the given mean and standard deviation.
virtual bool DoPopulateRandGaussian(Stream *stream, float mean, float stddev,
DeviceMemory<float> *v) {
LOG(ERROR)
<< "platform's random number generator does not support gaussian";
return false;
}
virtual bool DoPopulateRandGaussian(Stream *stream, double mean,
double stddev, DeviceMemory<double> *v) {
LOG(ERROR)
<< "platform's random number generator does not support gaussian";
return false;
}
// Specifies the seed used to initialize the RNG.
// This call does not transfer ownership of the buffer seed; its data should
// not be altered for the lifetime of this call. At least 16 bytes of seed
// data must be provided, but not all seed data will necessarily be used.
// seed: Pointer to seed data. Must not be null.
// seed_bytes: Size of seed buffer in bytes. Must be >= 16.
virtual bool SetSeed(Stream *stream, const uint8 *seed,
uint64 seed_bytes) = 0;
protected:
static bool CheckSeed(const uint8 *seed, uint64 seed_bytes);
};
} // namespace rng
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_RNG_H_
@@ -0,0 +1,81 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_SCRATCH_ALLOCATOR_H_
#define TENSORFLOW_STREAM_EXECUTOR_SCRATCH_ALLOCATOR_H_
#include <memory>
#include "tensorflow/stream_executor/device_memory.h"
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/temporary_device_memory.h"
namespace stream_executor {
class Stream;
// Interface that allows stream operations (e.g.
// Stream::ThenConvolveWithScratch) to optionally request scratch space be
// allocated in order to speed up the operation being enqueued.
//
// Note that the caller is responsible for deallocating the scratch space at a
// known-safe point, when all scratch-memory-consuming kernels are known for
// sure to have finished; e.g. at stream synchronization time. This is different
// from a traditional C++ object allocator, where the client is responsible for
// releasing. (Conceptually, scratch memory is a form of "temporary" device
// memory allocation.)
class ScratchAllocator {
public:
virtual ~ScratchAllocator();
// Returns a limit of memory this scratch allocator wants to produce, in
// bytes. This information may be used to help select an algorithm.
//
// Returns values < 0 to indicate that there is no recommended limit.
virtual int64 GetMemoryLimitInBytes(Stream* stream) = 0;
// Returns an allocation on byte_size bytes for use in an operation on stream.
//
// This is a temporary allocation, and the caller is responsible for
// deallocating at some known-safe point. See the class comment above.
virtual port::StatusOr<DeviceMemory<uint8>> AllocateBytes(
Stream* stream, int64 byte_size) = 0;
};
// Allocates a single temporary memory allocation -- this memory is deallocated
// at the next stream synchronization point after this object has gone out of
// scope. This satisfies the lifetime and deallocation properties given in the
// class comment above.
//
// Thread-compatible, but not thread-safe (use in scenarios where only one
// thread will request the scratch allocation).
class OneTimeScratchAllocator : public ScratchAllocator {
public:
OneTimeScratchAllocator();
~OneTimeScratchAllocator() override;
int64 GetMemoryLimitInBytes(Stream* stream) override;
port::StatusOr<DeviceMemory<uint8>> AllocateBytes(Stream* stream,
int64 byte_size) override;
private:
std::unique_ptr<TemporaryDeviceMemory<uint8>> temporary_;
SE_DISALLOW_COPY_AND_ASSIGN(OneTimeScratchAllocator);
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_SCRATCH_ALLOCATOR_H_
@@ -0,0 +1,34 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This file defines a uniform interface to configuration options for shared
// memory for supported devices. As with many StreamExecutor-supported features,
// support for the options defined herein is device-dependent.
#ifndef TENSORFLOW_STREAM_EXECUTOR_SHARED_MEMORY_CONFIG_H_
#define TENSORFLOW_STREAM_EXECUTOR_SHARED_MEMORY_CONFIG_H_
namespace stream_executor {
// SharedMemoryConfig enum describes potential widths of shared memory banks for
// a device or kernel.
enum class SharedMemoryConfig {
kDefault, // Use the device default configuration.
kFourByte, // Sets shared memory banks to be four bytes wide.
kEightByte, // Sets shared memory banks to be eight bytes wide.
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_SHARED_MEMORY_CONFIG_H_
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,49 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// The StreamExecutor is a single-device abstraction for:
//
// * Loading/launching data-parallel-kernels
// * Invoking pre-canned high-performance library routines (like matrix
// multiply)
#ifndef TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_H_
#define TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_H_
#include "tensorflow/stream_executor/device_description.h" // IWYU pragma: export
#include "tensorflow/stream_executor/device_memory.h" // IWYU pragma: export
#include "tensorflow/stream_executor/device_options.h" // IWYU pragma: export
#include "tensorflow/stream_executor/event.h" // IWYU pragma: export
#include "tensorflow/stream_executor/kernel.h" // IWYU pragma: export
#include "tensorflow/stream_executor/kernel_spec.h" // IWYU pragma: export
#include "tensorflow/stream_executor/launch_dim.h" // IWYU pragma: export
#include "tensorflow/stream_executor/multi_platform_manager.h" // IWYU pragma: export
#include "tensorflow/stream_executor/platform.h" // IWYU pragma: export
#include "tensorflow/stream_executor/stream.h" // IWYU pragma: export
#include "tensorflow/stream_executor/stream_executor_pimpl.h" // IWYU pragma: export
#include "tensorflow/stream_executor/timer.h" // IWYU pragma: export
namespace perftools {
namespace gputools {
// Temporarily pull stream_executor into perftools::gputools while we migrate
// code to the new namespace. TODO(b/77980417): Remove this once we've
// completed the migration.
using namespace stream_executor; // NOLINT[build/namespaces]
} // namespace gputools
} // namespace perftools
#endif // TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_H_
@@ -0,0 +1,388 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Interfaces for platform-dependent implementations to satisfy. This are
// delegated to from the StreamExecutor in pointer-to-implementation style; i.e.
// the StreamExecutor is just a husk that delegates calls to the
// platform-specific objects which implement the interfaces defined here.
#ifndef TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_INTERNAL_H_
#define TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_INTERNAL_H_
#include <functional>
#include <map>
#include <memory>
#include <utility>
#include <vector>
#include "tensorflow/stream_executor/device_description.h"
#include "tensorflow/stream_executor/device_memory.h"
#include "tensorflow/stream_executor/device_options.h"
#include "tensorflow/stream_executor/dnn.h"
#include "tensorflow/stream_executor/event.h"
#include "tensorflow/stream_executor/kernel.h"
#include "tensorflow/stream_executor/kernel_cache_config.h"
#include "tensorflow/stream_executor/kernel_spec.h"
#include "tensorflow/stream_executor/launch_dim.h"
#include "tensorflow/stream_executor/lib/inlined_vector.h"
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/module_spec.h"
#include "tensorflow/stream_executor/platform.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/plugin_registry.h"
#include "tensorflow/stream_executor/shared_memory_config.h"
#include "tensorflow/stream_executor/trace_listener.h"
namespace stream_executor {
class Stream;
class Timer;
// An opaque handle to a loaded module.
//
// An instance of this is returned from StreamExecutor::GetModule.
class ModuleHandle {
public:
/*implicit*/ ModuleHandle(void *id = nullptr) : id_(id) {}
// A ModuleHandle with id() == nullptr is an invalid module handle, akin to a
// null pointer.
void *id() const { return id_; }
explicit operator bool() const { return id() != nullptr; }
private:
void *id_;
};
namespace internal {
// Platform-dependent interface class for the generic Events interface, in
// the PIMPL style.
class EventInterface {
public:
EventInterface() {}
virtual ~EventInterface() {}
private:
SE_DISALLOW_COPY_AND_ASSIGN(EventInterface);
};
// Pointer-to-implementation object type (i.e. the KernelBase class delegates to
// this interface) with virtual destruction. This class exists for the
// platform-dependent code to hang any kernel data/resource info/functionality
// off of.
class KernelInterface {
public:
// Default constructor for the abstract interface.
KernelInterface() {}
// Default destructor for the abstract interface.
virtual ~KernelInterface() {}
// Returns the number of formal parameters that this kernel accepts.
virtual unsigned Arity() const = 0;
// Sets the preferred cache configuration.
virtual void SetPreferredCacheConfig(KernelCacheConfig config) = 0;
// Gets the preferred cache configuration.
virtual KernelCacheConfig GetPreferredCacheConfig() const = 0;
private:
SE_DISALLOW_COPY_AND_ASSIGN(KernelInterface);
};
// Pointer-to-implementation object type (i.e. the Stream class delegates to
// this interface) with virtual destruction. This class exists for the
// platform-dependent code to hang any kernel data/resource info/functionality
// off of.
class StreamInterface {
public:
// Default constructor for the abstract interface.
StreamInterface() {}
// Default destructor for the abstract interface.
virtual ~StreamInterface() {}
// Returns the GPU stream associated with this platform's stream
// implementation.
//
// WARNING: checks that the underlying platform is, in fact, CUDA or ROCm,
// causing a fatal error if it is not. This hack is made available solely for
// use from distbelief code, which temporarily has strong ties to CUDA or
// ROCm as a platform.
virtual void *GpuStreamHack() { return nullptr; }
// See the above comment on GpuStreamHack -- this further breaks abstraction
// for Eigen within distbelief, which has strong ties to CUDA or ROCm as a
// platform, and a historical attachment to a programming model which takes a
// stream-slot rather than a stream-value.
virtual void **GpuStreamMemberHack() { return nullptr; }
private:
SE_DISALLOW_COPY_AND_ASSIGN(StreamInterface);
};
// Pointer-to-implementation object type (i.e. the Timer class delegates to
// this interface) with virtual destruction. This class exists for the
// platform-dependent code to hang any timer data/resource info/functionality
// off of.
class TimerInterface {
public:
// Default constructor for the abstract interface.
TimerInterface() {}
// Default destructor for the abstract interface.
virtual ~TimerInterface() {}
// Returns the number of microseconds elapsed in a completed timer.
virtual uint64 Microseconds() const = 0;
// Returns the number of nanoseconds elapsed in a completed timer.
virtual uint64 Nanoseconds() const = 0;
private:
SE_DISALLOW_COPY_AND_ASSIGN(TimerInterface);
};
// Interface for the different StreamExecutor platforms (i.e. CUDA, OpenCL).
//
// Various platforms will provide an implementation that satisfy this interface.
class StreamExecutorInterface {
public:
// Default constructor for the abstract interface.
StreamExecutorInterface() {}
// Default destructor for the abstract interface.
virtual ~StreamExecutorInterface() {}
// Returns the (transitively) wrapped executor if this executor is
// wrapping another executor; otherwise, returns this.
virtual StreamExecutorInterface *GetUnderlyingExecutor() { return this; }
// See the StreamExecutor interface for comments on the same-named methods.
virtual port::Status Init(int device_ordinal,
DeviceOptions device_options) = 0;
virtual bool GetKernel(const MultiKernelLoaderSpec &spec,
KernelBase *kernel) {
return false;
}
virtual bool LoadModule(const MultiModuleLoaderSpec &spec,
ModuleHandle *module_handle) {
return false;
}
virtual bool UnloadModule(ModuleHandle module_handle) { return false; }
virtual bool Launch(Stream *stream, const ThreadDim &thread_dims,
const BlockDim &block_dims, const KernelBase &k,
const KernelArgsArrayBase &args) {
return false;
}
// Releases any state associated with the kernel.
virtual void UnloadKernel(const KernelBase *kernel) {}
virtual void *Allocate(uint64 size) = 0;
virtual void *AllocateSubBuffer(DeviceMemoryBase *parent, uint64 offset,
uint64 size) = 0;
virtual void Deallocate(DeviceMemoryBase *mem) = 0;
// Allocates unified memory space of the given size, if supported.
// See
// https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#um-unified-memory-programming-hd
// for more details on unified memory.
virtual void *UnifiedMemoryAllocate(uint64 size) { return nullptr; }
// Deallocates unified memory space previously allocated with
// UnifiedMemoryAllocate.
virtual void UnifiedMemoryDeallocate(void *mem) {}
virtual void *HostMemoryAllocate(uint64 size) = 0;
virtual void HostMemoryDeallocate(void *mem) = 0;
virtual bool HostMemoryRegister(void *mem, uint64 size) = 0;
virtual bool HostMemoryUnregister(void *mem) = 0;
virtual bool SynchronizeAllActivity() = 0;
virtual bool SynchronousMemZero(DeviceMemoryBase *location, uint64 size) = 0;
virtual bool SynchronousMemSet(DeviceMemoryBase *location, int value,
uint64 size) = 0;
virtual port::Status SynchronousMemcpy(DeviceMemoryBase *gpu_dst,
const void *host_src, uint64 size) = 0;
virtual port::Status SynchronousMemcpy(void *host_dst,
const DeviceMemoryBase &gpu_src,
uint64 size) = 0;
virtual port::Status SynchronousMemcpyDeviceToDevice(
DeviceMemoryBase *gpu_dst, const DeviceMemoryBase &gpu_src,
uint64 size) = 0;
virtual bool MemZero(Stream *stream, DeviceMemoryBase *location,
uint64 size) = 0;
virtual bool Memset(Stream *stream, DeviceMemoryBase *location, uint8 pattern,
uint64 size) {
return false;
}
virtual bool Memset32(Stream *stream, DeviceMemoryBase *location,
uint32 pattern, uint64 size) = 0;
virtual bool Memcpy(Stream *stream, void *host_dst,
const DeviceMemoryBase &gpu_src, uint64 size) = 0;
virtual bool Memcpy(Stream *stream, DeviceMemoryBase *gpu_dst,
const void *host_src, uint64 size) = 0;
virtual bool MemcpyDeviceToDevice(Stream *stream, DeviceMemoryBase *gpu_dst,
const DeviceMemoryBase &gpu_src,
uint64 size) = 0;
virtual bool HostCallback(Stream *stream, std::function<void()> callback) = 0;
virtual bool HostCallback(Stream *stream,
std::function<port::Status()> callback);
virtual port::Status AllocateEvent(Event *event) = 0;
virtual port::Status DeallocateEvent(Event *event) = 0;
virtual port::Status RecordEvent(Stream *stream, Event *event) = 0;
virtual port::Status WaitForEvent(Stream *stream, Event *event) = 0;
virtual Event::Status PollForEventStatus(Event *event) = 0;
virtual bool AllocateStream(Stream *stream) = 0;
virtual void DeallocateStream(Stream *stream) = 0;
virtual bool CreateStreamDependency(Stream *dependent, Stream *other) = 0;
virtual bool AllocateTimer(Timer *timer) = 0;
virtual void DeallocateTimer(Timer *timer) = 0;
virtual bool StartTimer(Stream *stream, Timer *timer) = 0;
virtual bool StopTimer(Stream *stream, Timer *timer) = 0;
virtual port::Status BlockHostUntilDone(Stream *stream) = 0;
virtual int PlatformDeviceCount() = 0;
virtual port::Status EnablePeerAccessTo(StreamExecutorInterface *other) = 0;
virtual bool CanEnablePeerAccessTo(StreamExecutorInterface *other) = 0;
virtual SharedMemoryConfig GetDeviceSharedMemoryConfig() = 0;
virtual port::Status SetDeviceSharedMemoryConfig(
SharedMemoryConfig config) = 0;
virtual int64 GetDeviceLoad() { return -1; }
virtual bool DeviceMemoryUsage(int64 *free, int64 *total) const {
return false;
}
// Retrieves device pointer and size for a symbol. The device pointer is
// stored at mem, and the size is stored at size. Either mem or bytes can be
// null, however, both of them cannot be null at the same time. To use
// constant memory in CUDA, GetSymbol has to be used. Returns true if symbol
// is found.
//
// If ModuleHandle is set then we search for `symbol_name` only within the
// module corresponding to `module_handle`. Otherwise all loaded modules are
// searched.
virtual bool GetSymbol(const string &symbol_name, ModuleHandle module_handle,
void **mem, size_t *bytes) {
return false;
}
// Creates a new DeviceDescription object. Ownership is transferred to the
// caller.
virtual DeviceDescription *PopulateDeviceDescription() const = 0;
// Attempts to register the provided TraceListener with the device-specific
// Executor implementation. When this is called, the PIMPL interface has
// already taken ownership of the object and is managing the generic tracing
// events. The device-specific implementation must determine if the passed
// listener is of a type appropriate for it to trace during registration (and
// before dispatching events to it).
// Returns true if the listener was successfully registered, false otherwise.
// Does not take ownership of listener.
virtual bool RegisterTraceListener(TraceListener* listener) { return false; }
// Unregisters the specified listener from the device-specific Executor.
// Returns true if the listener was successfully registered, false otherwise.
virtual bool UnregisterTraceListener(TraceListener* listener) {
return false;
}
// Returns whether this StreamExecutor has BLAS support for its underlying
// platform.
virtual bool SupportsBlas() const { return false; }
// Creates a new BlasSupport object, ownership is transferred to the caller.
// If SupportsBlas() is false, this will always return null.
//
// If SupportsBlas() is true, this may return null, for example, if the BLAS
// initialization fails.
virtual blas::BlasSupport *CreateBlas() { return nullptr; }
// Returns whether this StreamExecutor has FFT support for its underlying
// platform.
virtual bool SupportsFft() const { return false; }
// Creates a new fft::FftSupport object, ownership is transferred to the
// caller.
// If SupportsFft() is false, this will always return null.
//
// If SupportsFft() is true, this may return null, for example, if the FFT
// initialization fails.
virtual fft::FftSupport *CreateFft() { return nullptr; }
// Returns whether this StreamExecutor has Random Number Generation support
// for
// its underlying platform.
virtual bool SupportsRng() const { return false; }
// Returns whether this StreamExecutor has neural net support for its
// underlying
// platform.
virtual bool SupportsDnn() const { return false; }
// Creates a new RngSupport object, ownership is transferred to the caller.
// If SupportsRng() is false, this will always return null.
//
// If SupportsRng() is true, this may return null, for example, if the RNG
// initialization fails.
virtual rng::RngSupport *CreateRng() { return nullptr; }
// Creates a new DnnSupport object, ownership is transferred to the caller.
// If SupportsDnn() is false, this will always return null.
//
// If SupportsDnn() is true, this may return null, for example, if the DNN
// initialization fails.
virtual dnn::DnnSupport *CreateDnn() { return nullptr; }
// Each call creates a new instance of the platform-specific implementation of
// the corresponding interface type.
virtual std::unique_ptr<EventInterface> CreateEventImplementation() = 0;
virtual std::unique_ptr<KernelInterface> CreateKernelImplementation() = 0;
virtual std::unique_ptr<StreamInterface> GetStreamImplementation() = 0;
virtual std::unique_ptr<TimerInterface> GetTimerImplementation() = 0;
// Returns the CUDA or ROCm context associated with this StreamExecutor
// platform implementation.
//
// WARNING: checks that the underlying platform is, in fact, CUDA or ROCm,
// causing a fatal error if it is not. This hack is made available solely for
// use from distbelief code, which temporarily has strong ties to CUDA or ROCm
// as a platform.
virtual void *GpuContextHack() { return nullptr; }
private:
SE_DISALLOW_COPY_AND_ASSIGN(StreamExecutorInterface);
};
using StreamExecutorFactory =
std::function<StreamExecutorInterface *(const PluginConfig &)>;
using EventFactory = std::function<EventInterface *(StreamExecutor *)>;
using StreamFactory = std::function<StreamInterface *(StreamExecutor *)>;
using TimerFactory = std::function<TimerInterface *(StreamExecutor *)>;
using KernelFactory = std::function<KernelInterface*()>;
StreamExecutorFactory* MakeCUDAExecutorImplementation();
StreamExecutorFactory* MakeOpenCLExecutorImplementation();
extern StreamExecutorFactory MakeHostExecutorImplementation;
} // namespace internal
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_INTERNAL_H_
@@ -0,0 +1,881 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_PIMPL_H_
#define TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_PIMPL_H_
#include <atomic>
#include <memory>
#include <set>
#include <tuple>
#include <vector>
#include "absl/base/macros.h"
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/lib/strcat.h"
#include "tensorflow/stream_executor/lib/threadpool.h"
#include "tensorflow/stream_executor/platform.h"
#include "tensorflow/stream_executor/platform/logging.h"
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/platform/port.h"
#include "tensorflow/stream_executor/platform/thread_annotations.h"
#include "tensorflow/stream_executor/rng.h"
#include "tensorflow/stream_executor/shared_memory_config.h"
#include "tensorflow/stream_executor/stream.h"
#include "tensorflow/stream_executor/stream_executor_internal.h"
#include "tensorflow/stream_executor/trace_listener.h"
namespace stream_executor {
// Structure used for device memory leak checking.
struct AllocRecord {
// The requested allocation size of the buffer.
uint64 bytes;
// Holds a representation of the stack at the time the associated buffer was
// allocated. Produced in a form described in
// //util/symbolize/symbolized_stacktrace.h.
string stack_trace;
};
// Forward declaration of private friend class.
template <typename BeginCallT, typename CompleteCallT,
typename ReturnT, typename... BeginArgsT>
class ScopedTracer;
// A StreamExecutor manages a single device, in terms of executing work (kernel
// launches) and memory management (allocation/deallocation, memory copies to
// and from the device). It is conceptually the "handle" for a device -- Stream
// objects, which are used to enqueue work to run on the
// coprocessor have a StreamExecutor instance as their "parent" object.
//
// StreamExecutor objects have an underlying platform that is specified up
// front;
// e.g. either it is a CUDA or OpenCL executor.
//
// Thread-safe after initialization.
// StreamExecutor interface should not be invoked from a signal handler.
class StreamExecutor {
public:
explicit StreamExecutor(PlatformKind kind,
const PluginConfig &plugin_config = PluginConfig());
StreamExecutor(
const Platform *platform,
std::unique_ptr<internal::StreamExecutorInterface> implementation);
~StreamExecutor();
port::Status Init();
port::Status Init(int device_ordinal, DeviceOptions device_options);
// Returns the platform that this StreamExecutor is acting upon.
ABSL_DEPRECATED("Use platform() instead.")
PlatformKind platform_kind() const { return platform_kind_; }
// Returns a reference to the platform that created this executor.
const Platform *platform() const { return platform_; }
// Retrieves (loads) a kernel for the platform this StreamExecutor is acting
// upon, if one exists.
//
// Parameters:
// spec: The MultiKernelLoaderSpec is usually generated as a compile-time
// constant into an appropriate namespace. For example, see
// stream_executor::executor_sample::kKernelLoaderSpecs, from which a
// MultiKernelLoaderSpec is selected.
// kernel: Outparam that the kernel is loaded into. A given Kernel
// instantiation should not be loaded into more than once.
//
// If an error occurs, or there is no kernel available for the StreamExecutor
// platform, false is returned.
bool GetKernel(const MultiKernelLoaderSpec &spec, KernelBase *kernel);
// Releases any state associated with the previously loaded kernel.
void UnloadKernel(const KernelBase *kernel);
// Loads a module for the platform this StreamExecutor is acting upon.
//
// `spec` describes the module to be loaded. On success writes the handle for
// the loaded module to `module_handle` and returns true. Else returns false.
bool LoadModule(const MultiModuleLoaderSpec &spec,
ModuleHandle *module_handle);
// Unloads the module with handle `module_handle`.
bool UnloadModule(ModuleHandle module_handle);
// Synchronously allocates an array on the device of type T with element_count
// elements.
template <typename T>
DeviceMemory<T> AllocateArray(uint64 element_count);
// As AllocateArray(), but returns a ScopedDeviceMemory<T>.
template <typename T>
ScopedDeviceMemory<T> AllocateOwnedArray(uint64 element_count) {
return ScopedDeviceMemory<T>(this, AllocateArray<T>(element_count));
}
// Convenience wrapper that allocates space for a single element of type T in
// device memory.
template <typename T>
DeviceMemory<T> AllocateScalar() {
return AllocateArray<T>(1);
}
// As AllocateScalar(), but returns a ScopedDeviceMemory<T>.
template <typename T>
ScopedDeviceMemory<T> AllocateOwnedScalar() {
return AllocateOwnedArray<T>(1);
}
// Synchronously allocates a scalar of type T on the device that is (POD)
// zero-byte initialized.
template <typename T>
DeviceMemory<T> AllocateZeroed();
// As AllocateZeroed(), but returns a ScopedDeviceMemory<T>.
template <typename T>
ScopedDeviceMemory<T> AllocateOwnedZeroed() {
return ScopedDeviceMemory<T>(this, AllocateZeroed<T>());
}
// Allocate a memory region inside another allocated memory region.
// Offset and size are specified in terms of T elements.
// Warning: Do not free a parent buffer before its sub-buffers; this may cause
// use-after-free issues (the specific behavior is not consistent across
// platforms).
// - Note: OpenCL uses refcounting to manage buffer lifetimes, so use of a
// sub-buffer after parent deallocation is expected to be safe. This will
// render your code non-platform-portable, however.
template <typename T>
DeviceMemory<T> AllocateSubBuffer(DeviceMemory<T> *parent,
uint64 element_offset,
uint64 element_count);
// As AllocateSubBuffer(), but returns a ScopedDeviceMemory<T>.
template <typename T>
ScopedDeviceMemory<T> AllocateOwnedSubBuffer(DeviceMemory<T> *parent,
uint64 element_offset,
uint64 element_count) {
return ScopedDeviceMemory<T>(
this, AllocateSubBuffer<T>(parent, element_offset, element_count));
}
// Finds a symbol and returns device memory allocated to the symbol. The
// symbol is searched in any kernels that were previously loaded through
// GetKernel() before the GetSymbol() call. The user has to make sure that the
// type of symbol and T match.
// - Note: symbol_name should include its namespace as well. For example,
// pass "nms0::symbol" if referring to nms0::symbol.
//
// If `module_handle` is set then searches only within the module
// corresponding to `module_handle`.
template <typename T>
port::StatusOr<DeviceMemory<T>> GetSymbol(const string &symbol_name,
ModuleHandle module_handle = {});
// An untyped version of GetSymbol.
port::StatusOr<DeviceMemoryBase> GetUntypedSymbol(
const string &symbol_name, ModuleHandle module_handle = {});
// Deallocate the DeviceMemory previously allocated via this interface.
// Deallocation of a nullptr-representative value is permitted.
//
// Resets the internal contents of mem to be null-representative, but this
// null-out effect should not be relied upon in client code.
//
// TODO(jlebar): Change this to accept a DeviceMemoryBase by value, see
// discussion in cl/195744342.
void Deallocate(DeviceMemoryBase *mem);
// Retrieves a mapping of active opaque device memory pointer to a string
// representation of the [allocating thread's] stack at the time the pointer
// was allocated. Useful for tracking device memory leaks.
//
// Note: this will only be populated if --check_device_leaks flag is
// activated.
void GetMemAllocs(std::map<void *, AllocRecord> *records_out);
// Allocates unified memory space of the given size, if supported.
// See
// https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#um-unified-memory-programming-hd
// for more details on unified memory.
void *UnifiedMemoryAllocate(uint64 bytes);
// Deallocates unified memory space previously allocated with
// UnifiedMemoryAllocate.
void UnifiedMemoryDeallocate(void *location);
// Allocates a region of host memory and registers it with the platform API.
// Memory allocated in this manner (or allocated and registered with
// HostMemoryRegister() is required for use in asynchronous memcpy operations,
// such as Stream::ThenMemcpy.
void *HostMemoryAllocate(uint64 bytes);
// Deallocates a region of host memory allocated by HostMemoryAllocate().
void HostMemoryDeallocate(void *location);
// Registers a region of host memory with the platform API. Registered memory
// (or memory allocated with HostMemoryAllocate) is required for use with
// asynchronous memcpy operations, such as Stream::ThenMemcpy. This method
// is used to register memory allocated outside the StreamExecutor;
// HostMemoryAllocate implicitly registers its allocations and
// HostMemoryDeallocate implicitly deregisters on deallocation.
bool HostMemoryRegister(void *location, uint64 size) SE_MUST_USE_RESULT;
// Unregisters a region of host memory registered with HostMemoryRegister.
// This should be done before deallocating the region with delete[]/free/etc.
bool HostMemoryUnregister(void *location) SE_MUST_USE_RESULT;
// Synchronizes all activity occurring in the StreamExecutor's context (most
// likely a whole device).
bool SynchronizeAllActivity() SE_MUST_USE_RESULT;
// Blocks the caller while "size" bytes are zeroed out (in POD fashion) at the
// given location in device memory.
bool SynchronousMemZero(DeviceMemoryBase *location,
uint64 size) SE_MUST_USE_RESULT;
// Blocks the caller while "size" bytes are initialized to "value" (in POD
// fashion) at the given location in device memory.
bool SynchronousMemSet(DeviceMemoryBase *location, int value,
uint64 size) SE_MUST_USE_RESULT;
// [deprecated] Blocks the caller while a data segment of the given size is
// copied from the host source to the device destination.
ABSL_DEPRECATED(
"Prefer SynchronousMemcpyH2D, to avoid error-prone API usage.")
bool SynchronousMemcpy(DeviceMemoryBase *device_dst, const void *host_src,
uint64 size) SE_MUST_USE_RESULT;
// [deprecated] Blocks the caller while a data segment of the given size is
// copied from the device source to the host destination.
ABSL_DEPRECATED(
"Prefer SynchronousMemcpyD2H, to avoid error-prone API usage.")
bool SynchronousMemcpy(void *host_dst, const DeviceMemoryBase &device_src,
uint64 size) SE_MUST_USE_RESULT;
// Same as SynchronousMemcpy(DeviceMemoryBase*, ...) above.
port::Status SynchronousMemcpyH2D(const void *host_src, int64 size,
DeviceMemoryBase *device_dst);
// Alternative interface for memcpying from host to device that takes an
// array slice. Checks that the destination size can accommodate the host
// slice size.
template <class T>
port::Status SynchronousMemcpyH2D(port::ArraySlice<T> host_src,
DeviceMemoryBase *device_dst) {
auto host_size = host_src.size() * sizeof(T);
CHECK(device_dst->size() == 0 || device_dst->size() >= host_size);
return SynchronousMemcpyH2D(host_src.begin(), host_size, device_dst);
}
// Same as SynchronousMemcpy(void*, ...) above.
port::Status SynchronousMemcpyD2H(const DeviceMemoryBase &device_src,
int64 size, void *host_dst);
// Alternative interface for memcpying from device to host that takes an
// array slice. Checks that the destination size can accommodate the host
// slice size.
template <typename T>
port::Status SynchronousMemcpyD2H(const DeviceMemory<T> &device_src,
port::MutableArraySlice<T> host_dst) {
auto host_size = host_dst.size() * sizeof(T);
CHECK(device_src.size() == 0 || host_size >= device_src.size());
return SynchronousMemcpyD2H(device_src, host_size, host_dst.begin());
}
// Blocks the caller while a data segment of the given size is copied from the
// device source to the device destination.
bool SynchronousMemcpy(DeviceMemoryBase *device_dst,
const DeviceMemoryBase &device_src,
uint64 size) SE_MUST_USE_RESULT;
// Enqueues an operation onto stream to zero out size bytes at the given
// device memory location. Neither stream nor location may be null. Returns
// whether the operation was successfully enqueued onto the stream.
bool MemZero(Stream *stream, DeviceMemoryBase *location,
uint64 size) SE_MUST_USE_RESULT;
// Enqueues an operation onto stream to set 32-bit patterns starting at
// location, for byte count given by size. size must be 32-bit quantified
// (i.e. evently divisible by 4). Returns whether the operation was
// successfully enqueued onto the stream.
bool Memset32(Stream *stream, DeviceMemoryBase *location, uint32 pattern,
uint64 size) SE_MUST_USE_RESULT;
// Enables peer access from this StreamExecutor to memory
// allocated by other, such that launched device code, memcpies, etc may
// access it directly.
//
// Both this StreamExecutor and other must be backed by the same platform (as
// in
// CUDA vs OpenCL) implementation.
port::Status EnablePeerAccessTo(StreamExecutor *other);
// Returns whether it's possible to enable peer access from this
// StreamExecutor
// to memory allocated by another.
//
// Even when this returns true, EnablePeerAccessTo may fail for other reasons;
// this is more an up-front test as to whether it's expressly forbidden.
bool CanEnablePeerAccessTo(StreamExecutor *other);
// Gets the preferred shared memory configuration for the device to which this
// executor is bound.
SharedMemoryConfig GetDeviceSharedMemoryConfig();
// Sets the preferred shared memory configuration for the device to which this
// executor is bound.
port::Status SetDeviceSharedMemoryConfig(SharedMemoryConfig config);
// Obtains metadata about the underlying device.
// The value is cached on first use.
const DeviceDescription &GetDeviceDescription() const;
// If implemented, returns device specific measurement of load
// (e.g. pending requests).
int64 GetDeviceLoad() const;
// Returns the underlying device memory usage information, if it is available.
// If it is not available (false is returned), free/total may not be
// initialized.
//
// Note: "Free" reflects the amount of free memory on the underlying device,
// so allocations via other StreamExecutors that have the same underlying
// device
// will be reflected in "free".
bool DeviceMemoryUsage(int64 *free, int64 *total) const;
// The device count reported by this StreamExecutor's platform.
// Note: on OpenCL we implicitly select platform zero at the moment.
int PlatformDeviceCount() const;
// Returns whether the StreamExecutor supports BLAS routines for the platform
// that underlies this interface.
bool SupportsBlas() const;
// Returns whether the StreamExecutor supports FFT routines for the platform
// that underlies this interface.
bool SupportsFft() const;
// Returns whether the StreamExecutor supports RNG routines for the platform
// that underlies this interface.
bool SupportsRng() const;
// Returns whether the StreamExecutor support neural net routines for the
// platform that underlies this interface.
bool SupportsDnn() const;
// Returns the list of supported algorithms for the forward convolution
// operation.
bool GetConvolveAlgorithms(bool with_winograd_nonfused,
std::vector<dnn::AlgorithmDesc> *out_algorithms);
// Returns the list of supported algorithms for rnn operation.
bool GetRnnAlgorithms(std::vector<dnn::AlgorithmDesc> *out_algorithms);
// Get the list of supported algorithms for the backward convolution on data.
bool GetConvolveBackwardDataAlgorithms(
bool with_winograd_nonfused,
std::vector<dnn::AlgorithmDesc> *out_algorithms);
// Get the list of supported algorithms for the backward convolution on the
// filter.
bool GetConvolveBackwardFilterAlgorithms(
bool with_winograd_nonfused,
std::vector<dnn::AlgorithmDesc> *out_algorithms);
// Get the list of supported algorithms for BLAS gemm.
bool GetBlasGemmAlgorithms(std::vector<blas::AlgorithmType> *out_algorithms);
// Create an RNN descriptor based on model shapes and configurations.
// The caller retains the ownership of the descriptor.
port::StatusOr<std::unique_ptr<dnn::RnnDescriptor>> createRnnDescriptor(
int num_layers, int hidden_size, int input_size, int batch_size,
dnn::RnnInputMode input_mode, dnn::RnnDirectionMode direction_mode,
dnn::RnnMode rnn_mode, dnn::DataType data_type,
const dnn::AlgorithmConfig &algorithm_config, float dropout, uint64 seed,
ScratchAllocator *state_allocator);
// Create a RNN sequence descriptor that specifies either the input or output
// sequence. The caller retains the ownership of the returned descriptor.
port::StatusOr<std::unique_ptr<dnn::RnnSequenceTensorDescriptor>>
createRnnSequenceTensorDescriptor(int seq_length, int batch_size,
int data_size, dnn::DataType data_type);
// Create an RNN state descriptor that specifies the input or hidden state.
// The caller retains the ownership of the returned descriptor.
port::StatusOr<std::unique_ptr<dnn::RnnStateTensorDescriptor>>
createRnnStateTensorDescriptor(int num_layer, int batch_size, int data_size,
dnn::DataType data_type);
// Returns the device ordinal that this StreamExecutor was initialized with.
// Meaningless before initialization.
int device_ordinal() const { return device_ordinal_; }
// Returns a borrowed pointer to the underlying StreamExecutor implementation.
internal::StreamExecutorInterface *implementation();
// Warning: use Stream::ThenLaunch instead, this method is not for general
// consumption. However, this is the only way to launch a kernel for which
// the type signature is only known at runtime; say, if an application
// supports loading/launching kernels with arbitrary type signatures.
// In this case, the application is expected to know how to do parameter
// packing that obeys the contract of the underlying platform implementation.
//
// Launches a data parallel kernel with the given thread/block
// dimensionality and already-packed args/sizes to pass to the underlying
// platform driver.
//
// This is called by Stream::Launch() to delegate to the platform's launch
// implementation in StreamExecutorInterface::Launch().
bool Launch(Stream *stream, const ThreadDim &thread_dims,
const BlockDim &block_dims, const KernelBase &kernel,
const KernelArgsArrayBase &args);
// Gets-or-creates (creates with memoization) a FftSupport datatype that can
// be used to execute FFT routines on the current platform.
//
// Ownership and user-facing is the same as AsBlas() below.
//
// Returns null if there was an error initializing the FFT support for the
// underlying platform.
fft::FftSupport *AsFft();
// Gets-or-creates (creates with memoization) a DnnSupport datatype that can
// be used for neural network routines on the current platform.
//
// Ownership and user-facing is the same as AsBlas() below.
//
// Returns null if there was an error initializing the DNN support for the
// underlying platform.
dnn::DnnSupport *AsDnn();
// Turns StreamExecutor operation tracing on or off.
void EnableTracing(bool enable);
// Registers a trace listener to receive callbacks for only a single
// StreamExecutor instance.
// To register a listener for all executors for a given platform, see
// Platform::RegisterTraceListener().
// Does not take ownership of listener.
void RegisterTraceListener(TraceListener* listener);
// Removes a TraceListener from this StreamExecutor instance.
// Returns false (and logs) in cases where the argument listener was not
// previously registered.
bool UnregisterTraceListener(TraceListener* listener);
private:
template <typename BeginCallT, typename CompleteCallT,
typename ReturnT, typename... BeginArgsT>
friend class ScopedTracer;
friend class Event;
friend class Stream;
friend class Timer;
template <typename... Params>
friend class TypedKernel;
template <typename... Args>
friend struct ThenBlasImpl;
// Gets-or-creates (creates with memoization) a BlasSupport datatype that can
// be used to execute BLAS routines on the current platform. This is typically
// not user-facing, as users will use the Stream::ThenBlas* family of routines
// to entrain BLAS operations. See blas.h for additional details.
//
// Ownership is not transferred to the caller -- ownership is retained by this
// object for memoization. This BLAS interface is also only expected to be
// used by a Stream for entraining calls to BLAS functionality.
//
// Returns null if there was an error initializing the BLAS support for the
// underlying platform.
blas::BlasSupport *AsBlas();
// Gets-or-creates (creates with memoization) an RngSupport datatype that can
// be used for random-number-generation routines on the current platform.
//
// Ownership and user-facing is the same as AsBlas() above.
//
// Returns null if there was an error initializing the RNG support for the
// underlying platform.
rng::RngSupport *AsRng();
// Causes the host code to synchronously wait for operations entrained onto
// stream to complete. Effectively a join on the asynchronous device
// operations enqueued on the stream before this program point.
port::Status BlockHostUntilDone(Stream *stream);
// Synchronously allocates size bytes on the underlying platform and returns
// an opaque void* representing that allocation. In the case of failure,
// nullptr is returned.
void *Allocate(uint64 size);
// Finds and retrieves device memory for the symbol on the underlying
// platform.
bool GetSymbol(const string &symbol_name, ModuleHandle module_handle,
void **mem, size_t *bytes);
// Entrains a memcpy operation onto stream, with a host destination location
// host_dst and a device memory source, with target size size.
bool Memcpy(Stream *stream, void *host_dst,
const DeviceMemoryBase &device_src, uint64 size);
// Entrains a memcpy operation onto stream, with a device destination location
// and a host memory source, with target size size.
bool Memcpy(Stream *stream, DeviceMemoryBase *device_dst,
const void *host_src, uint64 size);
// Entrains a memcpy operation onto stream, with a device destination location
// and a device source location, with target size size. Peer access should
// have been enabled between the StreamExecutors owning the device memory
// regions.
bool MemcpyDeviceToDevice(Stream *stream, DeviceMemoryBase *device_dst,
const DeviceMemoryBase &device_src, uint64 size);
// Entrains on a stream a user-specified function to be run on the host.
// See Stream::ThenDoHostCallback for full details.
bool HostCallback(Stream *stream, std::function<void()> callback);
// Entrains on a stream a user-specified function to be run on the host.
// See Stream::ThenDoHostCallback for full details.
// This is the preferred form for a callback that may return an error.
bool HostCallback(Stream *stream, std::function<port::Status()> callback);
// Performs platform-specific allocation and initialization of an event.
port::Status AllocateEvent(Event *event);
// Performs platform-specific deallocation and cleanup of an event.
port::Status DeallocateEvent(Event *event);
// Inserts the specified event at the end of the specified stream.
port::Status RecordEvent(Stream *stream, Event *event);
// Wait for the specified event at the end of the specified stream.
port::Status WaitForEvent(Stream *stream, Event *event);
// Requests the current status of the event from the underlying platform.
Event::Status PollForEventStatus(Event *event);
// Allocates stream resources on the underlying platform for subject and
// initializes its internals.
bool AllocateStream(Stream *subject);
// Deallocates stream resources on the underlying platform.
void DeallocateStream(Stream *subject);
// Causes dependent to not begin execution until other has finished its
// last-enqueued work.
bool CreateStreamDependency(Stream *dependent, Stream *other);
// Allocates timer resources on the underlying platform for subject and
// initializes its internals.
bool AllocateTimer(Timer *subject);
// Deallocates timer resources on the underlying platform.
void DeallocateTimer(Timer *subject);
// Records a start event for an interval timer.
bool StartTimer(Stream *stream, Timer *timer);
// Records a stop event for an interval timer.
bool StopTimer(Stream *stream, Timer *timer);
// Allocates a new metadata object, appropriately populated, on the heap, with
// ownership transfer to caller.
DeviceDescription *PopulateDeviceDescription() const;
// Adds a task to the port::ThreadPool work queue. These tasks must be
// fire-and-forget and have no external data or timing dependencies; their
// execution order and completion time have no guarantees.
// For an example of an appropriate task, see HostBlas::DoBlasGemmInternal;
// there, temporary internal buffers are freed using this method.
void EnqueueOnBackgroundThread(std::function<void()> task);
// Adds an AllocRecord for 'opaque' of size 'bytes' to the record map, for
// leak checking. NULL buffer pointers and buffer sizes of 0 will not be
// tracked.
void CreateAllocRecord(void *opaque, uint64 size);
// Removes the AllocRecord keyed by 'opaque' from the record map. NULL
// pointers will not be erased (as they're not tracked, per above).
void EraseAllocRecord(void *opaque);
// Calls the relevant TraceListener routine to begin tracing for the specified
// asynchronous method.
template <typename TraceCallT, typename... ArgsT>
void SubmitTrace(TraceCallT trace_call, ArgsT&&... args);
// Reader/writer lock for class-static StreamExecutor members.
static mutex static_mu_;
// Reader/writer lock for mutable data structures on this StreamExecutor.
//
// Mutable so that caching functions (like DeviceDescription, AsBlas, etc.)
// can acquire the lock on their first (mutating) call as well.
mutable mutex mu_;
// Reference to the platform that created this executor.
const Platform *platform_;
// Pointer to the platform-specific-interface implementation. This is
// delegated to by the interface routines in pointer-to-implementation
// fashion.
std::unique_ptr<internal::StreamExecutorInterface> implementation_;
// A mapping of pointer (to device memory) to string representation of the
// stack (of the allocating thread) at the time at which the pointer was
// allocated.
std::map<void *, AllocRecord> mem_allocs_ GUARDED_BY(mu_);
// Memoized BLAS support object -- we only want to create this once when asked
// for a BLAS interface.
std::unique_ptr<blas::BlasSupport> blas_ GUARDED_BY(mu_);
// Memoized DNN support object -- we only want to create this once when asked
// for an DNN interface.
std::unique_ptr<dnn::DnnSupport> dnn_ GUARDED_BY(mu_);
// Memoized FFT support object -- we only want to create this once when asked
// for a FFT interface.
std::unique_ptr<fft::FftSupport> fft_;
// Memoized RNG support object -- we only want to create this once when asked
// for an RNG interface.
std::unique_ptr<rng::RngSupport> rng_ GUARDED_BY(mu_);
// Slot to cache the owned DeviceDescription for the underlying device
// once it has been quieried from DeviceDescription().
mutable std::unique_ptr<DeviceDescription> device_description_
GUARDED_BY(mu_);
// The kind of the underlying platform that is being targeted, as passed
// during construction.
//
// Immutable post-initialization.
PlatformKind platform_kind_;
// The device ordinal that this object was initialized with.
//
// Immutable post-initialization.
int device_ordinal_;
// Executor for handling host callback work that cannot be performed
// by a host callback thread - for example, cleanup after a host BLAS routine
// (which may make device API calls). This work cannot block the host
// callback thread, will be completed asynchronously, and should be treated
// as fire-and-forget. Assume no ordering guarantees WRT the tasks enqueued
// here.
//
// Immutable post-initialization. Object is thread-safe.
std::unique_ptr<port::ThreadPool> background_threads_;
// Counter for the current number of live streams. This is used to check
// for accidentally-outstanding streams at StreamExecutor teardown time, as
// well
// as to indicate leaks (via a large outstanding count being logged) in the
// case we can't allocate more streams.
std::atomic_int_fast32_t live_stream_count_;
// Only one worker thread is needed; little work will be done by the
// executor.
static const int kNumBackgroundThreads = 1;
// Indicates if StreamExecutor operation tracing should be performed.
bool tracing_enabled_;
// The set of TraceListeners registered for this StreamExecutor.
std::set<TraceListener*> listeners_ GUARDED_BY(mu_);
// Allocated memory in bytes.
int64 mem_alloc_bytes_;
// Memory limit in bytes. Value less or equal to 0 indicates there is no
// limit.
int64 memory_limit_bytes_;
SE_DISALLOW_COPY_AND_ASSIGN(StreamExecutor);
};
// A wrapper around ModuleHandle that uses RAII to manage its lifetime.
class ScopedModuleHandle {
public:
explicit ScopedModuleHandle(StreamExecutor *executor,
ModuleHandle module_handle)
: executor_(executor), module_handle_(module_handle) {}
ScopedModuleHandle(ScopedModuleHandle &&other) {
executor_ = other.executor_;
module_handle_ = other.module_handle_;
other.executor_ = nullptr;
other.module_handle_ = ModuleHandle();
}
ScopedModuleHandle &operator=(ScopedModuleHandle &&other) {
executor_ = other.executor_;
module_handle_ = other.module_handle_;
other.executor_ = nullptr;
other.module_handle_ = ModuleHandle();
return *this;
}
~ScopedModuleHandle() {
if (static_cast<bool>(module_handle_)) {
CHECK(executor_->UnloadModule(module_handle_));
}
}
private:
StreamExecutor *executor_;
ModuleHandle module_handle_;
TF_DISALLOW_COPY_AND_ASSIGN(ScopedModuleHandle);
};
////////////
// Inlines
template <typename T>
inline DeviceMemory<T> StreamExecutor::AllocateArray(uint64 element_count) {
uint64 bytes = sizeof(T) * element_count;
void *opaque = Allocate(bytes);
return DeviceMemory<T>::MakeFromByteSize(opaque, bytes);
}
template <typename T>
inline port::StatusOr<DeviceMemory<T>> StreamExecutor::GetSymbol(
const string &symbol_name, ModuleHandle module_handle) {
port::StatusOr<DeviceMemoryBase> untyped_symbol =
GetUntypedSymbol(symbol_name, module_handle);
if (!untyped_symbol.ok()) {
return untyped_symbol.status();
}
return DeviceMemory<T>(untyped_symbol.ValueOrDie());
}
template <typename ElemT>
ScopedDeviceMemory<ElemT>::ScopedDeviceMemory()
: wrapped_(DeviceMemoryBase()), parent_(nullptr) {}
template <typename ElemT>
ScopedDeviceMemory<ElemT>::ScopedDeviceMemory(StreamExecutor *parent,
DeviceMemoryBase value)
: wrapped_(value), parent_(parent) {}
template <typename ElemT>
ScopedDeviceMemory<ElemT>::ScopedDeviceMemory(
StreamExecutor *parent, std::initializer_list<ElemT> values)
: ScopedDeviceMemory(parent, parent->AllocateArray<ElemT>(values.size())) {
if (ptr() != nullptr) {
std::vector<ElemT> local(values);
if (!parent->SynchronousMemcpy(ptr(), const_cast<const ElemT *>(&local[0]),
ptr()->size())) {
Reset(nullptr);
}
}
}
template <typename ElemT>
ScopedDeviceMemory<ElemT>::~ScopedDeviceMemory() {
if (wrapped_ == nullptr) return;
DCHECK(parent_ != nullptr);
parent_->Deallocate(&wrapped_);
}
template <typename ElemT>
void ScopedDeviceMemory<ElemT>::Reset(DeviceMemory<ElemT> updated) {
if (wrapped_ != nullptr) {
DCHECK(parent_ != nullptr);
parent_->Deallocate(&wrapped_);
}
wrapped_ = updated;
}
template <typename ElemT>
void ScopedDeviceMemory<ElemT>::Reset(std::nullptr_t) {
if (wrapped_ != nullptr) {
DCHECK(parent_ != nullptr);
parent_->Deallocate(&wrapped_);
}
wrapped_ = DeviceMemory<ElemT>{};
}
template <typename T>
DeviceMemory<T> StreamExecutor::AllocateZeroed() {
void *opaque = Allocate(sizeof(T));
if (opaque == nullptr) {
return DeviceMemory<T>{};
}
DeviceMemory<T> result = DeviceMemory<T>::MakeFromByteSize(opaque, sizeof(T));
bool ok = SynchronousMemZero(&result, sizeof(T));
if (!ok) {
Deallocate(&result);
return DeviceMemory<T>{};
}
return result;
}
template <typename T>
DeviceMemory<T> StreamExecutor::AllocateSubBuffer(DeviceMemory<T> *parent,
uint64 element_offset,
uint64 element_count) {
if (element_offset + element_count > parent->ElementCount()) {
LOG(ERROR) << "requested sub-buffer allocation (offset + size) is greater "
<< "than parent allocation size: (" << element_offset << " + "
<< element_count << ") vs. (" << parent->ElementCount() << ")";
return DeviceMemory<T>{};
}
void *opaque = implementation_->AllocateSubBuffer(
parent, sizeof(T) * element_offset, sizeof(T) * element_count);
if (opaque == nullptr) {
return DeviceMemory<T>{};
}
CreateAllocRecord(opaque, sizeof(T) * element_count);
return DeviceMemory<T>(DeviceMemoryBase(opaque, sizeof(T) * element_count,
true /* = is_sub_buffer */));
}
template <typename... Params, typename... Args>
inline Stream &Stream::ThenLaunch(ThreadDim thread_dims, BlockDim block_dims,
const TypedKernel<Params...> &kernel,
Args... args) {
KernelInvocationChecker<std::tuple<Params...>,
std::tuple<Args...>>::CheckAllStaticAssert();
if (ok()) {
// This is the core that allows type-safe kernel launching.
// Since the platforms take kernel arguments as tuples of (void *, size),
// we pack the variadic parameters passed as ...args into the desired
// tuple form and pass that packed form to the StreamExecutor::Launch()
// implementation.
KernelArgsArray<sizeof...(args)> kernel_args;
kernel.PackParams(&kernel_args, args...);
DCHECK(parent_ != nullptr);
bool ok =
parent_->Launch(this, thread_dims, block_dims, kernel, kernel_args);
if (!ok) {
SetError();
LOG(WARNING) << "parent failed to launch kernel: " << &kernel;
}
}
return *this;
}
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_STREAM_EXECUTOR_PIMPL_H_
@@ -0,0 +1,136 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Temporary memories are used to allocate scratch space required by an
// operation about to be enqueued onto a stream.
//
// std::unique_ptr<TemporaryDeviceMemory<float>> temporary_memory =
// stream.AllocateTemporaryArray<float>(1024).ConsumeValueOrDie();
// // ... enqueue stuff onto the stream using the temporary memory ...
// // Note that the memory is accessible via
// // temporary_memory->device_memory() and similar.
//
// // Finalize the temporary memory. The underlying device memory may
// // be released any time after this program point, as another thread may
// // call Stream::BlockHostUntilDone, causing synchronization. This
// // finalization also happens automatically for the user if the unique_ptr
// // goes out of scope.
// temporary_memory.Finalize();
//
// WARNING: do NOT hold onto the device memory associated with temporary_memory
// after finalization. If temporary_memory->device_memory() is used after the
// temporary memory is finalized, it will cause a DCHECK failure.
//
// Note that standard usage takes advantage of the type-safe wrapper,
// TemporaryDeviceMemory<T>, defined below.
//
// Also see tests for executable sample usage.
#ifndef TENSORFLOW_STREAM_EXECUTOR_TEMPORARY_DEVICE_MEMORY_H_
#define TENSORFLOW_STREAM_EXECUTOR_TEMPORARY_DEVICE_MEMORY_H_
#include "tensorflow/stream_executor/device_memory.h"
namespace stream_executor {
class Stream;
namespace internal {
class TemporaryMemoryManager;
}
// Untyped base class (analogous to a void*) for temporary device memory
// allocations associated with a stream.
class TemporaryDeviceMemoryBase {
public:
// Marks the temporary memory as finalized if it is not already marked as
// such.
~TemporaryDeviceMemoryBase();
// Precondition: !IsFinalized()
DeviceMemoryBase* mutable_device_memory();
// Precondition: !IsFinalized()
const DeviceMemoryBase& device_memory() const;
// "Finalizes" this temporary memory, making it acceptable to release at the
// next stream synchronization point -- the device memory can be reclaimed at
// any time after the temporary memory is marked as finalized (e.g. if a
// separate thread is calls Stream::BlockHostUntilDone). This may only be
// called once -- see the precondition below.
//
// Precondition: !IsFinalized()
void Finalize();
// Returns true iff the temporary memory is finalized (that is, the user is
// done referring to the temporary device memory, and thus it can be released
// at the next stream synchronization point).
bool IsFinalized() const;
// Returns true iff the temporary memory is still allocated.
//
// Note: this is a polling call, no guarantee is made that the temporary
// memory is still allocated after the call has completed.
bool IsAllocated() const;
private:
friend class internal::TemporaryMemoryManager;
friend class TemporaryDeviceMemoryTest;
// Note: construction DCHECKs that the memory is known-allocated in the
// stream's temporary-allocation-manager.
TemporaryDeviceMemoryBase(Stream* parent, DeviceMemoryBase device_memory,
uint64 allocation_generation);
// The device memory region that has allocated.
DeviceMemoryBase device_memory_;
// The generation counter value for the temporary memory record in the
// temporary memory manager.
uint64 allocation_generation_;
// The stream that this temporary memory was allocated for.
Stream* parent_;
};
// Type-safe wrapper around the base type (which is analogous to a void*).
template <typename T>
class TemporaryDeviceMemory : public TemporaryDeviceMemoryBase {
public:
// Type-safe wrapper around TemporaryDeviceMemoryBase::mutable_device_memory.
DeviceMemory<T>* mutable_device_memory() {
StaticSlicingAssertionDummy();
return reinterpret_cast<DeviceMemory<T>*>(
TemporaryDeviceMemoryBase::mutable_device_memory());
}
// Type-safe wrapper around TemporaryDeviceMemoryBase::device_memory.
const DeviceMemory<T>& device_memory() const {
StaticSlicingAssertionDummy();
return reinterpret_cast<const DeviceMemory<T>&>(
TemporaryDeviceMemoryBase::device_memory());
}
private:
static void StaticSlicingAssertionDummy() {
static_assert(
sizeof(TemporaryDeviceMemory) == sizeof(TemporaryDeviceMemoryBase),
"derived class is simply a wrapper, no members may be added due to "
"slicing");
}
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_TEMPORARY_DEVICE_MEMORY_H_
@@ -0,0 +1,151 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// The temporary-memory-manager is a helper class for a Stream to keep track of
// temporary allocations. These allocations defer their deallocation to the next
// Stream::BlockHostUntilDone call for efficiency purposes (as deallocation
// itself generally forces synchronization to occur).
#ifndef TENSORFLOW_STREAM_EXECUTOR_TEMPORARY_MEMORY_MANAGER_H_
#define TENSORFLOW_STREAM_EXECUTOR_TEMPORARY_MEMORY_MANAGER_H_
#include <map>
#include <memory>
#include "tensorflow/stream_executor/device_memory.h"
#include "tensorflow/stream_executor/lib/status.h"
#include "tensorflow/stream_executor/lib/statusor.h"
#include "tensorflow/stream_executor/platform/mutex.h"
#include "tensorflow/stream_executor/platform/thread_annotations.h"
#include "tensorflow/stream_executor/temporary_device_memory.h"
namespace stream_executor {
namespace internal {
// Record used inside the TemporaryMemoryManager as metadata for a given device
// memory region.
struct TemporaryMemoryRecord {
// What "generation" this record was allocated in.
//
// Currently the generation counter is bumped for every allocation, but this
// could be made coarser if necessary.
uint64 allocation_generation;
// Notes whether the temporary memory has been marked as finalized, such that
// we can release the DeviceMemory associated with this record at
// synchronization time.
bool finalized;
};
// Manages temporary memories associated with a stream -- keeps records of
// outstanding temporaries and their state, and can deallocate them
// appropriately at points in the Stream lifecycle (e.g. BlockHostUntilDone,
// destruction).
class TemporaryMemoryManager {
public:
explicit TemporaryMemoryManager(Stream* stream) : stream_(stream) {}
// Allocates a temporary array that is then managed by this object.
template <typename T>
port::StatusOr<std::unique_ptr<TemporaryDeviceMemory<T>>> AllocateArray(
uint64 element_count);
// Forces deallocation of all managed temporary memory regions.
//
// Called, for example, when the Stream owning this temporary memory manager
// is destroyed.
//
// Note: These calls to Deallocate will likely force synchronization.
void ForceDeallocateAll();
// Marks the given memory region as finalized.
//
// If must_exist is set, this will check-fail if the temporary memory record
// is not found.
void MarkFinalized(const DeviceMemoryBase& device_memory, uint64 generation,
bool must_exist);
// Deallocates temporary memories that have been finalized.
//
// Note: These calls to Deallocate will likely force synchronization, so it is
// meant to be called before a "BlockHostUntilDone" is about to be performed.
void DeallocateFinalizedTemporaries();
// Returns whether the provided device_memory is finalized.
//
// In the vacuous case where the device memory doesn't appear in the temporary
// memory records, it is either not a temporary at all, or has already been
// deallocated, and thus returns true.
bool IsFinalized(const DeviceMemoryBase& device_memory,
uint64 allocation_generation) const;
// Returns whether the manager has a live allocation record for the given
// device memory pointer with the given generation counter.
//
// Note: this is a polling call -- there is no guarantee that the region is
// still allocated once the call has completed.
bool HasAllocated(const DeviceMemoryBase& device_memory,
uint64 generation) const;
private:
// Allocates an array without type parameterization, so that the
// implementation can live in the source file. Without this base allocation
// method, we incur a circular dependency between the StreamExecutor
// definition and this class' definition.
port::StatusOr<std::unique_ptr<TemporaryDeviceMemoryBase>> AllocateArrayBase(
uint64 element_count, uint64 element_size);
// Mutex to guard temporary record state.
mutable mutex mutex_;
// Mapping from device memory to the current (live) temporary memory record.
//
// If a device memory is not in this mapping, it is not a temporary currently
// allocated and owned by this temporary memory manager.
std::map<DeviceMemoryBase, TemporaryMemoryRecord> records_ GUARDED_BY(mutex_);
// Allocation generation -- we bump this counter to distinguish temporary
// memory handles that have been deallocated and later reallocated at the same
// device memory address.
uint64 generation_ GUARDED_BY(mutex_);
// The stream (parent object) for this temporary memory manager -- allocations
// are performed through this stream handle.
Stream* stream_;
SE_DISALLOW_COPY_AND_ASSIGN(TemporaryMemoryManager);
};
////////////
// Inlines
template <typename T>
port::StatusOr<std::unique_ptr<TemporaryDeviceMemory<T>>>
TemporaryMemoryManager::AllocateArray(uint64 element_count) {
port::StatusOr<std::unique_ptr<TemporaryDeviceMemoryBase>> temporary_memory =
AllocateArrayBase(element_count, sizeof(T));
if (!temporary_memory.ok()) {
return temporary_memory.status();
}
return std::unique_ptr<TemporaryDeviceMemory<T>>(
reinterpret_cast<TemporaryDeviceMemory<T>*>(
temporary_memory.ConsumeValueOrDie().release()));
}
} // namespace internal
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_TEMPORARY_MEMORY_MANAGER_H_
@@ -0,0 +1,73 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_STREAM_EXECUTOR_TIMER_H_
#define TENSORFLOW_STREAM_EXECUTOR_TIMER_H_
#include <memory>
#include "tensorflow/stream_executor/platform/port.h"
namespace stream_executor {
namespace internal {
class TimerInterface;
} // namespace internal
class StreamExecutor;
// An interval timer, suitable for use in timing the operations which occur in
// streams.
//
// Thread-hostile: CUDA associates a CUDA-context with a particular thread in
// the system. Any operation that a user attempts to perform by using a Timer
// on a thread not-associated with the CUDA-context has unknown behavior at the
// current time; see b/13176597
class Timer {
public:
// Instantiate a timer tied to parent as a platform executor.
explicit Timer(StreamExecutor *parent);
// Deallocates any timer resources that the parent StreamExecutor has bestowed
// upon this object.
~Timer();
// Returns the elapsed number of microseconds for a completed timer.
// Completed means has been through a start/stop lifecycle.
uint64 Microseconds() const;
// Returns the elapsed number of nanoseconds for a completed timer.
// Completed means has been through a start/stop lifecycle.
uint64 Nanoseconds() const;
// Returns the (opaque) backing platform ITimer instance. Ownership is
// not transferred to the caller.
internal::TimerInterface *implementation() { return implementation_.get(); }
private:
// The StreamExecutor that manages the platform-specific internals for this
// timer.
StreamExecutor *parent_;
// Platform-dependent implementation of the timer internals for the underlying
// platform. This class just delegates to this opaque instance.
std::unique_ptr<internal::TimerInterface> implementation_;
SE_DISALLOW_COPY_AND_ASSIGN(Timer);
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_TIMER_H_
@@ -0,0 +1,73 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This file defines the StreamExecutor trace listener, used for inserting
// non-device-specific instrumentation into the StreamExecutor.
#ifndef TENSORFLOW_STREAM_EXECUTOR_TRACE_LISTENER_H_
#define TENSORFLOW_STREAM_EXECUTOR_TRACE_LISTENER_H_
#include "tensorflow/stream_executor/device_memory.h"
#include "tensorflow/stream_executor/kernel.h"
#include "tensorflow/stream_executor/launch_dim.h"
#include "tensorflow/stream_executor/lib/status.h"
namespace stream_executor {
class Stream;
// Traces StreamExecutor PIMPL-level events.
// The few StreamExecutor interfaces that are synchronous have both Begin and
// Complete versions of their trace calls. Asynchronous operations only have
// Submit calls, as execution of the underlying operations is device-specific.
// As all tracing calls mirror StreamExecutor routines, documentation here is
// minimal.
//
// All calls have default implementations that perform no work; subclasses
// should override functionality of interest. Keep in mind that these routines
// are not called on a dedicated thread, so callbacks should execute quickly.
//
// Note: This API is constructed on an as-needed basis. Users should add
// support for further StreamExecutor operations as required. By enforced
// convention (see SCOPED_TRACE in stream_executor_pimpl.cc), synchronous
// tracepoints should be named NameBegin and NameComplete.
class TraceListener {
public:
virtual ~TraceListener() {}
virtual void LaunchSubmit(Stream* stream, const ThreadDim& thread_dims,
const BlockDim& block_dims,
const KernelBase& kernel,
const KernelArgsArrayBase& args) {}
virtual void SynchronousMemcpyH2DBegin(int64 correlation_id,
const void* host_src, int64 size,
DeviceMemoryBase* gpu_dst) {}
virtual void SynchronousMemcpyH2DComplete(int64 correlation_id,
const port::Status* result) {}
virtual void SynchronousMemcpyD2HBegin(int64 correlation_id,
const DeviceMemoryBase& gpu_src,
int64 size, void* host_dst) {}
virtual void SynchronousMemcpyD2HComplete(int64 correlation_id,
const port::Status* result) {}
virtual void BlockHostUntilDoneBegin(int64 correlation_id, Stream* stream) {}
virtual void BlockHostUntilDoneComplete(int64 correlation_id,
const port::Status* result) {}
};
} // namespace stream_executor
#endif // TENSORFLOW_STREAM_EXECUTOR_TRACE_LISTENER_H_